def get_assignment(result, skip=False, **kwargs):
# print(result)
result = iter(listize(result))
if skip:
return None, result
try:
first_result = next(result)
except StopIteration:
first_result = None
try:
second_result = next(result)
except StopIteration:
# pipe delivers one result, e.g., strconcat
result = chain([first_result], result)
multiple = False
else:
# pipe delivers multiple results, e.g., fetchpage/tokenizer
result = chain([first_result], [second_result], result)
multiple = True
first = kwargs.get('count') == 'first'
_all = kwargs.get('count') == 'all'
one = first or not (multiple or _all)
return one, iter([first_result]) if one else result
def of_seq(xs: Iterable[Tuple[Key, Value]]) -> MapTree[Key, Value]:
if isinstance(xs, FrozenList):
xs = cast(FrozenList[Tuple[Key, Value]], xs)
return of_list(xs)
else:
ie = builtins.iter(xs)
return mk_from_iterator(empty, ie)
def test_nonEmptyEnum(self) -> None:
class PlasmaTv(metaclass=Enum):
R = ...
G = 1
B = 11
Magenta = ...
Cyan = G + B
Velvet = ...
SkyBlue = ...
Grass = G
enumConstsFromIterator = builtins.list(builtins.iter(PlasmaTv))
enumConstsFromManualEntry = \
[
PlasmaTv.R,
PlasmaTv.G,
PlasmaTv.B,
PlasmaTv.Magenta,
PlasmaTv.Cyan,
PlasmaTv.Velvet,
PlasmaTv.SkyBlue,
PlasmaTv.Grass
]
self.assertEqual(enumConstsFromIterator, enumConstsFromManualEntry)
def __init__(self, *args: ty.Optional[int]):
s = builtins.slice(*args)
start, stop, step = (
s.start or 0,
s.stop or builtins.min(s.stop, MAX_RANGE),
s.step or 1,
)
self._it = builtins.iter(builtins.range(start, stop, step))
def test_next(self, next):
"""Calls __next__()."""
iterator = builtins.iter(range(3))
assert next(iterator) == 0
assert next(iterator) == 1
assert next(iterator) == 2
with pytest.raises(StopIteration):
next(iterator)
def test_default(self, next):
"""If 'default' is given, then it is returned when StopIteration is raised."""
iterator = builtins.iter(range(3))
default = object()
assert next(iterator, default) == 0
assert next(iterator, default) == 1
assert next(iterator, default) == 2
assert next(iterator, default) == default
def test_iterable(self, iter):
"""Return tp.__iter__(obj) when available."""
iterator = builtins.iter([])
class Iterable:
def __iter__(self):
return iterator
assert iter(Iterable()) is iterator
def accumulate(iterable, func=lambda x, y: x + y):
it = builtins.iter(iterable)
try:
acc = next(it)
except StopIteration:
return
yield acc
for element in it:
acc = func(acc, element)
yield acc
def accumulate(iterable, func=lambda x, y: x + y):
it = builtins.iter(iterable)
try:
acc = next(it)
except StopIteration:
return
yield acc
for element in it:
acc = func(acc, element)
yield acc
def __init__(self, title, iterable, length=None):
self._log = treelog.current
self._iter = builtins.iter(iterable)
self._title = title
self._length = length or _len(iterable)
self._index = 0
text = '{} 0'.format(self._title)
if self._length:
text += ' (0%)'
self._log.pushcontext(text)
self.closed = False
def __init__(self, *args: ty.Optional[int], elements: int = 1):
s = builtins.slice(*args)
start, stop, step = (
s.start or 0,
s.stop or builtins.min(s.stop, MAX_RANGE),
s.step or 1,
)
self._it = builtins.tuple(
builtins.iter(builtins.range(start, stop, step))
for _ in builtins.range(elements)
)
def __init__(
self,
obj: ty.Union[AsyncCallable, ty.Awaitable, ty.Any],
times: ty.Optional[int] = None,
):
"""Constructor for :class:`none.collection.a.repeat`."""
self._provider = obj
self._repeat = itertools.repeat(True)
if times is not None:
self._repeat = builtins.iter(range(times))
def __init__(self, title, iterable, length=None):
self._log = treelog.current
self._iter = builtins.iter(iterable)
self._title = title
self._length = length or _len(iterable)
self._index = 0
text = '{} 0'.format(self._title)
if self._length:
text += ' (0%)'
self._log.pushcontext(text)
self.closed = False
def __init__(self, title, iterable, length=None):
warnings.deprecation('log.iter is deprecated; use log.iter.percentage instead')
self._log = treelog.current
self._iter = builtins.iter(iterable)
self._title = title
self._length = length or _len(iterable)
self._index = 0
text = '{} 0'.format(self._title)
if self._length:
text += ' (0%)'
self._log.pushcontext(text)
self.closed = False
def get_assignment(result, skip, **kwargs):
result = iter(utils.listize(result))
if skip:
return None, result
first_result = next(result)
try:
second_result = next(result)
except StopIteration:
# pipe delivers one result, e.g., strconcat
result = chain([first_result], result)
multiple = False
else:
# pipe delivers multiple results, e.g., fetchpage/tokenizer
result = chain([first_result], [second_result], result)
multiple = True
first = kwargs.get('count') == 'first'
one = first or not multiple
return one, iter([first_result]) if one else result
def iter(title, iterable, length=None):
'''Progress logger identical to built in iter'''
if length is None:
length = _len(iterable)
it = builtins.iter(iterable)
for index in itertools.count():
text = '{} {}'.format(title, index)
if length:
text += ' ({:.0f}%)'.format(100 * index / length)
with context(text, mayskip=index):
try:
yield next(it)
except StopIteration:
break
def iter( title, iterable, length=None ):
'''Progress logger identical to built in iter'''
if length is None:
length = _len(iterable)
it = builtins.iter(iterable)
for index in itertools.count():
text = '{} {}'.format( title, index )
if length:
text += ' ({:.0f}%)'.format( 100 * index / length )
with _current_log.context( text ):
try:
yield next(it)
except StopIteration:
break
def iter(*args):
if len(args) == 1:
return builtins.iter(args[0])
class _iter:
def __init__(self, args):
self.f, self.sentinel = args
def __next__(self):
v = self.f()
if v == self.sentinel:
raise StopIteration
return v
return _iter(args)
def iter(*args):
if len(args) == 1:
return builtins.iter(args[0])
class _iter:
def __init__(self, args):
self.f, self.sentinel = args
def __next__(self):
v = self.f()
if v == self.sentinel:
raise StopIteration
return v
return _iter(args)
def islice(p, start, stop=(), step=1):
if stop == ():
stop = start
start = 0
# TODO: optimizing or breaking semantics?
if start >= stop:
return
it = builtins.iter(p)
for i in range(start):
next(it)
while True:
yield next(it)
for i in range(step - 1):
next(it)
start += step
if start >= stop:
return
def islice(p, start, stop=(), step=1):
if stop == ():
stop = start
start = 0
# TODO: optimizing or breaking semantics?
if start >= stop:
return
it = builtins.iter(p)
for i in range(start):
next(it)
while True:
yield next(it)
for i in range(step - 1):
next(it)
start += step
if start >= stop:
return
def iter(items, sentinel=None):
''' A more flexible and tolerant iter().
Accepts a single item. Treats None as an empty iterable. '''
if items is not None:
try:
if sentinel:
items = takewhile(lambda x: not sentinel(x), items)
for item in builtins.iter(items):
yield item
except TypeError:
# does this make any sense?
# does it belong above inside "if sentinel:"?
# 'items' may a sentinel. If so, it is just one item.
if sentinel:
if not sentinel(items):
yield items
else:
yield items
def __init__(self, iterable: ty.AsyncIterator[T], *args: ty.Optional[int]):
"""Constructor for :class:`none.collection.a.islice`."""
s = slice(*args)
if s.start is not None and s.start < 0:
raise ValueError("start must be a positive integer.")
if s.stop is not None and s.stop < 0:
raise ValueError("stop must be a positive integer.")
if s.step is not None and s.step < 1:
raise ValueError("step must be a non-null positive integer.")
self._next = s.start or 0
self._stop = s.stop if s.stop is not None else sys.maxsize
self._step = s.step or 1
self._cursor = 0
if self._next > self._stop:
raise ValueError("start cannot be higher than stop.")
self._it = iter(iterable)
self._nextit = builtins.iter(range(self._next, self._stop, self._step))
def iter(items, sentinel=None):
''' A more flexible and tolerant iter().
Accepts a single item. Treats None as an empty iterable. '''
if items != None:
try:
if sentinel:
items = takewhile(lambda x: not sentinel(x), items)
if IS_PY2:
for item in __builtin__.iter(items):
yield item
else:
for item in builtins.iter(items):
yield item
except TypeError:
# items is just one item
if sentinel:
if not sentinel(x):
yield items
else:
yield items
def tee(iterable, n=2):
return [builtins.iter(iterable)] * n
def wrapper(items=None, **kwargs):
module_name = wrapper.__module__.split('.')[-1]
wrapper.__dict__['name'] = module_name
defaults = {
'dictize': True, 'ftype': 'pass', 'ptype': 'pass',
'objectify': True, 'emit': True, 'assign': module_name}
combined = cdicts(self.defaults, defaults, self.opts, kwargs)
extracted = 'extract' in combined
pdictize = combined.get('listize') if extracted else True
combined.setdefault('pdictize', pdictize)
conf = {k: combined[k] for k in self.defaults}
conf.update(kwargs.get('conf', {}))
combined.update({'conf': conf})
# replace conf with dictized version so we can access its
# attributes even if we already extracted a value
updates = {'conf': DotDict(conf), 'assign': combined.get('assign')}
kwargs.update(updates)
items = items or iter([])
_INPUT = map(DotDict, items) if combined.get('dictize') else items
bfuncs = get_broadcast_funcs(**combined)
types = {combined['ftype'], combined['ptype']}
if types.difference({'pass', 'none'}):
dfuncs = get_dispatch_funcs(**combined)
else:
dfuncs = None
pairs = (dispatch(item, bfuncs, dfuncs=dfuncs) for item in _INPUT)
parsed, _ = dispatch(DotDict(), bfuncs, dfuncs=dfuncs)
# - operators can't skip items
# - purposely setting both variables to maps of the same iterable
# since only one is intended to be used at any given time
# - `tuples` is an iterator of tuples of the first two `parsed`
# elements
tuples = ((p[0][0], p[0][1]) for p in pairs)
orig_stream = (p[0][0] for p in pairs)
objconf = parsed[1]
if self.async:
stream = yield pipe(orig_stream, objconf, tuples, **kwargs)
else:
stream = pipe(orig_stream, objconf, tuples, **kwargs)
sub_type = 'aggregator' if hasattr(stream, 'keys') else 'composer'
wrapper.__dict__['sub_type'] = sub_type
# operators can only assign one value per item and can't skip items
_, assignment = get_assignment(stream, False, **combined)
if combined.get('emit'):
stream = assignment
else:
singles = (iter([v]) for v in assignment)
key = combined.get('assign')
assigned = (assign({}, s, key, one=True) for s in singles)
stream = utils.multiplex(assigned)
if self.async:
return_value(stream)
else:
for s in stream:
yield s
@Infix
def map(f, x):
"""
(map) :: Functor f => (a -> b) -> (f a -> f b)
This is infix and non-curried version of `fmap`.
Maps function over functor.
"""
return Functor[x].fmap(f, x)
@constraint(Functor(f))
def fmap(fn: a >> b, x: f(a)) -> f(b):
"""
fmap :: Functor f => (a -> b) -> (f a -> f b)
Maps function over functor.
"""
return Functor[x].fmap(fn, x)
@constraint(Functor(f))
def void(x: f(a)) -> f(Unit):
return fmap(const(Star), x)
instance(
Functor,
List).where(fmap=lambda fn, lst: L[builtins.map(fn, builtins.iter(lst))])
instance(Functor, TypedFunc).where(fmap=TypedFunc.__mul__)
def of_seq(xs: Iterable[Tuple[Key, Value]]) -> MapTree[Key, Value]:
ie = builtins.iter(xs)
return mk_from_iterator(empty, ie)
int._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.int)(*args, **kwargs), builtins.int)
isinstance = functools.update_wrapper(
lambda *args, **kwargs: builtins.isinstance(*args, **kwargs),
builtins.isinstance)
isinstance._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.isinstance)(*args, **kwargs),
builtins.isinstance)
issubclass = functools.update_wrapper(
lambda *args, **kwargs: builtins.issubclass(*args, **kwargs),
builtins.issubclass)
issubclass._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.issubclass)(*args, **kwargs),
builtins.issubclass)
iter = functools.update_wrapper(
lambda *args, **kwargs: builtins.iter(*args, **kwargs), builtins.iter)
iter._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.iter)(*args, **kwargs),
builtins.iter)
len = functools.update_wrapper(
lambda *args, **kwargs: builtins.len(*args, **kwargs), builtins.len)
len._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.len)(*args, **kwargs), builtins.len)
list = functools.update_wrapper(
lambda *args, **kwargs: builtins.list(*args, **kwargs), builtins.list)
list._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.list)(*args, **kwargs),
builtins.list)
locals = functools.update_wrapper(
lambda *args, **kwargs: builtins.locals(*args, **kwargs), builtins.locals)
locals._ = functools.update_wrapper(
def iter(node, tpl):
if not node.arguments:
raise MissingRequiredArguments(
"Missing required argument variable at file %s, line %d" %
(tpl.filename, node.line))
if 'ival' not in tpl.node_variables:
tpl.node_variables['ival'] = []
if 'iindex' not in tpl.node_variables:
tpl.node_variables['iindex'] = []
if 'iseq' not in tpl.node_variables:
tpl.node_variables['iseq'] = []
if 'ikey' not in tpl.node_variables:
tpl.node_variables['ikey'] = []
tpl.node_variables['ival'].append(None)
tpl.node_variables['iindex'].append(None)
tpl.node_variables['iseq'].append(None)
tpl.node_variables['ikey'].append(None)
value = ''
if node.arguments.startswith('ival'):
if node.arguments == 'ival':
value = tpl.node_variables['ival'][-2]
else:
# take from ival.foo.bar the foo.bar part
args = node.arguments.split('.', 1)[1]
if len(tpl.node_variables['ival']) >= 1:
value = tpl.variable_value('', args,
tpl.node_variables['ival'][-2])
elif node.arguments == 'ikey':
# iterable objects cannot be used as key in python
raise IteratingIkey("Cannot iterate over ikey at file %s, line %d" %
(tpl.filename, node.line))
else:
value = tpl.variable_value(node.arguments)
if not value:
return ''
tpl.node_variables['iseq'][-1] = value
# check if variable is iterable
try:
import builtins # we need this because this function is also called 'iter'
it = builtins.iter(value)
except TypeError:
raise VariableNotIterable(
'Variable "%s" not iterable in template "%s", at line %d' %
(node.arguments, tpl.filename, node.line))
evals = []
for index, item in enumerate(value):
tpl.node_variables['iindex'][-1] = index
tpl.node_variables['ikey'][-1] = item
if not isinstance(value, dict):
tpl.node_variables['ival'][-1] = item
else:
tpl.node_variables['ival'][-1] = value[item]
evals.append(''.join([x.evaluate(tpl) for x in node.nodes]))
tpl.node_variables['ival'].pop()
tpl.node_variables['iindex'].pop()
tpl.node_variables['iseq'].pop()
tpl.node_variables['ikey'].pop()
return ''.join(evals)
def __iter__(self) -> Iterator[TSource]:
"""Return iterator for sequence."""
return builtins.iter(self._value)
def wrapper(items=None, **kwargs):
module_name = wrapper.__module__.split('.')[-1]
wrapper.__dict__['name'] = module_name
defaults = {
'dictize': True,
'ftype': 'pass',
'ptype': 'pass',
'objectify': True,
'emit': True,
'assign': module_name
}
combined = cdicts(self.defaults, defaults, self.opts, kwargs)
extracted = 'extract' in combined
pdictize = combined.get('listize') if extracted else True
combined.setdefault('pdictize', pdictize)
conf = {k: combined[k] for k in self.defaults}
conf.update(kwargs.get('conf', {}))
combined.update({'conf': conf})
# replace conf with dictized version so we can access its
# attributes even if we already extracted a value
updates = {'conf': DotDict(conf), 'assign': combined.get('assign')}
kwargs.update(updates)
items = items or iter([])
_INPUT = map(DotDict, items) if combined.get('dictize') else items
bfuncs = get_broadcast_funcs(**combined)
types = {combined['ftype'], combined['ptype']}
if types.difference({'pass', 'none'}):
dfuncs = get_dispatch_funcs(**combined)
else:
dfuncs = None
pairs = (dispatch(item, bfuncs, dfuncs=dfuncs) for item in _INPUT)
parsed, _ = dispatch(DotDict(), bfuncs, dfuncs=dfuncs)
# - operators can't skip items
# - purposely setting both variables to maps of the same iterable
# since only one is intended to be used at any given time
# - `tuples` is an iterator of tuples of the first two `parsed`
# elements
tuples = ((p[0][0], p[0][1]) for p in pairs)
orig_stream = (p[0][0] for p in pairs)
objconf = parsed[1]
if self. async:
stream = yield pipe(orig_stream, objconf, tuples, **kwargs)
else:
stream = pipe(orig_stream, objconf, tuples, **kwargs)
sub_type = 'aggregator' if hasattr(stream, 'keys') else 'composer'
wrapper.__dict__['sub_type'] = sub_type
# operators can only assign one value per item and can't skip items
_, assignment = get_assignment(stream, False, **combined)
if combined.get('emit'):
stream = assignment
else:
singles = (iter([v]) for v in assignment)
key = combined.get('assign')
assigned = (assign({}, s, key, one=True) for s in singles)
stream = utils.multiplex(assigned)
if self. async:
return_value(stream)
else:
for s in stream:
yield s
def __iter__(self) -> Iterator[TResult]:
xs = self.gen()
return builtins.iter(xs)
return a_list[:i], a_list[i:]
def splitAt(n, a_list):
"""splitAt :: Int -> [a] -> ([a], [a])"""
return a_list[:n], a_list[n:]
def subtract(a, b):
"""subtract :: Num a => a -> a -> a"""
return a - b
def tail(a_list):
"""tail :: [a] -> [a]"""
try:
return a_list[1:]
except Exception: #assume an iterator (lazy) instead of a list
it = builtins.iter(a_list)
next(it)
return it
def take(i, a_list):
"""take :: Int -> [a] -> [a]"""
return a_list[:i]
def takewhile(fn, a_list):
"""takeWhile :: (a -> Bool) -> [a] -> [a]"""
for item in a_list:
if fn(item):
yield item
else:
break
def __iter__(self) -> Iterator[_TSource]:
xs = self.gen()
return builtins.iter(xs)
def __init__(self, titles, iterable): self._titles = builtins.iter(titles) self._iterable = builtins.iter(iterable) self._log = None self._warn = False
def __enter__(self):
if self._log is not None:
raise Exception('iter.wrap is not reentrant')
self._log = treelog.current
self._log.pushcontext(next(self._titles))
return builtins.iter(self)
def tee(iterable, n=2):
return [builtins.iter(iterable)] * n
