def cycle(iterable, n=None):
"""Cycle through given iterable specific (or infinite) number of times.
:param n: Number of cycles.
If None, result cycles through ``iterable`` indefinitely.
:return: Iterable that cycles through given one
.. note::
This is an extended version of ncycles() recipe
from the :module:`itertools` module documentation
that also has the functionality of standard :func:`itertools.cycle`.
"""
ensure_iterable(iterable)
if n is None:
return cycle_(iterable)
else:
if not isinstance(n, Integral):
raise TypeError("invalid number of cycles")
if n < 0:
raise ValueError("number of cycles cannot be negative")
return chain.from_iterable(repeat(tuple(iterable), n))
def batch(iterable, n, fillvalue=None):
"""Batches the elements of given iterable.
Resulting iterable will yield tuples containing at most ``n`` elements
(might be less if ``fillvalue`` isn't specified).
:param n: Number of items in every batch
:param fillvalue: Value to fill the last batch with. If None, last batch
might be shorter than ``n`` elements
:return: Iterable of batches
.. note::
This is an extended version of grouper() recipe
from the :module:`itertools` module documentation.
"""
ensure_iterable(iterable)
if not isinstance(n, Integral):
raise TypeError("invalid number of elements in a batch")
if not (n > 0):
raise ValueError("number of elements in a batch must be positive")
# since we must use ``izip_longest``
# (``izip`` fails if ``n`` is greater than length of ``iterable``),
# we will apply some 'trimming' to resulting tuples if necessary
if fillvalue is None:
fillvalue = object()
trimmer = lambda item: tuple(x for x in item if x is not fillvalue)
else:
trimmer = identity()
args = [iter(iterable)] * n
zipped = izip_longest(*args, fillvalue=fillvalue)
return imap(trimmer, zipped)
def omit(indices, from_, strict=False):
"""Returns a subsequence from given tuple, omitting specified indices.
:param indices: Iterable of indices to exclude
:param strict: Whether ``indices`` are required to exist in the tuple
:return: Tuple without elements of specified indices
:raise IndexError: If ``strict`` is True and one of ``indices``
is out of range.
.. versionadded:: 0.0.3
"""
from taipan.collections.sets import remove_subset
ensure_iterable(indices)
ensure_sequence(from_)
if strict:
remaining_indices = set(xrange(len(from_)))
try:
remove_subset(remaining_indices, indices)
except KeyError as e:
raise IndexError(int(str(e)))
else:
remaining_indices = set(xrange(len(from_))) - set(indices)
return from_.__class__(from_[index] for index in remaining_indices)
def concat(list_):
"""Concatenates a list of lists into a single resulting list."""
ensure_iterable(list_)
# we don't use ``itertools.chain.from_iterable``, because that would
# inadvertenly allow strings, treating them as lists of characters
# and potentially producing very difficult bugs
return sum(imap(ensure_sequence, list_), [])
def join(delimiter, iterable, **kwargs):
"""Returns a string which is a concatenation of strings in ``iterable``,
separated by given ``delimiter``.
:param delimiter: Delimiter to put between strings
:param iterable: Iterable to join
Optional keyword arguments control the exact joining strategy:
:param errors:
What to do with erroneous non-strings in the input.
Possible values include:
* ``'ignore'`` (or ``None``)
* ``'cast'`` (or ``False``) -- convert non-strings to strings
* ``'raise'`` (or ``True``) -- raise exception for any non-strings
* ``'replace'`` -- replace non-strings with alternative value
:param with_: Replacement used when ``errors == 'replace'``.
This can be a string, or a callable taking erroneous value
and returning a string replacement.
.. versionadded:: 0.0.3
Allow to specify error handling policy through ``errors`` parameter
"""
ensure_string(delimiter)
ensure_iterable(iterable)
ensure_keyword_args(kwargs, optional=('errors', 'with_'))
errors = kwargs.get('errors', True)
if errors in ('raise', True):
iterable = imap(ensure_string, iterable)
elif errors in ('ignore', None):
iterable = ifilter(is_string, iterable)
elif errors in ('cast', False):
iterable = imap(delimiter.__class__, iterable)
elif errors == 'replace':
if 'with_' not in kwargs:
raise ValueError("'replace' error policy requires specifying "
"replacement through with_=")
with_ = kwargs['with_']
if is_string(with_):
replacement = lambda x: with_
elif callable(with_):
replacement = with_
else:
raise TypeError("error replacement must be a string or function, "
"got %s" % type(with_).__name__)
iterable = (x if is_string(x) else ensure_string(replacement(x))
for x in iterable)
else:
raise TypeError("%r is not a valid error handling policy for join()" %
(errors, ))
return delimiter.join(iterable)
def join(delimiter, iterable, **kwargs):
"""Returns a string which is a concatenation of strings in ``iterable``,
separated by given ``delimiter``.
:param delimiter: Delimiter to put between strings
:param iterable: Iterable to join
Optional keyword arguments control the exact joining strategy:
:param errors:
What to do with erroneous non-strings in the input.
Possible values include:
* ``'ignore'`` (or ``None``)
* ``'cast'`` (or ``False``) -- convert non-strings to strings
* ``'raise'`` (or ``True``) -- raise exception for any non-strings
* ``'replace'`` -- replace non-strings with alternative value
:param with_: Replacement used when ``errors == 'replace'``.
This can be a string, or a callable taking erroneous value
and returning a string replacement.
.. versionadded:: 0.0.3
Allow to specify error handling policy through ``errors`` parameter
"""
ensure_string(delimiter)
ensure_iterable(iterable)
ensure_keyword_args(kwargs, optional=('errors', 'with_'))
errors = kwargs.get('errors', True)
if errors in ('raise', True):
iterable = imap(ensure_string, iterable)
elif errors in ('ignore', None):
iterable = ifilter(is_string, iterable)
elif errors in ('cast', False):
iterable = imap(delimiter.__class__, iterable)
elif errors == 'replace':
if 'with_' not in kwargs:
raise ValueError("'replace' error policy requires specifying "
"replacement through with_=")
with_ = kwargs['with_']
if is_string(with_):
replacement = lambda x: with_
elif callable(with_):
replacement = with_
else:
raise TypeError("error replacement must be a string or function, "
"got %s" % type(with_).__name__)
iterable = (x if is_string(x) else ensure_string(replacement(x))
for x in iterable)
else:
raise TypeError(
"%r is not a valid error handling policy for join()" % (errors,))
return delimiter.join(iterable)
def pad(iterable, with_=None):
"""Pad the end of given iterable with infinite sequence of given values.
:param with_: Object to pad the iterable with. None by default.
:return: New iterable yielding elements of given ``iterable``,
which are then followed by infinite number of ``with_`` values
"""
ensure_iterable(iterable)
return chain(iterable, repeat(with_))
def replace(needle, with_=None, in_=None):
"""Replace occurrences of string(s) with other string(s) in (a) string(s).
Unlike the built in :meth:`str.replace` method, this function provides
clean API that clearly distinguishes the "needle" (string to replace),
the replacement string, and the target string to perform replacement in
(the "haystack").
Additionally, a simultaneous replacement of several needles is possible.
Note that this is different from performing multiple separate replacements
one after another.
Examples::
replace('foo', with_='bar', in_=some_text)
replace('foo', with_='bar').in_(other_text)
replace('foo').with_('bar').in_(another_text)
replace(['foo', 'bar']).with_('baz').in_(perhaps_a_long_text)
replace({'foo': 'bar', 'baz': 'qud'}).in_(even_longer_text)
:param needle: String to replace, iterable thereof,
or a mapping from needles to corresponding replacements
:param with_: Replacement string, if ``needle`` was not a mapping
:param in_: Optional string to perform replacement in
:return: If all parameters were provided, result is the final string
after performing a specified replacement.
Otherwise, a :class:`Replacer` object is returned, allowing
e.g. to perform the same replacements in many haystacks.
"""
if needle is None:
raise TypeError("replacement needle cannot be None")
if not needle:
raise ValueError("replacement needle cannot be empty")
if is_string(needle):
replacer = Replacer((needle, ))
else:
ensure_iterable(needle)
if not is_mapping(needle):
if all(imap(is_pair, needle)):
needle = dict(needle)
elif not all(imap(is_string, needle)):
raise TypeError("invalid replacement needle")
replacer = Replacer(needle)
if with_ is not None:
ensure_string(with_)
replacer = replacer.with_(with_)
if in_ is not None:
ensure_string(in_)
return replacer.in_(in_)
return replacer
def replace(needle, with_=None, in_=None):
"""Replace occurrences of string(s) with other string(s) in (a) string(s).
Unlike the built in :meth:`str.replace` method, this function provides
clean API that clearly distinguishes the "needle" (string to replace),
the replacement string, and the target string to perform replacement in
(the "haystack").
Additionally, a simultaneous replacement of several needles is possible.
Note that this is different from performing multiple separate replacements
one after another.
Examples::
replace('foo', with_='bar', in_=some_text)
replace('foo', with_='bar').in_(other_text)
replace('foo').with_('bar').in_(another_text)
replace(['foo', 'bar']).with_('baz').in_(perhaps_a_long_text)
replace({'foo': 'bar', 'baz': 'qud'}).in_(even_longer_text)
:param needle: String to replace, iterable thereof,
or a mapping from needles to corresponding replacements
:param with_: Replacement string, if ``needle`` was not a mapping
:param in_: Optional string to perform replacement in
:return: If all parameters were provided, result is the final string
after performing a specified replacement.
Otherwise, a :class:`Replacer` object is returned, allowing
e.g. to perform the same replacements in many haystacks.
"""
if needle is None:
raise TypeError("replacement needle cannot be None")
if not needle:
raise ValueError("replacement needle cannot be empty")
if is_string(needle):
replacer = Replacer((needle,))
else:
ensure_iterable(needle)
if not is_mapping(needle):
if all(imap(is_pair, needle)):
needle = dict(needle)
elif not all(imap(is_string, needle)):
raise TypeError("invalid replacement needle")
replacer = Replacer(needle)
if with_ is not None:
ensure_string(with_)
replacer = replacer.with_(with_)
if in_ is not None:
ensure_string(in_)
return replacer.in_(in_)
return replacer
def iterate(iterator, n=None):
"""Efficiently advances the iterator N times; by default goes to its end.
The actual loop is done "in C" and hence it is faster than equivalent 'for'.
:param n: How much the iterator should be advanced.
If None, it will be advanced until the end.
"""
ensure_iterable(iterator)
if n is None:
deque(iterator, maxlen=0)
else:
next(islice(iterator, n, n), None)
def remove_subset(set_, subset):
"""Remove a subset from given set.
This is essentially an extension of :func:`set.remove`
to work with more than one set element.
:raise KeyError: If some element from ``subset`` is not present in ``set_``
.. versionadded:: 0.0.2
"""
ensure_set(set_)
ensure_iterable(subset)
for elem in subset:
set_.remove(elem)
def __init__(self, replacements):
"""Constructor.
:param replacements: Iterable of needles, or mapping of replacements.
If this is not a mapping, a :meth:`with_` method
must be called to provide target replacement(s)
before :meth:`in_` is called
"""
self._replacements = ensure_iterable(replacements)
def select(keys, from_, strict=False):
"""Selects a subset of given dictionary, including only the specified keys.
:param keys: Iterable of keys to include
:param strict: Whether ``keys`` are required to exist in the dictionary.
:return: Dictionary whose keys are a subset of given ``keys``
:raise KeyError: If ``strict`` is True and one of ``keys`` is not found
in the dictionary.
"""
ensure_iterable(keys)
ensure_mapping(from_)
if strict:
return from_.__class__((k, from_[k]) for k in keys)
else:
existing_keys = set(keys) & set(iterkeys(from_))
return from_.__class__((k, from_[k]) for k in existing_keys)
def __init__(self, replacements):
"""Constructor.
:param replacements: Iterable of needles, or mapping of replacements.
If this is not a mapping, a :meth:`with_` method
must be called to provide target replacement(s)
before :meth:`in_` is called
"""
self._replacements = ensure_iterable(replacements)
def unique(iterable, key=None):
"""Removes duplicates from given iterable, using given key as criterion.
:param key: Key function which returns a hashable,
uniquely identifying an object.
:return: Iterable with duplicates removed
"""
ensure_iterable(iterable)
key = hash if key is None else ensure_callable(key)
def generator():
seen = set()
for elem in iterable:
k = key(elem)
if k not in seen:
seen.add(k)
yield elem
return generator()
def get(dict_, keys=(), default=None):
"""Extensions of standard :meth:`dict.get`.
Retrieves an item from given dictionary, trying given keys in order.
:param dict_: Dictionary to perform the lookup(s) in
:param keys: Iterable of keys
:param default: Default value to return if no key is found
:return: Value for one of given ``keys``, or ``default``
"""
ensure_mapping(dict_)
ensure_iterable(keys)
for key in keys:
try:
return dict_[key]
except KeyError:
pass
return default
def select(indices, from_, strict=False):
"""Selects a subsequence of given tuple, including only specified indices.
:param indices: Iterable of indices to include
:param strict: Whether ``indices`` are required to exist in the tuple.
:return: Tuple with selected elements, in the order corresponding
to the order of ``indices``.
:raise IndexError: If ``strict`` is True and one of ``indices``
is out of range.
"""
ensure_iterable(indices)
ensure_sequence(from_)
if strict:
return from_.__class__(from_[index] for index in indices)
else:
len_ = len(from_)
return from_.__class__(from_[index] for index in indices
if 0 <= index < len_)
def ensure_keyword_args(kwargs, mandatory=(), optional=()):
"""Checks whether dictionary of keyword arguments satisfies conditions.
:param kwargs: Dictionary of keyword arguments, received via ``*kwargs``
:param mandatory: Iterable of mandatory argument names
:param optional: Iterable of optional argument names
:return: ``kwargs`` if the conditions are met:
all ``mandatory`` arguments are present, and besides that
no arguments outside of ``optional`` ones are.
:raise TypeError: When conditions are not met
"""
from taipan.strings import ensure_string
ensure_mapping(kwargs)
mandatory = list(map(ensure_string, ensure_iterable(mandatory)))
optional = list(map(ensure_string, ensure_iterable(optional)))
if not (mandatory or optional):
raise ValueError(
"mandatory and/or optional argument names must be provided")
names = set(kwargs)
for name in mandatory:
try:
names.remove(name)
except KeyError:
raise TypeError(
"no value for mandatory keyword argument '%s'" % name)
excess = names - set(optional)
if excess:
if len(excess) == 1:
raise TypeError("unexpected keyword argument '%s'" % excess.pop())
else:
raise TypeError(
"unexpected keyword arguments: %s" % (tuple(excess),))
return kwargs
def omit(keys, from_, strict=False):
"""Returns a subset of given dictionary, omitting specified keys.
:param keys: Iterable of keys to exclude
:param strict: Whether ``keys`` are required to exist in the dictionary
:return: Dictionary filtered by omitting ``keys``
:raise KeyError: If ``strict`` is True and one of ``keys`` is not found
in the dictionary
.. versionadded:: 0.0.2
"""
ensure_iterable(keys)
ensure_mapping(from_)
if strict:
remaining_keys = set(iterkeys(from_))
remove_subset(remaining_keys, keys) # raises KeyError if necessary
else:
remaining_keys = set(iterkeys(from_)) - set(keys)
return from_.__class__((k, from_[k]) for k in remaining_keys)
def topological_order(nodes, incoming):
"""Performs topological sort of a DAG-like structure
(directed acyclic graph).
:param nodes: Collection of nodes
:param incoming: Function taking node as an argument and returning iterable
of nodes with edges pointing _towards_ given one
:return: Iterable of nodes in the topological order
.. note::
``incoming`` function works in _reverse_ to the typical adjacency
relation in graphs: if ``A in incoming(B)``, it implies that ``A->B``
is among the graph edges (**not** ``B->A``!).
This reversal is useful in practice when dealing with graphs
representing dependencies, module imports, header includes, and so on.
Example::
for package in topological_order(packages, attr_func('dependencies')):
install(package)
.. versionadded:: 0.0.4
"""
ensure_iterable(nodes) ; ensure_countable(nodes)
ensure_callable(incoming)
# data structure for tracking node's visit state
NOT_VISITED, VISITING, VISITED = range(3)
visit_states = {}
visit_state = lambda node: visit_states.get(id(node), NOT_VISITED)
def visit(node):
"""Topological sort visitor function."""
if visit_state(node) == VISITING:
raise ValueError("cycle found on node %r" % (node,))
if visit_state(node) == NOT_VISITED:
visit_states[id(node)] = VISITING
for neighbor in incoming(node):
for n in visit(neighbor):
yield n
visit_states[id(node)] = VISITED
yield node
def generator():
"""Main generator function that loops through the nodes
until we've visited them all.
"""
visited_count = 0
while visited_count < len(nodes):
visited_count = 0
for node in nodes:
if visit_state(node) == VISITED:
visited_count += 1
else:
for n in visit(node):
yield n
return generator()
def try_(block, except_=None, else_=None, finally_=None):
"""Emulate a ``try`` block.
:param block: Function to be executed within the ``try`` statement
:param except_: Function to execute when an :class:`Exception` occurs.
It receives a single argument: the exception object.
Alternatively, a list of key-value pairs can be passed,
mapping exception types to their handler functions.
:param else_: Function to execute when ``block`` completes successfully.
Note that it requires ``except_`` to be provided as well
:param finally_: Function to execute at the end,
regardless of whether an exception occurred or not
:return: If no exception was raised while executing ``block``,
the result of ``else_`` (if provided) or ``block`` is returned.
Otherwise, the result of ``except_`` is returned.
Note that :func:`try_` can _still_ raise an exception if it occurs
while evaluating ``except_``, ``else_`` or ``finally_`` functions.
"""
ensure_callable(block)
if not (except_ or else_ or finally_):
raise TypeError("at least one of `except_`, `else_` or `finally_` "
"functions must be provided")
if else_ and not except_:
raise TypeError("`else_` can only be provided along with `except_`")
if except_:
if callable(except_):
except_ = [(Exception, except_)]
else:
ensure_iterable(except_)
if is_mapping(except_):
ensure_ordered_mapping(except_)
except_ = except_.items()
def handle_exception():
"""Dispatch current exception to proper handler in ``except_``."""
exc_type, exc_object = sys.exc_info()[:2]
for t, handler in except_:
if issubclass(exc_type, t):
return handler(exc_object)
raise
if else_:
ensure_callable(else_)
if finally_:
ensure_callable(finally_)
try:
block()
except:
return handle_exception()
else:
return else_()
finally:
finally_()
else:
try:
block()
except:
return handle_exception()
else:
return else_()
else:
if finally_:
ensure_callable(finally_)
try:
return block()
except:
return handle_exception()
finally:
finally_()
else:
try:
return block()
except:
return handle_exception()
elif finally_:
ensure_callable(finally_)
try:
return block()
finally:
finally_()
def try_(block, except_=None, else_=None, finally_=None):
"""Emulate a ``try`` block.
:param block: Function to be executed within the ``try`` statement
:param except_: Function to execute when an :class:`Exception` occurs.
It receives a single argument: the exception object.
Alternatively, a list of key-value pairs can be passed,
mapping exception types to their handler functions.
:param else_: Function to execute when ``block`` completes successfully.
Note that it requires ``except_`` to be provided as well
:param finally_: Function to execute at the end,
regardless of whether an exception occurred or not
:return: If no exception was raised while executing ``block``,
the result of ``else_`` (if provided) or ``block`` is returned.
Otherwise, the result of ``except_`` is returned.
Note that :func:`try_` can _still_ raise an exception if it occurs
while evaluating ``except_``, ``else_`` or ``finally_`` functions.
"""
ensure_callable(block)
if not (except_ or else_ or finally_):
raise TypeError("at least one of `except_`, `else_` or `finally_` "
"functions must be provided")
if else_ and not except_:
raise TypeError("`else_` can only be provided along with `except_`")
if except_:
if callable(except_):
except_ = [(Exception, except_)]
else:
ensure_iterable(except_)
if is_mapping(except_):
ensure_ordered_mapping(except_)
except_ = except_.items()
def handle_exception():
"""Dispatch current exception to proper handler in ``except_``."""
exc_type, exc_object = sys.exc_info()[:2]
for t, handler in except_:
if issubclass(exc_type, t):
return handler(exc_object)
raise
if else_:
ensure_callable(else_)
if finally_:
ensure_callable(finally_)
try:
block()
except:
return handle_exception()
else:
return else_()
finally:
finally_()
else:
try:
block()
except:
return handle_exception()
else:
return else_()
else:
if finally_:
ensure_callable(finally_)
try:
return block()
except:
return handle_exception()
finally:
finally_()
else:
try:
return block()
except:
return handle_exception()
elif finally_:
ensure_callable(finally_)
try:
return block()
finally:
finally_()