def cycle(iterable, n=None): """Cycle through given iterable specific (or infinite) number of times. :param n: Number of cycles. If None, result cycles through ``iterable`` indefinitely. :return: Iterable that cycles through given one .. note:: This is an extended version of ncycles() recipe from the :module:`itertools` module documentation that also has the functionality of standard :func:`itertools.cycle`. """ ensure_iterable(iterable) if n is None: return cycle_(iterable) else: if not isinstance(n, Integral): raise TypeError("invalid number of cycles") if n < 0: raise ValueError("number of cycles cannot be negative") return chain.from_iterable(repeat(tuple(iterable), n))
def batch(iterable, n, fillvalue=None): """Batches the elements of given iterable. Resulting iterable will yield tuples containing at most ``n`` elements (might be less if ``fillvalue`` isn't specified). :param n: Number of items in every batch :param fillvalue: Value to fill the last batch with. If None, last batch might be shorter than ``n`` elements :return: Iterable of batches .. note:: This is an extended version of grouper() recipe from the :module:`itertools` module documentation. """ ensure_iterable(iterable) if not isinstance(n, Integral): raise TypeError("invalid number of elements in a batch") if not (n > 0): raise ValueError("number of elements in a batch must be positive") # since we must use ``izip_longest`` # (``izip`` fails if ``n`` is greater than length of ``iterable``), # we will apply some 'trimming' to resulting tuples if necessary if fillvalue is None: fillvalue = object() trimmer = lambda item: tuple(x for x in item if x is not fillvalue) else: trimmer = identity() args = [iter(iterable)] * n zipped = izip_longest(*args, fillvalue=fillvalue) return imap(trimmer, zipped)
def omit(indices, from_, strict=False): """Returns a subsequence from given tuple, omitting specified indices. :param indices: Iterable of indices to exclude :param strict: Whether ``indices`` are required to exist in the tuple :return: Tuple without elements of specified indices :raise IndexError: If ``strict`` is True and one of ``indices`` is out of range. .. versionadded:: 0.0.3 """ from taipan.collections.sets import remove_subset ensure_iterable(indices) ensure_sequence(from_) if strict: remaining_indices = set(xrange(len(from_))) try: remove_subset(remaining_indices, indices) except KeyError as e: raise IndexError(int(str(e))) else: remaining_indices = set(xrange(len(from_))) - set(indices) return from_.__class__(from_[index] for index in remaining_indices)
def concat(list_): """Concatenates a list of lists into a single resulting list.""" ensure_iterable(list_) # we don't use ``itertools.chain.from_iterable``, because that would # inadvertenly allow strings, treating them as lists of characters # and potentially producing very difficult bugs return sum(imap(ensure_sequence, list_), [])
def join(delimiter, iterable, **kwargs): """Returns a string which is a concatenation of strings in ``iterable``, separated by given ``delimiter``. :param delimiter: Delimiter to put between strings :param iterable: Iterable to join Optional keyword arguments control the exact joining strategy: :param errors: What to do with erroneous non-strings in the input. Possible values include: * ``'ignore'`` (or ``None``) * ``'cast'`` (or ``False``) -- convert non-strings to strings * ``'raise'`` (or ``True``) -- raise exception for any non-strings * ``'replace'`` -- replace non-strings with alternative value :param with_: Replacement used when ``errors == 'replace'``. This can be a string, or a callable taking erroneous value and returning a string replacement. .. versionadded:: 0.0.3 Allow to specify error handling policy through ``errors`` parameter """ ensure_string(delimiter) ensure_iterable(iterable) ensure_keyword_args(kwargs, optional=('errors', 'with_')) errors = kwargs.get('errors', True) if errors in ('raise', True): iterable = imap(ensure_string, iterable) elif errors in ('ignore', None): iterable = ifilter(is_string, iterable) elif errors in ('cast', False): iterable = imap(delimiter.__class__, iterable) elif errors == 'replace': if 'with_' not in kwargs: raise ValueError("'replace' error policy requires specifying " "replacement through with_=") with_ = kwargs['with_'] if is_string(with_): replacement = lambda x: with_ elif callable(with_): replacement = with_ else: raise TypeError("error replacement must be a string or function, " "got %s" % type(with_).__name__) iterable = (x if is_string(x) else ensure_string(replacement(x)) for x in iterable) else: raise TypeError("%r is not a valid error handling policy for join()" % (errors, )) return delimiter.join(iterable)
def join(delimiter, iterable, **kwargs): """Returns a string which is a concatenation of strings in ``iterable``, separated by given ``delimiter``. :param delimiter: Delimiter to put between strings :param iterable: Iterable to join Optional keyword arguments control the exact joining strategy: :param errors: What to do with erroneous non-strings in the input. Possible values include: * ``'ignore'`` (or ``None``) * ``'cast'`` (or ``False``) -- convert non-strings to strings * ``'raise'`` (or ``True``) -- raise exception for any non-strings * ``'replace'`` -- replace non-strings with alternative value :param with_: Replacement used when ``errors == 'replace'``. This can be a string, or a callable taking erroneous value and returning a string replacement. .. versionadded:: 0.0.3 Allow to specify error handling policy through ``errors`` parameter """ ensure_string(delimiter) ensure_iterable(iterable) ensure_keyword_args(kwargs, optional=('errors', 'with_')) errors = kwargs.get('errors', True) if errors in ('raise', True): iterable = imap(ensure_string, iterable) elif errors in ('ignore', None): iterable = ifilter(is_string, iterable) elif errors in ('cast', False): iterable = imap(delimiter.__class__, iterable) elif errors == 'replace': if 'with_' not in kwargs: raise ValueError("'replace' error policy requires specifying " "replacement through with_=") with_ = kwargs['with_'] if is_string(with_): replacement = lambda x: with_ elif callable(with_): replacement = with_ else: raise TypeError("error replacement must be a string or function, " "got %s" % type(with_).__name__) iterable = (x if is_string(x) else ensure_string(replacement(x)) for x in iterable) else: raise TypeError( "%r is not a valid error handling policy for join()" % (errors,)) return delimiter.join(iterable)
def pad(iterable, with_=None): """Pad the end of given iterable with infinite sequence of given values. :param with_: Object to pad the iterable with. None by default. :return: New iterable yielding elements of given ``iterable``, which are then followed by infinite number of ``with_`` values """ ensure_iterable(iterable) return chain(iterable, repeat(with_))
def replace(needle, with_=None, in_=None): """Replace occurrences of string(s) with other string(s) in (a) string(s). Unlike the built in :meth:`str.replace` method, this function provides clean API that clearly distinguishes the "needle" (string to replace), the replacement string, and the target string to perform replacement in (the "haystack"). Additionally, a simultaneous replacement of several needles is possible. Note that this is different from performing multiple separate replacements one after another. Examples:: replace('foo', with_='bar', in_=some_text) replace('foo', with_='bar').in_(other_text) replace('foo').with_('bar').in_(another_text) replace(['foo', 'bar']).with_('baz').in_(perhaps_a_long_text) replace({'foo': 'bar', 'baz': 'qud'}).in_(even_longer_text) :param needle: String to replace, iterable thereof, or a mapping from needles to corresponding replacements :param with_: Replacement string, if ``needle`` was not a mapping :param in_: Optional string to perform replacement in :return: If all parameters were provided, result is the final string after performing a specified replacement. Otherwise, a :class:`Replacer` object is returned, allowing e.g. to perform the same replacements in many haystacks. """ if needle is None: raise TypeError("replacement needle cannot be None") if not needle: raise ValueError("replacement needle cannot be empty") if is_string(needle): replacer = Replacer((needle, )) else: ensure_iterable(needle) if not is_mapping(needle): if all(imap(is_pair, needle)): needle = dict(needle) elif not all(imap(is_string, needle)): raise TypeError("invalid replacement needle") replacer = Replacer(needle) if with_ is not None: ensure_string(with_) replacer = replacer.with_(with_) if in_ is not None: ensure_string(in_) return replacer.in_(in_) return replacer
def replace(needle, with_=None, in_=None): """Replace occurrences of string(s) with other string(s) in (a) string(s). Unlike the built in :meth:`str.replace` method, this function provides clean API that clearly distinguishes the "needle" (string to replace), the replacement string, and the target string to perform replacement in (the "haystack"). Additionally, a simultaneous replacement of several needles is possible. Note that this is different from performing multiple separate replacements one after another. Examples:: replace('foo', with_='bar', in_=some_text) replace('foo', with_='bar').in_(other_text) replace('foo').with_('bar').in_(another_text) replace(['foo', 'bar']).with_('baz').in_(perhaps_a_long_text) replace({'foo': 'bar', 'baz': 'qud'}).in_(even_longer_text) :param needle: String to replace, iterable thereof, or a mapping from needles to corresponding replacements :param with_: Replacement string, if ``needle`` was not a mapping :param in_: Optional string to perform replacement in :return: If all parameters were provided, result is the final string after performing a specified replacement. Otherwise, a :class:`Replacer` object is returned, allowing e.g. to perform the same replacements in many haystacks. """ if needle is None: raise TypeError("replacement needle cannot be None") if not needle: raise ValueError("replacement needle cannot be empty") if is_string(needle): replacer = Replacer((needle,)) else: ensure_iterable(needle) if not is_mapping(needle): if all(imap(is_pair, needle)): needle = dict(needle) elif not all(imap(is_string, needle)): raise TypeError("invalid replacement needle") replacer = Replacer(needle) if with_ is not None: ensure_string(with_) replacer = replacer.with_(with_) if in_ is not None: ensure_string(in_) return replacer.in_(in_) return replacer
def iterate(iterator, n=None): """Efficiently advances the iterator N times; by default goes to its end. The actual loop is done "in C" and hence it is faster than equivalent 'for'. :param n: How much the iterator should be advanced. If None, it will be advanced until the end. """ ensure_iterable(iterator) if n is None: deque(iterator, maxlen=0) else: next(islice(iterator, n, n), None)
def remove_subset(set_, subset): """Remove a subset from given set. This is essentially an extension of :func:`set.remove` to work with more than one set element. :raise KeyError: If some element from ``subset`` is not present in ``set_`` .. versionadded:: 0.0.2 """ ensure_set(set_) ensure_iterable(subset) for elem in subset: set_.remove(elem)
def __init__(self, replacements): """Constructor. :param replacements: Iterable of needles, or mapping of replacements. If this is not a mapping, a :meth:`with_` method must be called to provide target replacement(s) before :meth:`in_` is called """ self._replacements = ensure_iterable(replacements)
def select(keys, from_, strict=False): """Selects a subset of given dictionary, including only the specified keys. :param keys: Iterable of keys to include :param strict: Whether ``keys`` are required to exist in the dictionary. :return: Dictionary whose keys are a subset of given ``keys`` :raise KeyError: If ``strict`` is True and one of ``keys`` is not found in the dictionary. """ ensure_iterable(keys) ensure_mapping(from_) if strict: return from_.__class__((k, from_[k]) for k in keys) else: existing_keys = set(keys) & set(iterkeys(from_)) return from_.__class__((k, from_[k]) for k in existing_keys)
def __init__(self, replacements): """Constructor. :param replacements: Iterable of needles, or mapping of replacements. If this is not a mapping, a :meth:`with_` method must be called to provide target replacement(s) before :meth:`in_` is called """ self._replacements = ensure_iterable(replacements)
def unique(iterable, key=None): """Removes duplicates from given iterable, using given key as criterion. :param key: Key function which returns a hashable, uniquely identifying an object. :return: Iterable with duplicates removed """ ensure_iterable(iterable) key = hash if key is None else ensure_callable(key) def generator(): seen = set() for elem in iterable: k = key(elem) if k not in seen: seen.add(k) yield elem return generator()
def get(dict_, keys=(), default=None): """Extensions of standard :meth:`dict.get`. Retrieves an item from given dictionary, trying given keys in order. :param dict_: Dictionary to perform the lookup(s) in :param keys: Iterable of keys :param default: Default value to return if no key is found :return: Value for one of given ``keys``, or ``default`` """ ensure_mapping(dict_) ensure_iterable(keys) for key in keys: try: return dict_[key] except KeyError: pass return default
def select(indices, from_, strict=False): """Selects a subsequence of given tuple, including only specified indices. :param indices: Iterable of indices to include :param strict: Whether ``indices`` are required to exist in the tuple. :return: Tuple with selected elements, in the order corresponding to the order of ``indices``. :raise IndexError: If ``strict`` is True and one of ``indices`` is out of range. """ ensure_iterable(indices) ensure_sequence(from_) if strict: return from_.__class__(from_[index] for index in indices) else: len_ = len(from_) return from_.__class__(from_[index] for index in indices if 0 <= index < len_)
def ensure_keyword_args(kwargs, mandatory=(), optional=()): """Checks whether dictionary of keyword arguments satisfies conditions. :param kwargs: Dictionary of keyword arguments, received via ``*kwargs`` :param mandatory: Iterable of mandatory argument names :param optional: Iterable of optional argument names :return: ``kwargs`` if the conditions are met: all ``mandatory`` arguments are present, and besides that no arguments outside of ``optional`` ones are. :raise TypeError: When conditions are not met """ from taipan.strings import ensure_string ensure_mapping(kwargs) mandatory = list(map(ensure_string, ensure_iterable(mandatory))) optional = list(map(ensure_string, ensure_iterable(optional))) if not (mandatory or optional): raise ValueError( "mandatory and/or optional argument names must be provided") names = set(kwargs) for name in mandatory: try: names.remove(name) except KeyError: raise TypeError( "no value for mandatory keyword argument '%s'" % name) excess = names - set(optional) if excess: if len(excess) == 1: raise TypeError("unexpected keyword argument '%s'" % excess.pop()) else: raise TypeError( "unexpected keyword arguments: %s" % (tuple(excess),)) return kwargs
def omit(keys, from_, strict=False): """Returns a subset of given dictionary, omitting specified keys. :param keys: Iterable of keys to exclude :param strict: Whether ``keys`` are required to exist in the dictionary :return: Dictionary filtered by omitting ``keys`` :raise KeyError: If ``strict`` is True and one of ``keys`` is not found in the dictionary .. versionadded:: 0.0.2 """ ensure_iterable(keys) ensure_mapping(from_) if strict: remaining_keys = set(iterkeys(from_)) remove_subset(remaining_keys, keys) # raises KeyError if necessary else: remaining_keys = set(iterkeys(from_)) - set(keys) return from_.__class__((k, from_[k]) for k in remaining_keys)
def topological_order(nodes, incoming): """Performs topological sort of a DAG-like structure (directed acyclic graph). :param nodes: Collection of nodes :param incoming: Function taking node as an argument and returning iterable of nodes with edges pointing _towards_ given one :return: Iterable of nodes in the topological order .. note:: ``incoming`` function works in _reverse_ to the typical adjacency relation in graphs: if ``A in incoming(B)``, it implies that ``A->B`` is among the graph edges (**not** ``B->A``!). This reversal is useful in practice when dealing with graphs representing dependencies, module imports, header includes, and so on. Example:: for package in topological_order(packages, attr_func('dependencies')): install(package) .. versionadded:: 0.0.4 """ ensure_iterable(nodes) ; ensure_countable(nodes) ensure_callable(incoming) # data structure for tracking node's visit state NOT_VISITED, VISITING, VISITED = range(3) visit_states = {} visit_state = lambda node: visit_states.get(id(node), NOT_VISITED) def visit(node): """Topological sort visitor function.""" if visit_state(node) == VISITING: raise ValueError("cycle found on node %r" % (node,)) if visit_state(node) == NOT_VISITED: visit_states[id(node)] = VISITING for neighbor in incoming(node): for n in visit(neighbor): yield n visit_states[id(node)] = VISITED yield node def generator(): """Main generator function that loops through the nodes until we've visited them all. """ visited_count = 0 while visited_count < len(nodes): visited_count = 0 for node in nodes: if visit_state(node) == VISITED: visited_count += 1 else: for n in visit(node): yield n return generator()
def try_(block, except_=None, else_=None, finally_=None): """Emulate a ``try`` block. :param block: Function to be executed within the ``try`` statement :param except_: Function to execute when an :class:`Exception` occurs. It receives a single argument: the exception object. Alternatively, a list of key-value pairs can be passed, mapping exception types to their handler functions. :param else_: Function to execute when ``block`` completes successfully. Note that it requires ``except_`` to be provided as well :param finally_: Function to execute at the end, regardless of whether an exception occurred or not :return: If no exception was raised while executing ``block``, the result of ``else_`` (if provided) or ``block`` is returned. Otherwise, the result of ``except_`` is returned. Note that :func:`try_` can _still_ raise an exception if it occurs while evaluating ``except_``, ``else_`` or ``finally_`` functions. """ ensure_callable(block) if not (except_ or else_ or finally_): raise TypeError("at least one of `except_`, `else_` or `finally_` " "functions must be provided") if else_ and not except_: raise TypeError("`else_` can only be provided along with `except_`") if except_: if callable(except_): except_ = [(Exception, except_)] else: ensure_iterable(except_) if is_mapping(except_): ensure_ordered_mapping(except_) except_ = except_.items() def handle_exception(): """Dispatch current exception to proper handler in ``except_``.""" exc_type, exc_object = sys.exc_info()[:2] for t, handler in except_: if issubclass(exc_type, t): return handler(exc_object) raise if else_: ensure_callable(else_) if finally_: ensure_callable(finally_) try: block() except: return handle_exception() else: return else_() finally: finally_() else: try: block() except: return handle_exception() else: return else_() else: if finally_: ensure_callable(finally_) try: return block() except: return handle_exception() finally: finally_() else: try: return block() except: return handle_exception() elif finally_: ensure_callable(finally_) try: return block() finally: finally_()
def try_(block, except_=None, else_=None, finally_=None): """Emulate a ``try`` block. :param block: Function to be executed within the ``try`` statement :param except_: Function to execute when an :class:`Exception` occurs. It receives a single argument: the exception object. Alternatively, a list of key-value pairs can be passed, mapping exception types to their handler functions. :param else_: Function to execute when ``block`` completes successfully. Note that it requires ``except_`` to be provided as well :param finally_: Function to execute at the end, regardless of whether an exception occurred or not :return: If no exception was raised while executing ``block``, the result of ``else_`` (if provided) or ``block`` is returned. Otherwise, the result of ``except_`` is returned. Note that :func:`try_` can _still_ raise an exception if it occurs while evaluating ``except_``, ``else_`` or ``finally_`` functions. """ ensure_callable(block) if not (except_ or else_ or finally_): raise TypeError("at least one of `except_`, `else_` or `finally_` " "functions must be provided") if else_ and not except_: raise TypeError("`else_` can only be provided along with `except_`") if except_: if callable(except_): except_ = [(Exception, except_)] else: ensure_iterable(except_) if is_mapping(except_): ensure_ordered_mapping(except_) except_ = except_.items() def handle_exception(): """Dispatch current exception to proper handler in ``except_``.""" exc_type, exc_object = sys.exc_info()[:2] for t, handler in except_: if issubclass(exc_type, t): return handler(exc_object) raise if else_: ensure_callable(else_) if finally_: ensure_callable(finally_) try: block() except: return handle_exception() else: return else_() finally: finally_() else: try: block() except: return handle_exception() else: return else_() else: if finally_: ensure_callable(finally_) try: return block() except: return handle_exception() finally: finally_() else: try: return block() except: return handle_exception() elif finally_: ensure_callable(finally_) try: return block() finally: finally_()