def yield_blocks_reversed(self) -> Generator[Block, None, None]:
     # Highly recommended to override this method in the particular implementation of the blockchain for
     # performance reasons
     warnings.warn(
         'Using low performance implementation of yield_blocks_reversed() method (override it)'
     )
     yield from always_reversible(self.yield_blocks())
Example #2
0
 def yield_blocks_slice(self, from_block_number: int,
                        to_block_number: Optional[int]):
     # TODO(dmu) HIGH: Produce a better implementation of this method
     yield from always_reversible(
         self.yield_blocks_slice_reversed(
             from_block_number=to_block_number,
             to_block_number_exclusive=from_block_number - 1))
Example #3
0
def get_attribute_doc(clsasts: tp.Union[tp.Iterable[ast.ClassDef], ast.ClassDef], name) -> tp.Optional[str]:
    return re.sub(r'\n+', '\n', re.sub(r'([^\n])\n([^\n])', r'\1 \2', inspect.cleandoc(first((
        ret for clsast in always_iterable(clsasts, base_type=ast.ClassDef) if clsast is not None
        for members in (clsast.body,)
        for i, member in always_reversible(enumerate(members))
        if ((isinstance(member, ast.AnnAssign) and member.target.id == name
             or isinstance(member, ast.Assign) and name in (node.id for node in ast.walk(member) if isinstance(node, ast.Name)))
            and isinstance(members[i+1], ast.Expr) and isinstance(members[i+1].value, ast.Str))
        for ret in (members[i+1].value.s,)
    ), default='')))) or None
    def _yield_blocks_from_cache(self, start_block_number, end_block_number,
                                 direction):
        assert direction in (1, -1)

        iter_ = range(start_block_number, end_block_number + 1)
        if direction == -1:
            iter_ = always_reversible(iter_)

        for block_number in iter_:
            block = self.blocks_cache.get(block_number)
            if block is None:
                break

            yield block
Example #5
0
def read_transactions_history(repo: git.Repo) -> Iterator[Transaction]:
    all_transactions: Dict[date, List[Transaction]] = defaultdict(list)
    for commit in always_reversible(repo.iter_commits()):
        comm_transactions = list(read_transactions_at_commit(commit))
        for tr in comm_transactions:
            matched = _match_duplicate(all_transactions, tr)
            if matched is None:
                all_transactions[tr.on].append(tr)
            else:
                pass
                # ~170,000 logs, so not worth logging here
                # logger.debug(f"Matched duplicate:\n{tr}\n{matched}")

    # destructure defaultdict
    sorted_transactions = list(chain(*(v for v in all_transactions.values())))
    yield from sorted(sorted_transactions, key=lambda t: t.on)
    def _yield_blocks_from_file(self, file_path, direction, start=None):
        assert direction in (1, -1)
        storage = self.block_storage

        unpacker = msgpack.Unpacker()
        unpacker.feed(storage.load(file_path))
        if direction == -1:
            unpacker = always_reversible(unpacker)

        for block_compact_dict in unpacker:
            block = Block.from_compact_dict(block_compact_dict)
            block_number = block.message.block_number
            # TODO(dmu) HIGH: Implement a better skip
            if start is not None:
                if direction == 1 and block_number < start:
                    continue
                elif direction == -1 and block_number > start:
                    continue

            self.blocks_cache[block_number] = block
            yield block
Example #7
0
def groupby_records_iterable(records_pack_iterable,
                             sort_keys,
                             group_keys,
                             getter_func=itemgetter):
    """ группировка по ключам """

    ## Старая версия (стабильная но не работает сортировка с None)
    sorted_list = sorted(records_pack_iterable, key=getter_func(*sort_keys))

    # ### Новая НЕтестированная версия (устойчива к наличию None в полях записи)
    # sorted_list = sorted(
    #     records_pack_iterable,
    #     key=lambda x: list(map(bool, always_iterable(getter_func(*sort_keys)(x))))
    # )

    results = (tuple(always_reversible(group_values))
               for group_name, group_values in groupby(
                   sorted_list, key=getter_func(*group_keys))
               )  # например: [[1,2,3], [4,5,6]]

    results = zip_longest(*results,
                          fillvalue=[])  # например: [(3, 6), (2, 5), (1, 4)]

    return next(results, []), collapse(results)
Example #8
0
def select(
    src: Union[Iterable[ET], Callable[[], Iterable[ET]]],
    *,
    where: Optional[Where] = None,
    order_by: Optional[OrderFunc] = None,
    order_key: Optional[str] = None,
    order_value: Optional[Where] = None,
    default: Optional[U] = None,
    reverse: bool = False,
    limit: Optional[int] = None,
    drop_unsorted: bool = False,
    wrap_unsorted: bool = True,
    drop_exceptions: bool = False,
    raise_exceptions: bool = False,
) -> Iterator[ET]:
    """
    A function to query, order, sort and filter items from one or more sources
    This supports iterables and lists of mixed types (including handling errors),
    by allowing you to provide custom predicates (functions) which can sort
    by a function, an attribute, dict key, or by the attributes values.

    Since this supports mixed types, theres always a possibility
    of KeyErrors or AttributeErrors while trying to find some value to order by,
    so this provides multiple mechanisms to deal with that

    'where' lets you filter items before ordering, to remove possible errors
    or filter the iterator by some condition

    There are multiple ways to instruct select on how to order items. The most
    flexible is to provide an 'order_by' function, which takes an item in the
    iterator, does any custom checks you may want and then returns the value to sort by

    'order_key' is best used on items which have a similar structure, or have
    the same attribute name for every item in the iterator. If you have a
    iterator of objects whose datetime is accessed by the 'timestamp' attribute,
    supplying order_key='timestamp' would sort by that (dictionary or attribute) key

    'order_value' is the most confusing, but often the most useful. Instead of
    testing against the keys of an item, this allows you to write a predicate
    (function) to test against its values (dictionary, NamedTuple, dataclass, object).
    If you had an iterator of mixed types and wanted to sort by the datetime,
    but the attribute to access the datetime is different on each type, you can
    provide `order_value=lambda v: isinstance(v, datetime)`, and this will
    try to find that value for each type in the iterator, to sort it by
    the value which is received when the predicate is true

    'order_value' is often used in the 'hpi query' interface, because of its brevity.
    Just given the input function, this can typically sort it by timestamp with
    no human intervention. It can sort of be thought as an educated guess,
    but it can always be improved by providing a more complete guess function

    Note that 'order_value' is also the most computationally expensive, as it has
    to copy the iterator in memory (using itertools.tee) to determine how to order it
    in memory

    The 'drop_exceptions' and 'raise_exceptions' let you ignore or raise when the src contains exceptions

    src:            an iterable of mixed types, or a function to be called,
                    as the input to this function

    where:          a predicate which filters the results before sorting

    order_by:       a function which when given an item in the src,
                    returns the value to sort by. Similar to the 'key' value
                    typically passed directly to 'sorted'

    order_key:      a string which represents a dict key or attribute name
                    to use as they key to sort by

    order_value:    predicate which determines which attribute on an ADT-like item to sort by,
                    when given its value. lambda o: isinstance(o, datetime) is commonly passed to sort
                    by datetime, without knowing the attributes or interface for the items in the src

    default:        while ordering, if the order for an object cannot be determined,
                    use this as the default value

    reverse:        reverse the order of the resulting iterable

    limit:          limit the results to this many items

    drop_unsorted:  before ordering, drop any items from the iterable for which a
                    order could not be determined. False by default

    wrap_unsorted:  before ordering, wrap any items into an 'Unsortable' object. Place
                    them at the front of the list. True by default

    drop_exceptions: ignore any exceptions from the src

    raise_exceptions: raise exceptions when received from the input src
    """

    it: Iterable[ET] = []  # default
    if callable(src):
        # hopefully this returns an iterable and not something that causes a bunch of lag when its called?
        # should typically not be the common case, but giving the option to
        # provide a function as input anyways
        it = src()
    else:
        # assume it is already an iterable
        if not isinstance(src, Iterable):
            low(f"""Input was neither a function, or some iterable
Expected 'src' to be an Iterable, but found {type(src).__name__}...
Will attempt to call iter() on the value""")
        it = src

    # try/catch an explicit iter() call to making this an Iterator,
    # to validate the input as something other helpers here can work with,
    # else raise a QueryException
    try:
        itr: Iterator[ET] = iter(it)
    except TypeError as t:
        raise QueryException("Could not convert input src to an Iterator: " +
                             str(t))

    # if both drop_exceptions and drop_exceptions are provided for some reason,
    # should raise exceptions before dropping them
    if raise_exceptions:
        itr = _raise_exceptions(itr)

    if drop_exceptions:
        itr = _drop_exceptions(itr)

    if where is not None:
        itr = filter(where, itr)

    if order_by is not None or order_key is not None or order_value is not None:
        order_by_chosen, itr = _handle_generate_order_by(
            itr,
            order_by=order_by,
            order_key=order_key,
            order_value=order_value,
            default=default)

        # signifies itr was filtered down to no data
        if order_by_chosen is None:
            # previously would send an warning message here,
            # but sending the warning discourages this use-case
            # e.g. take this iterable and see if I've had an event in
            # the last week, else notify me to do something
            #
            # low("""While determining order_key, encountered empty iterable.
            # Your 'src' may have been empty of the 'where' clause filtered the iterable to nothing""")
            return itr

        assert order_by_chosen is not None
        # note: can't just attach sort unsortable values in the same iterable as the
        # other items because they don't have any lookups for order_key or functions
        # to handle items in the order_by_lookup dictionary
        unsortable, itr = _handle_unsorted(itr, order_by_chosen, drop_unsorted,
                                           wrap_unsorted)

        # run the sort, with the computed order by function
        itr = iter(sorted(itr, key=order_by_chosen,
                          reverse=reverse))  # type: ignore[arg-type, type-var]

        # re-attach unsortable values to the front/back of the list
        if reverse:
            itr = itertools.chain(itr, unsortable)
        else:
            itr = itertools.chain(unsortable, itr)
    else:
        # if not already done in the order_by block, reverse if specified
        if reverse:
            itr = more_itertools.always_reversible(itr)

    # apply limit argument
    if limit is not None:
        return itertools.islice(itr, limit)

    return itr
Example #9
0
def compose(
    *funcs: tp.Callable[[_Tin], tp.Union[_Tin, _Tout]]
) -> tp.Callable[[_Tin], _Tout]:
    return lambda arg: last(arg for arg in (arg, )
                            for f in always_reversible(flatten(funcs))
                            for arg in (f(arg), ))
Example #10
0
    def route() -> ResponseVal:
        # wrap TypeError, common for non-event-like functions to fail
        # when argument count doesnt match
        try:
            resp: Any = libfunc()
        except TypeError as e:
            return {
                "error":
                "TypeError calling HPI function, assuming not enough arguments passed",
                "exception": str(e),
            }, 400
        except Exception as e:
            return {
                "error": "Error calling HPI function",
                "exception": str(e)
            }, 400

        # if primitive, stats or dict, return directly
        if (any([isinstance(resp, _type) for _type in [int, float, str, bool]])
                or isinstance(resp, dict) or libfunc.__qualname__ == "stats"):
            return jsonsafe({"value": resp})

        # else, assume this returns event/namedtuple/object-like
        riter: Iterator[Any] = iter(resp)

        # parse GET arguments

        # parse limit GET arg
        limit_res: IntResult = parse_int_or_error(request.args.get("limit"),
                                                  50)
        if not isinstance(limit_res, int):
            return limit_res
        limit: int = limit_res
        # TODO: modularize into parse_int_or_error?
        if limit < 1:
            return {"error": "limit must be greater than or equal to 1"}, 400

        # parse page GET arg
        page_res: IntResult = parse_int_or_error(request.args.get("page"), 1)
        if not isinstance(page_res, int):
            return page_res
        page: int = page_res
        if page < 1:
            return {"error": "page must be greater than or equal to 1"}, 400

        # parse sort GET arg
        if "sort" in request.args:
            # peek at first item, to determine how to iterate over this
            # if values are a dict, should index, else use getattr
            val: Any = more_itertools.peekable(riter).peek()
            key: str = request.args["sort"]
            if isinstance(val, dict):
                # make sure dictionary has key
                if key not in val.keys():
                    return {
                        "error":
                        f"Value returned from iterator is a dictionary, but couldn't find key '{key}'"
                    }, 400
                riter = iter(sorted(
                    riter,
                    key=lambda v: v[key]))  # type: ignore[no-any-return]
            else:
                # if isn't a dict, assume namedtuple-like, or can use getattr on the object
                if not hasattr(val, key):
                    return {
                        "error":
                        f"Value returned from iterator doesn't have attribute '{key}'"
                    }, 400
                riter = iter(sorted(riter, key=lambda v: getattr(v, key))
                             )  # type: ignore[no-any-return]

        # parse order_by GET arg
        if "order_by" in request.args and request.args["order_by"] == "desc":
            riter = more_itertools.always_reversible(riter)

        # exhaust paginations for 'previous' pages
        for _ in range(page - 1):
            more_itertools.take(limit, riter)  # take into the void

        return jsonsafe({
            "page": page,
            "limit": limit,
            "items": more_itertools.take(limit, riter)
        })