def yield_blocks_reversed(self) -> Generator[Block, None, None]:
# Highly recommended to override this method in the particular implementation of the blockchain for
# performance reasons
warnings.warn(
'Using low performance implementation of yield_blocks_reversed() method (override it)'
)
yield from always_reversible(self.yield_blocks())
def yield_blocks_slice(self, from_block_number: int,
to_block_number: Optional[int]):
# TODO(dmu) HIGH: Produce a better implementation of this method
yield from always_reversible(
self.yield_blocks_slice_reversed(
from_block_number=to_block_number,
to_block_number_exclusive=from_block_number - 1))
def get_attribute_doc(clsasts: tp.Union[tp.Iterable[ast.ClassDef], ast.ClassDef], name) -> tp.Optional[str]:
return re.sub(r'\n+', '\n', re.sub(r'([^\n])\n([^\n])', r'\1 \2', inspect.cleandoc(first((
ret for clsast in always_iterable(clsasts, base_type=ast.ClassDef) if clsast is not None
for members in (clsast.body,)
for i, member in always_reversible(enumerate(members))
if ((isinstance(member, ast.AnnAssign) and member.target.id == name
or isinstance(member, ast.Assign) and name in (node.id for node in ast.walk(member) if isinstance(node, ast.Name)))
and isinstance(members[i+1], ast.Expr) and isinstance(members[i+1].value, ast.Str))
for ret in (members[i+1].value.s,)
), default='')))) or None
def _yield_blocks_from_cache(self, start_block_number, end_block_number,
direction):
assert direction in (1, -1)
iter_ = range(start_block_number, end_block_number + 1)
if direction == -1:
iter_ = always_reversible(iter_)
for block_number in iter_:
block = self.blocks_cache.get(block_number)
if block is None:
break
yield block
def read_transactions_history(repo: git.Repo) -> Iterator[Transaction]:
all_transactions: Dict[date, List[Transaction]] = defaultdict(list)
for commit in always_reversible(repo.iter_commits()):
comm_transactions = list(read_transactions_at_commit(commit))
for tr in comm_transactions:
matched = _match_duplicate(all_transactions, tr)
if matched is None:
all_transactions[tr.on].append(tr)
else:
pass
# ~170,000 logs, so not worth logging here
# logger.debug(f"Matched duplicate:\n{tr}\n{matched}")
# destructure defaultdict
sorted_transactions = list(chain(*(v for v in all_transactions.values())))
yield from sorted(sorted_transactions, key=lambda t: t.on)
def _yield_blocks_from_file(self, file_path, direction, start=None):
assert direction in (1, -1)
storage = self.block_storage
unpacker = msgpack.Unpacker()
unpacker.feed(storage.load(file_path))
if direction == -1:
unpacker = always_reversible(unpacker)
for block_compact_dict in unpacker:
block = Block.from_compact_dict(block_compact_dict)
block_number = block.message.block_number
# TODO(dmu) HIGH: Implement a better skip
if start is not None:
if direction == 1 and block_number < start:
continue
elif direction == -1 and block_number > start:
continue
self.blocks_cache[block_number] = block
yield block
def groupby_records_iterable(records_pack_iterable,
sort_keys,
group_keys,
getter_func=itemgetter):
""" группировка по ключам """
## Старая версия (стабильная но не работает сортировка с None)
sorted_list = sorted(records_pack_iterable, key=getter_func(*sort_keys))
# ### Новая НЕтестированная версия (устойчива к наличию None в полях записи)
# sorted_list = sorted(
# records_pack_iterable,
# key=lambda x: list(map(bool, always_iterable(getter_func(*sort_keys)(x))))
# )
results = (tuple(always_reversible(group_values))
for group_name, group_values in groupby(
sorted_list, key=getter_func(*group_keys))
) # например: [[1,2,3], [4,5,6]]
results = zip_longest(*results,
fillvalue=[]) # например: [(3, 6), (2, 5), (1, 4)]
return next(results, []), collapse(results)
def select(
src: Union[Iterable[ET], Callable[[], Iterable[ET]]],
*,
where: Optional[Where] = None,
order_by: Optional[OrderFunc] = None,
order_key: Optional[str] = None,
order_value: Optional[Where] = None,
default: Optional[U] = None,
reverse: bool = False,
limit: Optional[int] = None,
drop_unsorted: bool = False,
wrap_unsorted: bool = True,
drop_exceptions: bool = False,
raise_exceptions: bool = False,
) -> Iterator[ET]:
"""
A function to query, order, sort and filter items from one or more sources
This supports iterables and lists of mixed types (including handling errors),
by allowing you to provide custom predicates (functions) which can sort
by a function, an attribute, dict key, or by the attributes values.
Since this supports mixed types, theres always a possibility
of KeyErrors or AttributeErrors while trying to find some value to order by,
so this provides multiple mechanisms to deal with that
'where' lets you filter items before ordering, to remove possible errors
or filter the iterator by some condition
There are multiple ways to instruct select on how to order items. The most
flexible is to provide an 'order_by' function, which takes an item in the
iterator, does any custom checks you may want and then returns the value to sort by
'order_key' is best used on items which have a similar structure, or have
the same attribute name for every item in the iterator. If you have a
iterator of objects whose datetime is accessed by the 'timestamp' attribute,
supplying order_key='timestamp' would sort by that (dictionary or attribute) key
'order_value' is the most confusing, but often the most useful. Instead of
testing against the keys of an item, this allows you to write a predicate
(function) to test against its values (dictionary, NamedTuple, dataclass, object).
If you had an iterator of mixed types and wanted to sort by the datetime,
but the attribute to access the datetime is different on each type, you can
provide `order_value=lambda v: isinstance(v, datetime)`, and this will
try to find that value for each type in the iterator, to sort it by
the value which is received when the predicate is true
'order_value' is often used in the 'hpi query' interface, because of its brevity.
Just given the input function, this can typically sort it by timestamp with
no human intervention. It can sort of be thought as an educated guess,
but it can always be improved by providing a more complete guess function
Note that 'order_value' is also the most computationally expensive, as it has
to copy the iterator in memory (using itertools.tee) to determine how to order it
in memory
The 'drop_exceptions' and 'raise_exceptions' let you ignore or raise when the src contains exceptions
src: an iterable of mixed types, or a function to be called,
as the input to this function
where: a predicate which filters the results before sorting
order_by: a function which when given an item in the src,
returns the value to sort by. Similar to the 'key' value
typically passed directly to 'sorted'
order_key: a string which represents a dict key or attribute name
to use as they key to sort by
order_value: predicate which determines which attribute on an ADT-like item to sort by,
when given its value. lambda o: isinstance(o, datetime) is commonly passed to sort
by datetime, without knowing the attributes or interface for the items in the src
default: while ordering, if the order for an object cannot be determined,
use this as the default value
reverse: reverse the order of the resulting iterable
limit: limit the results to this many items
drop_unsorted: before ordering, drop any items from the iterable for which a
order could not be determined. False by default
wrap_unsorted: before ordering, wrap any items into an 'Unsortable' object. Place
them at the front of the list. True by default
drop_exceptions: ignore any exceptions from the src
raise_exceptions: raise exceptions when received from the input src
"""
it: Iterable[ET] = [] # default
if callable(src):
# hopefully this returns an iterable and not something that causes a bunch of lag when its called?
# should typically not be the common case, but giving the option to
# provide a function as input anyways
it = src()
else:
# assume it is already an iterable
if not isinstance(src, Iterable):
low(f"""Input was neither a function, or some iterable
Expected 'src' to be an Iterable, but found {type(src).__name__}...
Will attempt to call iter() on the value""")
it = src
# try/catch an explicit iter() call to making this an Iterator,
# to validate the input as something other helpers here can work with,
# else raise a QueryException
try:
itr: Iterator[ET] = iter(it)
except TypeError as t:
raise QueryException("Could not convert input src to an Iterator: " +
str(t))
# if both drop_exceptions and drop_exceptions are provided for some reason,
# should raise exceptions before dropping them
if raise_exceptions:
itr = _raise_exceptions(itr)
if drop_exceptions:
itr = _drop_exceptions(itr)
if where is not None:
itr = filter(where, itr)
if order_by is not None or order_key is not None or order_value is not None:
order_by_chosen, itr = _handle_generate_order_by(
itr,
order_by=order_by,
order_key=order_key,
order_value=order_value,
default=default)
# signifies itr was filtered down to no data
if order_by_chosen is None:
# previously would send an warning message here,
# but sending the warning discourages this use-case
# e.g. take this iterable and see if I've had an event in
# the last week, else notify me to do something
#
# low("""While determining order_key, encountered empty iterable.
# Your 'src' may have been empty of the 'where' clause filtered the iterable to nothing""")
return itr
assert order_by_chosen is not None
# note: can't just attach sort unsortable values in the same iterable as the
# other items because they don't have any lookups for order_key or functions
# to handle items in the order_by_lookup dictionary
unsortable, itr = _handle_unsorted(itr, order_by_chosen, drop_unsorted,
wrap_unsorted)
# run the sort, with the computed order by function
itr = iter(sorted(itr, key=order_by_chosen,
reverse=reverse)) # type: ignore[arg-type, type-var]
# re-attach unsortable values to the front/back of the list
if reverse:
itr = itertools.chain(itr, unsortable)
else:
itr = itertools.chain(unsortable, itr)
else:
# if not already done in the order_by block, reverse if specified
if reverse:
itr = more_itertools.always_reversible(itr)
# apply limit argument
if limit is not None:
return itertools.islice(itr, limit)
return itr
def compose(
*funcs: tp.Callable[[_Tin], tp.Union[_Tin, _Tout]]
) -> tp.Callable[[_Tin], _Tout]:
return lambda arg: last(arg for arg in (arg, )
for f in always_reversible(flatten(funcs))
for arg in (f(arg), ))
def route() -> ResponseVal:
# wrap TypeError, common for non-event-like functions to fail
# when argument count doesnt match
try:
resp: Any = libfunc()
except TypeError as e:
return {
"error":
"TypeError calling HPI function, assuming not enough arguments passed",
"exception": str(e),
}, 400
except Exception as e:
return {
"error": "Error calling HPI function",
"exception": str(e)
}, 400
# if primitive, stats or dict, return directly
if (any([isinstance(resp, _type) for _type in [int, float, str, bool]])
or isinstance(resp, dict) or libfunc.__qualname__ == "stats"):
return jsonsafe({"value": resp})
# else, assume this returns event/namedtuple/object-like
riter: Iterator[Any] = iter(resp)
# parse GET arguments
# parse limit GET arg
limit_res: IntResult = parse_int_or_error(request.args.get("limit"),
50)
if not isinstance(limit_res, int):
return limit_res
limit: int = limit_res
# TODO: modularize into parse_int_or_error?
if limit < 1:
return {"error": "limit must be greater than or equal to 1"}, 400
# parse page GET arg
page_res: IntResult = parse_int_or_error(request.args.get("page"), 1)
if not isinstance(page_res, int):
return page_res
page: int = page_res
if page < 1:
return {"error": "page must be greater than or equal to 1"}, 400
# parse sort GET arg
if "sort" in request.args:
# peek at first item, to determine how to iterate over this
# if values are a dict, should index, else use getattr
val: Any = more_itertools.peekable(riter).peek()
key: str = request.args["sort"]
if isinstance(val, dict):
# make sure dictionary has key
if key not in val.keys():
return {
"error":
f"Value returned from iterator is a dictionary, but couldn't find key '{key}'"
}, 400
riter = iter(sorted(
riter,
key=lambda v: v[key])) # type: ignore[no-any-return]
else:
# if isn't a dict, assume namedtuple-like, or can use getattr on the object
if not hasattr(val, key):
return {
"error":
f"Value returned from iterator doesn't have attribute '{key}'"
}, 400
riter = iter(sorted(riter, key=lambda v: getattr(v, key))
) # type: ignore[no-any-return]
# parse order_by GET arg
if "order_by" in request.args and request.args["order_by"] == "desc":
riter = more_itertools.always_reversible(riter)
# exhaust paginations for 'previous' pages
for _ in range(page - 1):
more_itertools.take(limit, riter) # take into the void
return jsonsafe({
"page": page,
"limit": limit,
"items": more_itertools.take(limit, riter)
})