def split_by(pred: Callable[[E], bool], it: Iterable[E]) -> Tuple[Iterable[E], Iterable[E]]: """ Split given items by a predicate into (positives, negatives). >>> even = lambda x: x % 2 == 0 >>> pos, neg = split_by(even, iter([1, 5, 4, 7, 2])) >>> list(pos), list(neg) ([4, 2], [1, 5, 7]) >>> pos, neg = split_by(even, iter([1, 3, 5])) >>> list(pos), list(neg) ([], [1, 3, 5]) >>> pos, neg = split_by(even, iter([])) >>> list(pos), list(neg) ([], []) This operation is terminal in given iterable. >>> it = iter([1, 2, 3]) >>> _ = split_by(even, it) >>> next(it) Traceback (most recent call last): ... StopIteration """ items = collect(it) rest = complement(pred) return filter(pred, items), filter(rest, items)
def filter(predicate: str, manifest: Pathlike, output_manifest: Pathlike): """ Filter a MANIFEST according to the rule specified in PREDICATE, and save the result to OUTPUT_MANIFEST. It is intended to work generically with most manifest types - it supports RecordingSet, SupervisionSet and CutSet. \b The PREDICATE specifies which attribute is used for item selection. Some examples: lhotse filter 'duration>4.5' supervision.json output.json lhotse filter 'num_frames<600' cuts.json output.json lhotse filter 'start=0' cuts.json output.json lhotse filter 'channel!=0' audio.json output.json It currently only supports comparison of numerical manifest item attributes, such as: start, duration, end, channel, num_frames, num_features, etc. """ data_set = load_manifest(manifest) predicate_pattern = re.compile( r'(?P<key>\w+)(?P<op>=|==|!=|>|<|>=|<=)(?P<value>[0-9.]+)') match = predicate_pattern.match(predicate) if match is None: raise ValueError( "Invalid predicate! Run with --help option to learn what predicates are allowed." ) compare = { '<': operator.lt, '>': operator.gt, '>=': operator.ge, '<=': operator.le, '=': isclose, '==': isclose, '!=': complement(isclose) }[match.group('op')] try: value = int(match.group('value')) except ValueError: value = float(match.group('value')) retained_items = [] try: for item in data_set: attr = getattr(item, match.group('key')) if compare(attr, value): retained_items.append(item) except AttributeError: click.echo( f'Invalid predicate! Items in "{manifest}" do not have the attribute "{match.group("key")}"', err=True) exit(1) filtered_data_set = to_manifest(retained_items) if filtered_data_set is None: click.echo('No items satisfying the predicate.', err=True) exit(0) filtered_data_set.to_json(output_manifest)
def test_complement(): # No args: assert complement(lambda: False)() assert not complement(lambda: True)() # Single arity: assert complement(iseven)(1) assert not complement(iseven)(2) assert complement(complement(iseven))(2) assert not complement(complement(isodd))(2) # Multiple arities: both_even = lambda a, b: iseven(a) and iseven(b) assert complement(both_even)(1, 2) assert not complement(both_even)(2, 2) # Generic truthiness: assert complement(lambda: "")() assert complement(lambda: 0)() assert complement(lambda: None)() assert complement(lambda: [])() assert not complement(lambda: "x")() assert not complement(lambda: 1)() assert not complement(lambda: [1])()
def is_testrpc_available(): try: import testrpc # noqa: F401 return True except ImportError: return False to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) TRANSACTION_FORMATTERS = { 'to': apply_formatter_if( compose(complement(bool), decode_hex), static_return(None), ), } def ethtestrpc_string_middleware(make_request, web3): def middleware(method, params): return force_obj_to_text(make_request(method, params)) return middleware ethtestrpc_middleware = construct_formatting_middleware( request_formatters={ 'eth_uninstallFilter': apply_formatter_at_index(to_integer_if_hex, 0), 'eth_getFilterChanges': apply_formatter_at_index(to_integer_if_hex, 0),
from .formatting import ( construct_formatting_middleware, ) def bytes_to_ascii(value): return codecs.decode(value, 'ascii') to_ascii_if_bytes = apply_formatter_if(is_bytes, bytes_to_ascii) to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) block_number_formatter = apply_formatter_if(is_integer, integer_to_hex) is_false = partial(operator.is_, False) is_not_false = complement(is_false) is_not_null = complement(is_null) @curry def to_hexbytes(num_bytes, val, variable_length=False): if isinstance(val, (str, int, bytes)): result = HexBytes(val) else: raise TypeError("Cannot convert %r to HexBytes" % val) extra_bytes = len(result) - num_bytes if extra_bytes == 0 or (variable_length and extra_bytes < 0): return result elif all(byte == 0 for byte in result[:extra_bytes]): return HexBytes(result[extra_bytes:])
apply_formatters_to_dict, apply_key_map, hex_to_integer, integer_to_hex, static_return, ) def is_named_block(value): return value in {"latest", "earliest", "pending"} to_integer_if_hex = apply_formatter_if(hex_to_integer, is_string) is_not_named_block = complement(is_named_block) TRANSACTION_KEY_MAPPINGS = { 'block_hash': 'blockHash', 'block_number': 'blockNumber', 'gas_price': 'gasPrice', 'transaction_hash': 'transactionHash', 'transaction_index': 'transactionIndex', } transaction_key_remapper = apply_key_map(TRANSACTION_KEY_MAPPINGS) RECEIPT_KEY_MAPPINGS = { 'block_hash': 'blockHash',
def is_testrpc_available(): try: import testrpc # noqa: F401 return True except ImportError: return False to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) TRANSACTION_FORMATTERS = { 'to': apply_formatter_if( compose(complement(bool), decode_hex), static_return(None), ), } def ethtestrpc_string_middleware(make_request, web3): def middleware(method, params): return force_obj_to_text(make_request(method, params)) return middleware ethtestrpc_middleware = construct_formatting_middleware( request_formatters={ 'eth_uninstallFilter': apply_formatter_at_index(to_integer_if_hex, 0),
construct_formatting_middleware, ) def bytes_to_ascii(value): return codecs.decode(value, 'ascii') to_ascii_if_bytes = apply_formatter_if(is_bytes, bytes_to_ascii) to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) block_number_formatter = apply_formatter_if(is_integer, integer_to_hex) is_false = partial(operator.is_, False) is_not_false = complement(is_false) is_not_null = complement(is_null) def is_array_of_strings(value): if not is_list_like(value): return False return all((is_string(item) for item in value)) def is_array_of_dicts(value): if not is_list_like(value): return False return all((is_dict(item) for item in value))
apply_formatter_at_index, ) from .formatting import ( construct_formatting_middleware, ) def bytes_to_ascii(value): return codecs.decode(value, 'ascii') to_ascii_if_bytes = apply_formatter_if(bytes_to_ascii, is_bytes) to_integer_if_hex = apply_formatter_if(hex_to_integer, is_string) block_number_formatter = apply_formatter_if(hex, is_integer) is_not_null = complement(is_null) # TODO: decide what inputs this allows. TRANSACTION_PARAMS_FORMATTERS = { 'value': hex, 'gas': hex, 'gasPrice': hex, 'nonce': hex, } transaction_params_formatter = apply_formatters_to_dict( TRANSACTION_PARAMS_FORMATTERS) def is_create_address(value): return decode_hex(value) == b""
apply_formatters_to_dict, apply_key_map, hex_to_integer, integer_to_hex, static_return, ) def is_named_block(value): return value in {"latest", "earliest", "pending"} to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) is_not_named_block = complement(is_named_block) TRANSACTION_KEY_MAPPINGS = { 'block_hash': 'blockHash', 'block_number': 'blockNumber', 'gas_price': 'gasPrice', 'transaction_hash': 'transactionHash', 'transaction_index': 'transactionIndex', } transaction_key_remapper = apply_key_map(TRANSACTION_KEY_MAPPINGS) LOG_KEY_MAPPINGS = { 'log_index': 'logIndex',