def split_by(pred: Callable[[E], bool],
it: Iterable[E]) -> Tuple[Iterable[E], Iterable[E]]:
"""
Split given items by a predicate into (positives, negatives).
>>> even = lambda x: x % 2 == 0
>>> pos, neg = split_by(even, iter([1, 5, 4, 7, 2]))
>>> list(pos), list(neg)
([4, 2], [1, 5, 7])
>>> pos, neg = split_by(even, iter([1, 3, 5]))
>>> list(pos), list(neg)
([], [1, 3, 5])
>>> pos, neg = split_by(even, iter([]))
>>> list(pos), list(neg)
([], [])
This operation is terminal in given iterable.
>>> it = iter([1, 2, 3])
>>> _ = split_by(even, it)
>>> next(it)
Traceback (most recent call last):
...
StopIteration
"""
items = collect(it)
rest = complement(pred)
return filter(pred, items), filter(rest, items)
def filter(predicate: str, manifest: Pathlike, output_manifest: Pathlike):
"""
Filter a MANIFEST according to the rule specified in PREDICATE, and save the result to OUTPUT_MANIFEST.
It is intended to work generically with most manifest types - it supports RecordingSet, SupervisionSet and CutSet.
\b
The PREDICATE specifies which attribute is used for item selection. Some examples:
lhotse filter 'duration>4.5' supervision.json output.json
lhotse filter 'num_frames<600' cuts.json output.json
lhotse filter 'start=0' cuts.json output.json
lhotse filter 'channel!=0' audio.json output.json
It currently only supports comparison of numerical manifest item attributes, such as:
start, duration, end, channel, num_frames, num_features, etc.
"""
data_set = load_manifest(manifest)
predicate_pattern = re.compile(
r'(?P<key>\w+)(?P<op>=|==|!=|>|<|>=|<=)(?P<value>[0-9.]+)')
match = predicate_pattern.match(predicate)
if match is None:
raise ValueError(
"Invalid predicate! Run with --help option to learn what predicates are allowed."
)
compare = {
'<': operator.lt,
'>': operator.gt,
'>=': operator.ge,
'<=': operator.le,
'=': isclose,
'==': isclose,
'!=': complement(isclose)
}[match.group('op')]
try:
value = int(match.group('value'))
except ValueError:
value = float(match.group('value'))
retained_items = []
try:
for item in data_set:
attr = getattr(item, match.group('key'))
if compare(attr, value):
retained_items.append(item)
except AttributeError:
click.echo(
f'Invalid predicate! Items in "{manifest}" do not have the attribute "{match.group("key")}"',
err=True)
exit(1)
filtered_data_set = to_manifest(retained_items)
if filtered_data_set is None:
click.echo('No items satisfying the predicate.', err=True)
exit(0)
filtered_data_set.to_json(output_manifest)
def test_complement():
# No args:
assert complement(lambda: False)()
assert not complement(lambda: True)()
# Single arity:
assert complement(iseven)(1)
assert not complement(iseven)(2)
assert complement(complement(iseven))(2)
assert not complement(complement(isodd))(2)
# Multiple arities:
both_even = lambda a, b: iseven(a) and iseven(b)
assert complement(both_even)(1, 2)
assert not complement(both_even)(2, 2)
# Generic truthiness:
assert complement(lambda: "")()
assert complement(lambda: 0)()
assert complement(lambda: None)()
assert complement(lambda: [])()
assert not complement(lambda: "x")()
assert not complement(lambda: 1)()
assert not complement(lambda: [1])()
def test_complement():
# No args:
assert complement(lambda: False)()
assert not complement(lambda: True)()
# Single arity:
assert complement(iseven)(1)
assert not complement(iseven)(2)
assert complement(complement(iseven))(2)
assert not complement(complement(isodd))(2)
# Multiple arities:
both_even = lambda a, b: iseven(a) and iseven(b)
assert complement(both_even)(1, 2)
assert not complement(both_even)(2, 2)
# Generic truthiness:
assert complement(lambda: "")()
assert complement(lambda: 0)()
assert complement(lambda: None)()
assert complement(lambda: [])()
assert not complement(lambda: "x")()
assert not complement(lambda: 1)()
assert not complement(lambda: [1])()
def is_testrpc_available():
try:
import testrpc # noqa: F401
return True
except ImportError:
return False
to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer)
TRANSACTION_FORMATTERS = {
'to': apply_formatter_if(
compose(complement(bool), decode_hex),
static_return(None),
),
}
def ethtestrpc_string_middleware(make_request, web3):
def middleware(method, params):
return force_obj_to_text(make_request(method, params))
return middleware
ethtestrpc_middleware = construct_formatting_middleware(
request_formatters={
'eth_uninstallFilter': apply_formatter_at_index(to_integer_if_hex, 0),
'eth_getFilterChanges': apply_formatter_at_index(to_integer_if_hex, 0),
from .formatting import (
construct_formatting_middleware, )
def bytes_to_ascii(value):
return codecs.decode(value, 'ascii')
to_ascii_if_bytes = apply_formatter_if(is_bytes, bytes_to_ascii)
to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer)
block_number_formatter = apply_formatter_if(is_integer, integer_to_hex)
is_false = partial(operator.is_, False)
is_not_false = complement(is_false)
is_not_null = complement(is_null)
@curry
def to_hexbytes(num_bytes, val, variable_length=False):
if isinstance(val, (str, int, bytes)):
result = HexBytes(val)
else:
raise TypeError("Cannot convert %r to HexBytes" % val)
extra_bytes = len(result) - num_bytes
if extra_bytes == 0 or (variable_length and extra_bytes < 0):
return result
elif all(byte == 0 for byte in result[:extra_bytes]):
return HexBytes(result[extra_bytes:])
apply_formatters_to_dict,
apply_key_map,
hex_to_integer,
integer_to_hex,
static_return,
)
def is_named_block(value):
return value in {"latest", "earliest", "pending"}
to_integer_if_hex = apply_formatter_if(hex_to_integer, is_string)
is_not_named_block = complement(is_named_block)
TRANSACTION_KEY_MAPPINGS = {
'block_hash': 'blockHash',
'block_number': 'blockNumber',
'gas_price': 'gasPrice',
'transaction_hash': 'transactionHash',
'transaction_index': 'transactionIndex',
}
transaction_key_remapper = apply_key_map(TRANSACTION_KEY_MAPPINGS)
RECEIPT_KEY_MAPPINGS = {
'block_hash': 'blockHash',
def is_testrpc_available():
try:
import testrpc # noqa: F401
return True
except ImportError:
return False
to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer)
TRANSACTION_FORMATTERS = {
'to':
apply_formatter_if(
compose(complement(bool), decode_hex),
static_return(None),
),
}
def ethtestrpc_string_middleware(make_request, web3):
def middleware(method, params):
return force_obj_to_text(make_request(method, params))
return middleware
ethtestrpc_middleware = construct_formatting_middleware(
request_formatters={
'eth_uninstallFilter': apply_formatter_at_index(to_integer_if_hex, 0),
construct_formatting_middleware,
)
def bytes_to_ascii(value):
return codecs.decode(value, 'ascii')
to_ascii_if_bytes = apply_formatter_if(is_bytes, bytes_to_ascii)
to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer)
block_number_formatter = apply_formatter_if(is_integer, integer_to_hex)
is_false = partial(operator.is_, False)
is_not_false = complement(is_false)
is_not_null = complement(is_null)
def is_array_of_strings(value):
if not is_list_like(value):
return False
return all((is_string(item) for item in value))
def is_array_of_dicts(value):
if not is_list_like(value):
return False
return all((is_dict(item) for item in value))
apply_formatter_at_index,
)
from .formatting import (
construct_formatting_middleware, )
def bytes_to_ascii(value):
return codecs.decode(value, 'ascii')
to_ascii_if_bytes = apply_formatter_if(bytes_to_ascii, is_bytes)
to_integer_if_hex = apply_formatter_if(hex_to_integer, is_string)
block_number_formatter = apply_formatter_if(hex, is_integer)
is_not_null = complement(is_null)
# TODO: decide what inputs this allows.
TRANSACTION_PARAMS_FORMATTERS = {
'value': hex,
'gas': hex,
'gasPrice': hex,
'nonce': hex,
}
transaction_params_formatter = apply_formatters_to_dict(
TRANSACTION_PARAMS_FORMATTERS)
def is_create_address(value):
return decode_hex(value) == b""
apply_formatters_to_dict,
apply_key_map,
hex_to_integer,
integer_to_hex,
static_return,
)
def is_named_block(value):
return value in {"latest", "earliest", "pending"}
to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer)
is_not_named_block = complement(is_named_block)
TRANSACTION_KEY_MAPPINGS = {
'block_hash': 'blockHash',
'block_number': 'blockNumber',
'gas_price': 'gasPrice',
'transaction_hash': 'transactionHash',
'transaction_index': 'transactionIndex',
}
transaction_key_remapper = apply_key_map(TRANSACTION_KEY_MAPPINGS)
LOG_KEY_MAPPINGS = {
'log_index': 'logIndex',
