async def read_meta(sftp, dir_meta): """ Read metadata from a reMarkable tablet. """ await sftp.mget(BASE_DIR + '/*.metadata', dir_meta) await sftp.mget(BASE_DIR + '/*.content', dir_meta) to_uuid = compose( operator.itemgetter(0), os.path.splitext, os.path.basename, ) load_json = compose(json.load, open) files_content = glob.glob(dir_meta + '/*.content') files_content = {to_uuid(fn): fn for fn in files_content} files = glob.glob(dir_meta + '/*.metadata') files = ((fn, to_uuid(fn)) for fn in files) files = ((fn, u) for fn, u in files if u in files_content) # load metadata file and content file data = ((u, load_json(fn), load_json(files_content[u])) for fn, u in files) meta = {u: assoc(m, 'content', c) for u, m, c in data} return resolve_uuid(meta)
def test_compose(): assert compose()(0) == 0 assert compose(inc)(0) == 1 assert compose(double, inc)(0) == 2 assert compose(str, iseven, inc, double)(3) == "False" assert compose(str, add)(1, 2) == '3' def f(a, b, c=10): return (a + b) * c assert compose(str, inc, f)(1, 2, c=3) == '10' # Define two functions with different names def f(a): return a def g(a): return a composed = compose(f, g) assert composed.__name__ == 'f_of_g' assert composed.__doc__ == 'lambda *args, **kwargs: f(g(*args, **kwargs))' # Create an object with no __name__. h = object() composed = compose(f, h) assert composed.__name__ == 'Compose' assert composed.__doc__ == 'A composition of functions'
def chain_transforms( curried_funcs: List[Callable[[Any, SparkDataFrame], SparkDataFrame]], df: SparkDataFrame, ) -> SparkDataFrame: """Chain multiple transformations to the source dataframe Python API for DataFrame doesn't support .transforms() so this helper makes composing multiple transformations easier. Args: curried_funcs (List[Callable[[Any, SparkDataFrame], SparkDataFrame]]): List of curried functions (annotated with cytoolz.curry) each is a transformation for source DataFrame df (SparkDataFrame) Returns: dataframe after being applied all transformations Examples: >>> @curry >>> def with_label(label, df): >>> return df.withColumn("label", lit(label)) >>> ... >>> actual_df = chain_transforms([with_label("offline")], source_df) """ chained = compose(*reversed(curried_funcs)) return chained(df)
def process(workbook: Any, contents: list) -> None: """Process Storage Inventory worksheet :param workbook: :param contents: """ worksheet_name = 'Storage Inventory' worksheet = workbook.get_sheet_by_name(worksheet_name) headers = list(concat([ ['Hostname', 'Model', 'OS', 'Nodes'], get_parser_header(DEDUPE_TMPL) ])) RowTuple = namedtuple('RowTuple', headers) build_header(worksheet, headers) rows = [] for content in contents: doc = xmltodict.parse(content) component_details = search_tag_value(doc, 'component_details') command_details = search_tag_value(doc, 'command_details') dedupe, nodes = [], 0 # type: (list, int) for entry in command_details: nodes_content = collected_data( entry, 'cmd', 'isi storagepool nodepools list') nodes = max(map(compose(int, itemgetter(0)), run_parser_over( nodes_content, NODES_TMPL))) if nodes_content else nodes dedupe_content = collected_data(entry, 'cmd', 'isi dedupe stats') dedupe = run_parser_over( dedupe_content, DEDUPE_TMPL) if dedupe_content else dedupe dedupe = dedupe if len(dedupe) > 1 else [['', '', '', '', '', '']] rows.append([ component_details['hostname'], component_details['model'], component_details['os'], str(nodes), *dedupe[0] ]) final_col, final_row = 0, 0 for row_n, row_tuple in enumerate(map(RowTuple._make, rows), 2): for col_n, col_value in \ enumerate(row_tuple._asdict().values(), ord('A')): cell = worksheet['{}{}'.format(chr(col_n), row_n)] cell.value = str.strip(col_value) style_value_cell(cell) set_cell_to_number(cell) final_col = col_n final_row = row_n sheet_process_output( worksheet, 'StorageInventoryTable', 'Storage Inventory', final_col, final_row)
def decode(value): return compose(dicts, fractions, lfe_decoders.dates, lfe_decoders.datetimes, lfe_decoders.times, lfe_decoders.timedeltas, interp1d)(value)
def combine_argument_formatters(*formatters): warnings.warn( DeprecationWarning( "combine_argument_formatters(formatter1, formatter2)([item1, item2])" "has been deprecated and will be removed in a subsequent major version " "release of the eth-utils library. Update your calls to use " "apply_formatters_to_sequence([formatter1, formatter2], [item1, item2]) " "instead.")) _formatter_at_index = curry(apply_formatter_at_index) return compose(*(_formatter_at_index(formatter, index) for index, formatter in enumerate(formatters)))
def _revert_pending_transaction_filter(self, filter): is_valid_transaction_hash = excepts( (TransactionNotFound,), compose( bool, self.get_transaction_by_hash, self.normalizer.normalize_outbound_transaction_hash, ), lambda v: False, ) values_to_remove = remove(is_valid_transaction_hash, filter.get_all()) filter.remove(*values_to_remove)
def apply_formatters_to_return(*formatters): formatter = compose(*formatters) def outer(fn): @functools.wraps(fn) def inner(*args, **kwargs): value = fn(*args, **kwargs) return formatter(value) return inner return outer
def _revert_block_filter(self, filter): is_valid_block_hash = excepts( (BlockNotFound,), compose( bool, self.get_block_by_hash, self.normalizer.normalize_outbound_block_hash, ), lambda v: False, ) values_to_remove = tuple(remove(is_valid_block_hash, filter.get_all())) filter.remove(*values_to_remove)
def test_currying(self): data = [("jose", 1), ("li", 2), ("liz", 3)] source_df = self.spark.createDataFrame(data, ["name", "age"]) pipeline = compose(with_stuff1("nice", "person"), with_stuff2("yoyo")) actual_df = pipeline(source_df) expected_data = [("jose", 1, "yoyo", "nice person"), ("li", 2, "yoyo", "nice person"), ("liz", 3, "yoyo", "nice person")] expected_df = self.spark.createDataFrame( expected_data, ["name", "age", "stuff2", "stuff1"]) assert expected_df.collect() == actual_df.collect()
def test_currying(spark): data = [("jose", 1), ("li", 2), ("liz", 3)] source_df = spark.createDataFrame(data, ["name", "age"]) pipeline = compose(with_stuff1("nice", "person"), with_stuff2("yoyo")) actual_df = pipeline(source_df) expected_data = [ ("jose", 1, "yoyo", "nice person"), ("li", 2, "yoyo", "nice person"), ("liz", 3, "yoyo", "nice person"), ] expected_df = spark.createDataFrame(expected_data, ["name", "age", "stuff2", "stuff1"]) chispa.assert_df_equality(actual_df, expected_df, ignore_nullable=True)
def link_bytecode(bytecode, link_reference_values): """ Given the bytecode for a contract, and it's dependencies in the form of {contract_name: address} this function returns the bytecode with all of the link references replaced with the dependency addresses. """ linker_fn = compose(*(functools.partial( insert_link_value, value=value, offset=link_reference['start'], ) for link_reference, value in link_reference_values)) linked_bytecode = linker_fn(bytecode) return linked_bytecode
def link_bytecode_by_name(bytecode, link_references, **link_names_and_values): """ Helper function for linking bytecode with a mapping of link reference names to their values. TODO: fix this as it now needs access to the source file paths which isn't ideal """ link_fn = compose(*(partial( insert_link_value, value=link_names_and_values[expand_placeholder( linkref['name'], link_names_and_values.keys())], offset=linkref['start'], ) for linkref in link_references)) linked_bytecode = link_fn(bytecode) return linked_bytecode
def mine_blocks(self, num_blocks=1, coinbase=None): for _ in range(num_blocks): block_to_mine = dissoc(self.block, 'hash') block_hash = fake_rlp_hash(block_to_mine) mined_block = assoc(block_to_mine, 'hash', block_hash) assign_block_info = compose( partial(assoc, key='block_number', value=mined_block['number']), partial(assoc, key='block_hash', value=mined_block['hash']), ) mined_block['transactions'] = tuple( assign_block_info(transaction) for transaction in mined_block['transactions']) self.blocks.append(mined_block) self.block = make_block_from_parent(mined_block) yield block_hash
def test_compose(): assert compose()(0) == 0 assert compose(inc)(0) == 1 assert compose(double, inc)(0) == 2 assert compose(str, iseven, inc, double)(3) == "False" assert compose(str, add)(1, 2) == '3' def f(a, b, c=10): return (a + b) * c assert compose(str, inc, f)(1, 2, c=3) == '10'
def link_bytecode(bytecode, link_reference_values): """ Given the bytecode for a contract, and it's dependencies in the form of {contract_name: address} this functino returns the bytecode with all of the link references replaced with the dependency addresses. TODO: validate that the provided values are of the appropriate length """ linker_fn = compose(*( functools.partial( insert_link_value, value=value, offset=link_reference['start'], ) for link_reference, value in link_reference_values )) linked_bytecode = linker_fn(bytecode) return linked_bytecode
def link_bytecode_by_name(bytecode, link_references, **link_names_and_values): """ Helper function for linking bytecode with a mapping of link reference names to their values. TODO: fix this as it now needs access to the source file paths which isn't ideal """ link_fn = compose(*( partial( insert_link_value, value=link_names_and_values[ expand_placeholder(linkref['name'], link_names_and_values.keys()) ], offset=linkref['start'], ) for linkref in link_references )) linked_bytecode = link_fn(bytecode) return linked_bytecode
def _find_matching_event_abi(cls, event_name=None, argument_names=None): filters = [ functools.partial(filter_by_type, 'event'), ] if event_name is not None: filters.append(functools.partial(filter_by_name, event_name)) if argument_names is not None: filters.append( functools.partial(filter_by_argument_name, argument_names)) filter_fn = compose(*filters) event_abi_candidates = filter_fn(cls.abi) if len(event_abi_candidates) == 1: return event_abi_candidates[0] elif not event_abi_candidates: raise ValueError("No matching functions found") else: raise ValueError("Multiple functions found")
from __future__ import unicode_literals from cytoolz.functoolz import ( compose, partial, ) from eth_utils import ( int_to_big_endian, pad_left, ) zpad = partial(pad_left, pad_with=b'\x00') zpad32 = partial(pad_left, to_size=32, pad_with=b'\x00') int_to_32byte_big_endian = compose( zpad32, int_to_big_endian, )
SYNCING_FORMATTERS = { 'startingBlock': to_integer_if_hex, 'currentBlock': to_integer_if_hex, 'highestBlock': to_integer_if_hex, 'knownStates': to_integer_if_hex, 'pulledStates': to_integer_if_hex, } syncing_formatter = apply_formatters_to_dict(SYNCING_FORMATTERS) TRANSACTION_POOL_CONTENT_FORMATTERS = { 'pending': compose( keymap(to_ascii_if_bytes), valmap(transaction_formatter), ), 'queued': compose( keymap(to_ascii_if_bytes), valmap(transaction_formatter), ), } transaction_pool_content_formatter = apply_formatters_to_dict( TRANSACTION_POOL_CONTENT_FORMATTERS ) TRANSACTION_POOL_INSPECT_FORMATTERS = { 'pending': keymap(to_ascii_if_bytes),
eth_tester.unlock_account(transaction['from'], password) transaction_hash = eth_tester.send_transaction(transaction) finally: eth_tester.lock_account(transaction['from']) return transaction_hash API_ENDPOINTS = { 'web3': { 'clientVersion': client_version, 'sha3': compose( encode_hex, keccak, decode_hex, without_eth_tester(operator.itemgetter(0)), ), }, 'net': { 'version': not_implemented, 'peerCount': not_implemented, 'listening': not_implemented, }, 'eth': { 'protocolVersion': not_implemented, 'syncing': not_implemented, 'coinbase': compose(
def is_testrpc_available(): try: import testrpc # noqa: F401 return True except ImportError: return False to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) TRANSACTION_FORMATTERS = { 'to': apply_formatter_if( compose(complement(bool), decode_hex), static_return(None), ), } def ethtestrpc_string_middleware(make_request, web3): def middleware(method, params): return force_obj_to_text(make_request(method, params)) return middleware ethtestrpc_middleware = construct_formatting_middleware( request_formatters={ 'eth_uninstallFilter': apply_formatter_at_index(to_integer_if_hex, 0), 'eth_getFilterChanges': apply_formatter_at_index(to_integer_if_hex, 0),
def encode(value): ret_value = value return compose(dicts, dates, datetimes, times, timedeltas)(ret_value)
try: eth_tester.unlock_account(transaction['from'], password) transaction_hash = eth_tester.send_transaction(transaction) finally: eth_tester.lock_account(transaction['from']) return transaction_hash API_ENDPOINTS = { 'web3': { 'clientVersion': client_version, 'sha3': compose( encode_hex, keccak, decode_hex, without_eth_tester(operator.itemgetter(0)), ), }, 'net': { 'version': not_implemented, 'peerCount': not_implemented, 'listening': not_implemented, }, 'eth': { 'protocolVersion': not_implemented, 'syncing': not_implemented, 'coinbase': compose( operator.itemgetter(0), call_eth_tester('get_accounts'), ),
block_formatter = apply_formatters_to_dict(BLOCK_FORMATTERS) SYNCING_FORMATTERS = { 'startingBlock': to_integer_if_hex, 'currentBlock': to_integer_if_hex, 'highestBlock': to_integer_if_hex, 'knownStates': to_integer_if_hex, 'pulledStates': to_integer_if_hex, } syncing_formatter = apply_formatters_to_dict(SYNCING_FORMATTERS) TRANSACTION_POOL_CONTENT_FORMATTERS = { 'pending': compose( keymap(to_ascii_if_bytes), valmap(transaction_formatter), ), 'queued': compose( keymap(to_ascii_if_bytes), valmap(transaction_formatter), ), } transaction_pool_content_formatter = apply_formatters_to_dict( TRANSACTION_POOL_CONTENT_FORMATTERS) TRANSACTION_POOL_INSPECT_FORMATTERS = { 'pending': keymap(to_ascii_if_bytes), 'queued': keymap(to_ascii_if_bytes), }
def test_compose_metadata(): # Define two functions with different names def f(a): return a def g(a): return a composed = compose(f, g) assert composed.__name__ == 'f_of_g' assert composed.__doc__ == 'lambda *args, **kwargs: f(g(*args, **kwargs))' # Create an object with no __name__. h = object() composed = compose(f, h) assert composed.__name__ == 'Compose' assert composed.__doc__ == 'A composition of functions' assert repr(composed) == 'Compose({!r}, {!r})'.format(f, h) assert composed == compose(f, h) assert composed == AlwaysEquals() assert not composed == compose(h, f) assert not composed == object() assert not composed == NeverEquals() assert composed != compose(h, f) assert composed != NeverEquals() assert composed != object() assert not composed != compose(f, h) assert not composed != AlwaysEquals() assert hash(composed) == hash(compose(f, h)) assert hash(composed) != hash(compose(h, f)) bindable = compose(str, lambda x: x*2, lambda x, y=0: int(x) + y) class MyClass: def __int__(self): return 8 my_method = bindable my_static_method = staticmethod(bindable) assert MyClass.my_method(3) == '6' assert MyClass.my_method(3, y=2) == '10' assert MyClass.my_static_method(2) == '4' assert MyClass().my_method() == '16' assert MyClass().my_method(y=3) == '22' assert MyClass().my_static_method(0) == '0' assert MyClass().my_static_method(0, 1) == '2' assert compose(f, h).__wrapped__ is h if hasattr(cytoolz, 'sandbox'): # only test this with Python version (i.e., not Cython) assert compose(f, h).__class__.__wrapped__ is None # __signature__ is python3 only def myfunc(a, b, c, *d, **e): return 4 def otherfunc(f): return 'result: {}'.format(f) # set annotations compatibly with python2 syntax myfunc.__annotations__ = { 'a': int, 'b': str, 'c': float, 'd': int, 'e': bool, 'return': int, } otherfunc.__annotations__ = {'f': int, 'return': str} composed = compose(otherfunc, myfunc) sig = inspect.signature(composed) assert sig.parameters == inspect.signature(myfunc).parameters assert sig.return_annotation == str class MyClass: method = composed assert len(inspect.signature(MyClass().method).parameters) == 4
def test_compose_metadata(): # Define two functions with different names def f(a): return a def g(a): return a composed = compose(f, g) assert composed.__name__ == "f_of_g" assert composed.__doc__ == "lambda *args, **kwargs: f(g(*args, **kwargs))" # Create an object with no __name__. h = object() composed = compose(f, h) assert composed.__name__ == "Compose" assert composed.__doc__ == "A composition of functions" assert repr(composed) == "Compose({!r}, {!r})".format(f, h) assert composed == compose(f, h) assert composed == AlwaysEquals() assert not composed == compose(h, f) assert not composed == object() assert not composed == NeverEquals() assert composed != compose(h, f) assert composed != NeverEquals() assert composed != object() assert not composed != compose(f, h) assert not composed != AlwaysEquals() assert hash(composed) == hash(compose(f, h)) assert hash(composed) != hash(compose(h, f)) bindable = compose(str, lambda x: x * 2, lambda x, y=0: int(x) + y) class MyClass: def __int__(self): return 8 my_method = bindable my_static_method = staticmethod(bindable) assert MyClass.my_method(3) == "6" assert MyClass.my_method(3, y=2) == "10" assert MyClass.my_static_method(2) == "4" assert MyClass().my_method() == "16" assert MyClass().my_method(y=3) == "22" assert MyClass().my_static_method(0) == "0" assert MyClass().my_static_method(0, 1) == "2" assert compose(f, h).__wrapped__ is h if hasattr(cytoolz, "sandbox" ): # only test this with Python version (i.e., not Cython) assert compose(f, h).__class__.__wrapped__ is None # __signature__ is python3 only if PY3: def myfunc(a, b, c, *d, **e): return 4 def otherfunc(f): return "result: {}".format(f) # set annotations compatibly with python2 syntax myfunc.__annotations__ = { "a": int, "b": str, "c": float, "d": int, "e": bool, "return": int, } otherfunc.__annotations__ = {"f": int, "return": str} composed = compose(otherfunc, myfunc) sig = inspect.signature(composed) assert sig.parameters == inspect.signature(myfunc).parameters assert sig.return_annotation == str class MyClass: method = composed assert len(inspect.signature(MyClass().method).parameters) == 4
def is_testrpc_available(): try: import testrpc # noqa: F401 return True except ImportError: return False to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) TRANSACTION_FORMATTERS = { 'to': apply_formatter_if( compose(complement(bool), decode_hex), static_return(None), ), } def ethtestrpc_string_middleware(make_request, web3): def middleware(method, params): return force_obj_to_text(make_request(method, params)) return middleware ethtestrpc_middleware = construct_formatting_middleware( request_formatters={ 'eth_uninstallFilter': apply_formatter_at_index(to_integer_if_hex, 0),
def encode(value): return compose(lfe_encode, fractions, interp1d)(value)
def combine_argument_formatters(*formatters): _formatter_at_index = curry(apply_formatter_at_index) return compose(*(_formatter_at_index(formatter, index) for index, formatter in enumerate(formatters)))
Constructs a decorator that will cause the underlying function to only be applied to the given value if the `test_fn` returns true for that value. """ def outer_fn(fn): @functools.wraps(fn) def inner(value): if test_fn(value): return fn(value) return value return inner return outer_fn apply_if_not_null = apply_if_passes_test(compose(operator.not_, is_null)) apply_if_string = apply_if_passes_test(is_string) apply_if_array = apply_if_passes_test(is_list_like) apply_if_dict = apply_if_passes_test(is_dict) apply_if_integer = apply_if_passes_test(is_integer) def apply_to_array(formatter_fn): # workaround for https://github.com/pytoolz/cytoolz/issues/103 # NOTE: must reverse arguments when migrating from _compose to cytoolz.compose return _compose( functools.partial(map, formatter_fn), list, )
def test_compose(): for (compose_args, args, kw, expected) in generate_compose_test_cases(): assert compose(*compose_args)(*args, **kw) == expected
def apply_formatters_to_args(*formatters): return compose(*( apply_formatter_at_index(formatter, index) for index, formatter in enumerate(formatters) ))
def encode(value): return compose(dicts, dates, datetimes, times, timedeltas)(value)
apply_formatter_if(to_integer_if_hex, is_not_named_block), to_integer_if_hex, ), 'eth_sendTransaction': apply_formatters_to_args( transaction_params_formatter, ), 'eth_estimateGas': apply_formatters_to_args( transaction_params_formatter, ), 'eth_call': apply_formatters_to_args( transaction_params_formatter, ), 'eth_uninstallFilter': apply_formatters_to_args(hex_to_integer), # Personal 'personal_sendTransaction': apply_formatters_to_args( compose(transaction_params_remapper, transaction_params_formatter), identity, ), }, result_formatters={ 'eth_getBlockByHash': apply_formatter_if( block_key_remapper, is_dict, ), 'eth_getBlockByNumber': apply_formatter_if( block_key_remapper, is_dict, ), 'eth_getBlockTransactionCountByHash': apply_formatter_if( transaction_key_remapper, is_dict,
apply_formatter_if(is_not_named_block, to_integer_if_hex), to_integer_if_hex, ), 'eth_sendTransaction': apply_formatters_to_args( transaction_params_formatter, ), 'eth_estimateGas': apply_formatters_to_args( transaction_params_formatter, ), 'eth_call': apply_formatters_to_args( transaction_params_formatter, ), 'eth_uninstallFilter': apply_formatters_to_args(hex_to_integer), # Personal 'personal_sendTransaction': apply_formatters_to_args( compose(transaction_params_remapper, transaction_params_formatter), identity, ), }, result_formatters={ 'eth_getBlockByHash': apply_formatter_if( is_dict, block_key_remapper, ), 'eth_getBlockByNumber': apply_formatter_if( is_dict, block_key_remapper, ), 'eth_getBlockTransactionCountByHash': apply_formatter_if( is_dict, transaction_key_remapper,