def export_tokens_command(execution_date, **kwargs):
with TemporaryDirectory() as tempdir:
copy_from_export_path(export_path("contracts", execution_date),
os.path.join(tempdir, "contracts.json"))
filter_items.callback(
input=os.path.join(tempdir, "contracts.json"),
output=os.path.join(tempdir, "token_contracts.json"),
predicate="item['is_erc20'] or item['is_erc721']",
)
extract_field.callback(
input=os.path.join(tempdir, "token_contracts.json"),
output=os.path.join(tempdir, "token_addresses.txt"),
field="address",
)
export_tokens.callback(
token_addresses=os.path.join(tempdir, "token_addresses.txt"),
output=os.path.join(tempdir, "tokens.csv"),
max_workers=export_max_workers,
provider_uri=web3_provider_uri,
)
copy_to_export_path(os.path.join(tempdir, "tokens.csv"),
export_path("tokens", execution_date))
def export_receipts_and_logs_command(execution_date, provider_uri,
**kwargs):
with TemporaryDirectory() as tempdir:
copy_from_export_path(export_path("transactions", execution_date),
os.path.join(tempdir, "transactions.json"))
logging.info('Calling extract_csv_column(...)')
extract_field.callback(
input=os.path.join(tempdir, "transactions.json"),
output=os.path.join(tempdir, "transaction_hashes.txt"),
field="hash",
)
logging.info(
'Calling export_receipts_and_logs({}, ..., {}, {}, ...)'.
format(export_batch_size, provider_uri, export_max_workers))
export_receipts_and_logs.callback(
batch_size=export_batch_size,
transaction_hashes=os.path.join(tempdir,
"transaction_hashes.txt"),
provider_uri=provider_uri,
max_workers=export_max_workers,
receipts_output=os.path.join(tempdir, "receipts.json"),
logs_output=os.path.join(tempdir, "logs.json"),
)
copy_to_export_path(os.path.join(tempdir, "receipts.json"),
export_path("receipts", execution_date))
copy_to_export_path(os.path.join(tempdir, "logs.json"),
export_path("logs", execution_date))
def export_contracts_command(execution_date, **kwargs):
task_instance = kwargs['ti']
live_uri = task_instance.xcom_pull(key='live_uri',
task_ids='node_watch')
with TemporaryDirectory() as tempdir:
copy_from_export_path(export_path("traces", execution_date),
os.path.join(tempdir, "traces.csv"))
logging.info('Calling filter_items(...)')
filter_items.callback(
input=os.path.join(tempdir, "traces.csv"),
output=os.path.join(tempdir, "traces_type_create.csv"),
predicate=
"item['trace_type']=='create' and item['to_address'] is not None and len(item['to_address']) > 0",
)
logging.info('Removing unneeded file traces.csv')
os.remove(os.path.join(tempdir, "traces.csv"))
logging.info('Calling extract_field(...)')
extract_field.callback(
input=os.path.join(tempdir, "traces_type_create.csv"),
output=os.path.join(tempdir, "contract_addresses.txt"),
field="to_address",
)
logging.info('Removing unneeded file traces_type_create.csv')
os.remove(os.path.join(tempdir, "traces_type_create.csv"))
logging.info('Calling export_contracts({}, ..., {}, {})'.format(
export_batch_size, export_max_workers, live_uri))
export_contracts.callback(
batch_size=export_batch_size,
contract_addresses=os.path.join(tempdir,
"contract_addresses.txt"),
output=os.path.join(tempdir, "contracts.json"),
max_workers=export_max_workers,
provider_uri=live_uri,
)
copy_to_export_path(os.path.join(tempdir, "contracts.json"),
export_path("contracts", execution_date))
def export_tokens_command(execution_date, **kwargs):
task_instance = kwargs['ti']
live_uri = task_instance.xcom_pull(key='live_uri',
task_ids='node_watch')
with TemporaryDirectory() as tempdir:
copy_from_export_path(export_path("contracts", execution_date),
os.path.join(tempdir, "contracts.json"))
logging.info('Calling filter_items(...)')
filter_items.callback(
input=os.path.join(tempdir, "contracts.json"),
output=os.path.join(tempdir, "token_contracts.json"),
predicate="item['is_erc20'] or item['is_erc721']",
)
logging.info('Removing unneeded file contracts.json')
os.remove(os.path.join(tempdir, "contracts.json"))
logging.info('Calling extract_field(...)')
extract_field.callback(
input=os.path.join(tempdir, "token_contracts.json"),
output=os.path.join(tempdir, "token_addresses.txt"),
field="address",
)
logging.info('Removing unneeded file token_contracts.json')
os.remove(os.path.join(tempdir, "token_contracts.json"))
logging.info('Calling export_tokens(..., {}, {})'.format(
export_max_workers, live_uri))
export_tokens.callback(
token_addresses=os.path.join(tempdir, "token_addresses.txt"),
output=os.path.join(tempdir, "tokens.csv"),
max_workers=export_max_workers,
provider_uri=live_uri,
)
copy_to_export_path(os.path.join(tempdir, "tokens.csv"),
export_path("tokens", execution_date))