class ExportErc20TokensJob(BaseJob):
def __init__(self, web3, item_exporter, token_addresses_iterable,
max_workers):
self.item_exporter = item_exporter
self.token_addresses_iterable = token_addresses_iterable
self.batch_work_executor = BatchWorkExecutor(1, max_workers)
self.erc20_token_service = EthErc20TokenService(
web3, clean_user_provided_content)
self.erc20_token_mapper = EthErc20TokenMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.token_addresses_iterable,
self._export_tokens)
def _export_tokens(self, token_addresses):
for token_address in token_addresses:
self._export_token(token_address)
def _export_token(self, token_address):
token = self.erc20_token_service.get_token(token_address)
token_dict = self.erc20_token_mapper.erc20_token_to_dict(token)
self.item_exporter.export_item(token_dict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportTokenTransfersJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
web3,
item_exporter,
max_workers,
tokens=None):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
self.tokens = tokens
self.item_exporter = item_exporter
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.receipt_log_mapper = EthReceiptLogMapper()
self.token_transfer_mapper = EthTokenTransferMapper()
self.token_transfer_extractor = EthTokenTransferExtractor()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
assert len(block_number_batch) > 0
# https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
filter_params = {
'fromBlock': block_number_batch[0],
'toBlock': block_number_batch[-1],
'topics': [TRANSFER_EVENT_TOPIC]
}
if self.tokens is not None and len(self.tokens) > 0:
filter_params['address'] = self.tokens
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
token_transfer = self.token_transfer_extractor.extract_transfer_from_log(
log)
if token_transfer is not None:
self.item_exporter.export_item(
self.token_transfer_mapper.token_transfer_to_dict(
token_transfer))
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExtractGethTracesJob(BaseJob):
def __init__(self, traces_iterable, batch_size, max_workers,
item_exporter):
self.traces_iterable = traces_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.trace_mapper = EthTraceMapper()
self.geth_trace_mapper = EthGethTraceMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.traces_iterable,
self._extract_geth_traces)
def _extract_geth_traces(self, geth_traces):
for geth_trace_dict in geth_traces:
geth_trace = self.geth_trace_mapper.json_dict_to_geth_trace(
geth_trace_dict)
traces = self.trace_mapper.geth_trace_to_traces(geth_trace)
for trace in traces:
self.item_exporter.export_item(
self.trace_mapper.trace_to_dict(trace))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExtractErc20TransfersJob(BaseJob):
def __init__(self, logs_iterable, batch_size, max_workers, item_exporter):
self.logs_iterable = logs_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_transfer_extractor = EthErc20TransferExtractor()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.logs_iterable,
self._extract_transfers)
def _extract_transfers(self, log_dicts):
for log_dict in log_dicts:
self._extract_transfer(log_dict)
def _extract_transfer(self, log_dict):
log = self.receipt_log_mapper.dict_to_receipt_log(log_dict)
erc20_transfer = self.erc20_transfer_extractor.extract_transfer_from_log(
log)
if erc20_transfer is not None:
self.item_exporter.export_item(
self.erc20_transfer_mapper.erc20_transfer_to_dict(
erc20_transfer))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportContractsJob(BaseJob):
def __init__(self, contract_addresses_iterable, batch_size,
batch_web3_provider, max_workers, item_exporter):
self.batch_web3_provider = batch_web3_provider
self.contract_addresses_iterable = contract_addresses_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.contract_service = EthContractService()
self.contract_mapper = EthContractMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.contract_addresses_iterable,
self._export_contracts)
def _export_contracts(self, contract_addresses):
contracts_code_rpc = list(
generate_get_code_json_rpc(contract_addresses))
response_batch = self.batch_web3_provider.make_batch_request(
json.dumps(contracts_code_rpc))
contracts = []
for response in response_batch:
# request id is the index of the contract address in contract_addresses list
request_id = response['id']
result = rpc_response_to_result(response)
contract_address = contract_addresses[request_id]
contract = self._get_contract(contract_address, result)
contracts.append(contract)
for contract in contracts:
self.item_exporter.export_item(
self.contract_mapper.contract_to_dict(contract))
def _get_contract(self, contract_address, rpc_result):
contract = self.contract_mapper.rpc_result_to_contract(
contract_address, rpc_result)
bytecode = contract.bytecode
function_sighashes = self.contract_service.get_function_sighashes(
bytecode)
contract.function_sighashes = function_sighashes
contract.is_erc20 = self.contract_service.is_erc20_contract(
function_sighashes)
contract.is_erc721 = self.contract_service.is_erc721_contract(
function_sighashes)
return contract
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportReceiptsJob(BaseJob):
def __init__(self,
tx_hashes_iterable,
batch_size,
ipc_wrapper,
max_workers,
item_exporter,
export_receipts=True,
export_logs=True):
self.ipc_wrapper = ipc_wrapper
self.tx_hashes_iterable = tx_hashes_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_receipts = export_receipts
self.export_logs = export_logs
if not self.export_receipts and not self.export_logs:
raise ValueError(
'At least one of export_receipts or export_logs must be True')
self.receipt_mapper = EthReceiptMapper()
self.receipt_log_mapper = EthReceiptLogMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.tx_hashes_iterable,
self._export_receipts)
def _export_receipts(self, tx_hashes):
receipts_rpc = list(
generate_get_receipt_by_tx_hash_json_rpc(tx_hashes))
response = self.ipc_wrapper.make_request(json.dumps(receipts_rpc))
results = rpc_response_batch_to_results(response)
receipts = [
self.receipt_mapper.json_dict_to_receipt(result)
for result in results
]
for receipt in receipts:
self._export_receipt(receipt)
def _export_receipt(self, receipt):
if self.export_receipts:
self.item_exporter.export_item(
self.receipt_mapper.receipt_to_dict(receipt))
if self.export_logs:
for log in receipt.logs:
self.item_exporter.export_item(
self.receipt_log_mapper.receipt_log_to_dict(log))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportErc20TokensJob(BaseJob):
def __init__(self, web3, item_exporter, token_addresses_iterable,
max_workers):
self.web3 = web3
self.item_exporter = item_exporter
self.token_addresses_iterable = token_addresses_iterable
self.batch_work_executor = BatchWorkExecutor(1, max_workers)
self.erc20_token_mapper = EthErc20TokenMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.token_addresses_iterable,
self._export_tokens)
def _export_tokens(self, token_addresses):
for token_address in token_addresses:
self._export_token(token_address)
def _export_token(self, token_address):
checksum_address = self.web3.toChecksumAddress(token_address)
contract = self.web3.eth.contract(address=checksum_address,
abi=EIP20_ABI)
symbol = self._call_contract_function(contract.functions.symbol())
name = self._call_contract_function(contract.functions.name())
decimals = self._call_contract_function(contract.functions.decimals())
total_supply = self._call_contract_function(
contract.functions.totalSupply())
token = EthErc20Token()
token.erc20_token_address = token_address
token.erc20_token_symbol = symbol
token.erc20_token_name = name
token.erc20_token_decimals = decimals
token.erc20_token_total_supply = total_supply
token_dict = self.erc20_token_mapper.erc20_token_to_dict(token)
self.item_exporter.export_item(token_dict)
def _call_contract_function(self, func):
# BadFunctionCallOutput exception happens if the token doesn't implement a particular function
# or was self-destructed
# OverflowError exception happens if the return type of the function doesn't match the expected type
result = call_contract_function(func=func,
ignore_errors=(BadFunctionCallOutput,
OverflowError),
default_value=None)
return clean_user_provided_content(result)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExtractContractsJob(BaseJob):
def __init__(
self,
traces_iterable,
batch_size,
max_workers,
item_exporter):
self.traces_iterable = traces_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.contract_service = EthContractService()
self.contract_mapper = EthContractMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.traces_iterable, self._extract_contracts)
def _extract_contracts(self, traces):
for trace in traces:
trace['status'] = to_int_or_none(trace.get('status'))
trace['block_number'] = to_int_or_none(trace.get('block_number'))
contract_creation_traces = [trace for trace in traces
if trace.get('trace_type') == 'create' and trace.get('to_address') is not None
and len(trace.get('to_address')) > 0 and trace.get('status') == 1]
contracts = []
for trace in contract_creation_traces:
contract = EthContract()
contract.address = trace.get('to_address')
bytecode = trace.get('output')
contract.bytecode = bytecode
contract.block_number = trace.get('block_number')
function_sighashes = self.contract_service.get_function_sighashes(bytecode)
contract.function_sighashes = function_sighashes
contract.is_erc20 = self.contract_service.is_erc20_contract(function_sighashes)
contract.is_erc721 = self.contract_service.is_erc721_contract(function_sighashes)
contracts.append(contract)
for contract in contracts:
self.item_exporter.export_item(self.contract_mapper.contract_to_dict(contract))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportTracesJob(BaseJob):
def __init__(
self,
start_block,
end_block,
batch_size,
web3,
item_exporter,
max_workers):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.trace_mapper = EthTraceMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1
)
def _export_batch(self, block_number_batch):
assert len(block_number_batch) > 0
filter_params = {
'fromBlock': hex(block_number_batch[0]),
'toBlock': hex(block_number_batch[-1]),
}
json_traces = self.web3.parity.traceFilter(filter_params)
for json_trace in json_traces:
trace = self.trace_mapper.json_dict_to_trace(json_trace)
self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportGethTracesJob(BaseJob):
def __init__(self, start_block, end_block, batch_size, batch_web3_provider,
max_workers, item_exporter):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_web3_provider = batch_web3_provider
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.geth_trace_mapper = EthGethTraceMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
trace_block_rpc = list(
generate_trace_block_by_number_json_rpc(block_number_batch))
response = self.batch_web3_provider.make_request(
json.dumps(trace_block_rpc))
for response_item in response:
block_number = response_item.get('id')
result = rpc_response_to_result(response_item)
geth_trace = self.geth_trace_mapper.json_dict_to_geth_trace({
'block_number':
block_number,
'transaction_traces':
[tx_trace.get('result') for tx_trace in result],
})
self.item_exporter.export_item(
self.geth_trace_mapper.geth_trace_to_dict(geth_trace))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportContractsJob(BaseJob):
def __init__(self, contract_addresses_iterable, batch_size,
batch_web3_provider, max_workers, item_exporter):
self.batch_web3_provider = batch_web3_provider
self.contract_addresses_iterable = contract_addresses_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.contract_mapper = EthContractMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.contract_addresses_iterable,
self._export_contracts)
def _export_contracts(self, contract_addresses):
contracts_code_rpc = list(
generate_get_code_json_rpc(contract_addresses))
response_batch = self.batch_web3_provider.make_request(
json.dumps(contracts_code_rpc))
contracts = []
for response in response_batch:
# request id is the index of the contract address in contract_addresses list
request_id = response['id']
result = response['result']
contract_address = contract_addresses[request_id]
contracts.append(
self.contract_mapper.rpc_result_to_receipt(
contract_address, result))
for contract in contracts:
self.item_exporter.export_item(
self.contract_mapper.contract_to_dict(contract))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_web3_provider = batch_web3_provider
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(block_number_batch,
self.export_transactions))
response = self.batch_web3_provider.make_request(
json.dumps(blocks_rpc))
results = rpc_response_batch_to_results(response)
blocks = [
self.block_mapper.json_dict_to_block(result) for result in results
]
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
print("block=", self.block_mapper.block_to_dict(block))
print("number=", self.block_mapper.block_to_dict(block)['number'])
if self.export_transactions:
for tx in block.transactions:
# print(self.transaction_mapper.transaction_to_dict(tx))
transactiondict = self.transaction_mapper.transaction_to_dict(
tx)
transactiondict['timestamp'] = self.block_mapper.block_to_dict(
block)['timestamp']
self.item_exporter.export_item(transactiondict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportTracesJob(BaseJob):
def __init__(
self,
start_block,
end_block,
batch_size,
web3,
item_exporter,
max_workers,
include_genesis_traces=False,
include_daofork_traces=False):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
# TODO: use batch_size when this issue is fixed https://github.com/paritytech/parity-ethereum/issues/9822
self.batch_work_executor = BatchWorkExecutor(1, max_workers)
self.item_exporter = item_exporter
self.trace_mapper = EthTraceMapper()
self.special_trace_service = EthSpecialTraceService()
self.include_genesis_traces = include_genesis_traces
self.include_daofork_traces = include_daofork_traces
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1
)
def _export_batch(self, block_number_batch):
# TODO: Change to len(block_number_batch) > 0 when this issue is fixed
# https://github.com/paritytech/parity-ethereum/issues/9822
assert len(block_number_batch) == 1
block_number = block_number_batch[0]
if self.include_genesis_traces and 0 in block_number_batch:
genesis_traces = self.special_trace_service.get_genesis_traces()
for trace in genesis_traces:
self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))
if self.include_daofork_traces and DAOFORK_BLOCK_NUMBER in block_number_batch:
daofork_traces = self.special_trace_service.get_daofork_traces()
for trace in daofork_traces:
self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))
# TODO: Change to traceFilter when this issue is fixed
# https://github.com/paritytech/parity-ethereum/issues/9822
json_traces = self.web3.parity.traceBlock(block_number)
if json_traces is None:
raise ValueError('Response from the node is None. Is the node fully synced?')
for json_trace in json_traces:
trace = self.trace_mapper.json_dict_to_trace(json_trace)
self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportOriginJob(BaseJob):
def __init__(self, start_block, end_block, batch_size, web3, ipfs_client,
marketplace_listing_exporter, shop_product_exporter,
max_workers):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
self.marketplace_listing_exporter = marketplace_listing_exporter
self.shop_product_exporter = shop_product_exporter
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.event_extractor = OriginEventExtractor(ipfs_client)
self.receipt_log_mapper = EthReceiptLogMapper()
self.marketplace_listing_mapper = OriginMarketplaceListingMapper()
self.shop_listing_mapper = OriginShopProductMapper()
def _start(self):
self.marketplace_listing_exporter.open()
self.shop_product_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
assert len(block_number_batch) > 0
from_block = block_number_batch[0]
to_block = block_number_batch[-1]
# Nothing to process if the block range is older than the V0 marketplace contract's epoch.
if to_block < ORIGIN_MARKETPLACE_V0_BLOCK_NUMBER_EPOCH:
return
# Determine the version and address of the marketplace contract to query based on the block range.
batches = []
if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH or from_block >= ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH:
# The block range falls within a single version of the marketplace contract.
version = '000' if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH else '001'
address = ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS if version == '000' else ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS
batches.append({
'contract_address': address,
'contract_version': version,
'from_block': from_block,
'to_block': to_block
})
else:
# The block range spans across 2 versions of the marketplace contract.
batches.append({
'contract_address':
ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS,
'contract_version':
'000',
'from_block':
from_block,
'to_block':
ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH - 1
})
batches.append({
'contract_address': ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS,
'contract_version': '001',
'from_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH,
'to_block': to_block
})
for batch in batches:
# https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
filter_params = {
'address': batch['contract_address'],
'fromBlock': batch['from_block'],
'toBlock': batch['to_block']
}
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
listing, shop_products = self.event_extractor.extract_event_from_log(
log, batch['contract_version'])
if listing:
item = self.marketplace_listing_mapper.listing_to_dict(
listing)
self.marketplace_listing_exporter.export_item(item)
for product in shop_products:
item = self.shop_listing_mapper.product_to_dict(product)
self.shop_product_exporter.export_item(item)
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
self.batch_work_executor.shutdown()
self.marketplace_listing_exporter.close()
self.shop_product_exporter.close()