def __init__(self,
start_block,
end_block,
batch_size,
web3,
output,
max_workers=5,
tokens=None,
fields_to_export=FIELDS_TO_EXPORT):
self.start_block = start_block
self.end_block = end_block
self.batch_size = batch_size
self.web3 = web3
self.output = output
self.max_workers = max_workers
self.tokens = tokens
self.fields_to_export = fields_to_export
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
self.output_file = None
self.exporter = None
self.executor: FailSafeExecutor = None
def __init__(self, logs_iterable, batch_size, max_workers, item_exporter):
self.logs_iterable = logs_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_transfer_extractor = EthErc20TransferExtractor()
class ExportTokenTransfersJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
web3,
item_exporter,
max_workers,
tokens=None):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
self.tokens = tokens
self.item_exporter = item_exporter
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.receipt_log_mapper = EthReceiptLogMapper()
self.token_transfer_mapper = EthTokenTransferMapper()
self.token_transfer_extractor = EthTokenTransferExtractor()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
assert len(block_number_batch) > 0
# https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
filter_params = {
'fromBlock': block_number_batch[0],
'toBlock': block_number_batch[-1],
'topics': [TRANSFER_EVENT_TOPIC]
}
if self.tokens is not None and len(self.tokens) > 0:
filter_params['address'] = self.tokens
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
token_transfer = self.token_transfer_extractor.extract_transfer_from_log(
log)
if token_transfer is not None:
self.item_exporter.export_item(
self.token_transfer_mapper.token_transfer_to_dict(
token_transfer))
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self,
start_block,
end_block,
batch_size,
web3,
item_exporter,
max_workers,
tokens=None):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.tokens = tokens
self.item_exporter = item_exporter
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
class ExtractErc20TransfersJob(BaseJob):
def __init__(self, logs_iterable, batch_size, max_workers, item_exporter):
self.logs_iterable = logs_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_transfer_extractor = EthErc20TransferExtractor()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.logs_iterable,
self._extract_transfers)
def _extract_transfers(self, log_dicts):
for log_dict in log_dicts:
self._extract_transfer(log_dict)
def _extract_transfer(self, log_dict):
log = self.receipt_log_mapper.dict_to_receipt_log(log_dict)
erc20_transfer = self.erc20_transfer_extractor.extract_transfer_from_log(
log)
if erc20_transfer is not None:
self.item_exporter.export_item(
self.erc20_transfer_mapper.erc20_transfer_to_dict(
erc20_transfer))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportErc20TransfersJob(BatchExportJob):
def __init__(self,
start_block,
end_block,
batch_size,
web3,
output,
max_workers=5,
tokens=None,
fields_to_export=FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.output = output
self.tokens = tokens
self.fields_to_export = fields_to_export
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
self.output_file = None
self.exporter = None
def _start(self):
super()._start()
self.output_file = get_file_handle(self.output,
binary=True,
create_parent_dirs=True)
self.exporter = CsvItemExporter(self.output_file,
fields_to_export=self.fields_to_export)
def _export_batch(self, batch_start, batch_end):
# https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
filter_params = {
'fromBlock': batch_start,
'toBlock': batch_end,
'topics': [TRANSFER_EVENT_TOPIC]
}
if self.tokens is not None and len(self.tokens) > 0:
filter_params['address'] = self.tokens
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
erc20_transfer = self.erc20_processor.filter_transfer_from_log(log)
if erc20_transfer is not None:
self.exporter.export_item(
self.erc20_transfer_mapper.erc20_transfer_to_dict(
erc20_transfer))
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
super()._end()
close_silently(self.output_file)
def __init__(self, start_block, end_block, batch_size, web3, ipfs_client,
marketplace_listing_exporter, shop_product_exporter,
max_workers):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
self.marketplace_listing_exporter = marketplace_listing_exporter
self.shop_product_exporter = shop_product_exporter
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.event_extractor = OriginEventExtractor(ipfs_client)
self.receipt_log_mapper = EthReceiptLogMapper()
self.marketplace_listing_mapper = OriginMarketplaceListingMapper()
self.shop_listing_mapper = OriginShopProductMapper()
class ExportReceiptsJob(BaseJob):
def __init__(self,
tx_hashes_iterable,
batch_size,
ipc_wrapper,
max_workers,
item_exporter,
export_receipts=True,
export_logs=True):
self.ipc_wrapper = ipc_wrapper
self.tx_hashes_iterable = tx_hashes_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_receipts = export_receipts
self.export_logs = export_logs
if not self.export_receipts and not self.export_logs:
raise ValueError(
'At least one of export_receipts or export_logs must be True')
self.receipt_mapper = EthReceiptMapper()
self.receipt_log_mapper = EthReceiptLogMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.tx_hashes_iterable,
self._export_receipts)
def _export_receipts(self, tx_hashes):
receipts_rpc = list(
generate_get_receipt_by_tx_hash_json_rpc(tx_hashes))
response = self.ipc_wrapper.make_request(json.dumps(receipts_rpc))
results = rpc_response_batch_to_results(response)
receipts = [
self.receipt_mapper.json_dict_to_receipt(result)
for result in results
]
for receipt in receipts:
self._export_receipt(receipt)
def _export_receipt(self, receipt):
if self.export_receipts:
self.item_exporter.export_item(
self.receipt_mapper.receipt_to_dict(receipt))
if self.export_logs:
for log in receipt.logs:
self.item_exporter.export_item(
self.receipt_log_mapper.receipt_log_to_dict(log))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self,
start_block,
end_block,
batch_size,
web3,
item_exporter,
max_workers,
tokens=None):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
self.tokens = tokens
self.item_exporter = item_exporter
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.receipt_log_mapper = EthReceiptLogMapper()
self.token_transfer_mapper = EthTokenTransferMapper()
self.token_transfer_extractor = EthTokenTransferExtractor()
def __init__(self,
tx_hashes_iterable,
batch_size,
ipc_wrapper,
max_workers,
item_exporter,
export_receipts=True,
export_logs=True):
self.ipc_wrapper = ipc_wrapper
self.tx_hashes_iterable = tx_hashes_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_receipts = export_receipts
self.export_logs = export_logs
if not self.export_receipts and not self.export_logs:
raise ValueError(
'At least one of export_receipts or export_logs must be True')
self.receipt_mapper = EthReceiptMapper()
self.receipt_log_mapper = EthReceiptLogMapper()
class EthReceiptMapper(object):
def __init__(self, receipt_log_mapper=None):
if receipt_log_mapper is None:
self.receipt_log_mapper = EthReceiptLogMapper()
else:
self.receipt_log_mapper = receipt_log_mapper
def json_dict_to_receipt(self, json_dict):
receipt = EthReceipt()
receipt.transaction_hash = json_dict.get('transactionHash')
receipt.transaction_index = hex_to_dec(
json_dict.get('transactionIndex'))
receipt.block_hash = json_dict.get('blockHash')
receipt.block_number = hex_to_dec(json_dict.get('blockNumber'))
receipt.cumulative_gas_used = hex_to_dec(
json_dict.get('cumulativeGasUsed'))
receipt.gas_used = hex_to_dec(json_dict.get('gasUsed'))
receipt.contract_address = to_normalized_address(
json_dict.get('contractAddress'))
receipt.root = json_dict.get('root')
receipt.status = hex_to_dec(json_dict.get('status'))
receipt.effective_gas_price = hex_to_dec(
json_dict.get('effectiveGasPrice'))
if 'logs' in json_dict:
receipt.logs = [
self.receipt_log_mapper.json_dict_to_receipt_log(log)
for log in json_dict['logs']
]
return receipt
def receipt_to_dict(self, receipt):
return {
'type': 'receipt',
'transaction_hash': receipt.transaction_hash,
'transaction_index': receipt.transaction_index,
'block_hash': receipt.block_hash,
'block_number': receipt.block_number,
'cumulative_gas_used': receipt.cumulative_gas_used,
'gas_used': receipt.gas_used,
'contract_address': receipt.contract_address,
'root': receipt.root,
'status': receipt.status,
'effective_gas_price': receipt.effective_gas_price
}
def __init__(self,
start_block,
end_block,
batch_size,
web3,
output,
max_workers=5,
tokens=None,
fields_to_export=FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.output = output
self.tokens = tokens
self.fields_to_export = fields_to_export
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
self.ethTransactionMapper = EthTransactionMapper()
self.output_file = None
# self.exporter = None
self.start_block = start_block
class ExportErc20TransfersJob(BatchExportJob):
def __init__(self,
start_block,
end_block,
batch_size,
web3,
item_exporter,
max_workers,
tokens=None):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.tokens = tokens
self.item_exporter = item_exporter
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
def _start(self):
super()._start()
self.item_exporter.open()
def _export_batch(self, batch_start, batch_end):
# https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
filter_params = {
'fromBlock': batch_start,
'toBlock': batch_end,
'topics': [TRANSFER_EVENT_TOPIC]
}
if self.tokens is not None and len(self.tokens) > 0:
filter_params['address'] = self.tokens
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
erc20_transfer = self.erc20_processor.filter_transfer_from_log(log)
if erc20_transfer is not None:
self.item_exporter.export_item(
self.erc20_transfer_mapper.erc20_transfer_to_dict(
erc20_transfer))
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
super()._end()
self.item_exporter.close()
class ExportOriginJob(BaseJob):
def __init__(self, start_block, end_block, batch_size, web3, ipfs_client,
marketplace_listing_exporter, shop_product_exporter,
max_workers):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.web3 = web3
self.marketplace_listing_exporter = marketplace_listing_exporter
self.shop_product_exporter = shop_product_exporter
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.event_extractor = OriginEventExtractor(ipfs_client)
self.receipt_log_mapper = EthReceiptLogMapper()
self.marketplace_listing_mapper = OriginMarketplaceListingMapper()
self.shop_listing_mapper = OriginShopProductMapper()
def _start(self):
self.marketplace_listing_exporter.open()
self.shop_product_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
assert len(block_number_batch) > 0
from_block = block_number_batch[0]
to_block = block_number_batch[-1]
# Nothing to process if the block range is older than the V0 marketplace contract's epoch.
if to_block < ORIGIN_MARKETPLACE_V0_BLOCK_NUMBER_EPOCH:
return
# Determine the version and address of the marketplace contract to query based on the block range.
batches = []
if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH or from_block >= ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH:
# The block range falls within a single version of the marketplace contract.
version = '000' if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH else '001'
address = ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS if version == '000' else ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS
batches.append({
'contract_address': address,
'contract_version': version,
'from_block': from_block,
'to_block': to_block
})
else:
# The block range spans across 2 versions of the marketplace contract.
batches.append({
'contract_address':
ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS,
'contract_version':
'000',
'from_block':
from_block,
'to_block':
ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH - 1
})
batches.append({
'contract_address': ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS,
'contract_version': '001',
'from_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH,
'to_block': to_block
})
for batch in batches:
# https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
filter_params = {
'address': batch['contract_address'],
'fromBlock': batch['from_block'],
'toBlock': batch['to_block']
}
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
listing, shop_products = self.event_extractor.extract_event_from_log(
log, batch['contract_version'])
if listing:
item = self.marketplace_listing_mapper.listing_to_dict(
listing)
self.marketplace_listing_exporter.export_item(item)
for product in shop_products:
item = self.shop_listing_mapper.product_to_dict(product)
self.shop_product_exporter.export_item(item)
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
self.batch_work_executor.shutdown()
self.marketplace_listing_exporter.close()
self.shop_product_exporter.close()
def __init__(self, receipt_log_mapper=None):
if receipt_log_mapper is None:
self.receipt_log_mapper = EthReceiptLogMapper()
else:
self.receipt_log_mapper = receipt_log_mapper
class ExportErc20TransfersJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
web3,
output,
max_workers=5,
tokens=None,
fields_to_export=FIELDS_TO_EXPORT):
self.start_block = start_block
self.end_block = end_block
self.batch_size = batch_size
self.web3 = web3
self.output = output
self.max_workers = max_workers
self.tokens = tokens
self.fields_to_export = fields_to_export
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
self.output_file = None
self.exporter = None
self.executor: FailSafeExecutor = None
def _start(self):
# Using bounded executor prevents unlimited queue growth
# and allows monitoring in-progress futures and failing fast in case of errors.
self.executor = FailSafeExecutor(BoundedExecutor(1, self.max_workers))
self.output_file = get_file_handle(self.output, binary=True)
self.exporter = CsvItemExporter(self.output_file,
fields_to_export=self.fields_to_export)
def _export(self):
for batch_start, batch_end in split_to_batches(self.start_block,
self.end_block,
self.batch_size):
self.executor.submit(self._fail_safe_export_batch, batch_start,
batch_end)
def _fail_safe_export_batch(self, batch_start, batch_end):
try:
self._export_batch(batch_start, batch_end)
except (Timeout, OSError):
# try exporting one by one
for block_number in range(batch_start, batch_end + 1):
self._export_batch(block_number, block_number)
def _export_batch(self, batch_start, batch_end):
filter_params = {
'fromBlock': batch_start,
'toBlock': batch_end,
'topics': [TRANSFER_EVENT_TOPIC]
}
if self.tokens is not None and len(self.tokens) > 0:
filter_params['address'] = self.tokens
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
erc20_transfer = self.erc20_processor.filter_transfer_from_log(log)
if erc20_transfer is not None:
self.exporter.export_item(
self.erc20_transfer_mapper.erc20_transfer_to_dict(
erc20_transfer))
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
self.executor.shutdown()
close_silently(self.output_file)
class ExportErc20TransfersJob(BatchExportJob):
def __init__(self,
start_block,
end_block,
batch_size,
web3,
output,
max_workers=5,
tokens=None,
fields_to_export=FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.output = output
self.tokens = tokens
self.fields_to_export = fields_to_export
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
self.ethTransactionMapper = EthTransactionMapper()
self.output_file = None
# self.exporter = None
self.start_block = start_block
def _start(self):
super()._start()
self.output_file = get_file_handle(self.output,
binary=True,
create_parent_dirs=True)
# self.exporter = CsvItemExporter(self.output_file, fields_to_export=self.fields_to_export)
def _export_batch(self, batch_start, batch_end):
# https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
filter_params = {
'fromBlock': batch_start,
'toBlock': batch_end,
'topics': [TRANSFER_EVENT_TOPIC]
}
if self.tokens is not None and len(self.tokens) > 0:
filter_params['address'] = self.tokens
event_filter = self.web3.eth.filter(filter_params)
events = event_filter.get_all_entries()
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
erc20_transfer = self.erc20_processor.filter_transfer_from_log(log)
if erc20_transfer is not None:
# self.exporter.export_item(self.erc20_transfer_mapper.erc20_transfer_to_dict(erc20_transfer))
erc20_transfer_dict = self.erc20_transfer_mapper.erc20_transfer_to_dict(
erc20_transfer)
print(erc20_transfer_dict)
erc20_transfer_dict["erc20_value"] = str(
erc20_transfer_dict["erc20_value"])
block = self.web3.eth.getBlock(
erc20_transfer_dict["erc20_block_number"])
erc20_transfer_dict["timestamp"] = block.timestamp
receipt = self.web3.eth.getTransactionReceipt(
erc20_transfer_dict.get("erc20_tx_hash"))
receipt_dict = self.erc20_transfer_mapper.erc20_receipt_transfer_to_dict(
receipt)
transaction = self.web3.eth.getTransaction(
erc20_transfer_dict.get("erc20_tx_hash"))
transaction_dict = self.erc20_transfer_mapper.transaction_to_dict(
transaction)
erc20_transfer_dict["gasUsed"] = receipt_dict.get("gasUsed")
erc20_transfer_dict["gasPrice"] = transaction_dict.get(
"gasPrice")
erc20_transfer_dict["status"] = 0 # 0 fail
if receipt_dict.get(
"blockNumber") is not None and receipt_dict.get(
"gasUsed") <= transaction_dict.get("gas"):
erc20_transfer_dict["status"] = 1 # 1 success
if erc20_transfers.find_one(
{"erc20_tx_hash": erc20_transfer_dict['erc20_tx_hash']
}) is None:
erc20_transfers.insert(erc20_transfer_dict)
erc20_receipt.insert(receipt_dict)
erc20_transaction.insert(transaction_dict)
self.web3.eth.uninstallFilter(event_filter.filter_id)
def _end(self):
super()._end()
close_silently(self.output_file)
blockConfig = eth_config.find_one({'config_id': 1})
blockConfig["export_flag"] = False
blockConfig["blockid"] = self.start_block
eth_config.save(blockConfig)