def __init__(
self,
start_block,
end_block,
eos_rpc,
max_workers,
item_exporter,
batch_size=1,
export_blocks=True,
export_transactions=True,
export_actions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
self.export_actions = export_actions
if not self.export_blocks and not self.export_transactions and not self.export_actions:
raise ValueError('At least one of export_blocks or export_transactions or export_actions must be True')
self.eos_service = EosService(eos_rpc)
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
def __init__(
self,
transactions_iterable,
batch_size,
eos_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = EosService(eos_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers, exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = EosTransactionMapper()
class EosBlockMapper(object):
def __init__(self, transaction_mapper=None):
if transaction_mapper is None:
self.transaction_mapper = EosTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
def json_dict_to_block(self, json_dict):
block = BtcBlock()
block.hash = json_dict.get('hash')
block.size = json_dict.get('size')
block.stripped_size = json_dict.get('strippedsize')
block.weight = json_dict.get('weight')
block.number = json_dict.get('height')
block.version = json_dict.get('version')
block.merkle_root = json_dict.get('merkleroot')
block.timestamp = json_dict.get('time')
# bitcoin and all clones except zcash return integer nonce, zcash return hex string
block.nonce = to_hex(json_dict.get('nonce'))
block.bits = json_dict.get('bits')
raw_transactions = json_dict.get('tx')
if raw_transactions is not None and len(raw_transactions) > 0:
if isinstance(raw_transactions[0], dict):
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(
tx, block) for tx in raw_transactions
]
else:
# Transaction hashes
block.transactions = raw_transactions
block.transaction_count = len(raw_transactions)
return block
def block_to_dict(self, block):
return {
'type': 'block',
'hash': block["id"],
'block_num': block["block_num"],
'ref_block_prefix': block["ref_block_prefix"],
'previous': block["previous"],
'action_mroot': block["action_mroot"],
'transaction_mroot': block["transaction_mroot"],
'new_producers': block["new_producers"],
'header_extensions': block["header_extensions"],
'block_extensions': block["block_extensions"],
'timestamp': block["timestamp"],
'producer': block["producer"],
'transaction_count': len(block["transactions"])
}
def __init__(self, transaction_mapper=None):
if transaction_mapper is None:
self.transaction_mapper = EosTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
def __init__(self, eos_rpc):
self.eos_rpc = eos_rpc
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
def __init__(self, eos_rpc, chain=Chain.BITCOIN):
self.eos_rpc = eos_rpc
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
self.chain = chain
class EosService(object):
def __init__(self, eos_rpc, chain=Chain.BITCOIN):
self.eos_rpc = eos_rpc
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
self.chain = chain
def get_block(self, block_number, with_transactions=False):
return self.eos_rpc.getblock(block_number)
def get_genesis_block(self, with_transactions=False):
return self.get_block(1, with_transactions)
def get_latest_block(self, with_transactions=False):
last_irreversible_block_id = self.eos_rpc.get_info(
)["last_irreversible_block_id"]
return self.get_block(last_irreversible_block_id, with_transactions)
def get_blocks(self, block_number_batch, with_transactions=False):
if not block_number_batch:
return []
return [self.get_block(x) for x in block_number_batch]
# block_hashes = self.get_block_hashes(block_number_batch)
# return self.get_blocks_by_hashes(block_hashes, with_transactions)
def get_blocks_by_hashes(self, block_hash_batch, with_transactions=True):
if not block_hash_batch:
return []
# get block details by hash
block_detail_rpc = list(
generate_get_block_by_hash_json_rpc(block_hash_batch,
with_transactions, self.chain))
block_detail_response = self.eos_rpc.batch(block_detail_rpc)
block_detail_results = list(
rpc_response_batch_to_results(block_detail_response))
blocks = [
self.block_mapper.json_dict_to_block(block_detail_result)
for block_detail_result in block_detail_results
]
if self.chain in Chain.HAVE_OLD_API and with_transactions:
self._fetch_transactions(blocks)
for block in blocks:
self._remove_coinbase_input(block)
if block.has_full_transactions():
for transaction in block.transactions:
self._add_non_standard_addresses(transaction)
if self.chain == Chain.ZCASH:
self._add_shielded_inputs_and_outputs(transaction)
return blocks
def get_block_hashes(self, block_number_batch):
block_hash_rpc = list(
generate_get_block_hash_by_number_json_rpc(block_number_batch))
block_hashes_response = self.eos_rpc.batch(block_hash_rpc)
block_hashes = rpc_response_batch_to_results(block_hashes_response)
return block_hashes
def get_transactions_by_hashes(self, hashes):
if hashes is None or len(hashes) == 0:
return []
raw_transactions = self._get_raw_transactions_by_hashes_batched(hashes)
transactions = [
self.transaction_mapper.json_dict_to_transaction(tx)
for tx in raw_transactions
]
for transaction in transactions:
self._add_non_standard_addresses(transaction)
if self.chain == Chain.ZCASH:
self._add_shielded_inputs_and_outputs(transaction)
return transactions
def _fetch_transactions(self, blocks):
all_transaction_hashes = [block.transactions for block in blocks]
flat_transaction_hashes = [
hash for transaction_hashes in all_transaction_hashes
for hash in transaction_hashes
]
raw_transactions = self._get_raw_transactions_by_hashes_batched(
flat_transaction_hashes)
for block in blocks:
raw_block_transactions = [
tx for tx in raw_transactions
if tx.get('txid') in block.transactions
]
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(tx, block)
for tx in raw_block_transactions
]
def _get_raw_transactions_by_hashes_batched(self, hashes):
if hashes is None or len(hashes) == 0:
return []
result = []
batch_size = 100
for batch in dynamic_batch_iterator(hashes, lambda: batch_size):
result.extend(self._get_raw_transactions_by_hashes(batch))
return result
def _get_raw_transactions_by_hashes(self, hashes):
if hashes is None or len(hashes) == 0:
return []
genesis_transaction = GENESIS_TRANSACTIONS.get(self.chain)
genesis_transaction_hash = genesis_transaction.get(
'txid') if genesis_transaction is not None else None
filtered_hashes = [
transaction_hash for transaction_hash in hashes
if transaction_hash != genesis_transaction_hash
]
transaction_detail_rpc = list(
generate_get_transaction_by_id_json_rpc(filtered_hashes))
transaction_detail_response = self.eos_rpc.batch(
transaction_detail_rpc)
transaction_detail_results = rpc_response_batch_to_results(
transaction_detail_response)
raw_transactions = list(transaction_detail_results)
if genesis_transaction_hash is not None and genesis_transaction_hash in hashes:
raw_transactions.append(genesis_transaction)
return raw_transactions
def _remove_coinbase_input(self, block):
if block.has_full_transactions():
for transaction in block.transactions:
coinbase_inputs = [
input for input in transaction.inputs
if input.is_coinbase()
]
if len(coinbase_inputs) > 1:
raise ValueError(
'There must be no more than 1 coinbase input in any transaction. Was {}, hash {}'
.format(len(coinbase_inputs), transaction.hash))
coinbase_input = coinbase_inputs[0] if len(
coinbase_inputs) > 0 else None
if coinbase_input is not None:
block.coinbase_param = coinbase_input.coinbase_param
transaction.inputs = [
input for input in transaction.inputs
if not input.is_coinbase()
]
transaction.is_coinbase = True
def _add_non_standard_addresses(self, transaction):
for output in transaction.outputs:
if output.addresses is None or len(output.addresses) == 0:
output.type = 'nonstandard'
output.addresses = [
script_hex_to_non_standard_address(output.script_hex)
]
def _add_shielded_inputs_and_outputs(self, transaction):
if transaction.join_splits is not None and len(
transaction.join_splits) > 0:
for join_split in transaction.join_splits:
input_value = join_split.public_input_value or 0
output_value = join_split.public_output_value or 0
if input_value > 0:
input = BtcTransactionInput()
input.type = ADDRESS_TYPE_SHIELDED
input.value = input_value
transaction.add_input(input)
if output_value > 0:
output = BtcTransactionOutput()
output.type = ADDRESS_TYPE_SHIELDED
output.value = output_value
transaction.add_output(output)
if transaction.value_balance is not None and transaction.value_balance != 0:
if transaction.value_balance > 0:
input = BtcTransactionInput()
input.type = ADDRESS_TYPE_SHIELDED
input.value = transaction.value_balance
transaction.add_input(input)
if transaction.value_balance < 0:
output = BtcTransactionOutput()
output.type = ADDRESS_TYPE_SHIELDED
output.value = transaction.value_balance
transaction.add_output(output)
class ExportBlocksJob(BaseJob):
def __init__(
self,
start_block,
end_block,
eos_rpc,
max_workers,
item_exporter,
batch_size=1,
export_blocks=True,
export_transactions=True,
export_actions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
self.export_actions = export_actions
if not self.export_blocks and not self.export_transactions and not self.export_actions:
raise ValueError('At least one of export_blocks or export_transactions or export_actions must be True')
self.eos_service = EosService(eos_rpc)
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1
)
def _export_batch(self, block_number_batch):
blocks = self.eos_service.get_blocks(block_number_batch)
for block in blocks:
self._export_block(block)
self._export_transactions(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(self.block_mapper.block_to_dict(block))
def _export_transactions(self, block):
if self.export_transactions:
for transaction in block['transactions']:
self._export_transaction(transaction, block)
def _export_transaction(self, transaction, block):
transaction_dict = self.transaction_mapper.transaction_to_dict(transaction, block)
self.item_exporter.export_item(transaction_dict)
if self.export_actions \
and isinstance(transaction.get('trx'), dict) \
and transaction.get('trx').get('transaction') is not None \
and transaction.get('trx').get('transaction').get('actions') is not None:
for action in transaction.get('trx').get('transaction').get('actions'):
action_dict = self.action_mapper.action_to_dict(action, transaction_dict)
self.item_exporter.export_item(action_dict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class EnrichTransactionsJob(BaseJob):
def __init__(
self,
transactions_iterable,
batch_size,
eos_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = EosService(eos_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers, exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = EosTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.transactions_iterable, self._enrich_transactions)
def _enrich_transactions(self, transactions):
transactions = [self.transaction_mapper.dict_to_transaction(transaction) for transaction in transactions]
all_inputs = [transaction.inputs for transaction in transactions]
flat_inputs = [input for inputs in all_inputs for input in inputs]
for transaction_input_batch in dynamic_batch_iterator(flat_inputs, lambda: self.batch_size):
input_transactions_map = self._get_input_transactions_as_map(transaction_input_batch)
for input in transaction_input_batch:
output = self._get_output_for_input(input, input_transactions_map) \
if input.spent_transaction_hash is not None else None
if output is not None:
input.required_signatures = output.required_signatures
input.type = output.type
input.addresses = output.addresses
input.value = output.value
for transaction in transactions:
tx_dict = self.transaction_mapper.transaction_to_dict(transaction)
if tx_dict: # skip in None returned
self.item_exporter.export_item(tx_dict)
def _get_input_transactions_as_map(self, transaction_inputs):
transaction_hashes = [input.spent_transaction_hash for input in transaction_inputs
if input.spent_transaction_hash is not None]
transaction_hashes = set(transaction_hashes)
if len(transaction_hashes) > 0:
transactions = self.btc_service.get_transactions_by_hashes(transaction_hashes)
return {transaction.hash: transaction for transaction in transactions}
else:
return {}
def _get_output_for_input(self, transaction_input, input_transactions_map):
spent_transaction_hash = transaction_input.spent_transaction_hash
input_transaction = input_transactions_map.get(spent_transaction_hash)
if input_transaction is None:
raise ValueError('Input transaction with hash {} not found'.format(spent_transaction_hash))
spent_output_index = transaction_input.spent_output_index
if input_transaction.outputs is None or len(input_transaction.outputs) < (spent_output_index + 1):
raise ValueError(
'There is no output with index {} in transaction with hash {}'.format(
spent_output_index, spent_transaction_hash))
output = input_transaction.outputs[spent_output_index]
return output
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
eos_rpc,
max_workers,
item_exporter,
chain,
export_blocks=True,
export_transactions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.eos_service = EosService(eos_rpc, chain)
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks = self.eos_service.get_blocks(block_number_batch,
self.export_transactions)
for block in blocks:
self._export_block(block)
self._export_txs(block)
def _export_block(self, block):
if not self.export_blocks:
return
self.item_exporter.export_item(self.block_mapper.block_to_dict(block))
def _export_txs(self, block):
if not self.export_transactions:
return
for tx in block["transactions"]:
self._export_tx(tx, block)
def _export_tx(self, tx, block):
tx_dict = self.transaction_mapper.transaction_to_dict(tx, block)
if not tx_dict: # skip in None returned
return
self.item_exporter.export_item(tx_dict)
if tx_dict.get("trx.transaction.actions") is None:
return
for action in tx_dict["trx.transaction.actions"]:
action_dict = self.action_mapper.action_to_dict(
action, tx_dict, block)
self.item_exporter.export_item(action_dict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()