def __init__(self,
start_block,
end_block,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain,
export_blocks=True,
export_transactions=True,
coin_price_type=CoinPriceType.empty):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
def __init__(self,
bitcoin_rpc,
chain=Chain.BITCOIN,
coin_price_type=CoinPriceType.empty):
self.bitcoin_rpc = bitcoin_rpc
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
self.chain = chain
self.coin_price_type = coin_price_type
self.cached_prices = {}
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain,
export_blocks=True,
export_transactions=True,
coin_price_type=CoinPriceType.empty):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks = self.btc_service.get_blocks(block_number_batch,
self.export_transactions)
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self,
transactions_iterable,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = BtcService(bitcoin_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size,
max_workers,
exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = BtcTransactionMapper()
class BtcBlockMapper(object):
def __init__(self, transaction_mapper=None):
if transaction_mapper is None:
self.transaction_mapper = BtcTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
def json_dict_to_block(self, json_dict):
block = BtcBlock()
block.hash = json_dict.get('hash')
block.size = json_dict.get('size')
block.stripped_size = json_dict.get('strippedsize')
block.weight = json_dict.get('weight')
block.number = json_dict.get('height')
block.version = json_dict.get('version')
block.merkle_root = json_dict.get('merkleroot')
block.timestamp = json_dict.get('time')
# bitcoin and all clones except zcash return integer nonce, zcash return hex string
block.nonce = to_hex(json_dict.get('nonce'))
block.bits = json_dict.get('bits')
raw_transactions = json_dict.get('tx')
if raw_transactions is not None and len(raw_transactions) > 0:
if isinstance(raw_transactions[0], dict):
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(
tx, block) for tx in raw_transactions
]
else:
# Transaction hashes
block.transactions = raw_transactions
block.transaction_count = len(raw_transactions)
return block
def block_to_dict(self, block):
return {
'type': 'block',
'hash': block.hash,
'size': block.size,
'stripped_size': block.stripped_size,
'weight': block.weight,
'number': block.number,
'version': block.version,
'merkle_root': block.merkle_root,
'timestamp': block.timestamp,
'nonce': block.nonce,
'bits': block.bits,
'coinbase_param': block.coinbase_param,
'transaction_count': len(block.transactions)
}
def __init__(self, transaction_mapper=None):
if transaction_mapper is None:
self.transaction_mapper = BtcTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
def __init__(self, bitcoin_rpc, chain=Chain.BITCOIN):
self.bitcoin_rpc = bitcoin_rpc
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
self.chain = chain
class BtcService(object):
def __init__(self, bitcoin_rpc, chain=Chain.BITCOIN):
self.bitcoin_rpc = bitcoin_rpc
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
self.chain = chain
def get_block(self, block_number, with_transactions=False):
block_hashes = self.get_block_hashes([block_number])
blocks = self.get_blocks_by_hashes(block_hashes, with_transactions)
return blocks[0] if len(blocks) > 0 else None
def get_genesis_block(self, with_transactions=False):
return self.get_block(0, with_transactions)
def get_latest_block(self, with_transactions=False):
block_number = self.bitcoin_rpc.getblockcount()
return self.get_block(block_number, with_transactions)
def get_blocks(self, block_number_batch, with_transactions=False):
if not block_number_batch:
return []
block_hashes = self.get_block_hashes(block_number_batch)
return self.get_blocks_by_hashes(block_hashes, with_transactions)
def get_blocks_by_hashes(self, block_hash_batch, with_transactions=True):
if not block_hash_batch:
return []
# get block details by hash
block_detail_rpc = list(
generate_get_block_by_hash_json_rpc(block_hash_batch,
with_transactions, self.chain))
block_detail_response = self.bitcoin_rpc.batch(block_detail_rpc)
block_detail_results = list(
rpc_response_batch_to_results(block_detail_response))
blocks = [
self.block_mapper.json_dict_to_block(block_detail_result)
for block_detail_result in block_detail_results
]
if self.chain in Chain.HAVE_OLD_API and with_transactions:
self._fetch_transactions(blocks)
for block in blocks:
self._remove_coinbase_input(block)
self._add_non_standard_addresses(block)
if self.chain == Chain.ZCASH:
self._add_shielded_inputs_and_outputs(block)
return blocks
def get_block_hashes(self, block_number_batch):
block_hash_rpc = list(
generate_get_block_hash_by_number_json_rpc(block_number_batch))
block_hashes_response = self.bitcoin_rpc.batch(block_hash_rpc)
block_hashes = rpc_response_batch_to_results(block_hashes_response)
return block_hashes
def _fetch_transactions(self, blocks):
all_transaction_hashes = [block.transactions for block in blocks]
flat_transaction_hashes = [
hash for transaction_hashes in all_transaction_hashes
for hash in transaction_hashes
]
raw_transactions = self._get_raw_transactions_by_hashes_batched(
flat_transaction_hashes)
for block in blocks:
raw_block_transactions = [
tx for tx in raw_transactions
if tx.get('txid') in block.transactions
]
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(tx, block)
for tx in raw_block_transactions
]
def _get_raw_transactions_by_hashes_batched(self, hashes):
if hashes is None or len(hashes) == 0:
return []
result = []
batch_size = 100
for batch in dynamic_batch_iterator(hashes, lambda: batch_size):
result.extend(self._get_raw_transactions_by_hashes(batch))
return result
def _get_raw_transactions_by_hashes(self, hashes):
if hashes is None or len(hashes) == 0:
return []
genesis_transaction = GENESIS_TRANSACTIONS.get(self.chain)
genesis_transaction_hash = genesis_transaction.get(
'txid') if genesis_transaction is not None else None
filtered_hashes = [
transaction_hash for transaction_hash in hashes
if transaction_hash != genesis_transaction_hash
]
transaction_detail_rpc = list(
generate_get_transaction_by_id_json_rpc(filtered_hashes))
transaction_detail_response = self.bitcoin_rpc.batch(
transaction_detail_rpc)
transaction_detail_results = rpc_response_batch_to_results(
transaction_detail_response)
raw_transactions = list(transaction_detail_results)
if genesis_transaction_hash is not None and genesis_transaction_hash in hashes:
raw_transactions.append(genesis_transaction)
return raw_transactions
def _remove_coinbase_input(self, block):
if block.has_full_transactions():
for transaction in block.transactions:
coinbase_inputs = [
input for input in transaction.inputs
if input.is_coinbase()
]
if len(coinbase_inputs) > 1:
raise ValueError(
'There must be no more than 1 coinbase input in any transaction. Was {}, hash {}'
.format(len(coinbase_inputs), transaction.hash))
coinbase_input = coinbase_inputs[0] if len(
coinbase_inputs) > 0 else None
if coinbase_input is not None:
block.coinbase_param = coinbase_input.coinbase_param
transaction.inputs = [
input for input in transaction.inputs
if not input.is_coinbase()
]
transaction.is_coinbase = True
def _add_non_standard_addresses(self, block):
if block.has_full_transactions():
for transaction in block.transactions:
for output in transaction.outputs:
if output.addresses is None or len(output.addresses) == 0:
output.type = 'nonstandard'
output.addresses = [
script_hex_to_non_standard_address(
output.script_hex)
]
def _add_shielded_inputs_and_outputs(self, block):
if block.has_full_transactions():
for transaction in block.transactions:
if transaction.join_splits is not None and len(
transaction.join_splits) > 0:
for join_split in transaction.join_splits:
input_value = join_split.public_input_value or 0
output_value = join_split.public_output_value or 0
if input_value > 0:
input = BtcTransactionInput()
input.type = ADDRESS_TYPE_SHIELDED
input.value = input_value
transaction.add_input(input)
if output_value > 0:
output = BtcTransactionOutput()
output.type = ADDRESS_TYPE_SHIELDED
output.value = output_value
transaction.add_output(output)
if transaction.value_balance is not None and transaction.value_balance != 0:
if transaction.value_balance > 0:
input = BtcTransactionInput()
input.type = ADDRESS_TYPE_SHIELDED
input.value = transaction.value_balance
transaction.add_input(input)
if transaction.value_balance < 0:
output = BtcTransactionOutput()
output.type = ADDRESS_TYPE_SHIELDED
output.value = transaction.value_balance
transaction.add_output(output)
class BtcService(object):
def __init__(self,
bitcoin_rpc,
chain=Chain.BITCOIN,
coin_price_type=CoinPriceType.empty):
self.bitcoin_rpc = bitcoin_rpc
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
self.chain = chain
self.coin_price_type = coin_price_type
self.cached_prices = {}
def get_block(self, block_number, with_transactions=False):
block_hashes = self.get_block_hashes([block_number])
blocks = self.get_blocks_by_hashes(block_hashes, with_transactions)
return blocks[0] if len(blocks) > 0 else None
def get_genesis_block(self, with_transactions=False):
return self.get_block(0, with_transactions)
def get_latest_block(self, with_transactions=False):
block_number = self.bitcoin_rpc.getblockcount()
return self.get_block(block_number, with_transactions)
def get_blocks(self, block_number_batch, with_transactions=False):
if not block_number_batch:
return []
block_hashes = self.get_block_hashes(block_number_batch)
return self.get_blocks_by_hashes(block_hashes, with_transactions)
def get_blocks_by_hashes(self, block_hash_batch, with_transactions=True):
if not block_hash_batch:
return []
# get block details by hash
block_detail_rpc = list(
generate_get_block_by_hash_json_rpc(block_hash_batch,
with_transactions, self.chain))
block_detail_response = self.bitcoin_rpc.batch(block_detail_rpc)
block_detail_results = list(
rpc_response_batch_to_results(block_detail_response))
blocks = [
self.block_mapper.json_dict_to_block(block_detail_result)
for block_detail_result in block_detail_results
]
if self.chain in Chain.HAVE_OLD_API and with_transactions:
self._fetch_transactions(blocks)
self._add_coin_price_to_blocks(blocks, self.coin_price_type)
for block in blocks:
self._remove_coinbase_input(block)
if block.has_full_transactions():
for transaction in block.transactions:
self._add_coin_price_to_transaction(
transaction, block.coin_price_usd)
self._add_non_standard_addresses(transaction)
if self.chain == Chain.ZCASH:
self._add_shielded_inputs_and_outputs(transaction)
return blocks
def get_block_hashes(self, block_number_batch):
block_hash_rpc = list(
generate_get_block_hash_by_number_json_rpc(block_number_batch))
block_hashes_response = self.bitcoin_rpc.batch(block_hash_rpc)
block_hashes = rpc_response_batch_to_results(block_hashes_response)
return block_hashes
def get_transactions_by_hashes(self, hashes):
if hashes is None or len(hashes) == 0:
return []
raw_transactions = self._get_raw_transactions_by_hashes_batched(hashes)
transactions = [
self.transaction_mapper.json_dict_to_transaction(tx)
for tx in raw_transactions
]
for transaction in transactions:
self._add_non_standard_addresses(transaction)
if self.chain == Chain.ZCASH:
self._add_shielded_inputs_and_outputs(transaction)
return transactions
def _fetch_transactions(self, blocks):
all_transaction_hashes = [block.transactions for block in blocks]
flat_transaction_hashes = [
hash for transaction_hashes in all_transaction_hashes
for hash in transaction_hashes
]
raw_transactions = self._get_raw_transactions_by_hashes_batched(
flat_transaction_hashes)
for block in blocks:
raw_block_transactions = [
tx for tx in raw_transactions
if tx.get('txid') in block.transactions
]
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(tx, block)
for tx in raw_block_transactions
]
def _get_raw_transactions_by_hashes_batched(self, hashes):
if hashes is None or len(hashes) == 0:
return []
result = []
batch_size = 100
for batch in dynamic_batch_iterator(hashes, lambda: batch_size):
result.extend(self._get_raw_transactions_by_hashes(batch))
return result
def _get_raw_transactions_by_hashes(self, hashes):
if hashes is None or len(hashes) == 0:
return []
genesis_transaction = GENESIS_TRANSACTIONS.get(self.chain)
genesis_transaction_hash = genesis_transaction.get(
'txid') if genesis_transaction is not None else None
filtered_hashes = [
transaction_hash for transaction_hash in hashes
if transaction_hash != genesis_transaction_hash
]
transaction_detail_rpc = list(
generate_get_transaction_by_id_json_rpc(filtered_hashes))
transaction_detail_response = self.bitcoin_rpc.batch(
transaction_detail_rpc)
transaction_detail_results = rpc_response_batch_to_results(
transaction_detail_response)
raw_transactions = list(transaction_detail_results)
if genesis_transaction_hash is not None and genesis_transaction_hash in hashes:
raw_transactions.append(genesis_transaction)
return raw_transactions
def _remove_coinbase_input(self, block):
if block.has_full_transactions():
for transaction in block.transactions:
coinbase_inputs = [
input for input in transaction.inputs
if input.is_coinbase()
]
if len(coinbase_inputs) > 1:
raise ValueError(
'There must be no more than 1 coinbase input in any transaction. Was {}, hash {}'
.format(len(coinbase_inputs), transaction.hash))
coinbase_input = coinbase_inputs[0] if len(
coinbase_inputs) > 0 else None
if coinbase_input is not None:
block.coinbase_param = coinbase_input.coinbase_param
transaction.inputs = [
input for input in transaction.inputs
if not input.is_coinbase()
]
transaction.is_coinbase = True
block.coinbase_param = coinbase_input.coinbase_param
block.coinbase_param_decoded = bytes.fromhex(
coinbase_input.coinbase_param).decode(
'utf-8', 'replace')
block.coinbase_tx = transaction
block.coinbase_txid = transaction.transaction_id
block.block_reward = self.get_block_reward(block)
transaction.input_count = 0
def _add_non_standard_addresses(self, transaction):
for output in transaction.outputs:
if output.addresses is None or len(output.addresses) == 0:
output.type = 'nonstandard'
output.addresses = [
script_hex_to_non_standard_address(output.script_hex)
]
def _add_shielded_inputs_and_outputs(self, transaction):
if transaction.join_splits is not None and len(
transaction.join_splits) > 0:
for join_split in transaction.join_splits:
input_value = join_split.public_input_value or 0
output_value = join_split.public_output_value or 0
if input_value > 0:
input = BtcTransactionInput()
input.type = ADDRESS_TYPE_SHIELDED
input.value = input_value
transaction.add_input(input)
if output_value > 0:
output = BtcTransactionOutput()
output.type = ADDRESS_TYPE_SHIELDED
output.value = output_value
transaction.add_output(output)
if transaction.value_balance is not None and transaction.value_balance != 0:
if transaction.value_balance > 0:
input = BtcTransactionInput()
input.type = ADDRESS_TYPE_SHIELDED
input.value = transaction.value_balance
transaction.add_input(input)
if transaction.value_balance < 0:
output = BtcTransactionOutput()
output.type = ADDRESS_TYPE_SHIELDED
output.value = -transaction.value_balance
transaction.add_output(output)
def get_block_reward(self, block):
return block.coinbase_tx.calculate_output_value()
def _add_coin_price_to_blocks(self, blocks, coin_price_type):
from_currency_code = Chain.ticker_symbol(self.chain)
if not from_currency_code or coin_price_type == CoinPriceType.empty:
return
elif coin_price_type == CoinPriceType.hourly:
block_hour_ids = list(
set([get_hour_id_from_ts(block.timestamp)
for block in blocks]))
block_hours_ts = {
hour_id: get_ts_from_hour_id(hour_id)
for hour_id in block_hour_ids
}
for hour_id, hour_ts in block_hours_ts.items():
if hour_id in self.cached_prices:
continue
self.cached_prices[hour_id] = get_coin_price(
from_currency_code=from_currency_code,
timestamp=hour_ts,
resource="histohour")
for block in blocks:
block_hour_id = get_hour_id_from_ts(block.timestamp)
block.coin_price_usd = self.cached_prices[block_hour_id]
elif coin_price_type == CoinPriceType.daily:
block_day_ids = list(
set([get_day_id_from_ts(block.timestamp) for block in blocks]))
block_days_ts = {
day_id: get_ts_from_day_id(day_id)
for day_id in block_day_ids
}
for day_id, day_ts in block_days_ts.items():
if day_id in self.cached_prices:
continue
self.cached_prices[day_id] = get_coin_price(
from_currency_code=from_currency_code,
timestamp=day_ts,
resource="histoday")
for block in blocks:
block_day_id = get_day_id_from_ts(block.timestamp)
block.coin_price_usd = self.cached_prices[block_day_id]
def _add_coin_price_to_transaction(self, transaction, coin_price_usd):
transaction.coin_price_usd = coin_price_usd
class EnrichTransactionsJob(BaseJob):
def __init__(self,
transactions_iterable,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = BtcService(bitcoin_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size,
max_workers,
exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = BtcTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.transactions_iterable,
self._enrich_transactions)
def _enrich_transactions(self, transactions):
transactions = [
self.transaction_mapper.dict_to_transaction(transaction)
for transaction in transactions
]
all_inputs = [transaction.inputs for transaction in transactions]
flat_inputs = [input for inputs in all_inputs for input in inputs]
for transaction_input_batch in dynamic_batch_iterator(
flat_inputs, lambda: self.batch_size):
input_transactions_map = self._get_input_transactions_as_map(
transaction_input_batch)
for input in transaction_input_batch:
output = self._get_output_for_input(input, input_transactions_map) \
if input.spent_transaction_hash is not None else None
if output is not None:
input.required_signatures = output.required_signatures
input.type = output.type
input.addresses = output.addresses
input.value = output.value
for transaction in transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(transaction))
def _get_input_transactions_as_map(self, transaction_inputs):
transaction_hashes = [
input.spent_transaction_hash for input in transaction_inputs
if input.spent_transaction_hash is not None
]
transaction_hashes = set(transaction_hashes)
if len(transaction_hashes) > 0:
transactions = self.btc_service.get_transactions_by_hashes(
transaction_hashes)
return {
transaction.hash: transaction
for transaction in transactions
}
else:
return {}
def _get_output_for_input(self, transaction_input, input_transactions_map):
spent_transaction_hash = transaction_input.spent_transaction_hash
input_transaction = input_transactions_map.get(spent_transaction_hash)
if input_transaction is None:
raise ValueError('Input transaction with hash {} not found'.format(
spent_transaction_hash))
spent_output_index = transaction_input.spent_output_index
if input_transaction.outputs is None or len(
input_transaction.outputs) < (spent_output_index + 1):
raise ValueError(
'There is no output with index {} in transaction with hash {}'.
format(spent_output_index, spent_transaction_hash))
output = input_transaction.outputs[spent_output_index]
return output
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class BtcBlockMapper(object):
def __init__(self, transaction_mapper=None):
if transaction_mapper is None:
self.transaction_mapper = BtcTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
def json_dict_to_block(self, json_dict):
block = BtcBlock()
block.hash = json_dict.get('hash')
block.size = json_dict.get('size')
block.stripped_size = json_dict.get('strippedsize')
block.weight = json_dict.get('weight')
block.number = json_dict.get('height')
block.version = json_dict.get('version')
block.merkle_root = json_dict.get('merkleroot')
block.timestamp = json_dict.get('time')
# bitcoin and all clones except zcash return integer nonce, zcash return hex string
block.nonce = to_hex(json_dict.get('nonce'))
block.bits = json_dict.get('bits')
raw_transactions = json_dict.get('tx')
if raw_transactions is not None and len(raw_transactions) > 0:
if isinstance(raw_transactions[0], dict):
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(
tx, block, idx)
for idx, tx in enumerate(raw_transactions)
]
block.transaction_ids = [
tx.transaction_id for tx in block.transactions
]
else:
# Transaction hashes
block.transactions = raw_transactions
block.transaction_ids = raw_transactions
block.transaction_count = len(raw_transactions)
# New fields
block.transaction_count = json_dict.get("nTx")
block.version_hex = json_dict.get("versionHex")
block.median_timestamp = json_dict.get("mediantime")
block.difficulty = int(json_dict.get("difficulty"))
block.chain_work = json_dict.get("chainwork")
block.coinbase_txid = json_dict.get("coinbase_txid")
block.previous_block_hash = json_dict.get("previousblockhash")
block.coin_price_usd = json_dict.get('coin_price_usd')
return block
def block_to_dict(self, block):
return {
'type': 'block',
'hash': block.hash,
'size': block.size,
'stripped_size': block.stripped_size,
'weight': block.weight,
'number': block.number,
'version': block.version,
'merkle_root': block.merkle_root,
'timestamp': block.timestamp,
'nonce': block.nonce,
'bits': block.bits,
'coinbase_param': block.coinbase_param,
'coinbase_param_decoded': block.coinbase_param_decoded,
'coinbase_txid': block.coinbase_txid,
'transaction_count': block.transaction_count,
'block_reward': block.block_reward,
'version_hex': block.version_hex,
'median_timestamp': block.median_timestamp,
'difficulty': block.difficulty,
'chain_work': block.chain_work,
'previous_block_hash': block.previous_block_hash,
"coin_price_usd": block.coin_price_usd,
"transaction_ids": block.transaction_ids
}
