def block_to_bx_block(
self, block_msg, tx_service, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
self._default_block_size = max(self._default_block_size,
len(block_msg.buf))
tsk = self.compression_tasks.borrow_task()
tsk.init(tpe.InputBytes(block_msg.buf), tx_service.proxy,
enable_block_compression, min_tx_age_seconds)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compression_tasks.return_task(tsk)
raise message_conversion_error.btc_block_compression_error(
block_msg.block_hash(), e)
bx_block = tsk.bx_block()
block = memoryview(bx_block)
compressed_size = len(block)
original_size = len(block_msg.rawbytes())
block_hash = OntObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string()))
block_info = BlockInfo(
block_hash, tsk.short_ids(), compress_start_datetime,
datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000,
tsk.txn_count(),
tsk.compressed_block_hash().hex_string(),
tsk.prev_block_hash().hex_string(), original_size, compressed_size,
100 - float(compressed_size) / original_size * 100,
tsk.ignored_short_ids())
self.compression_tasks.return_task(tsk)
return block, block_info
def contents_cleanup(transaction_service: TransactionService,
block_confirmation_message: AbstractCleanupMessage,
cleanup_tasks
):
start_datetime = datetime.utcnow()
start_time = time.time()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task = cleanup_tasks.borrow_task()
cleanup_task.init(tpe.InputBytes(block_confirmation_message.buf), tx_service.proxy)
task_pool_proxy.run_task(cleanup_task)
short_ids = cleanup_task.short_ids()
total_content_removed = cleanup_task.total_content_removed()
tx_count = cleanup_task.tx_count()
message_hash = block_confirmation_message.message_hash()
tx_service.update_removed_transactions(total_content_removed, short_ids)
transaction_service.on_block_cleaned_up(message_hash)
end_datetime = datetime.utcnow()
end_time = time.time()
duration = end_time - start_time
logger.statistics(
{
"type": "MemoryCleanup",
"event": "CacheStateAfterBlockCleanup",
"data": transaction_service.get_cache_state_json(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"duration": duration,
"total_content_removed": total_content_removed,
"tx_count": tx_count,
"short_ids_count": len(short_ids),
"message_hash": repr(message_hash),
}
)
cleanup_tasks.return_task(cleanup_task)
def compact_block_to_bx_block(
self, compact_block: CompactBlockBtcMessage,
transaction_service: ExtensionTransactionService
) -> CompactBlockCompressionResult:
compress_start_datetime = datetime.utcnow()
tsk = self.compact_mapping_tasks.borrow_task()
tsk.init(tpe.InputBytes(compact_block.buf), transaction_service.proxy,
compact_block.magic())
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compact_mapping_tasks.return_task(tsk)
raise message_conversion_error.btc_compact_block_compression_error(
compact_block.block_hash(), e)
success = tsk.success()
recovered_item = ExtensionCompactBlockRecoveryData(
transaction_service, tsk)
block_info = BlockInfo(compact_block.block_hash(), [],
compress_start_datetime,
compress_start_datetime, 0, None, None, None,
len(compact_block.rawbytes()), None, None, [])
if success:
result = CompactBlockCompressionResult(
False, block_info, None, None, [],
create_recovered_transactions())
return self._extension_recovered_compact_block_to_bx_block(
result, recovered_item)
else:
recovery_index = self._last_recovery_idx
self._extension_recovered_items[recovery_index] = recovered_item
self._last_recovery_idx += 1
return CompactBlockCompressionResult(
False, block_info, None, recovery_index, tsk.missing_indices(),
create_recovered_transactions())
def run_encryption(plain, et=None):
if et is None:
et = tpe.EncryptionTask(len(plain))
plain_text = tpe.InputBytes(bytearray(plain))
et.init(plain_text)
task_pool_proxy.run_task(et)
return et
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: ExtensionTransactionService
) -> Tuple[memoryview, BlockInfo]:
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
self._default_block_size = max(self._default_block_size,
len(block_msg.rawbytes()))
tsk = self.compression_tasks.borrow_task()
tsk.init(tpe.InputBytes(block_msg.rawbytes()), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compression_tasks.return_task(tsk)
raise message_conversion_error.eth_block_compression_error(
block_msg.block_hash(), e)
bx_block = tsk.bx_block()
starting_offset = tsk.starting_offset()
block = memoryview(bx_block)[starting_offset:]
compressed_size = len(block)
original_size = len(block_msg.rawbytes()) - starting_offset
block_hash = block_msg.block_hash()
block_info = BlockInfo(
block_hash, tsk.short_ids(), compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000, tsk.txn_count(),
tsk.compressed_block_hash().hex_string(),
tsk.prev_block_hash().hex_string(), original_size, compressed_size,
100 - float(compressed_size) / original_size * 100)
self.compression_tasks.return_task(tsk)
return block, block_info
def clean_block_transactions(
self, block_msg: BlockBtcMessage, transaction_service: TransactionService
) -> None:
start_datetime = datetime.utcnow()
start_time = time.time()
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len()
cleanup_task = self.block_cleanup_tasks.borrow_task()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task.init(tpe.InputBytes(block_msg.buf), tx_service.proxy)
init_time = time.time()
task_pool_proxy.run_task(cleanup_task)
task_run_time = time.time()
unknown_tx_hashes_count = len(cleanup_task.unknown_tx_hashes())
tx_property_fetch_time = time.time()
short_ids = cleanup_task.short_ids()
short_ids_fetch_time = time.time()
short_ids_count = len(short_ids)
tx_service.update_removed_transactions(cleanup_task.total_content_removed(), short_ids)
remove_from_tx_service_time = time.time()
# TODO : clean the short ids/transactions from the alarm queue after refactoring the transaction service
block_hash = block_msg.block_hash()
tx_service.on_block_cleaned_up(block_hash)
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len()
end_datetime = datetime.utcnow()
end_time = time.time()
logger.statistics(
{
"type": "BlockTransactionsCleanup",
"block_hash": repr(block_hash),
"unknown_tx_hashes_count": unknown_tx_hashes_count,
"short_ids_count": short_ids_count,
"block_transactions_count": cleanup_task.txn_count(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"task_init_time": init_time - start_time,
"task_run_time": task_run_time - init_time,
"tx_property_fetch_time": tx_property_fetch_time - task_run_time,
"short_ids_fetch_time": short_ids_fetch_time - tx_property_fetch_time,
"remove_from_tx_service_time": remove_from_tx_service_time - short_ids_fetch_time,
"duration": end_time - start_time,
"tx_hash_to_contents_len_before_cleanup": tx_hash_to_contents_len_before_cleanup,
"tx_hash_to_contents_len_after_cleanup": tx_hash_to_contents_len_after_cleanup,
}
)
self.block_cleanup_tasks.return_task(cleanup_task)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=short_ids,
unknown_tx_hashes=(
Sha256Hash(convert.hex_to_bytes(tx_hash.hex_string()))
for tx_hash in cleanup_task.unknown_tx_hashes()
)
)
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.decompression_tasks.return_task(tsk)
header_info = btc_normal_message_converter.parse_bx_block_header(bx_block_msg, deque())
raise message_conversion_error.btc_block_decompression_error(header_info.block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = BtcObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string())
)
if tsk.success():
btc_block_msg = BlockBtcMessage(buf=memoryview(tsk.block_message()))
logger.debug(
"Successfully parsed block broadcast message. {} transactions "
"in block {}",
total_tx_count,
block_hash
)
else:
btc_block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}",
block_hash,
len(unknown_tx_sids),
len(unknown_tx_hashes),
total_tx_count
)
block_info = get_block_info(
bx_block_msg,
block_hash,
tsk.short_ids(),
decompress_start_datetime,
decompress_start_timestamp,
total_tx_count,
btc_block_msg
)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(btc_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
def _extension_recovered_compact_block_to_bx_block(
self,
mapping_result: CompactBlockCompressionResult,
recovery_item: ExtensionCompactBlockRecoveryData
) -> CompactBlockCompressionResult:
mapping_task = recovery_item.mapping_task
compression_task: tpe.BtcCompactBlockCompressionTask = mapping_task.compression_task()
# pyre-fixme[16]: `List` has no attribute `vector`.
compression_task.add_recovered_transactions(mapping_result.recovered_transactions.vector)
mapping_block_info = mapping_result.block_info
try:
task_pool_proxy.run_task(compression_task)
except tpe.AggregatedException as e:
self.compact_mapping_tasks.return_task(mapping_task)
# pyre-fixme[16]: `Optional` has no attribute `block_hash`.
raise message_conversion_error.btc_compact_block_compression_error(mapping_block_info.block_hash, e)
bx_block = memoryview(compression_task.bx_block())
block_hash = mapping_block_info.block_hash
txn_count = compression_task.txn_count()
compressed_block_hash = compression_task.compressed_block_hash().hex_string()
prev_block_hash = compression_task.prev_block_hash().hex_string()
short_ids = compression_task.short_ids()
compress_end_datetime = datetime.utcnow()
# pyre-fixme[16]: `Optional` has no attribute `start_datetime`.
compress_start_datetime = mapping_block_info.start_datetime
# pyre-fixme[16]: `Optional` has no attribute `original_size`.
original_size = mapping_block_info.original_size
compressed_size = len(bx_block)
block_info = BlockInfo(
block_hash,
short_ids,
compress_start_datetime,
compress_end_datetime,
(compress_end_datetime - compress_start_datetime).total_seconds() * 1000,
txn_count,
compressed_block_hash,
prev_block_hash,
original_size,
compressed_size,
100 - float(compressed_size) / original_size * 100
)
self.compact_mapping_tasks.return_task(mapping_task)
return CompactBlockCompressionResult(True, block_info, bx_block, None, [], [])
def bx_block_to_block(self, bx_block_msg,
tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
block_hash = Sha256Hash(
convert.hex_to_bytes(tsk.block_hash().hex_string()))
self.decompression_tasks.return_task(tsk)
# TODO find a better solution
raise message_conversion_error.eth_block_decompression_error(
block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [
Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()
]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = Sha256Hash(
convert.hex_to_bytes(tsk.block_hash().hex_string()))
if tsk.success():
starting_offset = tsk.starting_offset()
block = memoryview(tsk.block_message())[starting_offset:]
block_msg = InternalEthBlockInfo(block)
content_size = len(block_msg.rawbytes())
logger.debug(
"Successfully parsed block broadcast message. {} "
"transactions in block {}", total_tx_count, block_hash)
bx_block_hash = convert.bytes_to_hex(
crypto.double_sha256(bx_block_msg))
compressed_size = len(bx_block_msg)
block_info = BlockInfo(
block_hash, tsk.short_ids(), decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
total_tx_count, bx_block_hash,
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()), compressed_size,
100 - float(compressed_size) / content_size * 100)
else:
block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}", block_hash, len(unknown_tx_sids),
len(unknown_tx_hashes), total_tx_count)
block_info = BlockInfo(
block_hash, tsk.short_ids(), decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000, None, None,
None, None, None, None)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(block_msg, block_info, unknown_tx_sids,
unknown_tx_hashes)
