def process_transactions_message_from_node(
self, msg) -> List[ProcessTransactionMessageFromNodeResult]:
opts = cast(GatewayOpts, self.node.opts)
msg_bytes = msg.rawbytes()
if isinstance(self.node,
OntGatewayNode) and opts.process_node_txs_in_extension:
ext_processing_results = memoryview(
self.proxy.process_gateway_transaction_from_node(
BlockchainProtocol.ONTOLOGY.value,
tpe.InputBytes(msg_bytes)))
elif isinstance(self.node,
EthGatewayNode) and opts.process_node_txs_in_extension:
ext_processing_results = memoryview(
self.proxy.process_gateway_transaction_from_node(
BlockchainProtocol.ETHEREUM.value,
tpe.InputBytes(msg_bytes)))
else:
return GatewayTransactionService.process_transactions_message_from_node(
self, msg)
result = []
offset = 0
txs_count, = struct.unpack_from("<I", ext_processing_results, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
while offset < len(ext_processing_results):
seen, = struct.unpack_from("<?", ext_processing_results, offset)
offset += 1
tx_hash = Sha256Hash(
ext_processing_results[offset:offset + crypto.SHA256_HASH_LEN])
offset += crypto.SHA256_HASH_LEN
tx_offset, = struct.unpack_from("<I", ext_processing_results,
offset)
offset += constants.UL_INT_SIZE_IN_BYTES
tx_len, = struct.unpack_from("<I", ext_processing_results, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
tx_contents = msg_bytes[tx_offset:tx_offset + tx_len]
result.append(
ProcessTransactionMessageFromNodeResult(
seen, tx_hash, tx_contents,
TxMessage(message_hash=tx_hash,
network_num=self.network_num,
tx_val=tx_contents)))
if not seen:
transaction_cache_key = self._tx_hash_to_cache_key(tx_hash)
self.set_transaction_contents_base(tx_hash,
transaction_cache_key,
tx_contents, False, 0,
False)
assert txs_count == len(result)
return result
def _tx_hash_to_cache_key(self,
transaction_hash) -> tpe.Sha256: # pyre-ignore
if isinstance(transaction_hash, Sha256Hash):
return tpe.Sha256(tpe.InputBytes(transaction_hash.binary))
if isinstance(transaction_hash, (bytes, bytearray, memoryview)):
return tpe.Sha256(tpe.InputBytes(transaction_hash))
if isinstance(transaction_hash, tpe.Sha256):
return transaction_hash
raise ValueError(
"Attempted to find cache entry with incorrect key type")
def contents_cleanup(transaction_service: TransactionService,
block_confirmation_message: AbstractCleanupMessage,
cleanup_tasks
):
start_datetime = datetime.utcnow()
start_time = time.time()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task = cleanup_tasks.borrow_task()
cleanup_task.init(tpe.InputBytes(block_confirmation_message.buf), tx_service.proxy)
task_pool_proxy.run_task(cleanup_task)
short_ids = cleanup_task.short_ids()
total_content_removed = cleanup_task.total_content_removed()
tx_count = cleanup_task.tx_count()
message_hash = block_confirmation_message.message_hash()
tx_service.update_removed_transactions(total_content_removed, short_ids)
transaction_service.on_block_cleaned_up(message_hash)
end_datetime = datetime.utcnow()
end_time = time.time()
duration = end_time - start_time
logger.statistics(
{
"type": "MemoryCleanup",
"event": "CacheStateAfterBlockCleanup",
"data": transaction_service.get_cache_state_json(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"duration": duration,
"total_content_removed": total_content_removed,
"tx_count": tx_count,
"short_ids_count": len(short_ids),
"message_hash": repr(message_hash),
}
)
cleanup_tasks.return_task(cleanup_task)
def block_to_bx_block(
self, block_msg, tx_service, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
self._default_block_size = max(self._default_block_size,
len(block_msg.buf))
tsk = self.compression_tasks.borrow_task()
tsk.init(tpe.InputBytes(block_msg.buf), tx_service.proxy,
enable_block_compression, min_tx_age_seconds)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compression_tasks.return_task(tsk)
raise message_conversion_error.btc_block_compression_error(
block_msg.block_hash(), e)
bx_block = tsk.bx_block()
block = memoryview(bx_block)
compressed_size = len(block)
original_size = len(block_msg.rawbytes())
block_hash = OntObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string()))
block_info = BlockInfo(
block_hash, tsk.short_ids(), compress_start_datetime,
datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000,
tsk.txn_count(),
tsk.compressed_block_hash().hex_string(),
tsk.prev_block_hash().hex_string(), original_size, compressed_size,
100 - float(compressed_size) / original_size * 100,
tsk.ignored_short_ids())
self.compression_tasks.return_task(tsk)
return block, block_info
def track_seen_short_ids(self, block_hash, short_ids: List[int]) -> None:
start_datetime = datetime.now()
super(ExtensionTransactionService,
self).track_seen_short_ids(block_hash, short_ids)
wrapped_block_hash = tpe.Sha256(
tpe.InputBytes(self._wrap_sha256(block_hash).binary))
proxy_start_datetime = datetime.now()
# TODO when refactoring add `block_hash` to proxy.track_seen_short_ids as first parameter and change ds type in cpp
result = self.proxy.track_seen_short_ids(wrapped_block_hash,
tpe.UIntList(short_ids))
removed_contents_size, dup_sids = result
self.update_removed_transactions(removed_contents_size, dup_sids)
logger_memory_cleanup.statistics({
"type":
"MemoryCleanup",
"event":
"ExtensionTransactionServiceTrackSeenSummary",
"seen_short_ids_count":
len(short_ids),
"total_content_size_removed":
removed_contents_size,
"total_duplicate_short_ids":
len(dup_sids),
"proxy_call_datetime":
proxy_start_datetime,
"data":
self.get_cache_state_json(),
"start_datetime":
start_datetime,
"block_hash":
repr(block_hash)
})
def process_transactions_message_from_node(
self,
msg,
min_tx_network_fee: int,
enable_transaction_validation: bool
) -> List[ProcessTransactionMessageFromNodeResult]:
opts = self.node.opts
msg_bytes = msg.rawbytes()
if (isinstance(self.node, OntGatewayNode) or isinstance(self.node, EthGatewayNode)) and \
opts.process_node_txs_in_extension:
ext_processing_results = memoryview(self.proxy.process_gateway_transaction_from_node(
tpe.InputBytes(msg_bytes),
min_tx_network_fee,
enable_transaction_validation
))
else:
return GatewayTransactionService.process_transactions_message_from_node(
self, msg, min_tx_network_fee, enable_transaction_validation
)
result = []
offset = 0
txs_count, = struct.unpack_from("<I", ext_processing_results, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
while offset < len(ext_processing_results):
seen, = struct.unpack_from("<?", ext_processing_results, offset)
offset += 1
tx_hash = Sha256Hash(ext_processing_results[offset:offset + crypto.SHA256_HASH_LEN])
offset += crypto.SHA256_HASH_LEN
tx_offset, = struct.unpack_from("<I", ext_processing_results, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
tx_len, = struct.unpack_from("<I", ext_processing_results, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
tx_validation_status, = struct.unpack_from("<I", ext_processing_results, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
tx_contents = msg_bytes[tx_offset:tx_offset + tx_len]
result.append(
ProcessTransactionMessageFromNodeResult(
seen,
tx_hash,
tx_contents,
TxMessage(message_hash=tx_hash, network_num=self.network_num, tx_val=tx_contents),
TxValidationStatus(tx_validation_status)
)
)
if not seen:
transaction_cache_key = self._tx_hash_to_cache_key(tx_hash)
self.set_transaction_contents_base(tx_hash, transaction_cache_key, False, 0, False, tx_contents, tx_len)
assert txs_count == len(result)
return result
def compact_block_to_bx_block(
self, compact_block: CompactBlockBtcMessage,
transaction_service: ExtensionTransactionService
) -> CompactBlockCompressionResult:
compress_start_datetime = datetime.utcnow()
tsk = self.compact_mapping_tasks.borrow_task()
tsk.init(tpe.InputBytes(compact_block.buf), transaction_service.proxy,
compact_block.magic())
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compact_mapping_tasks.return_task(tsk)
raise message_conversion_error.btc_compact_block_compression_error(
compact_block.block_hash(), e)
success = tsk.success()
recovered_item = ExtensionCompactBlockRecoveryData(
transaction_service, tsk)
block_info = BlockInfo(compact_block.block_hash(), [],
compress_start_datetime,
compress_start_datetime, 0, None, None, None,
len(compact_block.rawbytes()), None, None, [])
if success:
result = CompactBlockCompressionResult(
False, block_info, None, None, [],
create_recovered_transactions())
return self._extension_recovered_compact_block_to_bx_block(
result, recovered_item)
else:
recovery_index = self._last_recovery_idx
self._extension_recovered_items[recovery_index] = recovered_item
self._last_recovery_idx += 1
return CompactBlockCompressionResult(
False, block_info, None, recovery_index, tsk.missing_indices(),
create_recovered_transactions())
def run_encryption(plain, et=None):
if et is None:
et = tpe.EncryptionTask(len(plain))
plain_text = tpe.InputBytes(bytearray(plain))
et.init(plain_text)
task_pool_proxy.run_task(et)
return et
def process_gateway_transaction_from_bdn(
self, transaction_hash: Sha256Hash, short_id: int,
transaction_contents: Union[bytearray, memoryview],
is_compact: bool) -> TransactionFromBdnGatewayProcessingResult:
transaction_key = self.get_transaction_key(transaction_hash)
ext_result = self.proxy.process_gateway_transaction_from_bdn(
# pyre-fixme[6]: Expected `bxcommon.models.transaction_key.TransactionKey`
transaction_key.transaction_cache_key,
tpe.InputBytes(transaction_contents),
short_id,
is_compact)
result = TransactionFromBdnGatewayProcessingResult(
ext_result.get_ignore_seen(), ext_result.get_existing_short_id(),
ext_result.get_assigned_short_id(),
ext_result.get_existing_contents(), ext_result.get_set_contents())
if result.set_content:
has_short_id, previous_size = ext_result.get_set_contents_result()
self.set_transaction_contents_base_by_key(
transaction_key, has_short_id, previous_size, False,
transaction_contents, len(transaction_contents))
if result.assigned_short_id:
has_contents = result.existing_contents or result.set_content
self.assign_short_id_base_by_key(transaction_key, short_id,
has_contents, False)
return result
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: ExtensionTransactionService
) -> Tuple[memoryview, BlockInfo]:
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
self._default_block_size = max(self._default_block_size,
len(block_msg.rawbytes()))
tsk = self.compression_tasks.borrow_task()
tsk.init(tpe.InputBytes(block_msg.rawbytes()), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compression_tasks.return_task(tsk)
raise message_conversion_error.eth_block_compression_error(
block_msg.block_hash(), e)
bx_block = tsk.bx_block()
starting_offset = tsk.starting_offset()
block = memoryview(bx_block)[starting_offset:]
compressed_size = len(block)
original_size = len(block_msg.rawbytes()) - starting_offset
block_hash = block_msg.block_hash()
block_info = BlockInfo(
block_hash, tsk.short_ids(), compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000, tsk.txn_count(),
tsk.compressed_block_hash().hex_string(),
tsk.prev_block_hash().hex_string(), original_size, compressed_size,
100 - float(compressed_size) / original_size * 100)
self.compression_tasks.return_task(tsk)
return block, block_info
def process_tx_sync_message(
self,
msg: TxServiceSyncTxsMessage) -> List[TxSyncMsgProcessingItem]:
input_bytes = tpe.InputBytes(msg.rawbytes())
result_bytes = self.proxy.process_tx_sync_message(input_bytes)
assert result_bytes is not None
result_memory_view = memoryview(result_bytes)
result_items = []
offset = 0
txs_count, = struct.unpack_from("<L", result_memory_view, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
for _ in range(txs_count):
transaction_hash = Sha256Hash(
result_memory_view[offset:offset + crypto.SHA256_HASH_LEN])
transaction_key = self.get_transaction_key(transaction_hash)
offset += crypto.SHA256_HASH_LEN
content_len, = struct.unpack_from("<L", result_memory_view, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
short_id_count, = struct.unpack_from("<L", result_memory_view,
offset)
offset += constants.UL_INT_SIZE_IN_BYTES
if content_len > 0:
self.set_transaction_contents_base_by_key(
transaction_key, short_id_count > 0, 0, False, None,
content_len)
short_ids = []
for _ in range(short_id_count):
short_id, = struct.unpack_from("<L", result_memory_view,
offset)
offset += constants.UL_INT_SIZE_IN_BYTES
self.assign_short_id_base_by_key(transaction_key, short_id,
content_len > 0, False)
short_ids.append(short_id)
transaction_flags = []
for _ in range(short_id_count):
transaction_flag, = struct.unpack_from("<H",
result_memory_view,
offset)
offset += constants.TRANSACTION_FLAG_LEN
transaction_flags.append(TransactionFlag(transaction_flag))
result_items.append(
TxSyncMsgProcessingItem(transaction_hash, content_len,
short_ids, transaction_flags))
return result_items
def clean_block_transactions(
self, block_msg: BlockBtcMessage, transaction_service: TransactionService
) -> None:
start_datetime = datetime.utcnow()
start_time = time.time()
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len()
cleanup_task = self.block_cleanup_tasks.borrow_task()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task.init(tpe.InputBytes(block_msg.buf), tx_service.proxy)
init_time = time.time()
task_pool_proxy.run_task(cleanup_task)
task_run_time = time.time()
unknown_tx_hashes_count = len(cleanup_task.unknown_tx_hashes())
tx_property_fetch_time = time.time()
short_ids = cleanup_task.short_ids()
short_ids_fetch_time = time.time()
short_ids_count = len(short_ids)
tx_service.update_removed_transactions(cleanup_task.total_content_removed(), short_ids)
remove_from_tx_service_time = time.time()
# TODO : clean the short ids/transactions from the alarm queue after refactoring the transaction service
block_hash = block_msg.block_hash()
tx_service.on_block_cleaned_up(block_hash)
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len()
end_datetime = datetime.utcnow()
end_time = time.time()
logger.statistics(
{
"type": "BlockTransactionsCleanup",
"block_hash": repr(block_hash),
"unknown_tx_hashes_count": unknown_tx_hashes_count,
"short_ids_count": short_ids_count,
"block_transactions_count": cleanup_task.txn_count(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"task_init_time": init_time - start_time,
"task_run_time": task_run_time - init_time,
"tx_property_fetch_time": tx_property_fetch_time - task_run_time,
"short_ids_fetch_time": short_ids_fetch_time - tx_property_fetch_time,
"remove_from_tx_service_time": remove_from_tx_service_time - short_ids_fetch_time,
"duration": end_time - start_time,
"tx_hash_to_contents_len_before_cleanup": tx_hash_to_contents_len_before_cleanup,
"tx_hash_to_contents_len_after_cleanup": tx_hash_to_contents_len_after_cleanup,
}
)
self.block_cleanup_tasks.return_task(cleanup_task)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=short_ids,
unknown_tx_hashes=(
Sha256Hash(convert.hex_to_bytes(tx_hash.hex_string()))
for tx_hash in cleanup_task.unknown_tx_hashes()
)
)
def get_transactions(
self,
serialized_short_ids: Optional[bytearray] = None
) -> TransactionSearchResult:
"""
Fetches all transaction info for a set of short ids.
Short ids without a transaction entry will be omitted.
Function allows to pass a single short id or serialized list of short ids
:param serialized_short_ids: instance of get transactions message
:return: list of found and missing short ids
"""
assert serialized_short_ids is not None
input_bytes = tpe.InputBytes(serialized_short_ids)
result_bytes = self.proxy.get_transactions_by_short_ids(input_bytes)
assert result_bytes is not None
result_memory_view = memoryview(result_bytes)
found_txs_size, = struct.unpack_from("<L", result_memory_view, 0)
txs_bytes = result_memory_view[constants.UL_INT_SIZE_IN_BYTES:constants
.UL_INT_SIZE_IN_BYTES + found_txs_size]
found_txs_info = transactions_info_serializer \
.deserialize_transactions_info(txs_bytes)
missing_txs_info = []
offset = constants.UL_INT_SIZE_IN_BYTES + found_txs_size
missing_txs_count, = struct.unpack_from("<L", result_memory_view,
offset)
offset += constants.UL_INT_SIZE_IN_BYTES
for _ in range(missing_txs_count):
tx_sid, = struct.unpack_from("<L", result_memory_view, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
has_hash, = struct.unpack_from("<B", result_memory_view, offset)
offset += constants.UL_TINY_SIZE_IN_BYTES
if has_hash:
tx_hash = Sha256Hash(
result_memory_view[offset:offset + crypto.SHA256_HASH_LEN])
offset += crypto.SHA256_HASH_LEN
else:
tx_hash = None
missing_txs_info.append(TransactionInfo(tx_hash, None, tx_sid))
return TransactionSearchResult(found_txs_info, missing_txs_info)
def __init__(self, node, network_num):
super(ExtensionTransactionService, self).__init__(node, network_num)
self.proxy = tpe.TransactionService(
task_pool_proxy.get_pool_size(), node.opts.tx_mem_pool_bucket_size,
self._get_final_tx_confirmations_count())
raw_encoder = ObjectEncoder.raw_encoder()
self._tx_cache_key_to_short_ids = DefaultMapProxy(
self.proxy.tx_hash_to_short_ids(), raw_encoder, raw_encoder)
self._short_id_to_tx_cache_key = MapProxy(
self.proxy.short_id_to_tx_hash(), raw_encoder, raw_encoder)
content_encoder = ObjectEncoder(lambda buf_view: memoryview(buf_view),
lambda buf: tpe.InputBytes(buf))
self._tx_cache_key_to_contents = MapProxy(
self.proxy.tx_hash_to_contents(), raw_encoder, content_encoder)
self._tx_not_seen_in_blocks = self.proxy.tx_not_seen_in_blocks()
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.decompression_tasks.return_task(tsk)
header_info = btc_normal_message_converter.parse_bx_block_header(bx_block_msg, deque())
raise message_conversion_error.btc_block_decompression_error(header_info.block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = BtcObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string())
)
if tsk.success():
btc_block_msg = BlockBtcMessage(buf=memoryview(tsk.block_message()))
logger.debug(
"Successfully parsed block broadcast message. {} transactions "
"in block {}",
total_tx_count,
block_hash
)
else:
btc_block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}",
block_hash,
len(unknown_tx_sids),
len(unknown_tx_hashes),
total_tx_count
)
block_info = get_block_info(
bx_block_msg,
block_hash,
tsk.short_ids(),
decompress_start_datetime,
decompress_start_timestamp,
total_tx_count,
btc_block_msg
)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(btc_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
def set_transaction_contents_by_key(
self, transaction_key: TransactionKey,
transaction_contents: Union[bytearray, memoryview]):
"""
Adds transaction contents to transaction service cache with lookup key by transaction hash
:param transaction_key: transaction key object
:param transaction_contents: transaction contents bytes
"""
has_short_id, previous_size = self.proxy.set_transaction_contents(
# pyre-fixme[6]: Expected `tpe.Sha256` got `TransactionCacheKeyType`.
transaction_key.transaction_cache_key,
tpe.InputBytes(transaction_contents))
self.set_transaction_contents_base_by_key(transaction_key,
has_short_id, previous_size,
False, transaction_contents,
len(transaction_contents))
def process_txs_message(
self,
msg: TxsMessage
) -> Set[MissingTransactions]:
msg_bytes = msg.rawbytes()
missing_transactions: Set[MissingTransactions] = set()
result_buffer = memoryview(self.proxy.process_txs_msg(tpe.InputBytes(msg_bytes)))
# result_buffer is a bytearray that consists of
# number of missing transactions - 2 bytes
# a list of:
# short_id - 4 bytes
# Sha256Hash - 32 bytes
# content length that was added to transaction service - 4 bytes
offset = 0
missing_transactions_count, = struct.unpack_from("<H", result_buffer, offset)
offset += constants.UL_SHORT_SIZE_IN_BYTES
if missing_transactions_count == 0:
return missing_transactions
while offset < len(result_buffer):
short_id, = struct.unpack_from("<L", result_buffer, offset)
offset += constants.SID_LEN
transaction_hash = Sha256Hash(result_buffer[offset:offset + crypto.SHA256_HASH_LEN])
offset += crypto.SHA256_HASH_LEN
content_length, = struct.unpack_from("<L", result_buffer, offset)
offset += constants.UL_INT_SIZE_IN_BYTES
if content_length > 0:
self.set_transaction_contents_base(
transaction_hash,
transaction_hash,
True,
0,
False,
None,
content_length
)
missing_transactions.add(MissingTransactions(short_id, transaction_hash))
return missing_transactions
def create_recovered_transactions() -> VectorProxy[tpe.InputBytes, memoryview]:
encoder = ObjectEncoder(
lambda input_bytes: memoryview(input_bytes), lambda buf: tpe.InputBytes(buf)
)
return VectorProxy(tpe.TransactionList(), encoder)
def bx_block_to_block(self, bx_block_msg,
tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
block_hash = Sha256Hash(
convert.hex_to_bytes(tsk.block_hash().hex_string()))
self.decompression_tasks.return_task(tsk)
# TODO find a better solution
raise message_conversion_error.eth_block_decompression_error(
block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [
Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()
]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = Sha256Hash(
convert.hex_to_bytes(tsk.block_hash().hex_string()))
if tsk.success():
starting_offset = tsk.starting_offset()
block = memoryview(tsk.block_message())[starting_offset:]
block_msg = InternalEthBlockInfo(block)
content_size = len(block_msg.rawbytes())
logger.debug(
"Successfully parsed block broadcast message. {} "
"transactions in block {}", total_tx_count, block_hash)
bx_block_hash = convert.bytes_to_hex(
crypto.double_sha256(bx_block_msg))
compressed_size = len(bx_block_msg)
block_info = BlockInfo(
block_hash, tsk.short_ids(), decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
total_tx_count, bx_block_hash,
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()), compressed_size,
100 - float(compressed_size) / content_size * 100)
else:
block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}", block_hash, len(unknown_tx_sids),
len(unknown_tx_hashes), total_tx_count)
block_info = BlockInfo(
block_hash, tsk.short_ids(), decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000, None, None,
None, None, None, None)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(block_msg, block_info, unknown_tx_sids,
unknown_tx_hashes)
def on_block_cleaned_up(self, block_hash: Sha256Hash) -> None:
super(ExtensionTransactionService,
self).on_block_cleaned_up(block_hash)
wrapped_block_hash = tpe.Sha256(tpe.InputBytes(block_hash.binary))
self.proxy.on_block_cleaned_up(wrapped_block_hash)
def test_encode_sha256_tpe(self):
hash_ = "96e900d13d89eb12219e18ddc7aae8ec173a3cff196f09556b5d730df4a10732"
items = [tpe.Sha256(tpe.InputBytes(convert.hex_to_bytes(hash_)))]
js = json.dumps(items, cls=EnhancedJSONEncoder)
self.assertEqual(json.loads(js), [hash_])
