def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() self._default_block_size = max(self._default_block_size, len(block_msg.buf)) tsk = self.compression_tasks.borrow_task() tsk.init(tpe.InputBytes(block_msg.buf), tx_service.proxy, enable_block_compression, min_tx_age_seconds) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compression_tasks.return_task(tsk) raise message_conversion_error.btc_block_compression_error( block_msg.block_hash(), e) bx_block = tsk.bx_block() block = memoryview(bx_block) compressed_size = len(block) original_size = len(block_msg.rawbytes()) block_hash = OntObjectHash( binary=convert.hex_to_bytes(tsk.block_hash().hex_string())) block_info = BlockInfo( block_hash, tsk.short_ids(), compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tsk.txn_count(), tsk.compressed_block_hash().hex_string(), tsk.prev_block_hash().hex_string(), original_size, compressed_size, 100 - float(compressed_size) / original_size * 100, tsk.ignored_short_ids()) self.compression_tasks.return_task(tsk) return block, block_info
def contents_cleanup(transaction_service: TransactionService, block_confirmation_message: AbstractCleanupMessage, cleanup_tasks ): start_datetime = datetime.utcnow() start_time = time.time() tx_service = typing.cast(ExtensionTransactionService, transaction_service) cleanup_task = cleanup_tasks.borrow_task() cleanup_task.init(tpe.InputBytes(block_confirmation_message.buf), tx_service.proxy) task_pool_proxy.run_task(cleanup_task) short_ids = cleanup_task.short_ids() total_content_removed = cleanup_task.total_content_removed() tx_count = cleanup_task.tx_count() message_hash = block_confirmation_message.message_hash() tx_service.update_removed_transactions(total_content_removed, short_ids) transaction_service.on_block_cleaned_up(message_hash) end_datetime = datetime.utcnow() end_time = time.time() duration = end_time - start_time logger.statistics( { "type": "MemoryCleanup", "event": "CacheStateAfterBlockCleanup", "data": transaction_service.get_cache_state_json(), "start_datetime": start_datetime, "end_datetime": end_datetime, "duration": duration, "total_content_removed": total_content_removed, "tx_count": tx_count, "short_ids_count": len(short_ids), "message_hash": repr(message_hash), } ) cleanup_tasks.return_task(cleanup_task)
def compact_block_to_bx_block( self, compact_block: CompactBlockBtcMessage, transaction_service: ExtensionTransactionService ) -> CompactBlockCompressionResult: compress_start_datetime = datetime.utcnow() tsk = self.compact_mapping_tasks.borrow_task() tsk.init(tpe.InputBytes(compact_block.buf), transaction_service.proxy, compact_block.magic()) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compact_mapping_tasks.return_task(tsk) raise message_conversion_error.btc_compact_block_compression_error( compact_block.block_hash(), e) success = tsk.success() recovered_item = ExtensionCompactBlockRecoveryData( transaction_service, tsk) block_info = BlockInfo(compact_block.block_hash(), [], compress_start_datetime, compress_start_datetime, 0, None, None, None, len(compact_block.rawbytes()), None, None, []) if success: result = CompactBlockCompressionResult( False, block_info, None, None, [], create_recovered_transactions()) return self._extension_recovered_compact_block_to_bx_block( result, recovered_item) else: recovery_index = self._last_recovery_idx self._extension_recovered_items[recovery_index] = recovered_item self._last_recovery_idx += 1 return CompactBlockCompressionResult( False, block_info, None, recovery_index, tsk.missing_indices(), create_recovered_transactions())
def run_encryption(plain, et=None): if et is None: et = tpe.EncryptionTask(len(plain)) plain_text = tpe.InputBytes(bytearray(plain)) et.init(plain_text) task_pool_proxy.run_task(et) return et
def block_to_bx_block( self, block_msg: InternalEthBlockInfo, tx_service: ExtensionTransactionService ) -> Tuple[memoryview, BlockInfo]: compress_start_datetime = datetime.datetime.utcnow() compress_start_timestamp = time.time() self._default_block_size = max(self._default_block_size, len(block_msg.rawbytes())) tsk = self.compression_tasks.borrow_task() tsk.init(tpe.InputBytes(block_msg.rawbytes()), tx_service.proxy) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compression_tasks.return_task(tsk) raise message_conversion_error.eth_block_compression_error( block_msg.block_hash(), e) bx_block = tsk.bx_block() starting_offset = tsk.starting_offset() block = memoryview(bx_block)[starting_offset:] compressed_size = len(block) original_size = len(block_msg.rawbytes()) - starting_offset block_hash = block_msg.block_hash() block_info = BlockInfo( block_hash, tsk.short_ids(), compress_start_datetime, datetime.datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tsk.txn_count(), tsk.compressed_block_hash().hex_string(), tsk.prev_block_hash().hex_string(), original_size, compressed_size, 100 - float(compressed_size) / original_size * 100) self.compression_tasks.return_task(tsk) return block, block_info
def clean_block_transactions( self, block_msg: BlockBtcMessage, transaction_service: TransactionService ) -> None: start_datetime = datetime.utcnow() start_time = time.time() tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len() cleanup_task = self.block_cleanup_tasks.borrow_task() tx_service = typing.cast(ExtensionTransactionService, transaction_service) cleanup_task.init(tpe.InputBytes(block_msg.buf), tx_service.proxy) init_time = time.time() task_pool_proxy.run_task(cleanup_task) task_run_time = time.time() unknown_tx_hashes_count = len(cleanup_task.unknown_tx_hashes()) tx_property_fetch_time = time.time() short_ids = cleanup_task.short_ids() short_ids_fetch_time = time.time() short_ids_count = len(short_ids) tx_service.update_removed_transactions(cleanup_task.total_content_removed(), short_ids) remove_from_tx_service_time = time.time() # TODO : clean the short ids/transactions from the alarm queue after refactoring the transaction service block_hash = block_msg.block_hash() tx_service.on_block_cleaned_up(block_hash) tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len() end_datetime = datetime.utcnow() end_time = time.time() logger.statistics( { "type": "BlockTransactionsCleanup", "block_hash": repr(block_hash), "unknown_tx_hashes_count": unknown_tx_hashes_count, "short_ids_count": short_ids_count, "block_transactions_count": cleanup_task.txn_count(), "start_datetime": start_datetime, "end_datetime": end_datetime, "task_init_time": init_time - start_time, "task_run_time": task_run_time - init_time, "tx_property_fetch_time": tx_property_fetch_time - task_run_time, "short_ids_fetch_time": short_ids_fetch_time - tx_property_fetch_time, "remove_from_tx_service_time": remove_from_tx_service_time - short_ids_fetch_time, "duration": end_time - start_time, "tx_hash_to_contents_len_before_cleanup": tx_hash_to_contents_len_before_cleanup, "tx_hash_to_contents_len_after_cleanup": tx_hash_to_contents_len_after_cleanup, } ) self.block_cleanup_tasks.return_task(cleanup_task) self._block_hash_marked_for_cleanup.discard(block_hash) self.node.post_block_cleanup_tasks( block_hash=block_hash, short_ids=short_ids, unknown_tx_hashes=( Sha256Hash(convert.hex_to_bytes(tx_hash.hex_string())) for tx_hash in cleanup_task.unknown_tx_hashes() ) )
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult: decompress_start_datetime = datetime.utcnow() decompress_start_timestamp = time.time() tsk = self.decompression_tasks.borrow_task() tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.decompression_tasks.return_task(tsk) header_info = btc_normal_message_converter.parse_bx_block_header(bx_block_msg, deque()) raise message_conversion_error.btc_block_decompression_error(header_info.block_hash, e) total_tx_count = tsk.tx_count() unknown_tx_hashes = [Sha256Hash(bytearray(unknown_tx_hash.binary())) for unknown_tx_hash in tsk.unknown_tx_hashes()] unknown_tx_sids = tsk.unknown_tx_sids() block_hash = BtcObjectHash( binary=convert.hex_to_bytes(tsk.block_hash().hex_string()) ) if tsk.success(): btc_block_msg = BlockBtcMessage(buf=memoryview(tsk.block_message())) logger.debug( "Successfully parsed block broadcast message. {} transactions " "in block {}", total_tx_count, block_hash ) else: btc_block_msg = None logger.debug( "Block recovery needed for {}. Missing {} sids, {} tx hashes. " "Total txs in block: {}", block_hash, len(unknown_tx_sids), len(unknown_tx_hashes), total_tx_count ) block_info = get_block_info( bx_block_msg, block_hash, tsk.short_ids(), decompress_start_datetime, decompress_start_timestamp, total_tx_count, btc_block_msg ) self.decompression_tasks.return_task(tsk) return BlockDecompressionResult(btc_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
def _extension_recovered_compact_block_to_bx_block( self, mapping_result: CompactBlockCompressionResult, recovery_item: ExtensionCompactBlockRecoveryData ) -> CompactBlockCompressionResult: mapping_task = recovery_item.mapping_task compression_task: tpe.BtcCompactBlockCompressionTask = mapping_task.compression_task() # pyre-fixme[16]: `List` has no attribute `vector`. compression_task.add_recovered_transactions(mapping_result.recovered_transactions.vector) mapping_block_info = mapping_result.block_info try: task_pool_proxy.run_task(compression_task) except tpe.AggregatedException as e: self.compact_mapping_tasks.return_task(mapping_task) # pyre-fixme[16]: `Optional` has no attribute `block_hash`. raise message_conversion_error.btc_compact_block_compression_error(mapping_block_info.block_hash, e) bx_block = memoryview(compression_task.bx_block()) block_hash = mapping_block_info.block_hash txn_count = compression_task.txn_count() compressed_block_hash = compression_task.compressed_block_hash().hex_string() prev_block_hash = compression_task.prev_block_hash().hex_string() short_ids = compression_task.short_ids() compress_end_datetime = datetime.utcnow() # pyre-fixme[16]: `Optional` has no attribute `start_datetime`. compress_start_datetime = mapping_block_info.start_datetime # pyre-fixme[16]: `Optional` has no attribute `original_size`. original_size = mapping_block_info.original_size compressed_size = len(bx_block) block_info = BlockInfo( block_hash, short_ids, compress_start_datetime, compress_end_datetime, (compress_end_datetime - compress_start_datetime).total_seconds() * 1000, txn_count, compressed_block_hash, prev_block_hash, original_size, compressed_size, 100 - float(compressed_size) / original_size * 100 ) self.compact_mapping_tasks.return_task(mapping_task) return CompactBlockCompressionResult(True, block_info, bx_block, None, [], [])
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult: decompress_start_datetime = datetime.datetime.utcnow() decompress_start_timestamp = time.time() tsk = self.decompression_tasks.borrow_task() tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: block_hash = Sha256Hash( convert.hex_to_bytes(tsk.block_hash().hex_string())) self.decompression_tasks.return_task(tsk) # TODO find a better solution raise message_conversion_error.eth_block_decompression_error( block_hash, e) total_tx_count = tsk.tx_count() unknown_tx_hashes = [ Sha256Hash(bytearray(unknown_tx_hash.binary())) for unknown_tx_hash in tsk.unknown_tx_hashes() ] unknown_tx_sids = tsk.unknown_tx_sids() block_hash = Sha256Hash( convert.hex_to_bytes(tsk.block_hash().hex_string())) if tsk.success(): starting_offset = tsk.starting_offset() block = memoryview(tsk.block_message())[starting_offset:] block_msg = InternalEthBlockInfo(block) content_size = len(block_msg.rawbytes()) logger.debug( "Successfully parsed block broadcast message. {} " "transactions in block {}", total_tx_count, block_hash) bx_block_hash = convert.bytes_to_hex( crypto.double_sha256(bx_block_msg)) compressed_size = len(bx_block_msg) block_info = BlockInfo( block_hash, tsk.short_ids(), decompress_start_datetime, datetime.datetime.utcnow(), (time.time() - decompress_start_timestamp) * 1000, total_tx_count, bx_block_hash, convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), compressed_size, 100 - float(compressed_size) / content_size * 100) else: block_msg = None logger.debug( "Block recovery needed for {}. Missing {} sids, {} tx hashes. " "Total txs in block: {}", block_hash, len(unknown_tx_sids), len(unknown_tx_hashes), total_tx_count) block_info = BlockInfo( block_hash, tsk.short_ids(), decompress_start_datetime, datetime.datetime.utcnow(), (time.time() - decompress_start_timestamp) * 1000, None, None, None, None, None, None) self.decompression_tasks.return_task(tsk) return BlockDecompressionResult(block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)