def create_txs_service_msg( transaction_service: TransactionService, tx_service_snap: List[Sha256Hash], sync_tx_content: bool = True) -> List[TxContentShortIds]: task_start = time.time() txs_content_short_ids: List[TxContentShortIds] = [] txs_msg_len = 0 while tx_service_snap: transaction_key = transaction_service.get_transaction_key( tx_service_snap.pop()) short_ids = list( transaction_service.get_short_ids_by_key(transaction_key)) if sync_tx_content: tx_content = transaction_service.get_transaction_by_key( transaction_key) else: tx_content = bytearray(0) # TODO: evaluate short id quota type flag value short_id_flags = [ transaction_service.get_short_id_transaction_type(short_id) for short_id in short_ids ] tx_content_short_ids: TxContentShortIds = TxContentShortIds( transaction_key.transaction_hash, tx_content, short_ids, short_id_flags) txs_msg_len += txs_serializer.get_serialized_tx_content_short_ids_bytes_len( tx_content_short_ids) txs_content_short_ids.append(tx_content_short_ids) if txs_msg_len >= constants.TXS_MSG_SIZE or time.time( ) - task_start > constants.TXS_SYNC_TASK_DURATION: break return txs_content_short_ids
def create_txs_service_msg_from_time( transaction_service: TransactionService, start_time: float = 0, sync_tx_content: bool = True, snapshot_cache_keys: Optional[Set[TransactionCacheKeyType]] = None ) -> Tuple[List[TxContentShortIds], float, bool, Set[TransactionCacheKeyType]]: task_start = time.time() txs_content_short_ids: List[TxContentShortIds] = [] txs_msg_len = 0 if snapshot_cache_keys is None: snapshot_cache_keys = set() done = False timestamp = start_time expire_short_ids = [] for short_id, timestamp in transaction_service._tx_assignment_expire_queue.queue.items( ): if timestamp > start_time: cache_key = transaction_service._short_id_to_tx_cache_key.get( short_id, None) if cache_key is not None: transaction_key = transaction_service.get_transaction_key( None, cache_key) if cache_key not in snapshot_cache_keys: snapshot_cache_keys.add( transaction_key.transaction_cache_key) short_ids = list( transaction_service._tx_cache_key_to_short_ids[ transaction_key.transaction_cache_key]) if sync_tx_content and transaction_service.has_transaction_contents_by_key( transaction_key): tx_content = transaction_service._tx_cache_key_to_contents[ transaction_key.transaction_cache_key] else: tx_content = bytearray(0) short_id_flags = [ transaction_service.get_short_id_transaction_type( short_id) for short_id in short_ids ] tx_content_short_ids: TxContentShortIds = TxContentShortIds( transaction_key.transaction_hash, tx_content, short_ids, short_id_flags) txs_msg_len += txs_serializer.get_serialized_tx_content_short_ids_bytes_len( tx_content_short_ids) txs_content_short_ids.append(tx_content_short_ids) if txs_msg_len >= constants.TXS_MSG_SIZE or ( time.time() - task_start > constants.TXS_SYNC_TASK_DURATION): break else: expire_short_ids.append(short_id) else: done = True for short_id in expire_short_ids: transaction_service._tx_assignment_expire_queue.remove(short_id) return txs_content_short_ids, timestamp, done, snapshot_cache_keys
def contents_cleanup( transaction_service: TransactionService, block_confirmation_message: AbstractCleanupMessage ): message_hash = block_confirmation_message.message_hash() for short_id in block_confirmation_message.short_ids(): transaction_service.remove_transaction_by_short_id( short_id, remove_related_short_ids=True, force=True, removal_reason=TxRemovalReason.BLOCK_CLEANUP ) for tx_hash in block_confirmation_message.transaction_hashes(): transaction_service.remove_transaction_by_key(transaction_service.get_transaction_key(tx_hash), force=True) transaction_service.on_block_cleaned_up(message_hash) logger.statistics( { "type": "MemoryCleanup", "event": "CacheStateAfterBlockCleanup", "message_hash": repr(message_hash), "data": transaction_service.get_cache_state_json() } )
def block_to_bx_block( self, block_msg: InternalEthBlockInfo, tx_service: TransactionService, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: """ Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids The code is optimized and does not make copies of bytes :param block_msg: Ethereum new block message :param tx_service: Transactions service :param enable_block_compression :param min_tx_age_seconds :return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash) """ compress_start_datetime = datetime.datetime.utcnow() compress_start_timestamp = time.time() txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message( block_msg) used_short_ids = [] # creating transactions content content_size = 0 buf = deque() ignored_sids = [] tx_start_index = 0 tx_count = 0 original_size = len(block_msg.rawbytes()) max_timestamp_for_compression = time.time() - min_tx_age_seconds while True: if tx_start_index >= len(txs_bytes): break _, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix( txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length] tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes) tx_hash = Sha256Hash(tx_hash_bytes) tx_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(tx_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time( short_id) if short_id <= constants.NULL_TX_SID or \ not enable_block_compression or short_id_assign_time > max_timestamp_for_compression: if short_id > constants.NULL_TX_SID: ignored_sids.append(short_id) is_full_tx_bytes = rlp_utils.encode_int(1) tx_content_bytes = tx_bytes else: is_full_tx_bytes = rlp_utils.encode_int(0) used_short_ids.append(short_id) tx_content_bytes = bytes() tx_content_prefix = rlp_utils.get_length_prefix_str( len(tx_content_bytes)) short_tx_content_size = len(is_full_tx_bytes) + len( tx_content_prefix) + len(tx_content_bytes) short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list( short_tx_content_size) buf.append(short_tx_content_prefix_bytes) buf.append(is_full_tx_bytes) buf.append(tx_content_prefix) buf.append(tx_content_bytes) content_size += len( short_tx_content_prefix_bytes) + short_tx_content_size tx_start_index = tx_item_start + tx_item_length tx_count += 1 list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(list_of_txs_prefix_bytes) content_size += len(list_of_txs_prefix_bytes) buf.appendleft(block_hdr_full_bytes) content_size += len(block_hdr_full_bytes) buf.append(remaining_bytes) content_size += len(remaining_bytes) compact_block_msg_prefix = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(compact_block_msg_prefix) content_size += len(compact_block_msg_prefix) block = finalize_block_bytes(buf, content_size, used_short_ids) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo(block_msg.block_hash(), used_short_ids, compress_start_datetime, datetime.datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tx_count, bx_block_hash, convert.bytes_to_hex(prev_block_bytes), original_size, content_size, 100 - float(content_size) / original_size * 100, ignored_sids) return memoryview(block), block_info
def block_to_bx_block( self, block_msg: OntConsensusMessage, tx_service: TransactionService, enable_block_compression: bool, min_tx_age_seconds: float ) -> Tuple[memoryview, BlockInfo]: """ Pack an Ontology consensus message's transactions into a bloXroute block. """ consensus_msg = block_msg compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] ignored_sids = [] original_size = len(consensus_msg.rawbytes()) consensus_payload_header = consensus_msg.consensus_payload_header() consensus_payload_header_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_payload_header_len, 0, len(consensus_payload_header)) size += ont_constants.ONT_INT_LEN buf.append(consensus_payload_header_len) size += len(consensus_payload_header) buf.append(consensus_payload_header) consensus_data_type = bytearray(ont_constants.ONT_CHAR_LEN) struct.pack_into("<B", consensus_data_type, 0, consensus_msg.consensus_data_type()) size += ont_constants.ONT_CHAR_LEN buf.append(consensus_data_type) consensus_data_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_data_len, 0, consensus_msg.consensus_data_len()) size += ont_constants.ONT_INT_LEN buf.append(consensus_data_len) block_start_len = consensus_msg.block_start_len_memoryview() txn_header = consensus_msg.txn_header() block_start_len_and_txn_header_total_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", block_start_len_and_txn_header_total_len, 0, len(block_start_len) + len(txn_header)) size += ont_constants.ONT_INT_LEN buf.append(block_start_len_and_txn_header_total_len) size += len(block_start_len) buf.append(block_start_len) size += len(txn_header) buf.append(txn_header) max_timestamp_for_compression = time.time() - min_tx_age_seconds for tx in consensus_msg.txns(): tx_hash, _ = ont_messages_util.get_txid(tx) transaction_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(transaction_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time(short_id) if short_id == constants.NULL_TX_SID or \ not enable_block_compression or \ short_id_assign_time > max_timestamp_for_compression: if short_id != constants.NULL_TX_SIDS: ignored_sids.append(short_id) buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 # Prepend owner and signature, consensus payload tail, tx count and block hash to bx_block owner_and_signature = consensus_msg.owner_and_signature() owner_and_signature_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", owner_and_signature_len, 0, len(owner_and_signature)) size += len(owner_and_signature) buf.appendleft(owner_and_signature) size += ont_constants.ONT_INT_LEN buf.appendleft(owner_and_signature_len) payload_tail = consensus_msg.payload_tail() payload_tail_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", payload_tail_len, 0, len(payload_tail)) size += len(payload_tail) buf.appendleft(payload_tail) size += ont_constants.ONT_INT_LEN buf.appendleft(payload_tail_len) txn_count = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", txn_count, 0, consensus_msg.txn_count()) size += ont_constants.ONT_INT_LEN buf.appendleft(txn_count) block_hash = consensus_msg.block_hash().binary size += ont_constants.ONT_HASH_LEN buf.appendleft(block_hash) is_consensus_msg_buf = struct.pack("?", True) buf.appendleft(is_consensus_msg_buf) size += 1 block = finalize_block_bytes(buf, size, short_ids) prev_block_hash = convert.bytes_to_hex(consensus_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo( consensus_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, consensus_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100, ignored_sids ) return memoryview(block), block_info
def compact_block_to_bx_block( self, compact_block: CompactBlockBtcMessage, transaction_service: TransactionService ) -> CompactBlockCompressionResult: """ Handle decompression of Bitcoin compact block. Decompression converts compact block message to full block message. """ compress_start_datetime = datetime.utcnow() block_header = compact_block.block_header() sha256_hash = hashlib.sha256() sha256_hash.update(block_header) sha256_hash.update(compact_block.short_nonce_buf()) hex_digest = sha256_hash.digest() key = hex_digest[0:16] short_ids = compact_block.short_ids() short_id_to_tx_contents = {} for tx_hash in transaction_service.iter_transaction_hashes(): transaction_key = transaction_service.get_transaction_key(tx_hash) tx_hash_binary = tx_hash.binary[::-1] tx_short_id = compute_short_id(key, tx_hash_binary) if tx_short_id in short_ids: tx_content = transaction_service.get_transaction_by_key( transaction_key) if tx_content is None: logger.debug( "Hash {} is known by transactions service but content is missing.", tx_hash) else: short_id_to_tx_contents[tx_short_id] = tx_content if len(short_id_to_tx_contents) == len(short_ids): break block_transactions = [] missing_transactions_indices = [] pre_filled_transactions = compact_block.pre_filled_transactions() total_txs_count = len(pre_filled_transactions) + len(short_ids) size = 0 block_msg_parts = deque() block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size short_ids_iter = iter(short_ids.keys()) for index in range(total_txs_count): if index not in pre_filled_transactions: short_id = next(short_ids_iter) if short_id in short_id_to_tx_contents: short_tx = short_id_to_tx_contents[short_id] block_msg_parts.append(short_tx) block_transactions.append(short_tx) size += len(short_tx) else: missing_transactions_indices.append(index) block_transactions.append(None) else: pre_filled_transaction = pre_filled_transactions[index] block_msg_parts.append(pre_filled_transaction) block_transactions.append(pre_filled_transaction) size += len(pre_filled_transaction) recovered_item = CompactBlockRecoveryData(block_transactions, block_header, compact_block.magic(), transaction_service) block_info = BlockInfo(compact_block.block_hash(), [], compress_start_datetime, compress_start_datetime, 0, None, None, None, len(compact_block.rawbytes()), None, None, []) if len(missing_transactions_indices) > 0: recovery_index = self._last_recovery_idx self._last_recovery_idx += 1 self._recovery_items[ recovery_index] = recovered_item # pyre-ignore return CompactBlockCompressionResult(False, block_info, None, recovery_index, missing_transactions_indices, []) result = CompactBlockCompressionResult(False, block_info, None, None, [], []) return self._recovered_compact_block_to_bx_block( result, recovered_item)
class SyncTxServiceTest(MessageFactoryTestCase): NETWORK_NUM = 12345 def setUp(self) -> None: self.node = MockNode(helpers.get_common_opts(1234)) self.network_num = 4 self.transaction_service = TransactionService(self.node, self.network_num) def _add_transactions(self, tx_count, tx_size, short_id_offset=0): short_id = short_id_offset for i in range(int(tx_count)): tx_hash = Sha256Hash( binary=helpers.generate_bytearray(crypto.SHA256_HASH_LEN)) tx_content = helpers.generate_bytearray(tx_size) transaction_key = self.transaction_service.get_transaction_key( tx_hash) self.transaction_service.set_transaction_contents_by_key( transaction_key, tx_content) for _ in range(random.randrange(1, 10)): short_id += 1 self.transaction_service.assign_short_id(tx_hash, short_id) self.transaction_service.set_short_id_transaction_type( short_id, TransactionFlag.PAID_TX) if short_id % 7 < 2: self.transaction_service._short_id_to_tx_cache_key.pop( short_id, None) @skip("We don't sync tx service using time") def test_create_tx_service_msg(self): self._add_transactions(100000, tx_size=50) done = False msgs = [] timestamp = 0 snapshot_cache_keys = None total_time = 0 total_txs = 0 while not done: start_ = time.time() txs_content_short_ids, timestamp, done, snapshot_cache_keys = \ tx_sync_service_helpers.create_txs_service_msg_from_time( self.transaction_service, timestamp, False, snapshot_cache_keys ) duration = time.time() - start_ total_time += duration total_txs += len(txs_content_short_ids) msgs.append(txs_content_short_ids) # print(f"txs:{len(txs_content_short_ids)}, time: {duration}") print(f"total - msgs:{len(msgs)}, time:{total_time}") msg_build_time = 0 for txs_content_short_ids in msgs: start_ = time.time() msg = TxServiceSyncTxsMessage(self.network_num, txs_content_short_ids) duration = time.time() - start_ msg_build_time += duration print(f"total - message creation time: {msg_build_time}") self.assertTrue(True) @skip("We don't sync tx service using snapshot") def test_create_tx_service_msg_snapshot(self): self._add_transactions(100000, tx_size=50) total_time = 0 start_ = time.time() snapshot = self.transaction_service.get_snapshot(0) duration = time.time() - start_ print(f"snapshot creation: {duration}") total_time += duration msgs = [] while snapshot: start_ = time.time() txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg( self.transaction_service, snapshot, sync_tx_content=False) msgs.append(txs_content_short_ids) duration = time.time() - start_ total_time += duration # print(len(txs_content_short_ids), duration) print(f"total time: {total_time}") @skip("We don't sync tx service using snapshot") def test_create_tx_service_msg_snapshot_by_time(self): self._add_transactions(100000, tx_size=50) total_time = 0 start_ = time.time() snapshot = self.transaction_service.get_snapshot(1.0) duration = time.time() - start_ print(f"snapshot creation: {duration}") total_time += duration msgs = [] while snapshot: start_ = time.time() txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg( self.transaction_service, snapshot, sync_tx_content=False) msgs.append(txs_content_short_ids) duration = time.time() - start_ total_time += duration # print(len(txs_content_short_ids), duration) print(f"total time: {total_time}")
class SyncTxServiceTest(MessageFactoryTestCase): NETWORK_NUM = 12345 def setUp(self) -> None: self.node = MockNode(helpers.get_common_opts(1234)) self.network_num = 4 self.transaction_service = TransactionService(self.node, self.network_num) def get_message_factory(self): return bloxroute_message_factory def test_create_message_success_tx_service_sync_txs_msg(self): self._test_create_msg_success_tx_service_sync_with_tx_content_count( 100) def test_create_message_success_tx_service_sync_txs_msg_with_exceeded_buf( self): self._test_create_msg_success_tx_service_sync_with_tx_content_count( 1000) def _test_create_msg_success_tx_service_sync_with_tx_content_count( self, tx_content_count, sync_tx_content=True): short_ids = [ list(range(1, 6)), list(range(11, 15)), list(range(53, 250)), [31], list(range(41, 48)), [51, 52] ] transaction_hashes = list( map(crypto.double_sha256, map(bytes, short_ids))) for i in range(len(short_ids)): transaction_content = bytearray(tx_content_count) transaction_content[:32] = transaction_hashes[i] transaction_key = self.transaction_service.get_transaction_key( transaction_hashes[i]) self.transaction_service.set_transaction_contents_by_key( transaction_key, transaction_content) for short_id in short_ids[i]: self.transaction_service.assign_short_id_by_key( transaction_key, short_id) # Six blocks received after for i in range(len(short_ids)): self.transaction_service.track_seen_short_ids( Sha256Hash(helpers.generate_bytearray(32)), short_ids[i]) tx_service_snap = self.transaction_service.get_snapshot() txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg( self.transaction_service, tx_service_snap, sync_tx_content) if txs_content_short_ids: self._send_tx_msg(txs_content_short_ids, transaction_hashes) def _send_tx_msg(self, txs_content_short_ids, transaction_hashes): tx_service_sync_txs_msg: TxServiceSyncTxsMessage = \ self.create_message_successfully( TxServiceSyncTxsMessage( self.NETWORK_NUM, txs_content_short_ids ), TxServiceSyncTxsMessage ) self.assertEqual(self.NETWORK_NUM, tx_service_sync_txs_msg.network_num()) self.assertEqual(len(txs_content_short_ids), tx_service_sync_txs_msg.tx_count()) tx_service_txs_content_short_ids = tx_service_sync_txs_msg.txs_content_short_ids( ) tx_contents = [ self.transaction_service.get_transaction(short_id).contents for tx_content_short_id in tx_service_txs_content_short_ids for short_id in tx_content_short_id.short_ids ] for tx_content_short_id in tx_service_txs_content_short_ids: self.assertIn(bytearray(tx_content_short_id.tx_hash), transaction_hashes) self.assertIn(bytearray(tx_content_short_id.tx_content), tx_contents) self.assertEqual( tx_content_short_id.short_ids, list( self.transaction_service.get_short_ids_by_key( self.transaction_service.get_transaction_key( tx_content_short_id.tx_hash))))