def parse_bx_block_transactions_and_msg_tail(block_hash: Sha256Hash, bx_block: memoryview, offset: int, short_ids: List[int], block_offsets: BlockOffsets, tx_service: TransactionService, block_pieces: Deque[Union[bytearray, memoryview]]) -> \ Tuple[List[int], List[Sha256Hash], int]: has_missing, unknown_tx_sids, unknown_tx_hashes = tx_service.get_missing_transactions(short_ids) if has_missing: return unknown_tx_sids, unknown_tx_hashes, offset short_tx_index = 0 output_offset = offset while offset < block_offsets.short_id_offset: if bx_block[offset] == ont_constants.ONT_SHORT_ID_INDICATOR: try: sid = short_ids[short_tx_index] except IndexError: raise message_conversion_error.btc_block_decompression_error( block_hash, f"Message is improperly formatted, short id index ({short_tx_index}) " f"exceeded its array bounds (size: {len(short_ids)})" ) tx_hash, tx, _ = tx_service.get_transaction(sid) offset += ont_constants.ONT_SHORT_ID_INDICATOR_LENGTH short_tx_index += 1 else: tx_size = ont_messages_util.get_next_tx_size(bx_block, offset) tx = bx_block[offset:offset + tx_size] offset += tx_size assert tx is not None block_pieces.append(tx) output_offset += len(tx) # Add consensus payload tail and owner and signature to block_pieces offset = block_offsets.block_begin_offset + ont_constants.ONT_HASH_LEN + ont_constants.ONT_INT_LEN + 1 payload_tail_len, = struct.unpack_from("<L", bx_block, offset) offset += ont_constants.ONT_INT_LEN block_pieces.append(bx_block[offset: offset + payload_tail_len]) offset += payload_tail_len owner_and_signature_len, = struct.unpack_from("<L", bx_block, offset) offset += ont_constants.ONT_INT_LEN block_pieces.append(bx_block[offset: offset + owner_and_signature_len]) offset += owner_and_signature_len return unknown_tx_sids, unknown_tx_hashes, output_offset
def parse_bx_block_transactions( block_hash: Sha256Hash, bx_block: memoryview, offset: int, short_ids: List[int], block_offsets: BlockOffsets, tx_service: TransactionService, block_pieces: Deque[Union[bytearray, memoryview]] ) -> Tuple[List[int], List[Sha256Hash], int]: has_missing, unknown_tx_sids, unknown_tx_hashes = \ tx_service.get_missing_transactions(short_ids) if has_missing: return unknown_tx_sids, unknown_tx_hashes, offset short_tx_index = 0 output_offset = offset while offset < block_offsets.short_id_offset: if bx_block[offset] == btc_constants.BTC_SHORT_ID_INDICATOR: try: sid = short_ids[short_tx_index] except IndexError: raise message_conversion_error.btc_block_decompression_error( block_hash, f"Message is improperly formatted, short id index ({short_tx_index}) " f"exceeded its array bounds (size: {len(short_ids)})" ) tx_hash, tx, _ = tx_service.get_transaction(sid) offset += btc_constants.BTC_SHORT_ID_INDICATOR_LENGTH short_tx_index += 1 else: tx_size = btc_messages_util.get_next_tx_size(bx_block, offset) tx = bx_block[offset:offset + tx_size] offset += tx_size # pyre-fixme[6]: Expected `Union[bytearray, memoryview]` for 1st param but # got `Optional[Union[bytearray, memoryview]]`. block_pieces.append(tx) # pyre-fixme[6]: Expected `Sized` for 1st param but got # `Optional[Union[bytearray, memoryview]]`. output_offset += len(tx) return unknown_tx_sids, unknown_tx_hashes, output_offset
def _log_compressed_block_debug_info_eth(transaction_service: TransactionService, block_msg_bytes: Union[memoryview, bytearray]): is_block_relay = transaction_service.node.NODE_TYPE == NodeType.RELAY_BLOCK block_hash, short_ids, txs_bytes = _parse_block_eth(block_msg_bytes) # parse statistics variables short_tx_index = 0 tx_start_index = 0 tx_index_in_block = 0 txs_info = [] missing_short_ids = [] while True: if tx_start_index >= len(txs_bytes): break short_id = 0 has_contents = False assignmnet_time = 0 _, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len] is_full_tx_start = 0 is_full_tx, is_full_tx_len, = rlp_utils.decode_int(tx_bytes, is_full_tx_start) _, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix( tx_bytes, is_full_tx_start + is_full_tx_len) tx_content_bytes = tx_bytes[tx_content_start:tx_content_start + tx_content_len] if is_full_tx: tx_hash = Sha256Hash(eth_common_utils.keccak_hash(tx_content_bytes)) else: short_id = short_ids[short_tx_index] tx_hash, tx_bytes, _ = transaction_service.get_transaction(short_id) has_contents = tx_bytes is not None if tx_hash is not None: assignmnet_time = transaction_service.get_short_id_assign_time(short_id) short_tx_index += 1 if is_block_relay: txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash)) else: txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash, has_contents, assignmnet_time)) tx_index_in_block += 1 tx_start_index = tx_itm_start + tx_itm_len if not is_full_tx and not has_contents: missing_short_ids.append(short_id) if is_block_relay: log_message = \ "Block content (from block relay) {} from (index, is compressed, short id, hash is full) : {}" else: log_message = \ "Block content (full) {} (index, compressed, short id, hash, has contents, assignment time) : {}" logger.debug( log_message, block_hash, ",".join(str(tx_info) for tx_info in txs_info) ) node_type = transaction_service.node.NODE_TYPE assert node_type is not None log_can_decompress_block(node_type, block_hash, missing_short_ids)
class SyncTxServiceTest(MessageFactoryTestCase): NETWORK_NUM = 12345 def setUp(self) -> None: self.node = MockNode(helpers.get_common_opts(1234)) self.network_num = 4 self.transaction_service = TransactionService(self.node, self.network_num) def get_message_factory(self): return bloxroute_message_factory def test_create_message_success_tx_service_sync_txs_msg(self): self._test_create_msg_success_tx_service_sync_with_tx_content_count( 100) def test_create_message_success_tx_service_sync_txs_msg_with_exceeded_buf( self): self._test_create_msg_success_tx_service_sync_with_tx_content_count( 1000) def _test_create_msg_success_tx_service_sync_with_tx_content_count( self, tx_content_count, sync_tx_content=True): short_ids = [ list(range(1, 6)), list(range(11, 15)), list(range(53, 250)), [31], list(range(41, 48)), [51, 52] ] transaction_hashes = list( map(crypto.double_sha256, map(bytes, short_ids))) for i in range(len(short_ids)): transaction_content = bytearray(tx_content_count) transaction_content[:32] = transaction_hashes[i] transaction_key = self.transaction_service.get_transaction_key( transaction_hashes[i]) self.transaction_service.set_transaction_contents_by_key( transaction_key, transaction_content) for short_id in short_ids[i]: self.transaction_service.assign_short_id_by_key( transaction_key, short_id) # Six blocks received after for i in range(len(short_ids)): self.transaction_service.track_seen_short_ids( Sha256Hash(helpers.generate_bytearray(32)), short_ids[i]) tx_service_snap = self.transaction_service.get_snapshot() txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg( self.transaction_service, tx_service_snap, sync_tx_content) if txs_content_short_ids: self._send_tx_msg(txs_content_short_ids, transaction_hashes) def _send_tx_msg(self, txs_content_short_ids, transaction_hashes): tx_service_sync_txs_msg: TxServiceSyncTxsMessage = \ self.create_message_successfully( TxServiceSyncTxsMessage( self.NETWORK_NUM, txs_content_short_ids ), TxServiceSyncTxsMessage ) self.assertEqual(self.NETWORK_NUM, tx_service_sync_txs_msg.network_num()) self.assertEqual(len(txs_content_short_ids), tx_service_sync_txs_msg.tx_count()) tx_service_txs_content_short_ids = tx_service_sync_txs_msg.txs_content_short_ids( ) tx_contents = [ self.transaction_service.get_transaction(short_id).contents for tx_content_short_id in tx_service_txs_content_short_ids for short_id in tx_content_short_id.short_ids ] for tx_content_short_id in tx_service_txs_content_short_ids: self.assertIn(bytearray(tx_content_short_id.tx_hash), transaction_hashes) self.assertIn(bytearray(tx_content_short_id.tx_content), tx_contents) self.assertEqual( tx_content_short_id.short_ids, list( self.transaction_service.get_short_ids_by_key( self.transaction_service.get_transaction_key( tx_content_short_id.tx_hash))))