def parse_bx_block_transactions_and_msg_tail(block_hash: Sha256Hash, bx_block: memoryview, offset: int,
short_ids: List[int], block_offsets: BlockOffsets,
tx_service: TransactionService,
block_pieces: Deque[Union[bytearray, memoryview]]) -> \
Tuple[List[int], List[Sha256Hash], int]:
has_missing, unknown_tx_sids, unknown_tx_hashes = tx_service.get_missing_transactions(short_ids)
if has_missing:
return unknown_tx_sids, unknown_tx_hashes, offset
short_tx_index = 0
output_offset = offset
while offset < block_offsets.short_id_offset:
if bx_block[offset] == ont_constants.ONT_SHORT_ID_INDICATOR:
try:
sid = short_ids[short_tx_index]
except IndexError:
raise message_conversion_error.btc_block_decompression_error(
block_hash,
f"Message is improperly formatted, short id index ({short_tx_index}) "
f"exceeded its array bounds (size: {len(short_ids)})"
)
tx_hash, tx, _ = tx_service.get_transaction(sid)
offset += ont_constants.ONT_SHORT_ID_INDICATOR_LENGTH
short_tx_index += 1
else:
tx_size = ont_messages_util.get_next_tx_size(bx_block, offset)
tx = bx_block[offset:offset + tx_size]
offset += tx_size
assert tx is not None
block_pieces.append(tx)
output_offset += len(tx)
# Add consensus payload tail and owner and signature to block_pieces
offset = block_offsets.block_begin_offset + ont_constants.ONT_HASH_LEN + ont_constants.ONT_INT_LEN + 1
payload_tail_len, = struct.unpack_from("<L", bx_block, offset)
offset += ont_constants.ONT_INT_LEN
block_pieces.append(bx_block[offset: offset + payload_tail_len])
offset += payload_tail_len
owner_and_signature_len, = struct.unpack_from("<L", bx_block, offset)
offset += ont_constants.ONT_INT_LEN
block_pieces.append(bx_block[offset: offset + owner_and_signature_len])
offset += owner_and_signature_len
return unknown_tx_sids, unknown_tx_hashes, output_offset
def parse_bx_block_transactions(
block_hash: Sha256Hash,
bx_block: memoryview,
offset: int,
short_ids: List[int],
block_offsets: BlockOffsets,
tx_service: TransactionService,
block_pieces: Deque[Union[bytearray, memoryview]]
) -> Tuple[List[int], List[Sha256Hash], int]:
has_missing, unknown_tx_sids, unknown_tx_hashes = \
tx_service.get_missing_transactions(short_ids)
if has_missing:
return unknown_tx_sids, unknown_tx_hashes, offset
short_tx_index = 0
output_offset = offset
while offset < block_offsets.short_id_offset:
if bx_block[offset] == btc_constants.BTC_SHORT_ID_INDICATOR:
try:
sid = short_ids[short_tx_index]
except IndexError:
raise message_conversion_error.btc_block_decompression_error(
block_hash,
f"Message is improperly formatted, short id index ({short_tx_index}) "
f"exceeded its array bounds (size: {len(short_ids)})"
)
tx_hash, tx, _ = tx_service.get_transaction(sid)
offset += btc_constants.BTC_SHORT_ID_INDICATOR_LENGTH
short_tx_index += 1
else:
tx_size = btc_messages_util.get_next_tx_size(bx_block, offset)
tx = bx_block[offset:offset + tx_size]
offset += tx_size
# pyre-fixme[6]: Expected `Union[bytearray, memoryview]` for 1st param but
# got `Optional[Union[bytearray, memoryview]]`.
block_pieces.append(tx)
# pyre-fixme[6]: Expected `Sized` for 1st param but got
# `Optional[Union[bytearray, memoryview]]`.
output_offset += len(tx)
return unknown_tx_sids, unknown_tx_hashes, output_offset
def _log_compressed_block_debug_info_eth(transaction_service: TransactionService,
block_msg_bytes: Union[memoryview, bytearray]):
is_block_relay = transaction_service.node.NODE_TYPE == NodeType.RELAY_BLOCK
block_hash, short_ids, txs_bytes = _parse_block_eth(block_msg_bytes)
# parse statistics variables
short_tx_index = 0
tx_start_index = 0
tx_index_in_block = 0
txs_info = []
missing_short_ids = []
while True:
if tx_start_index >= len(txs_bytes):
break
short_id = 0
has_contents = False
assignmnet_time = 0
_, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len]
is_full_tx_start = 0
is_full_tx, is_full_tx_len, = rlp_utils.decode_int(tx_bytes, is_full_tx_start)
_, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix(
tx_bytes, is_full_tx_start + is_full_tx_len)
tx_content_bytes = tx_bytes[tx_content_start:tx_content_start + tx_content_len]
if is_full_tx:
tx_hash = Sha256Hash(eth_common_utils.keccak_hash(tx_content_bytes))
else:
short_id = short_ids[short_tx_index]
tx_hash, tx_bytes, _ = transaction_service.get_transaction(short_id)
has_contents = tx_bytes is not None
if tx_hash is not None:
assignmnet_time = transaction_service.get_short_id_assign_time(short_id)
short_tx_index += 1
if is_block_relay:
txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash))
else:
txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash, has_contents, assignmnet_time))
tx_index_in_block += 1
tx_start_index = tx_itm_start + tx_itm_len
if not is_full_tx and not has_contents:
missing_short_ids.append(short_id)
if is_block_relay:
log_message = \
"Block content (from block relay) {} from (index, is compressed, short id, hash is full) : {}"
else:
log_message = \
"Block content (full) {} (index, compressed, short id, hash, has contents, assignment time) : {}"
logger.debug(
log_message,
block_hash,
",".join(str(tx_info) for tx_info in txs_info)
)
node_type = transaction_service.node.NODE_TYPE
assert node_type is not None
log_can_decompress_block(node_type, block_hash, missing_short_ids)
class SyncTxServiceTest(MessageFactoryTestCase):
NETWORK_NUM = 12345
def setUp(self) -> None:
self.node = MockNode(helpers.get_common_opts(1234))
self.network_num = 4
self.transaction_service = TransactionService(self.node,
self.network_num)
def get_message_factory(self):
return bloxroute_message_factory
def test_create_message_success_tx_service_sync_txs_msg(self):
self._test_create_msg_success_tx_service_sync_with_tx_content_count(
100)
def test_create_message_success_tx_service_sync_txs_msg_with_exceeded_buf(
self):
self._test_create_msg_success_tx_service_sync_with_tx_content_count(
1000)
def _test_create_msg_success_tx_service_sync_with_tx_content_count(
self, tx_content_count, sync_tx_content=True):
short_ids = [
list(range(1, 6)),
list(range(11, 15)),
list(range(53, 250)), [31],
list(range(41, 48)), [51, 52]
]
transaction_hashes = list(
map(crypto.double_sha256, map(bytes, short_ids)))
for i in range(len(short_ids)):
transaction_content = bytearray(tx_content_count)
transaction_content[:32] = transaction_hashes[i]
transaction_key = self.transaction_service.get_transaction_key(
transaction_hashes[i])
self.transaction_service.set_transaction_contents_by_key(
transaction_key, transaction_content)
for short_id in short_ids[i]:
self.transaction_service.assign_short_id_by_key(
transaction_key, short_id)
# Six blocks received after
for i in range(len(short_ids)):
self.transaction_service.track_seen_short_ids(
Sha256Hash(helpers.generate_bytearray(32)), short_ids[i])
tx_service_snap = self.transaction_service.get_snapshot()
txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg(
self.transaction_service, tx_service_snap, sync_tx_content)
if txs_content_short_ids:
self._send_tx_msg(txs_content_short_ids, transaction_hashes)
def _send_tx_msg(self, txs_content_short_ids, transaction_hashes):
tx_service_sync_txs_msg: TxServiceSyncTxsMessage = \
self.create_message_successfully(
TxServiceSyncTxsMessage(
self.NETWORK_NUM, txs_content_short_ids
),
TxServiceSyncTxsMessage
)
self.assertEqual(self.NETWORK_NUM,
tx_service_sync_txs_msg.network_num())
self.assertEqual(len(txs_content_short_ids),
tx_service_sync_txs_msg.tx_count())
tx_service_txs_content_short_ids = tx_service_sync_txs_msg.txs_content_short_ids(
)
tx_contents = [
self.transaction_service.get_transaction(short_id).contents
for tx_content_short_id in tx_service_txs_content_short_ids
for short_id in tx_content_short_id.short_ids
]
for tx_content_short_id in tx_service_txs_content_short_ids:
self.assertIn(bytearray(tx_content_short_id.tx_hash),
transaction_hashes)
self.assertIn(bytearray(tx_content_short_id.tx_content),
tx_contents)
self.assertEqual(
tx_content_short_id.short_ids,
list(
self.transaction_service.get_short_ids_by_key(
self.transaction_service.get_transaction_key(
tx_content_short_id.tx_hash))))