def test_bdn_stats_block_new_from_bdn(self):
block_msg = mock_eth_messages.new_block_eth_protocol_message(21, 1017)
internal_new_block_msg = InternalEthBlockInfo.from_new_block_msg(
block_msg)
msg_bytes, block_info = self.node.message_converter.block_to_bx_block(
internal_new_block_msg, self.node._tx_service, True,
self.node.network.min_tx_age_seconds)
msg_hash = Sha256Hash(crypto.double_sha256(msg_bytes))
broadcast_msg = BroadcastMessage(message_hash=msg_hash,
network_num=1,
is_encrypted=False,
blob=msg_bytes)
self.relay_connection.msg_broadcast(broadcast_msg)
self.assertEqual(
3,
len(gateway_bdn_performance_stats_service.interval_data.
blockchain_node_to_bdn_stats))
for stats in gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats.values(
):
self.assertEqual(1, stats.new_blocks_received_from_bdn)
def test_msg_get_block_headers_missing_block(self):
self.node.opts.max_block_interval_s = 0
block_hashes = []
for i in (j for j in range(20) if j != 17):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(i, i + 1000)
)
block_hash = block_message.block_hash()
block_hashes.append(block_hash)
self.node.block_queuing_service_manager.push(block_hash, block_message)
self.enqueued_messages.clear()
self.sut.msg_proxy_request = MagicMock()
message = GetBlockHeadersEthProtocolMessage(
None, block_hashes[-1].binary, 10, 0, 1
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_called_once()
self.assertEqual(0, len(self.enqueued_messages))
def test_bdn_stats_block_new_from_node_ignore_from_bdn(self):
block_msg = NewBlockEthProtocolMessage(
None,
_block_with_timestamp(
time.time() + 1 -
self.node.opts.blockchain_ignore_block_interval_count *
self.node.opts.blockchain_block_interval), 10)
block_msg.serialize()
self.block_blockchain_connection_protocol.msg_block(block_msg)
internal_new_block_msg = InternalEthBlockInfo.from_new_block_msg(
block_msg)
msg_bytes, block_info = self.node.message_converter.block_to_bx_block(
internal_new_block_msg, self.node._tx_service, True,
self.node.network.min_tx_age_seconds)
msg_hash = Sha256Hash(crypto.double_sha256(msg_bytes))
broadcast_msg = BroadcastMessage(message_hash=msg_hash,
network_num=1,
is_encrypted=False,
blob=msg_bytes)
self.relay_connection.msg_broadcast(broadcast_msg)
self.assertEqual(
3,
len(gateway_bdn_performance_stats_service.interval_data.
blockchain_node_to_bdn_stats))
node_1_stats = gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats[
self.node_1_endpoint]
for endpoint, stats in gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats.items(
):
if endpoint == self.node_1_endpoint:
continue
self.assertEqual(1, stats.new_blocks_received_from_bdn)
self.assertEqual(1,
node_1_stats.new_blocks_received_from_blockchain_node)
self.assertEqual(0, node_1_stats.new_blocks_received_from_bdn)
def setUp(self) -> None:
self.node = MockGatewayNode(
gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0))
self.node.broadcast = MagicMock()
self.block_queuing_service = EthBlockQueuingService(self.node)
self.node.block_queuing_service = self.block_queuing_service
self.block_hashes = []
self.block_messages = []
self.block_headers = []
self.block_bodies = []
# block numbers: 1000-1019
prev_block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
i, i + 1000, prev_block_hash=prev_block_hash))
block_hash = block_message.block_hash()
self.block_hashes.append(block_hash)
self.block_messages.append(block_message)
block_parts = block_message.to_new_block_parts()
self.block_headers.append(
BlockHeadersEthProtocolMessage.from_header_bytes(
block_parts.block_header_bytes).get_block_headers()[0])
self.block_bodies.append(
BlockBodiesEthProtocolMessage.from_body_bytes(
block_parts.block_body_bytes).get_blocks()[0])
self.block_queuing_service.push(block_hash, block_message)
prev_block_hash = block_hash
self.block_processing_service = EthBlockProcessingService(self.node)
self.node.broadcast.reset_mock()
def test_header_body_fetch_abort_from_bdn(self):
self.node.block_processing_service.queue_block_for_processing = MagicMock()
self.sut.is_valid_block_timestamp = MagicMock(return_value=True)
header = mock_eth_messages.get_dummy_block_header(1)
block = mock_eth_messages.get_dummy_block(1, header)
block_hash = header.hash_object()
new_block_hashes_message = NewBlockHashesEthProtocolMessage.from_block_hash_number_pair(
block_hash, 1
)
header_message = BlockHeadersEthProtocolMessage(None, [header])
bodies_message = BlockBodiesEthProtocolMessage(None, [TransientBlockBody(block.transactions, block.uncles)])
internal_block_info = InternalEthBlockInfo.from_new_block_msg(NewBlockEthProtocolMessage(None, block, 1))
self.sut.msg_new_block_hashes(new_block_hashes_message)
self.assertEqual(1, len(self.sut.pending_new_block_parts.contents))
self.assertEqual(2, len(self.enqueued_messages))
self.node.on_block_received_from_bdn(block_hash, internal_block_info)
self.sut.msg_block_headers(header_message)
self.sut.msg_block_bodies(bodies_message)
self.node.block_processing_service.queue_block_for_processing.assert_not_called()
def test_msg_get_block_headers_known_many(self):
self.node.opts.max_block_interval_s = 0
block_hashes = []
block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
i, i + 1000, block_hash
)
)
block_hash = block_message.block_hash()
block_hashes.append(block_hash)
self.node.block_queuing_service_manager.push(block_hash, block_message)
self.block_queuing_service.best_sent_block = (1019, block_hashes[-1], 0)
self.enqueued_messages.clear()
self.sut.msg_proxy_request = MagicMock()
message = GetBlockHeadersEthProtocolMessage(
None, block_hashes[10].binary, 10, 0, 0
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_not_called()
self.assertEqual(1, len(self.enqueued_messages))
headers_sent = self.enqueued_messages[0]
self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage)
self.assertEqual(10, len(headers_sent.get_block_headers()))
for i in range(10):
self.assertEqual(
block_hashes[i + 10],
headers_sent.get_block_headers()[i].hash_object(),
)
def test_block_to_bx_block__empty_block_success(self):
block = Block(mock_eth_messages.get_dummy_block_header(8), [], [])
dummy_chain_difficulty = 10
block_msg = NewBlockEthProtocolMessage(None, block,
dummy_chain_difficulty)
self.assertTrue(block_msg.rawbytes())
internal_new_block_msg = InternalEthBlockInfo.from_new_block_msg(
block_msg)
bx_block_msg, block_info = self.eth_message_converter.block_to_bx_block(
internal_new_block_msg, self.tx_service, True, 0)
self.assertEqual(0, block_info.txn_count)
self.assertEqual(convert.bytes_to_hex(block.header.prev_hash),
block_info.prev_block_hash)
self.assertTrue(bx_block_msg)
self.assertIsInstance(bx_block_msg, memoryview)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block_msg)
_, short_ids_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block_msg, block_offsets.short_id_offset)
compact_block = rlp.decode(
bx_block_msg[block_offsets.block_begin_offset:block_offsets.
short_id_offset].tobytes(), CompactBlock)
self.assertTrue(compact_block)
self.assertIsInstance(compact_block, CompactBlock)
self._assert_values_equal(compact_block.header, block.header)
self.assertEqual(0, len(compact_block.uncles))
self.assertEqual(0, len(compact_block.transactions))
self.assertEqual(compact_block.chain_difficulty,
block_msg.chain_difficulty)
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:param enable_block_compression
:param min_tx_age_seconds
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message(
block_msg)
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
ignored_sids = []
tx_start_index = 0
tx_count = 0
original_size = len(block_msg.rawbytes())
max_timestamp_for_compression = time.time() - min_tx_age_seconds
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
tx_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(tx_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(
short_id)
if short_id <= constants.NULL_TX_SID or \
not enable_block_compression or short_id_assign_time > max_timestamp_for_compression:
if short_id > constants.NULL_TX_SID:
ignored_sids.append(short_id)
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
block = finalize_block_bytes(buf, content_size, used_short_ids)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100,
ignored_sids)
return memoryview(block), block_info
def bx_block_to_block(self, bx_block_msg,
tx_service) -> BlockDecompressionResult:
"""
Converts internal broadcast message to Ethereum new block message
The code is optimized and does not make copies of bytes
:param bx_block_msg: internal broadcast message bytes
:param tx_service: Transactions service
:return: tuple (new block message, block hash, unknown transaction short id, unknown transaction hashes)
"""
if not isinstance(bx_block_msg, (bytearray, memoryview)):
raise TypeError(
"Type bytearray is expected for arg block_bytes but was {0}".
format(type(bx_block_msg)))
decompress_start_datetime = datetime.datetime.utcnow()
decompress_start_timestamp = time.time()
block_msg_bytes = bx_block_msg if isinstance(
bx_block_msg, memoryview) else memoryview(bx_block_msg)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block_msg)
short_ids, short_ids_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block_msg, block_offsets.short_id_offset)
block_bytes = block_msg_bytes[
block_offsets.block_begin_offset:block_offsets.short_id_offset]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_bytes, 0)
block_itm_bytes = block_bytes[block_itm_start:]
_, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
full_hdr_bytes = block_itm_bytes[0:block_hdr_start + block_hdr_len]
block_hash_bytes = eth_common_utils.keccak_hash(full_hdr_bytes)
block_hash = Sha256Hash(block_hash_bytes)
_, block_txs_len, block_txs_start = rlp_utils.consume_length_prefix(
block_itm_bytes, block_hdr_start + block_hdr_len)
txs_bytes = block_itm_bytes[block_txs_start:block_txs_start +
block_txs_len]
remaining_bytes = block_itm_bytes[block_txs_start + block_txs_len:]
# parse statistics variables
short_tx_index = 0
unknown_tx_sids = []
unknown_tx_hashes = []
# creating transactions content
content_size = 0
buf = deque()
tx_count = 0
tx_start_index = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len]
is_full_tx_start = 0
is_full_tx, is_full_tx_len, = rlp_utils.decode_int(
tx_bytes, is_full_tx_start)
_, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix(
tx_bytes, is_full_tx_start + is_full_tx_len)
tx_content_bytes = tx_bytes[tx_content_start:tx_content_start +
tx_content_len]
if is_full_tx:
tx_bytes = tx_content_bytes
else:
short_id = short_ids[short_tx_index]
tx_hash, tx_bytes, _ = tx_service.get_transaction(short_id)
if tx_hash is None:
unknown_tx_sids.append(short_id)
elif tx_bytes is None:
unknown_tx_hashes.append(tx_hash)
short_tx_index += 1
if tx_bytes is not None and not unknown_tx_sids and not unknown_tx_hashes:
buf.append(tx_bytes)
content_size += len(tx_bytes)
tx_count += 1
tx_start_index = tx_itm_start + tx_itm_len
if not unknown_tx_sids and not unknown_tx_hashes:
txs_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(txs_prefix)
content_size += len(txs_prefix)
buf.appendleft(full_hdr_bytes)
content_size += len(full_hdr_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
msg_len_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(msg_len_prefix)
block_msg_bytes = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block_msg_bytes[off:next_off] = blob
off = next_off
block_msg = InternalEthBlockInfo(block_msg_bytes)
logger.debug(
"Successfully parsed block broadcast message. {} "
"transactions in block {}", tx_count, block_hash)
bx_block_hash = convert.bytes_to_hex(
crypto.double_sha256(bx_block_msg))
compressed_size = len(bx_block_msg)
block_info = BlockInfo(
block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000, tx_count,
bx_block_hash,
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()), compressed_size,
100 - float(compressed_size) / content_size * 100, [])
return BlockDecompressionResult(block_msg, block_info,
unknown_tx_sids, unknown_tx_hashes)
else:
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}", block_hash, len(unknown_tx_sids),
len(unknown_tx_hashes), tx_count)
return BlockDecompressionResult(
None,
BlockInfo(block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
None, None, None, None, None, None, []),
unknown_tx_sids, unknown_tx_hashes)
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
msg_bytes = memoryview(block_msg.rawbytes())
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
msg_bytes, 0)
block_msg_bytes = msg_bytes[block_msg_itm_start:block_msg_itm_start +
block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_full_bytes = block_msg_bytes[0:block_hdr_itm_start +
block_hdr_itm_len]
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
_, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix(
block_hdr_bytes, 0)
prev_block_bytes = block_hdr_bytes[
prev_block_itm_start:prev_block_itm_start + prev_block_itm_len]
_, txs_itm_len, txs_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_hdr_itm_start + block_hdr_itm_len)
txs_bytes = block_msg_bytes[txs_itm_start:txs_itm_start + txs_itm_len]
remaining_bytes = block_msg_bytes[txs_itm_start + txs_itm_len:]
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
tx_start_index = 0
tx_count = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
short_id = tx_service.get_short_id(tx_hash)
if short_id <= 0:
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
short_ids_bytes = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(
used_short_ids)
buf.append(short_ids_bytes)
content_size += constants.UL_ULL_SIZE_IN_BYTES
offset_buf = struct.pack("<Q", content_size)
buf.appendleft(offset_buf)
content_size += len(short_ids_bytes)
# Parse it into the bloXroute message format and send it along
block = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block[off:next_off] = blob
off = next_off
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
original_size = len(block_msg.rawbytes())
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100)
return memoryview(block), block_info
def test_handle_single_block_fork_already_accepted(self):
block_1 = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(1, 1))
block_hash_1 = block_1.block_hash()
block_2a = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
2, 2, block_hash_1))
block_hash_2a = block_2a.block_hash()
block_2b = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
3, 2, block_hash_1))
block_hash_2b = block_2b.block_hash()
block_3b = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
4, 3, block_hash_2b))
block_hash_3b = block_3b.block_hash()
block_4b = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
5, 4, block_hash_3b))
block_hash_4b = block_4b.block_hash()
# accept block 1
self.block_queuing_service.push(block_hash_1, block_1)
self._assert_block_sent(block_hash_1)
self.block_queuing_service.mark_block_seen_by_blockchain_node(
block_hash_1, block_1)
# accept block 2a
self.block_queuing_service.push(block_hash_2a, block_2a)
self._assert_block_sent(block_hash_2a)
self.block_queuing_service.mark_block_seen_by_blockchain_node(
block_hash_2a, block_2a)
# block 2b will never be sent
self.block_queuing_service.push(block_hash_2b, block_2b)
self._assert_no_blocks_sent()
# block 3b will be sent
self.block_queuing_service.push(block_hash_3b, block_3b)
self._assert_block_sent(block_hash_3b)
# sync triggered, requesting 2a by hash
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_hash_2a.binary, 1, 0,
0))
# response is empty
self._assert_headers_sent([])
# request block 1 to establish common ancestor
block_number_bytes = struct.pack(">I", 1)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 1, 0,
0))
self._assert_headers_sent([block_hash_1])
# request block 193, 193 + 191, etc to determine chain state
block_number_bytes = struct.pack(">I", 193)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 128,
191, 0))
self._assert_headers_sent([])
# request block 2, 3, 4, ... to compare state
block_number_bytes = struct.pack(">I", 2)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0,
0))
self._assert_headers_sent([block_hash_2b, block_hash_3b])
# request block 4, 5, 6, ... to compare state
block_number_bytes = struct.pack(">I", 4)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0,
0))
self._assert_headers_sent([])
# 4b is sent after timeout (presumably Ethereum node didn't send back acceptance
# because it's resolving chainstate)
self.block_queuing_service.push(block_hash_4b, block_4b)
self._assert_no_blocks_sent()
self._progress_time()
self._assert_block_sent(block_hash_4b)
def test_block_to_bx_block__no_compressed_block(self):
txs = []
txs_bytes = []
txs_hashes = []
short_ids = []
used_short_ids = []
tx_count = 150
for i in range(1, tx_count):
tx = mock_eth_messages.get_dummy_transaction(1)
txs.append(tx)
tx_bytes = rlp.encode(tx, Transaction)
txs_bytes.append(tx_bytes)
tx_hash = tx.hash()
txs_hashes.append(tx_hash)
short_ids.append(0)
block = Block(
mock_eth_messages.get_dummy_block_header(1),
txs,
[
mock_eth_messages.get_dummy_block_header(2),
mock_eth_messages.get_dummy_block_header(3),
]
)
dummy_chain_difficulty = 10
block_msg = NewBlockEthProtocolMessage(None, block, dummy_chain_difficulty)
self.assertTrue(block_msg.rawbytes())
internal_new_block_msg = InternalEthBlockInfo.from_new_block_msg(block_msg)
bx_block_msg, block_info = self.eth_message_converter.block_to_bx_block(
internal_new_block_msg, self.tx_service, False, 0
)
self.assertTrue(len(bx_block_msg) >= len(internal_new_block_msg.rawbytes()))
self.assertEqual(len(txs), block_info.txn_count)
self.assertEqual(convert.bytes_to_hex(block.header.prev_hash), block_info.prev_block_hash)
self.assertEqual(used_short_ids, list(block_info.short_ids))
self.assertTrue(bx_block_msg)
self.assertIsInstance(bx_block_msg, memoryview)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(bx_block_msg)
compact_block = rlp.decode(
bx_block_msg[block_offsets.block_begin_offset: block_offsets.short_id_offset].tobytes(),
CompactBlock
)
self.assertTrue(compact_block)
self.assertIsInstance(compact_block, CompactBlock)
self._assert_values_equal(compact_block.header, block.header)
self._assert_values_equal(compact_block.uncles, block.uncles)
self.assertEqual(len(compact_block.transactions), len(block.transactions))
for tx, short_tx, i in zip(block.transactions, compact_block.transactions, range(1, tx_count)):
self.assertIsInstance(tx, Transaction)
self.assertIsInstance(short_tx, ShortTransaction)
self.assertEqual(1, short_tx.full_transaction)
self.assertEqual(short_tx.transaction_bytes, txs_bytes[i - 1])
self.assertEqual(compact_block.chain_difficulty, block_msg.chain_difficulty)
converted_block_msg, _, _, _ = self.eth_message_converter.bx_block_to_block(bx_block_msg, self.tx_service)
self.assertIsNotNone(converted_block_msg)
def bx_block_to_block(self, bx_block_msg,
tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
block_hash = Sha256Hash(
convert.hex_to_bytes(tsk.block_hash().hex_string()))
self.decompression_tasks.return_task(tsk)
# TODO find a better solution
raise message_conversion_error.eth_block_decompression_error(
block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [
Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()
]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = Sha256Hash(
convert.hex_to_bytes(tsk.block_hash().hex_string()))
if tsk.success():
starting_offset = tsk.starting_offset()
block = memoryview(tsk.block_message())[starting_offset:]
block_msg = InternalEthBlockInfo(block)
content_size = len(block_msg.rawbytes())
logger.debug(
"Successfully parsed block broadcast message. {} "
"transactions in block {}", total_tx_count, block_hash)
bx_block_hash = convert.bytes_to_hex(
crypto.double_sha256(bx_block_msg))
compressed_size = len(bx_block_msg)
block_info = BlockInfo(
block_hash, tsk.short_ids(), decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
total_tx_count, bx_block_hash,
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()), compressed_size,
100 - float(compressed_size) / content_size * 100)
else:
block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}", block_hash, len(unknown_tx_sids),
len(unknown_tx_hashes), total_tx_count)
block_info = BlockInfo(
block_hash, tsk.short_ids(), decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000, None, None,
None, None, None, None)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(block_msg, block_info, unknown_tx_sids,
unknown_tx_hashes)
