def test_block_bodies_msg_from_bodies_bytes(self):
txs = []
txs_bytes = []
txs_hashes = []
tx_count = 10
for i in range(1, tx_count):
tx = mock_eth_messages.get_dummy_transaction(1)
txs.append(tx)
tx_bytes = rlp.encode(tx, Transaction)
txs_bytes.append(tx_bytes)
tx_hash = tx.hash()
txs_hashes.append(tx_hash)
uncles = [
mock_eth_messages.get_dummy_block_header(2),
mock_eth_messages.get_dummy_block_header(3),
]
block_body = TransientBlockBody(txs, uncles)
block_body_bytes = memoryview(
rlp.encode(TransientBlockBody.serialize(block_body)))
block_bodies_msg = BlockBodiesEthProtocolMessage.from_body_bytes(
block_body_bytes)
self.assertEqual(1, len(block_bodies_msg.get_blocks()))
self.assertEqual(1, len(block_bodies_msg.get_block_bodies_bytes()))
self.assertEqual(block_body, block_bodies_msg.get_blocks()[0])
self.assertEqual(block_body_bytes,
block_bodies_msg.get_block_bodies_bytes()[0])
def test_complete_header_body_recovery(self):
self.node.block_processing_service.queue_block_for_processing = MagicMock()
self.node.block_queuing_service_manager.push = MagicMock()
self.node.on_block_seen_by_blockchain_node = MagicMock(return_value=False)
self.sut.is_valid_block_timestamp = MagicMock(return_value=True)
header = mock_eth_messages.get_dummy_block_header(1)
block = mock_eth_messages.get_dummy_block(1, header)
block_hash = header.hash_object()
new_block_hashes_message = NewBlockHashesEthProtocolMessage.from_block_hash_number_pair(
block_hash, 1
)
header_message = BlockHeadersEthProtocolMessage(None, [header])
bodies_message = BlockBodiesEthProtocolMessage(None, [TransientBlockBody(block.transactions, block.uncles)])
self.sut.msg_new_block_hashes(new_block_hashes_message)
self.node.on_block_seen_by_blockchain_node.assert_called_once()
self.assertEqual(1, len(self.sut.pending_new_block_parts.contents))
self.assertEqual(2, len(self.enqueued_messages))
self.node.on_block_seen_by_blockchain_node.reset_mock()
self.node.on_block_seen_by_blockchain_node = MagicMock(return_value=True)
self.sut.msg_block_headers(header_message)
self.sut.msg_block_bodies(bodies_message)
self.assertEqual(1, len(gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats))
for stats in gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats.values():
self.assertEqual(1, stats.new_blocks_received_from_blockchain_node)
self.assertEqual(0, stats.new_blocks_received_from_bdn)
self.node.on_block_seen_by_blockchain_node.assert_called_once()
self.node.block_queuing_service_manager.push.assert_not_called()
self.node.block_processing_service.queue_block_for_processing.assert_not_called()
def msg_block_bodies(self, msg: BlockBodiesEthProtocolMessage):
if not self.node.should_process_block_hash():
return
if self._block_bodies_requests:
requested_hashes = self._block_bodies_requests.popleft()
bodies_bytes = msg.get_block_bodies_bytes()
if len(requested_hashes) != len(bodies_bytes):
logger.debug(
"Expected {} bodies in response but received {}. Proxy message to remote node.",
len(requested_hashes), len(bodies_bytes))
self._block_bodies_requests.clear()
return
logger.debug(
"Processing expected block bodies messages for blocks [{}]",
", ".join([
convert.bytes_to_hex(block_hash.binary)
for block_hash in requested_hashes
]))
for block_hash, block_body_bytes in zip(requested_hashes,
bodies_bytes):
if block_hash in self.pending_new_block_parts.contents:
logger.debug(
"Received block body for pending new block {}",
convert.bytes_to_hex(block_hash.binary))
self.pending_new_block_parts.contents[
block_hash].block_body_bytes = block_body_bytes
self._check_pending_new_block(block_hash)
elif self.node.block_cleanup_service.is_marked_for_cleanup(
block_hash):
transactions_hashes = \
BlockBodiesEthProtocolMessage.from_body_bytes(block_body_bytes).get_block_transaction_hashes(0)
# pyre-fixme[16]: `AbstractBlockCleanupService` has no attribute
# `clean_block_transactions_by_block_components`.
self.node.block_cleanup_service.clean_block_transactions_by_block_components(
transaction_service=self.node.get_tx_service(),
block_hash=block_hash,
transactions_list=transactions_hashes)
else:
logger.warning(log_messages.REDUNDANT_BLOCK_BODY,
convert.bytes_to_hex(block_hash.binary))
self._process_ready_new_blocks()
def get_block_body_from_message(
self,
block_hash: Sha256Hash) -> Optional[BlockBodiesEthProtocolMessage]:
block_parts = self.get_block_parts(block_hash)
if block_parts is None:
return None
return BlockBodiesEthProtocolMessage.from_body_bytes(
block_parts.block_body_bytes)
def test_try_process_get_block_bodies_request(self):
success = self.block_processing_service.try_process_get_block_bodies_request(
GetBlockBodiesEthProtocolMessage(
None,
[block_hash.binary for block_hash in self.block_hashes[:2]]))
self.assertTrue(success)
self.node.send_msg_to_node.assert_called_once_with(
BlockBodiesEthProtocolMessage(None, self.block_bodies[:2]))
def test_try_send_bodies_to_node_success(self):
self.node_conn.enqueue_msg.reset_mock()
result = self.block_queuing_service.try_send_bodies_to_node(
self.block_hashes[:4])
self.assertTrue(result)
self.node_conn.enqueue_msg.assert_called_once_with(
BlockBodiesEthProtocolMessage(None, self.block_bodies[:4]))
def test_try_send_bodies_to_node_success(self):
self.node.broadcast = MagicMock()
result = self.block_queuing_service.try_send_bodies_to_node(
self.block_hashes[:4])
self.assertTrue(result)
self.node.broadcast.assert_called_once_with(
BlockBodiesEthProtocolMessage(None, self.block_bodies[:4]),
connection_types=[ConnectionType.BLOCKCHAIN_NODE])
def setUp(self):
self.node = MockGatewayNode(
gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0))
self.node.block_parts_storage = ExpiringDict(
self.node.alarm_queue,
gateway_constants.MAX_BLOCK_CACHE_TIME_S,
"eth_block_queue_parts",
)
self.node_connection = Mock()
self.node_connection.is_active = MagicMock(return_value=True)
self.node.set_known_total_difficulty = MagicMock()
self.node_conn = self.node_connection
self.block_queuing_service = EthBlockQueuingService(
self.node, self.node_conn)
self.node.block_queuing_service_manager.add_block_queuing_service(
self.node_conn, self.block_queuing_service)
self.node_conn.enqueue_msg = MagicMock()
self.block_hashes = []
self.block_messages = []
self.block_headers = []
self.block_bodies = []
# block numbers: 1000-1019
prev_block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
i, i + 1000, prev_block_hash=prev_block_hash))
block_hash = block_message.block_hash()
self.block_hashes.append(block_hash)
self.block_messages.append(block_message)
block_parts = block_message.to_new_block_parts()
self.block_headers.append(
BlockHeadersEthProtocolMessage.from_header_bytes(
block_parts.block_header_bytes).get_block_headers()[0])
self.block_bodies.append(
BlockBodiesEthProtocolMessage.from_body_bytes(
block_parts.block_body_bytes).get_blocks()[0])
self.node.block_queuing_service_manager.push(
block_hash, block_message)
prev_block_hash = block_hash
for block_hash in self.block_hashes:
self.block_queuing_service.remove_from_queue(block_hash)
self.block_queuing_service.best_sent_block = (1019,
self.block_hashes[-1],
time.time())
self.block_queuing_service.best_accepted_block = (
1019, self.block_hashes[-1])
def try_send_bodies_to_node(self, block_hashes: List[Sha256Hash]) -> bool:
"""
Creates and sends block bodies to blockchain connection.
"""
bodies = []
for block_hash in block_hashes:
if block_hash not in self._blocks:
self.connection.log_debug(
"{} was not found in queuing service. Aborting attempt to send bodies.",
block_hash)
return False
if not self.node.block_queuing_service_manager.is_in_common_block_storage(
block_hash):
self.connection.log_debug(
"{} was not in the block storage. Aborting attempt to send bodies.",
block_hash)
return False
block_message = cast(InternalEthBlockInfo,
self.node.block_storage[block_hash])
if block_hash in self.node.block_parts_storage:
block_body_bytes = self.node.block_parts_storage[
block_hash].block_body_bytes
else:
block_body_bytes = (
block_message.to_new_block_parts().block_body_bytes)
partial_message = BlockBodiesEthProtocolMessage.from_body_bytes(
block_body_bytes)
block_bodies = partial_message.get_blocks()
assert len(block_bodies) == 1
bodies.append(block_bodies[0])
height = self._height_by_block_hash.contents.get(block_hash, None)
self.connection.log_debug(
"Appending {} body ({}) for sending to blockchain node.",
block_hash, height)
full_message = BlockBodiesEthProtocolMessage(None, bodies)
self.connection.enqueue_msg(full_message)
return True
def try_send_bodies_to_node(self, block_hashes: List[Sha256Hash]) -> bool:
"""
Creates and sends block bodies to blockchain connection.
"""
bodies = []
for block_hash in block_hashes:
if block_hash not in self._blocks:
logger.debug("{} was not found in queue. Aborting.",
block_hash)
return False
block_message = self._blocks[block_hash]
if block_message is None:
logger.debug("{} was not ready in the queue. Aborting",
block_hash)
return False
if block_hash in self._block_parts:
block_body_bytes = self._block_parts[
block_hash].block_body_bytes
else:
block_body_bytes = (
block_message.to_new_block_parts().block_body_bytes)
partial_message = BlockBodiesEthProtocolMessage.from_body_bytes(
block_body_bytes)
block_bodies = partial_message.get_blocks()
assert len(block_bodies) == 1
bodies.append(block_bodies[0])
height = self._height_by_block_hash.contents.get(block_hash, None)
logger.debug(
"Appending {} body ({}) for sending to blockchain node.",
block_hash, height)
full_message = BlockBodiesEthProtocolMessage(None, bodies)
# TODO: Should only be sent to the node that requested (https://bloxroute.atlassian.net/browse/BX-1922)
self.node.broadcast(full_message,
connection_types=[ConnectionType.BLOCKCHAIN_NODE])
return True
def msg_get_block_bodies(self,
msg: GetBlockBodiesEthProtocolMessage) -> None:
block_hashes = msg.get_block_hashes()
logger.debug(
"Received unexpected get block bodies message from remote "
"blockchain node for {} blocks: {}. Replying with empty block bodies.",
len(block_hashes), ", ".join([
convert.bytes_to_hex(block_hash.binary)
for block_hash in block_hashes[:10]
]))
empty_block_bodies_msg = BlockBodiesEthProtocolMessage(None, [])
self.connection.enqueue_msg(empty_block_bodies_msg)
def test_try_process_get_block_bodies_request(self):
success = self.block_processing_service.try_process_get_block_bodies_request(
GetBlockBodiesEthProtocolMessage(
None,
[block_hash.binary for block_hash in self.block_hashes[:2]]))
self.assertTrue(success)
self.node.broadcast.assert_called_once_with(
BlockBodiesEthProtocolMessage(
None,
self.block_bodies[:2],
),
connection_types=[ConnectionType.BLOCKCHAIN_NODE])
def try_send_bodies_to_node(self, block_hashes: List[Sha256Hash]) -> bool:
"""
Creates and sends block bodies to blockchain connection.
"""
bodies = []
for block_hash in block_hashes:
if block_hash not in self._blocks:
logger.debug("{} was not found in queue. Aborting.",
block_hash)
return False
block_message = self._blocks[block_hash]
if block_message is None:
logger.debug("{} was not ready in the queue. Aborting",
block_hash)
return False
if block_hash in self._block_parts:
block_body_bytes = self._block_parts[
block_hash].block_body_bytes
else:
block_body_bytes = (
block_message.to_new_block_parts().block_body_bytes)
partial_message = BlockBodiesEthProtocolMessage.from_body_bytes(
block_body_bytes)
block_bodies = partial_message.get_blocks()
assert len(block_bodies) == 1
bodies.append(block_bodies[0])
height = self._height_by_block_hash.contents.get(block_hash, None)
logger.debug(
"Appending {} body ({}) for sending to blockchain node.",
block_hash, height)
full_message = BlockBodiesEthProtocolMessage(None, bodies)
self.node.send_msg_to_node(full_message)
return True
def test_block_bodies_msg__get_block_transactions(self):
txs = []
txs_bytes = []
txs_hashes = []
tx_count = 10
for i in range(1, tx_count):
tx = mock_eth_messages.get_dummy_transaction(1)
txs.append(tx)
tx_bytes = rlp.encode(LegacyTransaction.serialize(tx))
txs_bytes.append(tx_bytes)
tx_hash = tx.hash()
txs_hashes.append(tx_hash)
al_tx = mock_eth_messages.get_dummy_access_list_transaction(11)
txs.append(al_tx)
txs_bytes.append(Transaction.serialize(al_tx))
txs_hashes.append(al_tx.hash())
uncles = [
mock_eth_messages.get_dummy_block_header(2),
mock_eth_messages.get_dummy_block_header(3),
]
block_body = TransientBlockBody(txs, uncles)
block_body_bytes = rlp.encode(block_body)
block_bodies_msg = BlockBodiesEthProtocolMessage.from_body_bytes(
block_body_bytes)
parsed_txs_bytes = block_bodies_msg.get_block_transaction_bytes(0)
self.assertEqual(len(parsed_txs_bytes), len(txs_hashes))
for index, parsed_tx_bytes in enumerate(parsed_txs_bytes):
self.assertEqual(
convert.bytes_to_hex(txs_bytes[index]),
convert.bytes_to_hex(parsed_tx_bytes),
f"failed at index {index}",
)
parsed_txs_hashes = block_bodies_msg.get_block_transaction_hashes(0)
self.assertEqual(len(parsed_txs_hashes), len(txs_hashes))
for index, parsed_tx_hash in enumerate(parsed_txs_hashes):
self.assertEqual(parsed_tx_hash, txs_hashes[index])
def test_complete_header_body_fetch(self):
self.node.block_processing_service.queue_block_for_processing = MagicMock()
self.sut.is_valid_block_timestamp = MagicMock(return_value=True)
header = mock_eth_messages.get_dummy_block_header(1)
block = mock_eth_messages.get_dummy_block(1, header)
block_hash = header.hash_object()
new_block_hashes_message = NewBlockHashesEthProtocolMessage.from_block_hash_number_pair(
block_hash, 1
)
header_message = BlockHeadersEthProtocolMessage(None, [header])
bodies_message = BlockBodiesEthProtocolMessage(None, [TransientBlockBody(block.transactions, block.uncles)])
self.sut.msg_new_block_hashes(new_block_hashes_message)
self.assertEqual(1, len(self.sut.pending_new_block_parts.contents))
self.assertEqual(2, len(self.enqueued_messages))
self.sut.msg_block_headers(header_message)
self.sut.msg_block_bodies(bodies_message)
self.node.block_processing_service.queue_block_for_processing.assert_called_once()
def setUp(self) -> None:
self.node = MockGatewayNode(
gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0))
self.node.broadcast = MagicMock()
self.block_queuing_service = EthBlockQueuingService(self.node)
self.node.block_queuing_service = self.block_queuing_service
self.block_hashes = []
self.block_messages = []
self.block_headers = []
self.block_bodies = []
# block numbers: 1000-1019
prev_block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
i, i + 1000, prev_block_hash=prev_block_hash))
block_hash = block_message.block_hash()
self.block_hashes.append(block_hash)
self.block_messages.append(block_message)
block_parts = block_message.to_new_block_parts()
self.block_headers.append(
BlockHeadersEthProtocolMessage.from_header_bytes(
block_parts.block_header_bytes).get_block_headers()[0])
self.block_bodies.append(
BlockBodiesEthProtocolMessage.from_body_bytes(
block_parts.block_body_bytes).get_blocks()[0])
self.block_queuing_service.push(block_hash, block_message)
prev_block_hash = block_hash
self.block_processing_service = EthBlockProcessingService(self.node)
self.node.broadcast.reset_mock()
def test_request_block_bodies(self):
self.cleanup_service.clean_block_transactions_by_block_components = (
MagicMock()
)
block_hashes_sets = [
[helpers.generate_object_hash(), helpers.generate_object_hash()],
[helpers.generate_object_hash()],
[helpers.generate_object_hash()],
]
block_bodies_messages = [
BlockBodiesEthProtocolMessage(
None,
[
mock_eth_messages.get_dummy_transient_block_body(1),
mock_eth_messages.get_dummy_transient_block_body(2),
],
),
BlockBodiesEthProtocolMessage(
None, [mock_eth_messages.get_dummy_transient_block_body(3)]
),
BlockBodiesEthProtocolMessage(
None, [mock_eth_messages.get_dummy_transient_block_body(4)]
),
]
transactions = [
[
[
tx.hash()
for tx in BlockBodiesEthProtocolMessage.from_body_bytes(
block_body_bytes
)
.get_blocks()[0]
.transactions
]
for block_body_bytes in msg.get_block_bodies_bytes()
]
for msg in block_bodies_messages
]
for block_bodies_message in block_bodies_messages:
block_bodies_message.serialize()
for block_hashes_set in block_hashes_sets:
for block_hash in block_hashes_set:
self.node.block_cleanup_service._block_hash_marked_for_cleanup.add(
block_hash
)
self.sut.request_block_body(block_hashes_set)
self.sut.msg_block_bodies(block_bodies_messages[0])
call_args_list = (
self.cleanup_service.clean_block_transactions_by_block_components.call_args_list
)
first_call = call_args_list[0]
first_kwargs = first_call[1]
self.assertEqual(
first_kwargs["transaction_service"], self.node.get_tx_service()
)
self.assertEqual(first_kwargs["block_hash"], block_hashes_sets[0][0])
self.assertEqual(
list(first_kwargs["transactions_list"]), transactions[0][0]
)
second_call = call_args_list[1]
second_kwargs = second_call[1]
self.assertEqual(
second_kwargs["transaction_service"], self.node.get_tx_service()
)
self.assertEqual(second_kwargs["block_hash"], block_hashes_sets[0][1])
self.assertEqual(
list(second_kwargs["transactions_list"]), transactions[0][1]
)
self.cleanup_service.clean_block_transactions_by_block_components.reset_mock()
self.sut.msg_block_bodies(block_bodies_messages[1])
call_args_list = (
self.cleanup_service.clean_block_transactions_by_block_components.call_args_list
)
first_call = call_args_list[0]
first_kwargs = first_call[1]
self.assertEqual(
first_kwargs["transaction_service"], self.node.get_tx_service()
)
self.assertEqual(first_kwargs["block_hash"], block_hashes_sets[1][0])
self.assertEqual(
list(first_kwargs["transactions_list"]), transactions[1][0]
)
self.cleanup_service.clean_block_transactions_by_block_components.reset_mock()
self.sut.msg_block_bodies(block_bodies_messages[2])
call_args_list = (
self.cleanup_service.clean_block_transactions_by_block_components.call_args_list
)
first_call = call_args_list[0]
first_kwargs = first_call[1]
self.assertEqual(
first_kwargs["transaction_service"], self.node.get_tx_service()
)
self.assertEqual(first_kwargs["block_hash"], block_hashes_sets[2][0])
self.assertEqual(
list(first_kwargs["transactions_list"]), transactions[2][0]
)
def msg_block_bodies(self, msg: BlockBodiesEthProtocolMessage) -> None:
self.node.log_received_remote_blocks(len(msg.get_block_bodies_bytes()))
self.msg_proxy_response(msg)