def test_block_headers_request(self):
get_headers = GetBlockHeadersEthProtocolMessage(None, self.BLOCK_HASH, 111, 222, 0)
# Reply with empty headers to the first get headers request for fast sync mode support
get_headers_frames = map(self.eth_node_cipher.encrypt_frame,
frame_utils.get_frames(get_headers.msg_type, get_headers.rawbytes()))
for get_headers_frame in get_headers_frames:
helpers.receive_node_message(self.gateway_node, self.local_node_fileno, get_headers_frame)
self.eth_remote_node_connection.enqueue_msg.assert_not_called()
self.eth_node_connection.enqueue_msg.assert_called_once_with(BlockHeadersEthProtocolMessage(None, []))
# The second get headers message should be proxied to remote blockchain node
get_headers_frames = map(self.eth_node_cipher.encrypt_frame,
frame_utils.get_frames(get_headers.msg_type, get_headers.rawbytes()))
for get_headers_frame in get_headers_frames:
helpers.receive_node_message(self.gateway_node, self.local_node_fileno, get_headers_frame)
self.eth_remote_node_connection.enqueue_msg.assert_called_once_with(get_headers)
headers = BlockHeadersEthProtocolMessage(None, [
mock_eth_messages.get_dummy_block_header(1), mock_eth_messages.get_dummy_block_header(2)
])
headers_frames = map(self.eth_node_cipher.encrypt_frame,
frame_utils.get_frames(headers.msg_type, headers.rawbytes()))
for headers_frame in headers_frames:
helpers.receive_node_message(self.gateway_node, self.remote_node_fileno, headers_frame)
self.eth_node_connection.enqueue_msg.assert_called_with(headers)
def test_complete_header_body_recovery(self):
self.node.block_processing_service.queue_block_for_processing = MagicMock()
self.node.block_queuing_service_manager.push = MagicMock()
self.node.on_block_seen_by_blockchain_node = MagicMock(return_value=False)
self.sut.is_valid_block_timestamp = MagicMock(return_value=True)
header = mock_eth_messages.get_dummy_block_header(1)
block = mock_eth_messages.get_dummy_block(1, header)
block_hash = header.hash_object()
new_block_hashes_message = NewBlockHashesEthProtocolMessage.from_block_hash_number_pair(
block_hash, 1
)
header_message = BlockHeadersEthProtocolMessage(None, [header])
bodies_message = BlockBodiesEthProtocolMessage(None, [TransientBlockBody(block.transactions, block.uncles)])
self.sut.msg_new_block_hashes(new_block_hashes_message)
self.node.on_block_seen_by_blockchain_node.assert_called_once()
self.assertEqual(1, len(self.sut.pending_new_block_parts.contents))
self.assertEqual(2, len(self.enqueued_messages))
self.node.on_block_seen_by_blockchain_node.reset_mock()
self.node.on_block_seen_by_blockchain_node = MagicMock(return_value=True)
self.sut.msg_block_headers(header_message)
self.sut.msg_block_bodies(bodies_message)
self.assertEqual(1, len(gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats))
for stats in gateway_bdn_performance_stats_service.interval_data.blockchain_node_to_bdn_stats.values():
self.assertEqual(1, stats.new_blocks_received_from_blockchain_node)
self.assertEqual(0, stats.new_blocks_received_from_bdn)
self.node.on_block_seen_by_blockchain_node.assert_called_once()
self.node.block_queuing_service_manager.push.assert_not_called()
self.node.block_processing_service.queue_block_for_processing.assert_not_called()
def try_send_headers_to_node(self, block_hashes: List[Sha256Hash]) -> bool:
"""
Creates and sends a block headers message to blockchain connection.
In most cases, this method should be called with block hashes that are confirmed to
exist in the block queuing service, but contains checks for safety for otherwise,
and aborts the function if any headers are not found.
"""
headers = []
for block_hash in block_hashes:
if block_hash not in self._blocks:
logger.debug("{} was not found in queue. Aborting.",
block_hash)
return False
block_message = self._blocks[block_hash]
if block_message is None:
logger.debug("{} was not ready in the queue. Aborting",
block_hash)
return False
partial_headers_message = self.build_block_header_message(
block_hash, block_message)
block_headers = partial_headers_message.get_block_headers()
assert len(block_headers) == 1
headers.append(block_headers[0])
height = self._height_by_block_hash.contents.get(block_hash, None)
logger.debug(
"Appending {} header ({}) for sending to blockchain node.",
block_hash, height)
full_header_message = BlockHeadersEthProtocolMessage(None, headers)
self.node.send_msg_to_node(full_header_message)
return True
def test_try_send_headers_to_node_success(self):
self.node_conn.enqueue_msg.reset_mock()
result = self.block_queuing_service.try_send_headers_to_node(
self.block_hashes[:4])
self.assertTrue(result)
self.node_conn.enqueue_msg.assert_called_once_with(
BlockHeadersEthProtocolMessage(None, self.block_headers[:4]))
def test_try_send_headers_to_node_success(self):
self.node.broadcast = MagicMock()
result = self.block_queuing_service.try_send_headers_to_node(
self.block_hashes[:4])
self.assertTrue(result)
self.node.broadcast.assert_called_once_with(
BlockHeadersEthProtocolMessage(None, self.block_headers[:4]),
connection_types=[ConnectionType.BLOCKCHAIN_NODE])
def setUp(self):
self.node = MockGatewayNode(
gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0))
self.node.block_parts_storage = ExpiringDict(
self.node.alarm_queue,
gateway_constants.MAX_BLOCK_CACHE_TIME_S,
"eth_block_queue_parts",
)
self.node_connection = Mock()
self.node_connection.is_active = MagicMock(return_value=True)
self.node.set_known_total_difficulty = MagicMock()
self.node_conn = self.node_connection
self.block_queuing_service = EthBlockQueuingService(
self.node, self.node_conn)
self.node.block_queuing_service_manager.add_block_queuing_service(
self.node_conn, self.block_queuing_service)
self.node_conn.enqueue_msg = MagicMock()
self.block_hashes = []
self.block_messages = []
self.block_headers = []
self.block_bodies = []
# block numbers: 1000-1019
prev_block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
i, i + 1000, prev_block_hash=prev_block_hash))
block_hash = block_message.block_hash()
self.block_hashes.append(block_hash)
self.block_messages.append(block_message)
block_parts = block_message.to_new_block_parts()
self.block_headers.append(
BlockHeadersEthProtocolMessage.from_header_bytes(
block_parts.block_header_bytes).get_block_headers()[0])
self.block_bodies.append(
BlockBodiesEthProtocolMessage.from_body_bytes(
block_parts.block_body_bytes).get_blocks()[0])
self.node.block_queuing_service_manager.push(
block_hash, block_message)
prev_block_hash = block_hash
for block_hash in self.block_hashes:
self.block_queuing_service.remove_from_queue(block_hash)
self.block_queuing_service.best_sent_block = (1019,
self.block_hashes[-1],
time.time())
self.block_queuing_service.best_accepted_block = (
1019, self.block_hashes[-1])
def msg_block_headers(self, msg: BlockHeadersEthProtocolMessage):
if not self.node.should_process_block_hash():
return
block_headers = msg.get_block_headers()
if self.pending_new_block_parts.contents and len(block_headers) == 1:
header_bytes = msg.get_block_headers_bytes()[0]
block_hash_bytes = eth_common_utils.keccak_hash(header_bytes)
block_hash = Sha256Hash(block_hash_bytes)
if block_hash in self.pending_new_block_parts.contents:
logger.debug("Received block header for new block {}",
convert.bytes_to_hex(block_hash.binary))
self.pending_new_block_parts.contents[
block_hash].block_header_bytes = header_bytes
self._check_pending_new_block(block_hash)
self._process_ready_new_blocks()
return
if len(block_headers) > 0:
block_hashes = [blk.hash_object() for blk in block_headers]
block_hashes.insert(0, Sha256Hash(block_headers[0].prev_hash))
self.node.block_cleanup_service.mark_blocks_and_request_cleanup(
block_hashes)
latest_block_number = 0
latest_block_difficulty = 0
block_queuing_service = self.node.block_queuing_service_manager.get_block_queuing_service(
self.connection)
if block_queuing_service is not None:
for block_header in block_headers:
block_queuing_service.mark_block_seen_by_blockchain_node(
block_header.hash_object(), None, block_header.number)
if block_header.number > latest_block_number:
latest_block_number = block_header.number
latest_block_difficulty = block_header.difficulty
self.node.block_processing_service.set_last_confirmed_block_parameters(
latest_block_number, latest_block_difficulty)
def build_block_header_message(
self, block_hash: Sha256Hash, block_message: InternalEthBlockInfo
) -> BlockHeadersEthProtocolMessage:
if block_hash in self.node.block_parts_storage:
block_header_bytes = self.node.block_parts_storage[
block_hash].block_header_bytes
else:
block_header_bytes = (
block_message.to_new_block_parts().block_header_bytes)
return BlockHeadersEthProtocolMessage.from_header_bytes(
block_header_bytes)
def msg_block_headers(self, msg: BlockHeadersEthProtocolMessage):
if not self.node.should_process_block_hash():
return
block_headers = msg.get_block_headers()
if self._pending_new_blocks_parts.contents and len(block_headers) == 1:
header_bytes = msg.get_block_headers_bytes()[0]
block_hash_bytes = eth_common_utils.keccak_hash(header_bytes)
block_hash = Sha256Hash(block_hash_bytes)
if block_hash in self._pending_new_blocks_parts.contents:
logger.debug("Received block header for new block {}", convert.bytes_to_hex(block_hash.binary))
self._pending_new_blocks_parts.contents[block_hash].block_header_bytes = header_bytes
self._check_pending_new_block(block_hash)
self._process_ready_new_blocks()
return
if len(block_headers) > 0:
block_hashes = [blk.hash_object() for blk in block_headers]
block_hashes.insert(0, Sha256Hash(block_headers[0].prev_hash))
self.node.block_cleanup_service.mark_blocks_and_request_cleanup(block_hashes)
self.node.block_queuing_service.mark_blocks_seen_by_blockchain_node(block_hashes)
def test_block_headers_msg_from_header_bytes(self):
block_header = mock_eth_messages.get_dummy_block_header(1)
block_header_bytes = memoryview(
rlp.encode(BlockHeader.serialize(block_header)))
block_headers_msg = BlockHeadersEthProtocolMessage.from_header_bytes(
block_header_bytes)
raw_headers = block_headers_msg.get_block_headers()
headers_list = list(raw_headers)
self.assertEqual(len(headers_list), 1)
self.assertTrue(headers_list)
self.assertEqual(1, len(block_headers_msg.get_block_headers()))
self.assertEqual(1, len(block_headers_msg.get_block_headers_bytes()))
self.assertEqual(block_header,
block_headers_msg.get_block_headers()[0])
self.assertEqual(
block_header_bytes.tobytes(),
block_headers_msg.get_block_headers_bytes()[0].tobytes())
def test_complete_header_body_fetch(self):
self.node.block_processing_service.queue_block_for_processing = MagicMock()
self.sut.is_valid_block_timestamp = MagicMock(return_value=True)
header = mock_eth_messages.get_dummy_block_header(1)
block = mock_eth_messages.get_dummy_block(1, header)
block_hash = header.hash_object()
new_block_hashes_message = NewBlockHashesEthProtocolMessage.from_block_hash_number_pair(
block_hash, 1
)
header_message = BlockHeadersEthProtocolMessage(None, [header])
bodies_message = BlockBodiesEthProtocolMessage(None, [TransientBlockBody(block.transactions, block.uncles)])
self.sut.msg_new_block_hashes(new_block_hashes_message)
self.assertEqual(1, len(self.sut.pending_new_block_parts.contents))
self.assertEqual(2, len(self.enqueued_messages))
self.sut.msg_block_headers(header_message)
self.sut.msg_block_bodies(bodies_message)
self.node.block_processing_service.queue_block_for_processing.assert_called_once()
def try_send_headers_to_node(self, block_hashes: List[Sha256Hash]) -> bool:
"""
Creates and sends a block headers message to blockchain connection.
In most cases, this method should be called with block hashes that are confirmed to
exist in the block queuing service, but contains checks for safety for otherwise,
and aborts the function if any headers are not found.
"""
headers = []
for block_hash in block_hashes:
if block_hash not in self._blocks:
self.connection.log_debug(
"{} was not found in queuing service. Aborting attempt to send headers.",
block_hash)
return False
if not self.node.block_queuing_service_manager.is_in_common_block_storage(
block_hash):
self.connection.log_debug(
"{} was not in block storage. Aborting attempt to send headers",
block_hash)
return False
block_message = cast(InternalEthBlockInfo,
self.node.block_storage[block_hash])
partial_headers_message = self.build_block_header_message(
block_hash, block_message)
block_headers = partial_headers_message.get_block_headers()
assert len(block_headers) == 1
headers.append(block_headers[0])
height = self._height_by_block_hash.contents.get(block_hash, None)
self.connection.log_debug(
"Appending {} header ({}) for sending to blockchain node.",
block_hash, height)
full_header_message = BlockHeadersEthProtocolMessage(None, headers)
self.connection.enqueue_msg(full_header_message)
return True
def setUp(self) -> None:
self.node = MockGatewayNode(
gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0))
self.node.broadcast = MagicMock()
self.block_queuing_service = EthBlockQueuingService(self.node)
self.node.block_queuing_service = self.block_queuing_service
self.block_hashes = []
self.block_messages = []
self.block_headers = []
self.block_bodies = []
# block numbers: 1000-1019
prev_block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
i, i + 1000, prev_block_hash=prev_block_hash))
block_hash = block_message.block_hash()
self.block_hashes.append(block_hash)
self.block_messages.append(block_message)
block_parts = block_message.to_new_block_parts()
self.block_headers.append(
BlockHeadersEthProtocolMessage.from_header_bytes(
block_parts.block_header_bytes).get_block_headers()[0])
self.block_bodies.append(
BlockBodiesEthProtocolMessage.from_body_bytes(
block_parts.block_body_bytes).get_blocks()[0])
self.block_queuing_service.push(block_hash, block_message)
prev_block_hash = block_hash
self.block_processing_service = EthBlockProcessingService(self.node)
self.node.broadcast.reset_mock()
def msg_get_block_headers(self, msg):
self.connection.log_trace(
"Replying with empty headers message to the get headers request")
block_headers_msg = BlockHeadersEthProtocolMessage(None, [])
self.connection.enqueue_msg(block_headers_msg)
self._waiting_checkpoint_headers_request = False