def test_block_headers_request(self):
get_headers = GetBlockHeadersEthProtocolMessage(None, self.BLOCK_HASH, 111, 222, 0)
# Reply with empty headers to the first get headers request for fast sync mode support
get_headers_frames = map(self.eth_node_cipher.encrypt_frame,
frame_utils.get_frames(get_headers.msg_type, get_headers.rawbytes()))
for get_headers_frame in get_headers_frames:
helpers.receive_node_message(self.gateway_node, self.local_node_fileno, get_headers_frame)
self.eth_remote_node_connection.enqueue_msg.assert_not_called()
self.eth_node_connection.enqueue_msg.assert_called_once_with(BlockHeadersEthProtocolMessage(None, []))
# The second get headers message should be proxied to remote blockchain node
get_headers_frames = map(self.eth_node_cipher.encrypt_frame,
frame_utils.get_frames(get_headers.msg_type, get_headers.rawbytes()))
for get_headers_frame in get_headers_frames:
helpers.receive_node_message(self.gateway_node, self.local_node_fileno, get_headers_frame)
self.eth_remote_node_connection.enqueue_msg.assert_called_once_with(get_headers)
headers = BlockHeadersEthProtocolMessage(None, [
mock_eth_messages.get_dummy_block_header(1), mock_eth_messages.get_dummy_block_header(2)
])
headers_frames = map(self.eth_node_cipher.encrypt_frame,
frame_utils.get_frames(headers.msg_type, headers.rawbytes()))
for headers_frame in headers_frames:
helpers.receive_node_message(self.gateway_node, self.remote_node_fileno, headers_frame)
self.eth_node_connection.enqueue_msg.assert_called_with(headers)
def test_msg_get_block_headers_future_block(self):
self.node.opts.max_block_interval_s = 0
block_hashes = []
block_heights = []
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(i, i + 1000)
)
block_hash = block_message.block_hash()
block_heights.append(block_message.block_number())
block_hashes.append(block_hash)
self.node.block_queuing_service_manager.push(block_hash, block_message)
self.enqueued_messages.clear()
self.sut.msg_proxy_request = MagicMock()
block_height_bytes = block_heights[-1] + 4
block_height_bytes = block_height_bytes.to_bytes(8, "big")
message = GetBlockHeadersEthProtocolMessage(
None, block_height_bytes, 1, 0, 0
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_not_called()
self.assertEqual(1, len(self.enqueued_messages))
headers_sent = self.enqueued_messages[0]
self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage)
self.assertEqual(0, len(headers_sent.get_block_headers()))
def test_msg_get_block_headers_fork(self):
self.node.opts.max_block_interval_s = 0
block_hashes = []
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(i, i + 1000)
)
block_hash = block_message.block_hash()
block_hashes.append(block_hash)
self.node.block_queuing_service_manager.push(block_hash, block_message)
# create fork
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(34, 1017)
)
block_hash = block_message.block_hash()
block_hashes.append(block_hash)
self.node.block_queuing_service_manager.push(block_hash, block_message)
self.enqueued_messages.clear()
self.sut.msg_proxy_request = MagicMock()
message = GetBlockHeadersEthProtocolMessage(
None, block_hashes[-1].binary, 10, 0, 1
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_called_once()
self.assertEqual(0, len(self.enqueued_messages))
def test_msg_get_block_headers_block_number(self):
block_number = 123456
header = mock_eth_messages.get_dummy_block_header(
12, block_number=block_number
)
block = mock_eth_messages.get_dummy_block(1, header)
block_hash = header.hash_object()
new_block_message = NewBlockEthProtocolMessage(None, block, 10)
eth_block_info = InternalEthBlockInfo.from_new_block_msg(
new_block_message
)
self.node.block_queuing_service_manager.push(block_hash, eth_block_info)
self.enqueued_messages.clear()
self.sut.msg_proxy_request = MagicMock()
block_number_bytes = struct.pack(">I", block_number)
message = GetBlockHeadersEthProtocolMessage(
None, block_number_bytes, 1, 0, 0
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_not_called()
self.assertEqual(1, len(self.enqueued_messages))
headers_sent = self.enqueued_messages[0]
self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage)
self.assertEqual(1, len(headers_sent.get_block_headers()))
self.assertEqual(
block_hash, headers_sent.get_block_headers()[0].hash_object()
)
def test_msg_get_block_headers_known_single(self):
header = mock_eth_messages.get_dummy_block_header(12)
block = mock_eth_messages.get_dummy_block(1, header)
raw_hash = header.hash()
block_hash = header.hash_object()
new_block_message = NewBlockEthProtocolMessage(None, block, 10)
eth_block_info = InternalEthBlockInfo.from_new_block_msg(
new_block_message
)
self.node.block_queuing_service_manager.push(block_hash, eth_block_info)
self.enqueued_messages.clear()
self.sut.msg_proxy_request = MagicMock()
message = GetBlockHeadersEthProtocolMessage(None, raw_hash, 1, 0, 0)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_not_called()
self.assertEqual(1, len(self.enqueued_messages))
headers_sent = self.enqueued_messages[0]
self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage)
self.assertEqual(1, len(headers_sent.get_block_headers()))
self.assertEqual(
block_hash, headers_sent.get_block_headers()[0].hash_object()
)
self.block_queuing_service.mark_block_seen_by_blockchain_node(
block_hash, eth_block_info
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_not_called()
self.assertEqual(2, len(self.enqueued_messages))
def test_msg_get_block_headers_unknown(self):
block_hash = helpers.generate_hash()
self.sut.msg_proxy_request = MagicMock()
message = GetBlockHeadersEthProtocolMessage(None, block_hash, 1, 0, 0)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_called_once()
def msg_new_block_hashes(self, msg: NewBlockHashesEthProtocolMessage):
if not self.node.should_process_block_hash(msg.block_hash()):
return
block_hash_number_pairs = []
for block_hash, block_number in msg.get_block_hash_number_pairs():
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_ANNOUNCED_BY_BLOCKCHAIN_NODE,
network_num=self.connection.network_num,
more_info="Protocol: {}, Network: {}. {}".format(
self.node.opts.blockchain_protocol,
self.node.opts.blockchain_network, msg.extra_stats_data()),
block_height=block_number,
)
gateway_bdn_performance_stats_service.log_block_message_from_blockchain_node(
False)
if block_hash in self.node.blocks_seen.contents:
self.node.on_block_seen_by_blockchain_node(
block_hash, block_number=block_number)
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.
BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN,
network_num=self.connection.network_num,
block_height=block_number,
)
self.connection.log_info(
"Ignoring duplicate block {} from local blockchain node.",
block_hash)
continue
recovery_cancelled = self.node.on_block_seen_by_blockchain_node(
block_hash, block_number=block_number)
if recovery_cancelled:
continue
self.node.track_block_from_node_handling_started(block_hash)
block_hash_number_pairs.append((block_hash, block_number))
self.connection.log_info(
"Fetching block {} from local Ethereum node.", block_hash)
if not block_hash_number_pairs:
return
for block_hash, block_number in block_hash_number_pairs:
# pyre-fixme[6]: Expected `memoryview` for 1st param but got `None`.
self.pending_new_block_parts.add(
block_hash, NewBlockParts(None, None, block_number))
self.connection.enqueue_msg(
GetBlockHeadersEthProtocolMessage(None, block_hash.binary, 1,
0, False))
self.request_block_body(
[block_hash for block_hash, _ in block_hash_number_pairs])
def try_process_get_block_headers_request(
self, msg: GetBlockHeadersEthProtocolMessage
) -> bool:
block_queuing_service = self._node.block_queuing_service
block_hash = msg.get_block_hash()
if block_hash is not None:
logger.trace(
"Checking for headers by hash ({}) in local block cache...",
block_hash,
)
(
success, requested_block_hashes
) = block_queuing_service.get_block_hashes_starting_from_hash(
block_hash,
msg.get_amount(),
msg.get_skip(),
bool(msg.get_reverse()),
)
else:
block_number = msg.get_block_number()
if block_number:
logger.trace(
"Checking for headers by block number ({}) "
"in local block cache",
block_number,
)
(
success, requested_block_hashes
) = block_queuing_service.get_block_hashes_starting_from_height(
block_number,
msg.get_amount(),
msg.get_skip(),
bool(msg.get_reverse()),
)
else:
logger.debug(
"Unexpectedly, request for headers did not contain "
"block hash or block number. Skipping."
)
return False
if success:
return block_queuing_service.try_send_headers_to_node(
requested_block_hashes
)
else:
logger.trace(
"Could not find requested block hashes. "
"Forwarding to remote blockchain connection."
)
return False
def _check_for_block_on_repeat(self, block_hash: Sha256Hash) -> float:
get_confirmation_message = GetBlockHeadersEthProtocolMessage(
None, block_hash.binary, 1, 0, 0)
self.connection.enqueue_msg(get_confirmation_message)
if self.block_check_repeat_count[
block_hash] < eth_common_constants.CHECK_BLOCK_RECEIPT_MAX_COUNT:
self.block_check_repeat_count[block_hash] += 1
return eth_common_constants.CHECK_BLOCK_RECEIPT_INTERVAL_S
else:
del self.block_check_repeat_count[block_hash]
del self.block_checking_alarms[block_hash]
return constants.CANCEL_ALARMS
def test_msg_block_adds_headers_and_bodies(self):
self.node.opts.max_block_interval = 0
self.sut.is_valid_block_timestamp = MagicMock(return_value=True)
block_hashes = []
block_messages = []
block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(i, i + 1000, block_hash)
)
block_hash = block_message.block_hash()
block_hashes.append(block_hash)
block_messages.append(block_message)
# push all blocks, except for # 17
for i in (j for j in range(20) if j != 17):
self.node.block_queuing_service.push(
block_hashes[i], block_messages[i]
)
self.node.block_queuing_service.remove_from_queue(block_hashes[i])
self.node.block_queuing_service.best_sent_block = (1019, block_hashes[-1], 0)
self.node.send_to_node_messages.clear()
self.sut.msg_proxy_request = MagicMock()
message = GetBlockHeadersEthProtocolMessage(
None, block_hashes[10].binary, 10, 0, 0
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_called_once()
self.assertEqual(0, len(self.node.send_to_node_messages))
self.sut.msg_proxy_request.reset_mock()
self.sut.msg_block(block_messages[17].to_new_block_msg())
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_not_called()
self.assertEqual(1, len(self.node.send_to_node_messages))
headers_sent = self.node.send_to_node_messages[0]
self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage)
self.assertEqual(10, len(headers_sent.get_block_headers()))
for i in range(10):
self.assertEqual(
block_hashes[i + 10],
headers_sent.get_block_headers()[i].hash_object(),
)
def _check_for_block_on_repeat(self, block_hash: Sha256Hash) -> float:
get_confirmation_message = GetBlockHeadersEthProtocolMessage(
None, block_hash.binary, 1, 0, 0)
# TODO: Should only be sent to the node that requested (https://bloxroute.atlassian.net/browse/BX-1922)
self.node.broadcast(get_confirmation_message,
connection_types=[ConnectionType.BLOCKCHAIN_NODE])
if self.block_check_repeat_count[
block_hash] < eth_common_constants.CHECK_BLOCK_RECEIPT_MAX_COUNT:
self.block_check_repeat_count[block_hash] += 1
return eth_common_constants.CHECK_BLOCK_RECEIPT_INTERVAL_S
else:
del self.block_check_repeat_count[block_hash]
del self.block_checking_alarms[block_hash]
return constants.CANCEL_ALARMS
def _build_get_blocks_message_for_block_confirmation(self, hashes: List[Sha256Hash]) -> AbstractMessage:
# TODO: remove unused method
block_hash = NULL_SHA256_HASH
for block_hash in hashes:
last_attempt = self.requested_blocks_for_confirmation.contents.get(block_hash, 0)
if time() - last_attempt > \
self.block_cleanup_poll_interval_s * eth_common_constants.BLOCK_CONFIRMATION_REQUEST_INTERVALS:
break
else:
block_hash = hashes[0]
self.requested_blocks_for_confirmation.add(block_hash, time())
return GetBlockHeadersEthProtocolMessage(
None,
block_hash=bytes(block_hash.binary),
amount=100,
skip=0,
reverse=0
)
def test_msg_get_block_headers_known_many(self):
self.node.opts.max_block_interval_s = 0
block_hashes = []
block_hash = None
for i in range(20):
block_message = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
i, i + 1000, block_hash
)
)
block_hash = block_message.block_hash()
block_hashes.append(block_hash)
self.node.block_queuing_service_manager.push(block_hash, block_message)
self.block_queuing_service.best_sent_block = (1019, block_hashes[-1], 0)
self.enqueued_messages.clear()
self.sut.msg_proxy_request = MagicMock()
message = GetBlockHeadersEthProtocolMessage(
None, block_hashes[10].binary, 10, 0, 0
)
self.sut.msg_get_block_headers(message)
self.sut.msg_proxy_request.assert_not_called()
self.assertEqual(1, len(self.enqueued_messages))
headers_sent = self.enqueued_messages[0]
self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage)
self.assertEqual(10, len(headers_sent.get_block_headers()))
for i in range(10):
self.assertEqual(
block_hashes[i + 10],
headers_sent.get_block_headers()[i].hash_object(),
)
def test_handle_single_block_fork_already_accepted(self):
block_1 = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(1, 1))
block_hash_1 = block_1.block_hash()
block_2a = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
2, 2, block_hash_1))
block_hash_2a = block_2a.block_hash()
block_2b = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
3, 2, block_hash_1))
block_hash_2b = block_2b.block_hash()
block_3b = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
4, 3, block_hash_2b))
block_hash_3b = block_3b.block_hash()
block_4b = InternalEthBlockInfo.from_new_block_msg(
mock_eth_messages.new_block_eth_protocol_message(
5, 4, block_hash_3b))
block_hash_4b = block_4b.block_hash()
# accept block 1
self.block_queuing_service.push(block_hash_1, block_1)
self._assert_block_sent(block_hash_1)
self.block_queuing_service.mark_block_seen_by_blockchain_node(
block_hash_1, block_1)
# accept block 2a
self.block_queuing_service.push(block_hash_2a, block_2a)
self._assert_block_sent(block_hash_2a)
self.block_queuing_service.mark_block_seen_by_blockchain_node(
block_hash_2a, block_2a)
# block 2b will never be sent
self.block_queuing_service.push(block_hash_2b, block_2b)
self._assert_no_blocks_sent()
# block 3b will be sent
self.block_queuing_service.push(block_hash_3b, block_3b)
self._assert_block_sent(block_hash_3b)
# sync triggered, requesting 2a by hash
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_hash_2a.binary, 1, 0,
0))
# response is empty
self._assert_headers_sent([])
# request block 1 to establish common ancestor
block_number_bytes = struct.pack(">I", 1)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 1, 0,
0))
self._assert_headers_sent([block_hash_1])
# request block 193, 193 + 191, etc to determine chain state
block_number_bytes = struct.pack(">I", 193)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 128,
191, 0))
self._assert_headers_sent([])
# request block 2, 3, 4, ... to compare state
block_number_bytes = struct.pack(">I", 2)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0,
0))
self._assert_headers_sent([block_hash_2b, block_hash_3b])
# request block 4, 5, 6, ... to compare state
block_number_bytes = struct.pack(">I", 4)
self.block_processing_service.try_process_get_block_headers_request(
GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0,
0))
self._assert_headers_sent([])
# 4b is sent after timeout (presumably Ethereum node didn't send back acceptance
# because it's resolving chainstate)
self.block_queuing_service.push(block_hash_4b, block_4b)
self._assert_no_blocks_sent()
self._progress_time()
self._assert_block_sent(block_hash_4b)
