def test_block_headers_request(self): get_headers = GetBlockHeadersEthProtocolMessage(None, self.BLOCK_HASH, 111, 222, 0) # Reply with empty headers to the first get headers request for fast sync mode support get_headers_frames = map(self.eth_node_cipher.encrypt_frame, frame_utils.get_frames(get_headers.msg_type, get_headers.rawbytes())) for get_headers_frame in get_headers_frames: helpers.receive_node_message(self.gateway_node, self.local_node_fileno, get_headers_frame) self.eth_remote_node_connection.enqueue_msg.assert_not_called() self.eth_node_connection.enqueue_msg.assert_called_once_with(BlockHeadersEthProtocolMessage(None, [])) # The second get headers message should be proxied to remote blockchain node get_headers_frames = map(self.eth_node_cipher.encrypt_frame, frame_utils.get_frames(get_headers.msg_type, get_headers.rawbytes())) for get_headers_frame in get_headers_frames: helpers.receive_node_message(self.gateway_node, self.local_node_fileno, get_headers_frame) self.eth_remote_node_connection.enqueue_msg.assert_called_once_with(get_headers) headers = BlockHeadersEthProtocolMessage(None, [ mock_eth_messages.get_dummy_block_header(1), mock_eth_messages.get_dummy_block_header(2) ]) headers_frames = map(self.eth_node_cipher.encrypt_frame, frame_utils.get_frames(headers.msg_type, headers.rawbytes())) for headers_frame in headers_frames: helpers.receive_node_message(self.gateway_node, self.remote_node_fileno, headers_frame) self.eth_node_connection.enqueue_msg.assert_called_with(headers)
def test_msg_get_block_headers_future_block(self): self.node.opts.max_block_interval_s = 0 block_hashes = [] block_heights = [] for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(i, i + 1000) ) block_hash = block_message.block_hash() block_heights.append(block_message.block_number()) block_hashes.append(block_hash) self.node.block_queuing_service_manager.push(block_hash, block_message) self.enqueued_messages.clear() self.sut.msg_proxy_request = MagicMock() block_height_bytes = block_heights[-1] + 4 block_height_bytes = block_height_bytes.to_bytes(8, "big") message = GetBlockHeadersEthProtocolMessage( None, block_height_bytes, 1, 0, 0 ) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_not_called() self.assertEqual(1, len(self.enqueued_messages)) headers_sent = self.enqueued_messages[0] self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage) self.assertEqual(0, len(headers_sent.get_block_headers()))
def test_msg_get_block_headers_fork(self): self.node.opts.max_block_interval_s = 0 block_hashes = [] for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(i, i + 1000) ) block_hash = block_message.block_hash() block_hashes.append(block_hash) self.node.block_queuing_service_manager.push(block_hash, block_message) # create fork block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(34, 1017) ) block_hash = block_message.block_hash() block_hashes.append(block_hash) self.node.block_queuing_service_manager.push(block_hash, block_message) self.enqueued_messages.clear() self.sut.msg_proxy_request = MagicMock() message = GetBlockHeadersEthProtocolMessage( None, block_hashes[-1].binary, 10, 0, 1 ) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_called_once() self.assertEqual(0, len(self.enqueued_messages))
def test_msg_get_block_headers_block_number(self): block_number = 123456 header = mock_eth_messages.get_dummy_block_header( 12, block_number=block_number ) block = mock_eth_messages.get_dummy_block(1, header) block_hash = header.hash_object() new_block_message = NewBlockEthProtocolMessage(None, block, 10) eth_block_info = InternalEthBlockInfo.from_new_block_msg( new_block_message ) self.node.block_queuing_service_manager.push(block_hash, eth_block_info) self.enqueued_messages.clear() self.sut.msg_proxy_request = MagicMock() block_number_bytes = struct.pack(">I", block_number) message = GetBlockHeadersEthProtocolMessage( None, block_number_bytes, 1, 0, 0 ) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_not_called() self.assertEqual(1, len(self.enqueued_messages)) headers_sent = self.enqueued_messages[0] self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage) self.assertEqual(1, len(headers_sent.get_block_headers())) self.assertEqual( block_hash, headers_sent.get_block_headers()[0].hash_object() )
def test_msg_get_block_headers_known_single(self): header = mock_eth_messages.get_dummy_block_header(12) block = mock_eth_messages.get_dummy_block(1, header) raw_hash = header.hash() block_hash = header.hash_object() new_block_message = NewBlockEthProtocolMessage(None, block, 10) eth_block_info = InternalEthBlockInfo.from_new_block_msg( new_block_message ) self.node.block_queuing_service_manager.push(block_hash, eth_block_info) self.enqueued_messages.clear() self.sut.msg_proxy_request = MagicMock() message = GetBlockHeadersEthProtocolMessage(None, raw_hash, 1, 0, 0) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_not_called() self.assertEqual(1, len(self.enqueued_messages)) headers_sent = self.enqueued_messages[0] self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage) self.assertEqual(1, len(headers_sent.get_block_headers())) self.assertEqual( block_hash, headers_sent.get_block_headers()[0].hash_object() ) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash, eth_block_info ) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_not_called() self.assertEqual(2, len(self.enqueued_messages))
def test_msg_get_block_headers_unknown(self): block_hash = helpers.generate_hash() self.sut.msg_proxy_request = MagicMock() message = GetBlockHeadersEthProtocolMessage(None, block_hash, 1, 0, 0) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_called_once()
def msg_new_block_hashes(self, msg: NewBlockHashesEthProtocolMessage): if not self.node.should_process_block_hash(msg.block_hash()): return block_hash_number_pairs = [] for block_hash, block_number in msg.get_block_hash_number_pairs(): block_stats.add_block_event_by_block_hash( block_hash, BlockStatEventType.BLOCK_ANNOUNCED_BY_BLOCKCHAIN_NODE, network_num=self.connection.network_num, more_info="Protocol: {}, Network: {}. {}".format( self.node.opts.blockchain_protocol, self.node.opts.blockchain_network, msg.extra_stats_data()), block_height=block_number, ) gateway_bdn_performance_stats_service.log_block_message_from_blockchain_node( False) if block_hash in self.node.blocks_seen.contents: self.node.on_block_seen_by_blockchain_node( block_hash, block_number=block_number) block_stats.add_block_event_by_block_hash( block_hash, BlockStatEventType. BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN, network_num=self.connection.network_num, block_height=block_number, ) self.connection.log_info( "Ignoring duplicate block {} from local blockchain node.", block_hash) continue recovery_cancelled = self.node.on_block_seen_by_blockchain_node( block_hash, block_number=block_number) if recovery_cancelled: continue self.node.track_block_from_node_handling_started(block_hash) block_hash_number_pairs.append((block_hash, block_number)) self.connection.log_info( "Fetching block {} from local Ethereum node.", block_hash) if not block_hash_number_pairs: return for block_hash, block_number in block_hash_number_pairs: # pyre-fixme[6]: Expected `memoryview` for 1st param but got `None`. self.pending_new_block_parts.add( block_hash, NewBlockParts(None, None, block_number)) self.connection.enqueue_msg( GetBlockHeadersEthProtocolMessage(None, block_hash.binary, 1, 0, False)) self.request_block_body( [block_hash for block_hash, _ in block_hash_number_pairs])
def try_process_get_block_headers_request( self, msg: GetBlockHeadersEthProtocolMessage ) -> bool: block_queuing_service = self._node.block_queuing_service block_hash = msg.get_block_hash() if block_hash is not None: logger.trace( "Checking for headers by hash ({}) in local block cache...", block_hash, ) ( success, requested_block_hashes ) = block_queuing_service.get_block_hashes_starting_from_hash( block_hash, msg.get_amount(), msg.get_skip(), bool(msg.get_reverse()), ) else: block_number = msg.get_block_number() if block_number: logger.trace( "Checking for headers by block number ({}) " "in local block cache", block_number, ) ( success, requested_block_hashes ) = block_queuing_service.get_block_hashes_starting_from_height( block_number, msg.get_amount(), msg.get_skip(), bool(msg.get_reverse()), ) else: logger.debug( "Unexpectedly, request for headers did not contain " "block hash or block number. Skipping." ) return False if success: return block_queuing_service.try_send_headers_to_node( requested_block_hashes ) else: logger.trace( "Could not find requested block hashes. " "Forwarding to remote blockchain connection." ) return False
def _check_for_block_on_repeat(self, block_hash: Sha256Hash) -> float: get_confirmation_message = GetBlockHeadersEthProtocolMessage( None, block_hash.binary, 1, 0, 0) self.connection.enqueue_msg(get_confirmation_message) if self.block_check_repeat_count[ block_hash] < eth_common_constants.CHECK_BLOCK_RECEIPT_MAX_COUNT: self.block_check_repeat_count[block_hash] += 1 return eth_common_constants.CHECK_BLOCK_RECEIPT_INTERVAL_S else: del self.block_check_repeat_count[block_hash] del self.block_checking_alarms[block_hash] return constants.CANCEL_ALARMS
def test_msg_block_adds_headers_and_bodies(self): self.node.opts.max_block_interval = 0 self.sut.is_valid_block_timestamp = MagicMock(return_value=True) block_hashes = [] block_messages = [] block_hash = None for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(i, i + 1000, block_hash) ) block_hash = block_message.block_hash() block_hashes.append(block_hash) block_messages.append(block_message) # push all blocks, except for # 17 for i in (j for j in range(20) if j != 17): self.node.block_queuing_service.push( block_hashes[i], block_messages[i] ) self.node.block_queuing_service.remove_from_queue(block_hashes[i]) self.node.block_queuing_service.best_sent_block = (1019, block_hashes[-1], 0) self.node.send_to_node_messages.clear() self.sut.msg_proxy_request = MagicMock() message = GetBlockHeadersEthProtocolMessage( None, block_hashes[10].binary, 10, 0, 0 ) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_called_once() self.assertEqual(0, len(self.node.send_to_node_messages)) self.sut.msg_proxy_request.reset_mock() self.sut.msg_block(block_messages[17].to_new_block_msg()) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_not_called() self.assertEqual(1, len(self.node.send_to_node_messages)) headers_sent = self.node.send_to_node_messages[0] self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage) self.assertEqual(10, len(headers_sent.get_block_headers())) for i in range(10): self.assertEqual( block_hashes[i + 10], headers_sent.get_block_headers()[i].hash_object(), )
def _check_for_block_on_repeat(self, block_hash: Sha256Hash) -> float: get_confirmation_message = GetBlockHeadersEthProtocolMessage( None, block_hash.binary, 1, 0, 0) # TODO: Should only be sent to the node that requested (https://bloxroute.atlassian.net/browse/BX-1922) self.node.broadcast(get_confirmation_message, connection_types=[ConnectionType.BLOCKCHAIN_NODE]) if self.block_check_repeat_count[ block_hash] < eth_common_constants.CHECK_BLOCK_RECEIPT_MAX_COUNT: self.block_check_repeat_count[block_hash] += 1 return eth_common_constants.CHECK_BLOCK_RECEIPT_INTERVAL_S else: del self.block_check_repeat_count[block_hash] del self.block_checking_alarms[block_hash] return constants.CANCEL_ALARMS
def _build_get_blocks_message_for_block_confirmation(self, hashes: List[Sha256Hash]) -> AbstractMessage: # TODO: remove unused method block_hash = NULL_SHA256_HASH for block_hash in hashes: last_attempt = self.requested_blocks_for_confirmation.contents.get(block_hash, 0) if time() - last_attempt > \ self.block_cleanup_poll_interval_s * eth_common_constants.BLOCK_CONFIRMATION_REQUEST_INTERVALS: break else: block_hash = hashes[0] self.requested_blocks_for_confirmation.add(block_hash, time()) return GetBlockHeadersEthProtocolMessage( None, block_hash=bytes(block_hash.binary), amount=100, skip=0, reverse=0 )
def test_msg_get_block_headers_known_many(self): self.node.opts.max_block_interval_s = 0 block_hashes = [] block_hash = None for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( i, i + 1000, block_hash ) ) block_hash = block_message.block_hash() block_hashes.append(block_hash) self.node.block_queuing_service_manager.push(block_hash, block_message) self.block_queuing_service.best_sent_block = (1019, block_hashes[-1], 0) self.enqueued_messages.clear() self.sut.msg_proxy_request = MagicMock() message = GetBlockHeadersEthProtocolMessage( None, block_hashes[10].binary, 10, 0, 0 ) self.sut.msg_get_block_headers(message) self.sut.msg_proxy_request.assert_not_called() self.assertEqual(1, len(self.enqueued_messages)) headers_sent = self.enqueued_messages[0] self.assertIsInstance(headers_sent, BlockHeadersEthProtocolMessage) self.assertEqual(10, len(headers_sent.get_block_headers())) for i in range(10): self.assertEqual( block_hashes[i + 10], headers_sent.get_block_headers()[i].hash_object(), )
def test_handle_single_block_fork_already_accepted(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2a = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2a = block_2a.block_hash() block_2b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 2, block_hash_1)) block_hash_2b = block_2b.block_hash() block_3b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 4, 3, block_hash_2b)) block_hash_3b = block_3b.block_hash() block_4b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 5, 4, block_hash_3b)) block_hash_4b = block_4b.block_hash() # accept block 1 self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) # accept block 2a self.block_queuing_service.push(block_hash_2a, block_2a) self._assert_block_sent(block_hash_2a) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2a, block_2a) # block 2b will never be sent self.block_queuing_service.push(block_hash_2b, block_2b) self._assert_no_blocks_sent() # block 3b will be sent self.block_queuing_service.push(block_hash_3b, block_3b) self._assert_block_sent(block_hash_3b) # sync triggered, requesting 2a by hash self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_hash_2a.binary, 1, 0, 0)) # response is empty self._assert_headers_sent([]) # request block 1 to establish common ancestor block_number_bytes = struct.pack(">I", 1) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 1, 0, 0)) self._assert_headers_sent([block_hash_1]) # request block 193, 193 + 191, etc to determine chain state block_number_bytes = struct.pack(">I", 193) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 128, 191, 0)) self._assert_headers_sent([]) # request block 2, 3, 4, ... to compare state block_number_bytes = struct.pack(">I", 2) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0, 0)) self._assert_headers_sent([block_hash_2b, block_hash_3b]) # request block 4, 5, 6, ... to compare state block_number_bytes = struct.pack(">I", 4) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0, 0)) self._assert_headers_sent([]) # 4b is sent after timeout (presumably Ethereum node didn't send back acceptance # because it's resolving chainstate) self.block_queuing_service.push(block_hash_4b, block_4b) self._assert_no_blocks_sent() self._progress_time() self._assert_block_sent(block_hash_4b)