class EthBlockProcessingServiceTest(AbstractTestCase): def setUp(self) -> None: self.node = MockGatewayNode( gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0)) self.node.broadcast = MagicMock() self.block_queuing_service = EthBlockQueuingService(self.node) self.node.block_queuing_service = self.block_queuing_service self.block_hashes = [] self.block_messages = [] self.block_headers = [] self.block_bodies = [] # block numbers: 1000-1019 prev_block_hash = None for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( i, i + 1000, prev_block_hash=prev_block_hash)) block_hash = block_message.block_hash() self.block_hashes.append(block_hash) self.block_messages.append(block_message) block_parts = block_message.to_new_block_parts() self.block_headers.append( BlockHeadersEthProtocolMessage.from_header_bytes( block_parts.block_header_bytes).get_block_headers()[0]) self.block_bodies.append( BlockBodiesEthProtocolMessage.from_body_bytes( block_parts.block_body_bytes).get_blocks()[0]) self.block_queuing_service.push(block_hash, block_message) prev_block_hash = block_hash self.block_processing_service = EthBlockProcessingService(self.node) self.node.broadcast.reset_mock() def test_try_process_get_block_bodies_request(self): success = self.block_processing_service.try_process_get_block_bodies_request( GetBlockBodiesEthProtocolMessage( None, [block_hash.binary for block_hash in self.block_hashes[:2]])) self.assertTrue(success) self.node.broadcast.assert_called_once_with( BlockBodiesEthProtocolMessage( None, self.block_bodies[:2], ), connection_types=[ConnectionType.BLOCKCHAIN_NODE]) def test_try_process_get_block_bodies_request_not_found(self): success = self.block_processing_service.try_process_get_block_bodies_request( GetBlockBodiesEthProtocolMessage( None, [self.block_hashes[0].binary, bytes(helpers.generate_hash())])) self.assertFalse(success) self.node.broadcast.assert_not_called()
def setUp(self): self.node = MockGatewayNode( gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0)) self.node.block_parts_storage = ExpiringDict( self.node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, "eth_block_queue_parts", ) self.node_connection = Mock() self.node_connection.is_active = MagicMock(return_value=True) self.node.set_known_total_difficulty = MagicMock() self.node_conn = self.node_connection self.block_queuing_service = EthBlockQueuingService( self.node, self.node_conn) self.node.block_queuing_service_manager.add_block_queuing_service( self.node_conn, self.block_queuing_service) self.node_conn.enqueue_msg = MagicMock() self.block_hashes = [] self.block_messages = [] self.block_headers = [] self.block_bodies = [] # block numbers: 1000-1019 prev_block_hash = None for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( i, i + 1000, prev_block_hash=prev_block_hash)) block_hash = block_message.block_hash() self.block_hashes.append(block_hash) self.block_messages.append(block_message) block_parts = block_message.to_new_block_parts() self.block_headers.append( BlockHeadersEthProtocolMessage.from_header_bytes( block_parts.block_header_bytes).get_block_headers()[0]) self.block_bodies.append( BlockBodiesEthProtocolMessage.from_body_bytes( block_parts.block_body_bytes).get_blocks()[0]) self.node.block_queuing_service_manager.push( block_hash, block_message) prev_block_hash = block_hash for block_hash in self.block_hashes: self.block_queuing_service.remove_from_queue(block_hash) self.block_queuing_service.best_sent_block = (1019, self.block_hashes[-1], time.time()) self.block_queuing_service.best_accepted_block = ( 1019, self.block_hashes[-1])
def setUp(self) -> None: opts = gateway_helpers.get_gateway_opts(8000, include_default_eth_args=True) self.node = MockGatewayNode(opts) self.connection = MagicMock(spec=AbstractGatewayBlockchainConnection) self.connection.node = self.node self.connection.is_active = MagicMock(return_value=True) self.connection.network_num = 5 self.connection.format_connection_desc = "127.0.0.1:12345 - B" self.node.set_known_total_difficulty = MagicMock() self.node.node_conn = self.connection self.node.message_converter = EthNormalMessageConverter() self.cleanup_service = EthNormalBlockCleanupService( self.node, NETWORK_NUM ) self.node.block_cleanup_service = self.cleanup_service self.node.block_processing_service = EthBlockProcessingService( self.node ) self.node.block_processing_service.queue_block_for_processing = MagicMock() self.node.block_queuing_service = EthBlockQueuingService(self.node) dummy_private_key = crypto_utils.make_private_key( helpers.generate_bytearray(111) ) dummy_public_key = crypto_utils.private_to_public_key(dummy_private_key) self.sut = EthNodeConnectionProtocol( self.connection, True, dummy_private_key, dummy_public_key ) self.sut._waiting_checkpoint_headers_request = False
def __init__(self, opts, node_ssl_service: NodeSSLService) -> None: super(EthGatewayNode, self).__init__( opts, node_ssl_service, eth_common_constants.TRACKED_BLOCK_CLEANUP_INTERVAL_S) self._node_public_key = None self._remote_public_key = None if opts.node_public_key is not None: self._node_public_key = convert.hex_to_bytes(opts.node_public_key) elif opts.blockchain_peers is None: raise RuntimeError( "128 digit public key must be included with command-line specified blockchain peer." ) if opts.remote_blockchain_peer is not None: if opts.remote_public_key is None: raise RuntimeError( "128 digit public key must be included with command-line specified remote blockchain peer." ) else: self._remote_public_key = convert.hex_to_bytes( opts.remote_public_key) self.block_processing_service: EthBlockProcessingService = EthBlockProcessingService( self) self.block_queuing_service: EthBlockQueuingService = EthBlockQueuingService( self) # List of know total difficulties, tuples of values (block hash, total difficulty) self._last_known_difficulties = deque( maxlen=eth_common_constants.LAST_KNOWN_TOTAL_DIFFICULTIES_MAX_COUNT ) # queue of the block hashes requested from remote blockchain node during sync self._requested_remote_blocks_queue = deque() # number of remote block requests to skip in case if requests and responses got out of sync self._skip_remote_block_requests_stats_count = 0 self.init_eth_gateway_stat_logging() self.init_eth_on_block_feed_stat_logging() self.message_converter = converter_factory.create_eth_message_converter( self.opts) self.eth_ws_proxy_publisher = EthWsProxyPublisher( opts.eth_ws_uri, self.feed_manager, self._tx_service, self) if self.opts.ws and not self.opts.eth_ws_uri: logger.warning(log_messages.ETH_WS_SUBSCRIBER_NOT_STARTED) self.average_block_gas_price = RunningAverage( gateway_constants.ETH_GAS_RUNNING_AVERAGE_SIZE) self.min_tx_from_node_gas_price = IntervalMinimum( gateway_constants.ETH_MIN_GAS_INTERVAL_S, self.alarm_queue) logger.info("Gateway enode url: {}", self.get_enode())
def setUp(self): self.node = MockGatewayNode( gateway_helpers.get_gateway_opts( 8000, max_block_interval_s=BLOCK_INTERVAL)) self.node.block_parts_storage = ExpiringDict( self.node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, "eth_block_queue_parts", ) self.node.alarm_queue = AlarmQueue() self.node.set_known_total_difficulty = MagicMock() self.block_processing_service = EthBlockProcessingService(self.node) self.node.block_processing_service = self.block_processing_service self.node_connection = MockConnection( MockSocketConnection(1, self.node, ip_address=LOCALHOST, port=8002), self.node) self.node_connection.is_active = MagicMock(return_value=True) self.block_queuing_service = EthBlockQueuingService( self.node, self.node_connection) self.node.block_queuing_service_manager.add_block_queuing_service( self.node_connection, self.block_queuing_service) self.node_connection.enqueue_msg = MagicMock() self.node_connection_2 = MockConnection( MockSocketConnection(1, self.node, ip_address=LOCALHOST, port=8003), self.node) self.node_connection_2.is_active = MagicMock(return_value=True) self.block_queuing_service_2 = EthBlockQueuingService( self.node, self.node_connection_2) self.node.block_queuing_service_manager.add_block_queuing_service( self.node_connection_2, self.block_queuing_service_2) self.node_connection_2.enqueue_msg = MagicMock() self.blockchain_connections = [ self.node_connection, self.node_connection_2 ] self.block_queuing_services = [ self.block_queuing_service, self.block_queuing_service_2 ] time.time = MagicMock(return_value=time.time())
def setUp(self): self.node = MockGatewayNode( gateway_helpers.get_gateway_opts( 8000, max_block_interval=BLOCK_INTERVAL)) self.node.alarm_queue = AlarmQueue() self.node_connection = MagicMock() self.node_connection.is_active = MagicMock(return_value=True) self.node.set_known_total_difficulty = MagicMock() self.node.node_conn = self.node_connection self.block_queuing_service = EthBlockQueuingService(self.node) self.block_processing_service = EthBlockProcessingService(self.node) self.node.block_queuing_service = self.block_queuing_service self.node.block_processing_service = self.block_processing_service self.node.send_msg_to_node = MagicMock() time.time = MagicMock(return_value=time.time())
def clean_block_transactions_from_block_queue( self, block_hash: Sha256Hash, block_queuing_service: EthBlockQueuingService) -> None: try: block_body = block_queuing_service.get_block_body_from_message( block_hash) assert block_body is not None transactions_hashes = block_body.get_block_transaction_hashes(0) self.clean_block_transactions_by_block_components( transaction_service=self.node.get_tx_service(), block_hash=block_hash, transactions_list=transactions_hashes) except Exception as error: logger.error( "clean_block_transactions_from_block_queue operation for block {} failed with error {}.", block_hash, error) block_body_parts = block_queuing_service.get_block_parts( block_hash) logger.debug( "Failed block body bytes: {}", "Empty" if block_body_parts is None else convert.bytes_to_hex( block_body_parts.block_body_bytes))
def setUp(self) -> None: self.node = MockGatewayNode( gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0)) self.node.broadcast = MagicMock() self.block_queuing_service = EthBlockQueuingService(self.node) self.node.block_queuing_service = self.block_queuing_service self.block_hashes = [] self.block_messages = [] self.block_headers = [] self.block_bodies = [] # block numbers: 1000-1019 prev_block_hash = None for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( i, i + 1000, prev_block_hash=prev_block_hash)) block_hash = block_message.block_hash() self.block_hashes.append(block_hash) self.block_messages.append(block_message) block_parts = block_message.to_new_block_parts() self.block_headers.append( BlockHeadersEthProtocolMessage.from_header_bytes( block_parts.block_header_bytes).get_block_headers()[0]) self.block_bodies.append( BlockBodiesEthProtocolMessage.from_body_bytes( block_parts.block_body_bytes).get_blocks()[0]) self.block_queuing_service.push(block_hash, block_message) prev_block_hash = block_hash self.block_processing_service = EthBlockProcessingService(self.node) self.node.broadcast.reset_mock()
class EthBlockQueuingServiceFetchingTest(AbstractTestCase): def setUp(self): self.node = MockGatewayNode( gateway_helpers.get_gateway_opts(8000, max_block_interval_s=0)) self.node.block_parts_storage = ExpiringDict( self.node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, "eth_block_queue_parts", ) self.node_connection = Mock() self.node_connection.is_active = MagicMock(return_value=True) self.node.set_known_total_difficulty = MagicMock() self.node_conn = self.node_connection self.block_queuing_service = EthBlockQueuingService( self.node, self.node_conn) self.node.block_queuing_service_manager.add_block_queuing_service( self.node_conn, self.block_queuing_service) self.node_conn.enqueue_msg = MagicMock() self.block_hashes = [] self.block_messages = [] self.block_headers = [] self.block_bodies = [] # block numbers: 1000-1019 prev_block_hash = None for i in range(20): block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( i, i + 1000, prev_block_hash=prev_block_hash)) block_hash = block_message.block_hash() self.block_hashes.append(block_hash) self.block_messages.append(block_message) block_parts = block_message.to_new_block_parts() self.block_headers.append( BlockHeadersEthProtocolMessage.from_header_bytes( block_parts.block_header_bytes).get_block_headers()[0]) self.block_bodies.append( BlockBodiesEthProtocolMessage.from_body_bytes( block_parts.block_body_bytes).get_blocks()[0]) self.node.block_queuing_service_manager.push( block_hash, block_message) prev_block_hash = block_hash for block_hash in self.block_hashes: self.block_queuing_service.remove_from_queue(block_hash) self.block_queuing_service.best_sent_block = (1019, self.block_hashes[-1], time.time()) self.block_queuing_service.best_accepted_block = ( 1019, self.block_hashes[-1]) def test_get_block_hashes_from_hash(self): # request: start: 1019, count: 1 # result: 1019 success, hashes = self.block_queuing_service.get_block_hashes_starting_from_hash( self.block_hashes[19], 1, 0, False) self.assertTrue(success) self.assertEqual(1, len(hashes)) self.assertEqual(self.block_hashes[19], hashes[0]) # request: start: 1010, count: 10 # result: 1010, 1011, .. 1019 success, hashes = self.block_queuing_service.get_block_hashes_starting_from_hash( self.block_hashes[10], 10, 0, False) self.assertTrue(success) self.assertEqual(10, len(hashes)) for i, block_hash in enumerate(hashes): self.assertEqual(self.block_hashes[i + 10], hashes[i]) # request: start: 1010, count: 2, skip: 5 # result: 1010, 1016 success, hashes = self.block_queuing_service.get_block_hashes_starting_from_hash( self.block_hashes[10], 2, 5, False) self.assertTrue(success) self.assertEqual(2, len(hashes)) self.assertEqual(self.block_hashes[10], hashes[0]) self.assertEqual(self.block_hashes[16], hashes[1]) # request: start: 1010, count: 2, skip: 5, reverse = True # result: 1010, 1004 success, hashes = self.block_queuing_service.get_block_hashes_starting_from_hash( self.block_hashes[10], 2, 5, True) self.assertTrue(success) self.assertEqual(2, len(hashes)) self.assertEqual(self.block_hashes[10], hashes[0]) self.assertEqual(self.block_hashes[4], hashes[1]) def test_get_block_hashes_from_height(self): success, hashes = self.block_queuing_service.get_block_hashes_starting_from_height( 1019, 1, 0, False) self.assertTrue(success) self.assertEqual(1, len(hashes)) self.assertEqual(self.block_hashes[19], hashes[0]) def test_get_block_by_hash_orphaned_not_found(self): # create fork at block 17 block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(21, 1017)) block_hash = block_message.block_hash() self.block_queuing_service.push(block_hash, block_message) success, hashes = self.block_queuing_service.get_block_hashes_starting_from_hash( block_hash, 1, 0, False) self.assertTrue(success) self.assertEqual(0, len(hashes)) def test_get_block_by_number_ambiguous_succeeds(self): # create fork at block 17 block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(21, 1017)) block_hash = block_message.block_hash() self.block_queuing_service.push(block_hash, block_message) success, hashes = self.block_queuing_service.get_block_hashes_starting_from_height( 1017, 1, 0, False) self.assertTrue(success) self.assertEqual(self.block_hashes[17], hashes[0]) def test_get_block_hashes_fork_follows_last_sent(self): # create fork at block 17 block_message = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(21, 1017)) block_hash = block_message.block_hash() self.block_queuing_service.push(block_hash, block_message) success, hashes = self.block_queuing_service.get_block_hashes_starting_from_hash( self.block_hashes[15], 5, 0, False) self.assertTrue(success) self.assertEqual(5, len(hashes)) self.assertEqual(self.block_hashes[15:20], hashes) def test_get_blocks_from_height_not_all_found_at_end(self): success, hashes = self.block_queuing_service.get_block_hashes_starting_from_height( 1018, 5, 0, False) self.assertTrue(success) self.assertEqual(2, len(hashes)) self.assertEqual(self.block_hashes[18], hashes[0]) self.assertEqual(self.block_hashes[19], hashes[1]) def test_get_blocks_from_height_not_all_found_at_beginning(self): success, hashes = self.block_queuing_service.get_block_hashes_starting_from_height( 950, 5, 0, False) self.assertFalse(success) self.assertEqual(0, len(hashes)) def test_iterate_recent_block_hashes(self): top_blocks = list( self.block_queuing_service.iterate_recent_block_hashes( max_count=10)) block_hash = top_blocks[0] self.assertEqual( self.block_queuing_service._height_by_block_hash[block_hash], self.block_queuing_service._highest_block_number) self.assertEqual(10, len(top_blocks)) def test_get_transactions_hashes_from_message(self): last_block_hash = list( self.block_queuing_service._block_hashes_by_height[ self.block_queuing_service._highest_block_number])[0] self.assertIsNotNone( self.block_queuing_service.get_block_body_from_message( last_block_hash)) self.assertIsNone( self.block_queuing_service.get_block_body_from_message(bytes(64))) def test_tracked_block_cleanup(self): from bxgateway.services.eth.eth_normal_block_cleanup_service import EthNormalBlockCleanupService self.node.block_cleanup_service = EthNormalBlockCleanupService( self.node, 1) tx_service = self.node.get_tx_service() for block_hash in self.block_queuing_service.iterate_recent_block_hashes( self.node.network.block_confirmations_count + 2): tx_service.track_seen_short_ids(block_hash, []) self.node.block_cleanup_service.block_cleanup_request = MagicMock() self.node._tracked_block_cleanup() self.node.block_cleanup_service.block_cleanup_request.assert_called_once( ) def test_try_send_headers_to_node_success(self): self.node_conn.enqueue_msg.reset_mock() result = self.block_queuing_service.try_send_headers_to_node( self.block_hashes[:4]) self.assertTrue(result) self.node_conn.enqueue_msg.assert_called_once_with( BlockHeadersEthProtocolMessage(None, self.block_headers[:4])) def test_try_send_headers_to_node_unknown_block(self): self.node.broadcast = MagicMock() result = self.block_queuing_service.try_send_headers_to_node( [self.block_hashes[0], helpers.generate_object_hash()]) self.assertFalse(result) self.node.broadcast.assert_not_called() def test_try_send_bodies_to_node_success(self): self.node_conn.enqueue_msg.reset_mock() result = self.block_queuing_service.try_send_bodies_to_node( self.block_hashes[:4]) self.assertTrue(result) self.node_conn.enqueue_msg.assert_called_once_with( BlockBodiesEthProtocolMessage(None, self.block_bodies[:4])) def test_try_send_bodies_to_node_unknown_block(self): self.node_conn.enqueue_msg.reset_mock() result = self.block_queuing_service.try_send_bodies_to_node( [self.block_hashes[0], helpers.generate_object_hash()]) self.assertFalse(result) self.node_conn.enqueue_msg.assert_not_called()
def build_block_queuing_service(self) -> EthBlockQueuingService: return EthBlockQueuingService(self)
class EthBlockQueuingServicePushingTest(AbstractTestCase): def setUp(self): self.node = MockGatewayNode( gateway_helpers.get_gateway_opts( 8000, max_block_interval=BLOCK_INTERVAL)) self.node.alarm_queue = AlarmQueue() self.node_connection = MagicMock() self.node_connection.is_active = MagicMock(return_value=True) self.node.set_known_total_difficulty = MagicMock() self.node.node_conn = self.node_connection self.block_queuing_service = EthBlockQueuingService(self.node) self.block_processing_service = EthBlockProcessingService(self.node) self.node.block_queuing_service = self.block_queuing_service self.node.block_processing_service = self.block_processing_service self.node.send_msg_to_node = MagicMock() time.time = MagicMock(return_value=time.time()) def test_accepted_blocks_pushes_next_immediately(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self.block_queuing_service.push(block_hash_2, block_2) self.block_queuing_service.push(block_hash_3, block_3) self._assert_block_sent(block_hash_1) # after block 1 is accepted, block 2 is immediately pushed self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) self._assert_block_sent(block_hash_2) self._progress_time() self._assert_block_sent(block_hash_3) # receive 1, 2a, 2b, 3b, 4b # confirm 1, 2a # send 1 (instant), 2a (instant), 3b (instant), 4b (after delay) def test_handle_single_block_fork_already_accepted(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2a = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2a = block_2a.block_hash() block_2b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 2, block_hash_1)) block_hash_2b = block_2b.block_hash() block_3b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 4, 3, block_hash_2b)) block_hash_3b = block_3b.block_hash() block_4b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 5, 4, block_hash_3b)) block_hash_4b = block_4b.block_hash() # accept block 1 self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) # accept block 2a self.block_queuing_service.push(block_hash_2a, block_2a) self._assert_block_sent(block_hash_2a) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2a, block_2a) # block 2b will never be sent self.block_queuing_service.push(block_hash_2b, block_2b) self._assert_no_blocks_sent() # block 3b will be sent self.block_queuing_service.push(block_hash_3b, block_3b) self._assert_block_sent(block_hash_3b) # sync triggered, requesting 2a by hash self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_hash_2a.binary, 1, 0, 0)) # response is empty self._assert_headers_sent([]) # request block 1 to establish common ancestor block_number_bytes = struct.pack(">I", 1) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 1, 0, 0)) self._assert_headers_sent([block_hash_1]) # request block 193, 193 + 191, etc to determine chain state block_number_bytes = struct.pack(">I", 193) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 128, 191, 0)) self._assert_headers_sent([]) # request block 2, 3, 4, ... to compare state block_number_bytes = struct.pack(">I", 2) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0, 0)) self._assert_headers_sent([block_hash_2b, block_hash_3b]) # request block 4, 5, 6, ... to compare state block_number_bytes = struct.pack(">I", 4) self.block_processing_service.try_process_get_block_headers_request( GetBlockHeadersEthProtocolMessage(None, block_number_bytes, 192, 0, 0)) self._assert_headers_sent([]) # 4b is sent after timeout (presumably Ethereum node didn't send back acceptance # because it's resolving chainstate) self.block_queuing_service.push(block_hash_4b, block_4b) self._assert_no_blocks_sent() self._progress_time() self._assert_block_sent(block_hash_4b) def test_handle_single_block_fork_accepted_later(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2a = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2a = block_2a.block_hash() block_2b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 2, block_hash_1)) block_hash_2b = block_2b.block_hash() block_3b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 4, 3, block_hash_2b)) block_hash_3b = block_3b.block_hash() block_4b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 5, 4, block_hash_3b)) block_hash_4b = block_4b.block_hash() # send + accept block 1 self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) # send block 2a self.block_queuing_service.push(block_hash_2a, block_2a) self._assert_block_sent(block_hash_2a) # block 2b will never be sent (despite no confirmation) self.block_queuing_service.push(block_hash_2b, block_2b) self._assert_no_blocks_sent() self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2a, block_2a) # block 3b will be sent immediately (block at height 2 confirmed) self.block_queuing_service.push(block_hash_3b, block_3b) self._assert_block_sent(block_hash_3b) # block 4b send later (no confirmation for height 3) self.block_queuing_service.push(block_hash_4b, block_4b) self._assert_no_blocks_sent() self._progress_time() self._assert_block_sent(block_hash_4b) def test_handle_out_of_order_blocks(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() # second block won't be sent, was somehow not received from BDN block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() # third block should still be queued and sent after timeout block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) # block 3 sent after a delay, since no block 2 was ever seen self.block_queuing_service.push(block_hash_3, block_3) self._assert_no_blocks_sent() self._progress_time() self._assert_block_sent(block_hash_3) def test_handle_out_of_order_blocks_confirmed_ancestor(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() # block 2 will be received after block 3, but still should be pushed to # blockchain node first since block 1 is confirmed and timeout not reached block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) self.block_queuing_service.push(block_hash_3, block_3) self._assert_no_blocks_sent() self.block_queuing_service.push(block_hash_2, block_2) self._assert_block_sent(block_hash_2) self._progress_time() self._assert_block_sent(block_hash_3) def test_handle_out_of_order_blocks_unconfirmed_ancestor(self): self._set_block_queue_in_progress() block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() # block 2 will be received after block 3, but still should be pushed to # blockchain node first after timeout (no confirmation on block 1) block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() # after another timeout, block 3 will be pushed block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.push(block_hash_3, block_3) self._assert_no_blocks_sent() self.block_queuing_service.push(block_hash_2, block_2) self._assert_no_blocks_sent() self._progress_time() self._assert_block_sent(block_hash_2) self._progress_time() self._assert_block_sent(block_hash_3) def test_handle_recovering_block_timeout(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) self.block_queuing_service.push(block_hash_2, waiting_for_recovery=True) self.block_queuing_service.push(block_hash_3, block_3) # sent block 3 anyway, block 2 is taking too long to recover self._progress_time() self._assert_block_sent(block_hash_3) # recovery times out in the end, nothing sent and queue cleared self._progress_recovery_timeout() self._assert_no_blocks_sent() self.assertEqual(0, len(self.block_queuing_service)) def test_handle_recovering_block_recovered_quickly(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) self.block_queuing_service.push(block_hash_2, waiting_for_recovery=True) self.block_queuing_service.push(block_hash_3, block_3) self._assert_no_blocks_sent() # block 2 recovers quickly enough, is sent ahead of block 3 self.block_queuing_service.update_recovered_block( block_hash_2, block_2) self._assert_block_sent(block_hash_2) self._progress_time() self._assert_block_sent(block_hash_3) def test_handle_recovering_block_recovered_ordering(self): self._set_block_queue_in_progress() block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() # blocks in recovery get added out of order self.block_queuing_service.push(block_hash_2, waiting_for_recovery=True) self.block_queuing_service.push(block_hash_1, waiting_for_recovery=True) self.block_queuing_service.push(block_hash_3, block_3) self._assert_no_blocks_sent() # everything in recovery recovers quickly, so correct order re-established self.block_queuing_service.update_recovered_block( block_hash_2, block_2) self.block_queuing_service.update_recovered_block( block_hash_1, block_1) self._assert_block_sent(block_hash_1) self._progress_time() self._assert_block_sent(block_hash_2) self._progress_time() self._assert_block_sent(block_hash_3) def test_handle_recovering_block_recovered_too_slow(self): self._set_block_queue_in_progress() block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() self.block_queuing_service.push(block_hash_1, waiting_for_recovery=True) self.block_queuing_service.push(block_hash_2, block_2) self._assert_no_blocks_sent() # block 1 takes too long to recover, so block 2 is just sent self._progress_time() self._assert_block_sent(block_hash_2) # block 1 is now stale, queue should be empty self.block_queuing_service.update_recovered_block( block_hash_1, block_1) self._assert_no_blocks_sent() self._progress_time() self._assert_no_blocks_sent() def test_dont_send_stale_blocks_newer_block_confirmed(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2, block_2) # block 1 will be ignored by queue, since block 2 already confirmed self.block_queuing_service.push(block_hash_1, block_1) self._assert_no_blocks_sent() self._progress_time() self._assert_no_blocks_sent() self.assertEqual(0, len(self.block_queuing_service)) def test_dont_send_stale_blocks_newer_block_sent(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() self.block_queuing_service.push(block_hash_2, block_2) self._assert_block_sent(block_hash_2) # block 1 will be ignored by queue, since block 2 already sent self.block_queuing_service.push(block_hash_1, block_1) self._assert_no_blocks_sent() self._progress_time() self._assert_no_blocks_sent() self.assertEqual(0, len(self.block_queuing_service)) def test_dont_send_stale_recovered_block(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2, block_2) self.block_queuing_service.push(block_hash_1, waiting_for_recovery=True) # block 1 will be ignored by queue after recovery (didn't know block # number before then), since block 2 already confirmed self.block_queuing_service.update_recovered_block( block_hash_1, block_1) self._assert_no_blocks_sent() self._progress_time() self._assert_no_blocks_sent() self.assertEqual(0, len(self.block_queuing_service)) def test_dont_send_seen_block(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self.block_queuing_service.push(block_hash_2, block_2) self._assert_block_sent(block_hash_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2, block_2) # block 2 marked seen by blockchain node, so eject block 2 self._progress_time() self._assert_no_blocks_sent() self.assertEqual(0, len(self.block_queuing_service)) def test_clear_out_stale_blocks(self): self._set_block_queue_in_progress() block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_2, block_2) self.block_queuing_service.push(block_hash_3, block_3) self._assert_no_blocks_sent() # block 3 marked seen by blockchain node, so eject all earlier blocks (2, 3) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_3, block_3) self._assert_no_blocks_sent() self.assertEqual(0, len(self.block_queuing_service)) def test_partial_chainstate(self): self.assertEqual(deque(), self.block_queuing_service.partial_chainstate(10)) block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) self.block_queuing_service.push(block_hash_2, block_2) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2, block_2) self.block_queuing_service.push(block_hash_3, block_3) self.assertEqual(0, len(self.block_queuing_service)) expected_result = deque([ EthBlockInfo(1, block_hash_1), EthBlockInfo(2, block_hash_2), EthBlockInfo(3, block_hash_3), ]) self.assertEqual(expected_result, self.block_queuing_service.partial_chainstate(3)) # returns full chain if requested length is shorter self.assertEqual(expected_result, self.block_queuing_service.partial_chainstate(1)) # not more data, so can't meet asked length next_expected_result = deque([ EthBlockInfo(1, block_hash_1), EthBlockInfo(2, block_hash_2), EthBlockInfo(3, block_hash_3), ]) self.assertEqual(next_expected_result, self.block_queuing_service.partial_chainstate(10)) def test_partial_chainstate_reorganizes(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_2b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 2, block_hash_1)) block_hash_2b = block_2b.block_hash() block_3b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 4, 3, block_hash_2b)) block_hash_3b = block_3b.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) self.block_queuing_service.push(block_hash_2, block_2) self.block_queuing_service.push(block_hash_2b, block_2b) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2, block_2) expected_result = deque([ EthBlockInfo(1, block_hash_1), EthBlockInfo(2, block_hash_2), ]) self.assertEqual(expected_result, self.block_queuing_service.partial_chainstate(10)) self.block_queuing_service.push(block_hash_3b, block_3b) next_expected_result = deque([ EthBlockInfo(1, block_hash_1), EthBlockInfo(2, block_hash_2b), EthBlockInfo(3, block_hash_3b), ]) self.assertEqual(next_expected_result, self.block_queuing_service.partial_chainstate(10)) def test_partial_chainstate_missing_entries(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() # 2b will never be pushed to block queue block_2b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 2, block_hash_1)) block_hash_2b = block_2b.block_hash() block_3b = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 4, 3, block_hash_2b)) block_hash_3b = block_3b.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) self.block_queuing_service.push(block_hash_2, block_2) # establish an earlier chainstate first, just for testing self.block_queuing_service.partial_chainstate(10) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_2, block_2) self.block_queuing_service.push(block_hash_3b, block_3b) expected_result = deque([ EthBlockInfo(3, block_hash_3b), ]) self.assertEqual(expected_result, self.block_queuing_service.partial_chainstate(10)) def test_partial_chainstate_missing_entries_in_head(self): block_1 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message(1, 1)) block_hash_1 = block_1.block_hash() block_2 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 2, 2, block_hash_1)) block_hash_2 = block_2.block_hash() block_3 = InternalEthBlockInfo.from_new_block_msg( mock_eth_messages.new_block_eth_protocol_message( 3, 3, block_hash_2)) block_hash_3 = block_3.block_hash() self.block_queuing_service.push(block_hash_1, block_1) self.block_queuing_service.mark_block_seen_by_blockchain_node( block_hash_1, block_1) # establish an earlier chainstate first, just for extending later self.block_queuing_service.partial_chainstate(10) self.block_queuing_service.push(block_hash_3, block_3) self._progress_time() expected_result = deque([ EthBlockInfo(3, block_hash_3), ]) self.assertEqual(expected_result, self.block_queuing_service.partial_chainstate(10)) def _assert_block_sent(self, block_hash: Sha256Hash) -> None: self.node.send_msg_to_node.assert_called() num_calls = 0 for call_args in self.node.send_msg_to_node.call_args_list: ((block_message, ), _) = call_args if isinstance(block_message, NewBlockEthProtocolMessage): self.assertEqual(block_hash, block_message.block_hash()) num_calls += 1 self.assertEqual(1, num_calls, "No blocks were sent") self.node.send_msg_to_node.reset_mock() def _assert_no_blocks_sent(self) -> None: for call_args in self.node.send_msg_to_node.call_args_list: ((block_message, ), _) = call_args if isinstance(block_message, NewBlockEthProtocolMessage): self.fail( f"Unexpected block sent: {block_message.block_hash()}") def _assert_headers_sent(self, hashes: List[Sha256Hash]): self.node.send_msg_to_node.assert_called_once() ((headers_message, ), _) = self.node.send_msg_to_node.call_args assert isinstance(headers_message, BlockHeadersEthProtocolMessage) headers = headers_message.get_block_headers() self.assertEqual(len(hashes), len(headers)) for expected_hash, header in zip(hashes, headers): self.assertEqual(expected_hash, header.hash_object()) self.node.send_msg_to_node.reset_mock() def _progress_time(self): time.time = MagicMock(return_value=time.time() + BLOCK_INTERVAL) self.node.alarm_queue.fire_alarms() def _progress_recovery_timeout(self): time.time = MagicMock(return_value=time.time() + gateway_constants.BLOCK_RECOVERY_MAX_QUEUE_TIME) self.node.alarm_queue.fire_alarms() def _set_block_queue_in_progress(self): # pretend block queuing service is in progress self.block_queuing_service.best_sent_block = (0, NULL_SHA256_HASH, time.time()) self.block_queuing_service.best_accepted_block = (0, NULL_SHA256_HASH)
def setUp(self) -> None: super().setUp() self.node.block_queuing_service = EthBlockQueuingService(self.node) node_conn = MagicMock() self.node.connection_pool.add(1, "127.0.0.0", 8002, node_conn)
def build_block_queuing_service( self, connection: AbstractGatewayBlockchainConnection ) -> EthBlockQueuingService: return EthBlockQueuingService(self, connection)
def setUp(self) -> None: super().setUp() self.node.block_queuing_service = EthBlockQueuingService(self.node)