def test_get_object_size(self): mock_node = MockNode(get_common_opts(1234)) object_size = memory_utils.get_object_size(mock_node) self.assertIsInstance(object_size, ObjectSize) self.assertTrue(object_size.size > 0) self.assertTrue(object_size.flat_size > 0) self.assertTrue(object_size.is_actual_size) self.assertEqual(0, len(object_size.references)) ex_set = ExpiringSet(AlarmQueue(), 10, "testset") s = set() h1 = Sha256Hash(b"1" * 32) h2 = Sha256Hash(b"0" * 32) print(memory_utils.get_object_size(ex_set).size) print(memory_utils.get_object_size(s).size) print(memory_utils.get_object_size(h1).size) print(memory_utils.get_object_size(h2).size) print(memory_utils.get_special_size(ex_set).size) print(memory_utils.get_special_size(s).size) print(memory_utils.get_special_size(h1).size) print(memory_utils.get_special_size(h2).size) ex_set.add(h1) ex_set.add(h2) s.add(h1) s.add(h2) print(memory_utils.get_object_size(ex_set).size) print(memory_utils.get_special_size(ex_set).size) print(memory_utils.get_object_size(s).size)
def __init__( self, node: "AbstractGatewayNode", connection: AbstractGatewayBlockchainConnection ): self.node = node self.connection = connection self._blocks: ExpiringSet[Sha256Hash] = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, f"block_queuing_service_hashes_{self.connection.endpoint}" ) # queue of tuple (block hash, timestamp) for blocks that need to be # sent to blockchain node self._block_queue: Deque[BlockQueueEntry] = deque( maxlen=gateway_constants.BLOCK_QUEUE_LENGTH_LIMIT ) self._blocks_waiting_for_recovery: Dict[Sha256Hash, bool] = {} self._blocks_seen_by_blockchain_node: ExpiringSet[ Sha256Hash ] = ExpiringSet( node.alarm_queue, gateway_constants.GATEWAY_BLOCKS_SEEN_EXPIRATION_TIME_S, f"block_queuing_service_blocks_seen_{self.connection.endpoint}", )
class EthPendingTransactionFeed(Feed[EthTransactionFeedEntry, EthRawTransaction]): NAME = rpc_constants.ETH_PENDING_TRANSACTION_FEED_NAME FIELDS = ["tx_hash", "tx_contents"] published_transactions: ExpiringSet[Sha256Hash] def __init__(self, alarm_queue: AlarmQueue) -> None: super().__init__(self.NAME) # enforce uniqueness, since multiple sources can publish to # pending transactions (eth ws + remote) self.published_transactions = ExpiringSet( alarm_queue, EXPIRATION_TIME_S, "pendingTxs" ) def subscribe( self, options: Dict[str, Any] ) -> Subscriber[EthTransactionFeedEntry]: duplicates = options.get("duplicates", None) if duplicates is not None: if not isinstance(duplicates, bool): raise RpcInvalidParams( "\"duplicates\" must be a boolean" ) return super().subscribe(options) def publish(self, raw_message: EthRawTransaction) -> None: if ( raw_message.tx_hash in self.published_transactions and not self.any_subscribers_want_duplicates() ): return super().publish(raw_message) self.published_transactions.add(raw_message.tx_hash) def serialize(self, raw_message: EthRawTransaction) -> EthTransactionFeedEntry: return EthTransactionFeedEntry(raw_message.tx_hash, raw_message.tx_contents) def any_subscribers_want_duplicates(self) -> bool: for subscriber in self.subscribers.values(): if subscriber.options.get("duplicates", False): return True return False def should_publish_message_to_subscriber( self, subscriber: Subscriber[EthTransactionFeedEntry], raw_message: EthRawTransaction, serialized_message: EthTransactionFeedEntry ) -> bool: if ( raw_message.tx_hash in self.published_transactions and not subscriber.options.get("duplicates", False) ): return False else: return True
def __init__(self, alarm_queue: AlarmQueue) -> None: super().__init__(self.NAME) # enforce uniqueness, since multiple sources can publish to # pending transactions (eth ws + remote) self.published_transactions = ExpiringSet( alarm_queue, EXPIRATION_TIME_S, "pendingTxs" )
class ExpiringSetTest(AbstractTestCase): ALARM_QUEUE = AlarmQueue() EXPIRATION_TIME_S = 1 def setUp(self): self.sut = ExpiringSet(self.ALARM_QUEUE, self.EXPIRATION_TIME_S) def test_cleanup(self): test_item = "dummy_text" self.sut.add(test_item) self.assertTrue(test_item in self.sut.contents) time.time = MagicMock(return_value=time.time() + self.EXPIRATION_TIME_S + 1) self.ALARM_QUEUE.fire_alarms() self.assertFalse(test_item in self.sut.contents) def test_cleanup__not_existing_item(self): test_item = "dummy_text" self.sut.add(test_item) self.assertTrue(test_item in self.sut.contents) self.sut.contents.remove(test_item) self.assertFalse(test_item in self.sut.contents) time.time = MagicMock(return_value=time.time() + self.EXPIRATION_TIME_S + 1) self.ALARM_QUEUE.fire_alarms() self.assertFalse(test_item in self.sut.contents) def test_get_recent(self): for i in range(5): self.sut.add(i) self.assertEqual([4, 3, 2], self.sut.get_recent_items(3)) self.assertEqual([4, 3, 2, 1, 0], self.sut.get_recent_items(6)) self.assertEqual([4], self.sut.get_recent_items(1))
def __init__( self, alarm_queue: AlarmQueue, network_num: int = constants.ALL_NETWORK_NUM, ) -> None: super().__init__(self.NAME, network_num=network_num) # enforce uniqueness, since multiple sources can publish to # pending transactions (eth ws + remote) self.published_transactions = ExpiringSet(alarm_queue, EXPIRATION_TIME_S, "pendingTxs")
def __init__(self, node: "EthGatewayNode") -> None: super().__init__(self.NAME) self.last_block_number = 0 self.hash_for_last_block_number = set() self.node = node self.published_blocks = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="published_blocks") self.published_blocks_height = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="published_blocks_height")
def __init__(self, node: "EthGatewayNode", network_num: int = constants.ALL_NETWORK_NUM, ) -> None: super().__init__(self.NAME, network_num) self.node = node self.last_block_number = 0 self.published_blocks = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_published_blocks" ) self.published_blocks_height = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_published_blocks_height" ) self.blocks_confirmed_by_new_heads_notification = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_newHeads_confirmed_blocks" )
def test_request_block_propagation(self): block = btc_block().rawbytes() # propagate block helpers.receive_node_message(self.node1, self.blockchain_fileno, block) block_hold_request_relay = self.node1.get_bytes_to_send( self.relay_fileno) self.assertIn(BlockHoldingMessage.MESSAGE_TYPE, block_hold_request_relay.tobytes()) self.node1.on_bytes_sent(self.relay_fileno, len(block_hold_request_relay)) relayed_block = self.node1.get_bytes_to_send(self.relay_fileno) self.assertIn(BroadcastMessage.MESSAGE_TYPE, relayed_block.tobytes()) block_hold_request_gateway = self.node1.get_bytes_to_send( self.gateway_fileno) self.assertIn(BlockHoldingMessage.MESSAGE_TYPE, block_hold_request_gateway.tobytes()) self.clear_all_buffers() # receipt timeout time.time = MagicMock( return_value=time.time() + gateway_constants.NEUTRALITY_BROADCAST_BLOCK_TIMEOUT_S) self.node1.alarm_queue.fire_alarms() key_msg_gateway = self.node1.get_bytes_to_send(self.gateway_fileno) self.assertIn(KeyMessage.MESSAGE_TYPE, key_msg_gateway.tobytes()) self.node1.on_bytes_sent(self.gateway_fileno, len(key_msg_gateway)) block_prop_request = self.node1.get_bytes_to_send(self.gateway_fileno) self.assertIn(BlockPropagationRequestMessage.MESSAGE_TYPE, block_prop_request.tobytes()) self.clear_all_buffers() # get new block to send helpers.receive_node_message(self.node2, self.gateway_fileno, block_prop_request) new_relayed_block = self.node2.get_bytes_to_send(self.relay_fileno) self.assertIn(BroadcastMessage.MESSAGE_TYPE, new_relayed_block.tobytes()) helpers.clear_node_buffer(self.node2, self.relay_fileno) # receive new block helpers.receive_node_message(self.node1, self.relay_fileno, new_relayed_block) block_receipt = self.node1.get_bytes_to_send(self.gateway_fileno) self.assertIn(BlockReceivedMessage.MESSAGE_TYPE, block_receipt.tobytes()) # receive block receipt helpers.receive_node_message(self.node2, self.gateway_fileno, block_receipt) key_message = self.node2.get_bytes_to_send(self.relay_fileno) self.assertIn(KeyMessage.MESSAGE_TYPE, key_message.tobytes()) # receive key, but already seen so dont forward to blockchain helpers.receive_node_message(self.node1, self.relay_fileno, key_message) bytes_to_blockchain = self.node1.get_bytes_to_send( self.blockchain_fileno) self.assertEqual(OutputBuffer.EMPTY, bytes_to_blockchain) # clear blocks seen, rereceive self.node1.blocks_seen = ExpiringSet( self.node1.alarm_queue, gateway_constants.GATEWAY_BLOCKS_SEEN_EXPIRATION_TIME_S, "testset") helpers.receive_node_message(self.node1, self.relay_fileno, key_message) # ignore key message even if block is not in "blocks_seen" bytes_to_blockchain = self.node1.get_bytes_to_send( self.blockchain_fileno) self.assertEqual(OutputBuffer.EMPTY, bytes_to_blockchain)
def setUp(self): self.sut = ExpiringSet(self.ALARM_QUEUE, self.EXPIRATION_TIME_S, "testset")
class EthNewBlockFeed(Feed[EthBlockFeedEntry, EthRawBlock]): NAME = "newBlocks" FIELDS = ["hash", "header", "transactions", "uncles"] ALL_FIELDS = FIELDS VALID_SOURCES = { FeedSource.BLOCKCHAIN_SOCKET, FeedSource.BLOCKCHAIN_RPC, FeedSource.BDN_SOCKET, FeedSource.BDN_INTERNAL } published_blocks: ExpiringSet[Sha256Hash] published_blocks_height: ExpiringSet[int] def __init__( self, node: "EthGatewayNode", network_num: int = constants.ALL_NETWORK_NUM, ) -> None: super().__init__(self.NAME, network_num=network_num) self.last_block_number = 0 self.hash_for_last_block_number = set() self.node = node self.published_blocks = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="published_blocks") self.published_blocks_height = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="published_blocks_height") def serialize(self, raw_message: EthRawBlock) -> EthBlockFeedEntry: block_message = raw_message.block assert block_message is not None return EthBlockFeedEntry(raw_message.block_hash, block_message) def publish_blocks_from_queue(self, start_block_height, end_block_height) -> Set[int]: missing_blocks = set() block_queuing_service = cast( EthBlockQueuingService, self.node.block_queuing_service_manager. get_designated_block_queuing_service()) if block_queuing_service is None: return missing_blocks for block_number in range(start_block_height, end_block_height): block_hash = block_queuing_service.accepted_block_hash_at_height.contents.get( block_number) if block_hash: self.publish( EthRawBlock( block_number, block_hash, FeedSource.BDN_INTERNAL, self.node._get_block_message_lazy(None, block_hash))) else: missing_blocks.add(block_number) return missing_blocks def publish(self, raw_message: EthRawBlock) -> None: logger.trace("attempting to publish message: {} for feed {}", raw_message, self.name) if raw_message.source not in self.VALID_SOURCES: return if self.subscriber_count() == 0: return block_hash = raw_message.block_hash block_number = raw_message.block_number if block_number < self.last_block_number - gateway_constants.MAX_BLOCK_BACKLOG_TO_PUBLISH: # published block is too far behind ignore return if block_hash in self.published_blocks: # already published ignore return if raw_message.block is None: block_queuing_service = cast( EthBlockQueuingService, self.node.block_queuing_service_manager. get_designated_block_queuing_service()) best_accepted_height, _ = block_queuing_service.best_accepted_block logger.warning( "{} Feed Failed to recover block for message: {}," "last_block_published {} last block in queueing service {}", self.name, raw_message, self.last_block_number, best_accepted_height) return self.published_blocks.add(block_hash) self.published_blocks_height.add(block_number) if self.last_block_number and block_number > self.last_block_number + 1: # try to publish all intermediate blocks first missing_blocks = self.publish_blocks_from_queue( self.last_block_number + 1, block_number - 1) if missing_blocks: logger.info( "Attempting to publish to feed block: {}, missing previous blocks {} ", block_number, missing_blocks) logger.debug("{} Processing new block message: {}", self.name, raw_message) super(EthNewBlockFeed, self).publish(raw_message) if block_number in self.published_blocks_height and block_number <= self.last_block_number: # possible fork, try to republish all later blocks _missing_blocks = self.publish_blocks_from_queue( block_number + 1, self.last_block_number) if block_number > self.last_block_number: self.last_block_number = block_number
class EthTransactionReceiptsFeed(Feed[TransactionReceiptsFeedEntry, Union[EthRawBlock, Dict]]): NAME = rpc_constants.ETH_TRANSACTION_RECEIPTS_FEED_NAME FIELDS = [ "receipt", "receipt.block_hash", "receipt.block_number", "receipt.contract_address", "receipt.cumulative_gas_used", "receipt.from", "receipt.gas_used", "receipt.logs", "receipt.logs_bloom", "receipt.status", "receipt.to", "receipt.transaction_hash", "receipt.transaction_index" ] ALL_FIELDS = ["receipt"] VALID_SOURCES = { FeedSource.BLOCKCHAIN_SOCKET, FeedSource.BLOCKCHAIN_RPC, FeedSource.BDN_SOCKET } def __init__(self, node: "EthGatewayNode", network_num: int = constants.ALL_NETWORK_NUM, ) -> None: super().__init__(self.NAME, network_num) self.node = node self.last_block_number = 0 self.published_blocks = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_published_blocks" ) self.published_blocks_height = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_published_blocks_height" ) self.blocks_confirmed_by_new_heads_notification = ExpiringSet( node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_newHeads_confirmed_blocks" ) def serialize(self, raw_message: Union[EthRawBlock, Dict]) -> TransactionReceiptsFeedEntry: # only receipts are serialized for publishing if isinstance(raw_message, Dict): return TransactionReceiptsFeedEntry(raw_message["result"]) else: raise NotImplementedError def publish(self, raw_message: Union[EthRawBlock, Dict]) -> None: logger.trace( "attempting to publish message: {} for feed {}", raw_message, self.name ) if raw_message.source not in self.VALID_SOURCES or self.subscriber_count() == 0: return if isinstance(raw_message, Dict): # transaction receipts published via parent publish method raise NotImplementedError block_hash = raw_message.block_hash block_number = raw_message.block_number block = raw_message.block # receipts won't be available until NewHeads feed notification if raw_message.source in [FeedSource.BLOCKCHAIN_RPC, FeedSource.BLOCKCHAIN_SOCKET]: self.blocks_confirmed_by_new_heads_notification.add(block_hash) if block_number < self.last_block_number - gateway_constants.MAX_BLOCK_BACKLOG_TO_PUBLISH: # published block is too far behind, ignore return if block_hash in self.published_blocks: # already published ignore return if block is None: block = cast(InternalEthBlockInfo, self.node.block_queuing_service_manager.get_block_data(block_hash)) if block is None: return assert block is not None if block_hash not in self.blocks_confirmed_by_new_heads_notification: return if raw_message.source == FeedSource.BDN_SOCKET: block = block.to_new_block_msg() self.published_blocks.add(block_hash) self.published_blocks_height.add(block_number) if self.last_block_number and block_number > self.last_block_number + 1: # try to publish all intermediate blocks first missing_blocks = self._publish_blocks_from_queue(self.last_block_number + 1, block_number - 1) if missing_blocks: logger.info( "Attempting to publish to feed block: {}, missing previous blocks {} ", block_number, missing_blocks ) logger.debug("{} Attempting to fetch transaction receipts for block {}", self.name, block_hash) block_hash_str = block_hash.to_string(True) for tx in block.txns(): asyncio.create_task(self._publish(tx.hash().to_string(True), block_hash_str)) if block_number in self.published_blocks_height and block_number <= self.last_block_number: # possible fork, try to republish all later blocks _missing_blocks = self._publish_blocks_from_queue(block_number + 1, self.last_block_number) if block_number > self.last_block_number: self.last_block_number = block_number async def _publish( self, transaction_hash: str, block_hash: str, retry_count: int = 0 ) -> None: response = None try: response = await self.node.eth_ws_proxy_publisher.call_rpc( rpc_constants.ETH_GET_TRANSACTION_RECEIPT_RPC_METHOD, [transaction_hash], ) except RpcError as e: error_response = e.to_json() logger.warning( "Failed to fetch transaction receipt for {} in block {}: {}. Ceasing attempts.", transaction_hash, block_hash, error_response ) assert response is not None if response.result is None: if retry_count == 0 or retry_count == RETRIES_MAX_ATTEMPTS: logger.debug( "Failed to fetch transaction receipt for tx {} in block {}: not found. " "Attempt: {}. Max attempts: {}.", transaction_hash, block_hash, retry_count + 1, RETRIES_MAX_ATTEMPTS + 1 ) if retry_count < RETRIES_MAX_ATTEMPTS: sleep_time = utils.fibonacci(retry_count + 1) * 0.1 await asyncio.sleep(sleep_time) asyncio.create_task(self._publish(transaction_hash, block_hash, retry_count + 1)) return response.result = humps.decamelize(response.result) json_response = response.to_json() if json_response["result"]["block_hash"] != block_hash: return super().publish(json_response) if retry_count > 0: logger.debug( "Succeeded in fetching receipt for tx {} in block {} after {} attempts.", transaction_hash, block_hash, retry_count ) def _publish_blocks_from_queue(self, start_block_height, end_block_height) -> Set[int]: missing_blocks = set() block_queuing_service = cast( EthBlockQueuingService, self.node.block_queuing_service_manager.get_designated_block_queuing_service() ) if block_queuing_service is None: return missing_blocks for block_number in range(start_block_height, end_block_height): block_hash = block_queuing_service.accepted_block_hash_at_height.contents.get(block_number) if block_hash: block = self.node.block_queuing_service_manager.get_block_data(block_hash) if block is not None: block_hash_str = block_hash.to_string(True) for tx in block.txns(): asyncio.create_task(self._publish(tx.hash().to_string(True), block_hash_str)) else: missing_blocks.add(block_number) return missing_blocks
class EthPendingTransactionFeed(Feed[EthTransactionFeedEntry, EthRawTransaction]): NAME = rpc_constants.ETH_PENDING_TRANSACTION_FEED_NAME FIELDS = [ "tx_hash", "tx_contents", "tx_contents.nonce", "tx_contents.gas_price", "tx_contents.gas", "tx_contents.to", "tx_contents.value", "tx_contents.input", "tx_contents.v", "tx_contents.r", "tx_contents.s", "tx_contents.from", "local_region", ] FILTERS = {"value", "from", "to"} ALL_FIELDS = ["tx_hash", "tx_contents", "local_region"] published_transactions: ExpiringSet[Sha256Hash] def __init__( self, alarm_queue: AlarmQueue, network_num: int = constants.ALL_NETWORK_NUM, ) -> None: super().__init__(self.NAME, network_num=network_num) # enforce uniqueness, since multiple sources can publish to # pending transactions (eth ws + remote) self.published_transactions = ExpiringSet(alarm_queue, EXPIRATION_TIME_S, "pendingTxs") def subscribe( self, options: Dict[str, Any]) -> Subscriber[EthTransactionFeedEntry]: duplicates = options.get("duplicates", None) if duplicates is not None: if not isinstance(duplicates, bool): raise RpcInvalidParams('"duplicates" must be a boolean') return super().subscribe(options) def publish(self, raw_message: EthRawTransaction) -> None: if (raw_message.tx_hash in self.published_transactions and not self.any_subscribers_want_duplicates()): return super().publish(raw_message) self.published_transactions.add(raw_message.tx_hash) def serialize(self, raw_message: EthRawTransaction) -> EthTransactionFeedEntry: return EthTransactionFeedEntry(raw_message.tx_hash, raw_message.tx_contents, raw_message.local_region) def any_subscribers_want_duplicates(self) -> bool: for subscriber in self.subscribers.values(): if subscriber.options.get("duplicates", False): return True return False def should_publish_message_to_subscriber( self, subscriber: Subscriber[EthTransactionFeedEntry], raw_message: EthRawTransaction, serialized_message: EthTransactionFeedEntry, ) -> bool: if (raw_message.tx_hash in self.published_transactions and not subscriber.options.get("duplicates", False)): return False should_publish = True if subscriber.filters: logger_filters.trace( "checking if should publish to {} with filters {}", subscriber.subscription_id, subscriber.filters, ) contents = serialized_message.tx_contents state = { "value": eth_filter_handlers.reformat_tx_value(contents["value"]), "to": eth_filter_handlers.reformat_address(contents["to"]), "from": eth_filter_handlers.reformat_address(contents["from"]), } should_publish = subscriber.validator(state) logger_filters.trace("should publish: {}", should_publish) return should_publish
class EthPendingTransactionFeed(Feed[EthTransactionFeedEntry, EthRawTransaction]): NAME = rpc_constants.ETH_PENDING_TRANSACTION_FEED_NAME FIELDS = ["tx_hash", "tx_contents"] FILTERS = {"transaction_value_range_eth", "from", "to"} published_transactions: ExpiringSet[Sha256Hash] def __init__(self, alarm_queue: AlarmQueue) -> None: super().__init__(self.NAME) # enforce uniqueness, since multiple sources can publish to # pending transactions (eth ws + remote) self.published_transactions = ExpiringSet(alarm_queue, EXPIRATION_TIME_S, "pendingTxs") def subscribe( self, options: Dict[str, Any]) -> Subscriber[EthTransactionFeedEntry]: duplicates = options.get("duplicates", None) if duplicates is not None: if not isinstance(duplicates, bool): raise RpcInvalidParams('"duplicates" must be a boolean') return super().subscribe(options) def publish(self, raw_message: EthRawTransaction) -> None: if (raw_message.tx_hash in self.published_transactions and not self.any_subscribers_want_duplicates()): return super().publish(raw_message) self.published_transactions.add(raw_message.tx_hash) def serialize(self, raw_message: EthRawTransaction) -> EthTransactionFeedEntry: return EthTransactionFeedEntry(raw_message.tx_hash, raw_message.tx_contents) def any_subscribers_want_duplicates(self) -> bool: for subscriber in self.subscribers.values(): if subscriber.options.get("duplicates", False): return True return False def should_publish_message_to_subscriber( self, subscriber: Subscriber[EthTransactionFeedEntry], raw_message: EthRawTransaction, serialized_message: EthTransactionFeedEntry, ) -> bool: if (raw_message.tx_hash in self.published_transactions and not subscriber.options.get("duplicates", False)): return False should_publish = True if subscriber.filters: logger_filters.trace( "checking if should publish to {} with filters {}", subscriber.subscription_id, subscriber.filters, ) should_publish = filter_dsl.handle( subscriber.filters, eth_filter_handlers.handle_filter, serialized_message, ) logger_filters.trace("should publish: {}", should_publish) return should_publish def reformat_filters(self, filters: Dict[str, Any]) -> Dict[str, Any]: return filter_dsl.reformat(filters, eth_filter_handlers.reformat_filter)