class EthPendingTransactionFeed(Feed[EthTransactionFeedEntry, EthRawTransaction]):
NAME = rpc_constants.ETH_PENDING_TRANSACTION_FEED_NAME
FIELDS = ["tx_hash", "tx_contents"]
published_transactions: ExpiringSet[Sha256Hash]
def __init__(self, alarm_queue: AlarmQueue) -> None:
super().__init__(self.NAME)
# enforce uniqueness, since multiple sources can publish to
# pending transactions (eth ws + remote)
self.published_transactions = ExpiringSet(
alarm_queue, EXPIRATION_TIME_S, "pendingTxs"
)
def subscribe(
self, options: Dict[str, Any]
) -> Subscriber[EthTransactionFeedEntry]:
duplicates = options.get("duplicates", None)
if duplicates is not None:
if not isinstance(duplicates, bool):
raise RpcInvalidParams(
"\"duplicates\" must be a boolean"
)
return super().subscribe(options)
def publish(self, raw_message: EthRawTransaction) -> None:
if (
raw_message.tx_hash in self.published_transactions
and not self.any_subscribers_want_duplicates()
):
return
super().publish(raw_message)
self.published_transactions.add(raw_message.tx_hash)
def serialize(self, raw_message: EthRawTransaction) -> EthTransactionFeedEntry:
return EthTransactionFeedEntry(raw_message.tx_hash, raw_message.tx_contents)
def any_subscribers_want_duplicates(self) -> bool:
for subscriber in self.subscribers.values():
if subscriber.options.get("duplicates", False):
return True
return False
def should_publish_message_to_subscriber(
self,
subscriber: Subscriber[EthTransactionFeedEntry],
raw_message: EthRawTransaction,
serialized_message: EthTransactionFeedEntry
) -> bool:
if (
raw_message.tx_hash in self.published_transactions
and not subscriber.options.get("duplicates", False)
):
return False
else:
return True
def test_get_object_size(self):
mock_node = MockNode(get_common_opts(1234))
object_size = memory_utils.get_object_size(mock_node)
self.assertIsInstance(object_size, ObjectSize)
self.assertTrue(object_size.size > 0)
self.assertTrue(object_size.flat_size > 0)
self.assertTrue(object_size.is_actual_size)
self.assertEqual(0, len(object_size.references))
ex_set = ExpiringSet(AlarmQueue(), 10, "testset")
s = set()
h1 = Sha256Hash(b"1" * 32)
h2 = Sha256Hash(b"0" * 32)
print(memory_utils.get_object_size(ex_set).size)
print(memory_utils.get_object_size(s).size)
print(memory_utils.get_object_size(h1).size)
print(memory_utils.get_object_size(h2).size)
print(memory_utils.get_special_size(ex_set).size)
print(memory_utils.get_special_size(s).size)
print(memory_utils.get_special_size(h1).size)
print(memory_utils.get_special_size(h2).size)
ex_set.add(h1)
ex_set.add(h2)
s.add(h1)
s.add(h2)
print(memory_utils.get_object_size(ex_set).size)
print(memory_utils.get_special_size(ex_set).size)
print(memory_utils.get_object_size(s).size)
class ExpiringSetTest(AbstractTestCase):
ALARM_QUEUE = AlarmQueue()
EXPIRATION_TIME_S = 1
def setUp(self):
self.sut = ExpiringSet(self.ALARM_QUEUE, self.EXPIRATION_TIME_S)
def test_cleanup(self):
test_item = "dummy_text"
self.sut.add(test_item)
self.assertTrue(test_item in self.sut.contents)
time.time = MagicMock(return_value=time.time() + self.EXPIRATION_TIME_S + 1)
self.ALARM_QUEUE.fire_alarms()
self.assertFalse(test_item in self.sut.contents)
def test_cleanup__not_existing_item(self):
test_item = "dummy_text"
self.sut.add(test_item)
self.assertTrue(test_item in self.sut.contents)
self.sut.contents.remove(test_item)
self.assertFalse(test_item in self.sut.contents)
time.time = MagicMock(return_value=time.time() + self.EXPIRATION_TIME_S + 1)
self.ALARM_QUEUE.fire_alarms()
self.assertFalse(test_item in self.sut.contents)
def test_get_recent(self):
for i in range(5):
self.sut.add(i)
self.assertEqual([4, 3, 2], self.sut.get_recent_items(3))
self.assertEqual([4, 3, 2, 1, 0], self.sut.get_recent_items(6))
self.assertEqual([4], self.sut.get_recent_items(1))
class EthNewBlockFeed(Feed[EthBlockFeedEntry, EthRawBlock]):
NAME = "newBlocks"
FIELDS = ["hash", "header", "transactions", "uncles"]
ALL_FIELDS = FIELDS
VALID_SOURCES = {
FeedSource.BLOCKCHAIN_SOCKET, FeedSource.BLOCKCHAIN_RPC,
FeedSource.BDN_SOCKET, FeedSource.BDN_INTERNAL
}
published_blocks: ExpiringSet[Sha256Hash]
published_blocks_height: ExpiringSet[int]
def __init__(
self,
node: "EthGatewayNode",
network_num: int = constants.ALL_NETWORK_NUM,
) -> None:
super().__init__(self.NAME, network_num=network_num)
self.last_block_number = 0
self.hash_for_last_block_number = set()
self.node = node
self.published_blocks = ExpiringSet(
node.alarm_queue,
gateway_constants.MAX_BLOCK_CACHE_TIME_S,
name="published_blocks")
self.published_blocks_height = ExpiringSet(
node.alarm_queue,
gateway_constants.MAX_BLOCK_CACHE_TIME_S,
name="published_blocks_height")
def serialize(self, raw_message: EthRawBlock) -> EthBlockFeedEntry:
block_message = raw_message.block
assert block_message is not None
return EthBlockFeedEntry(raw_message.block_hash, block_message)
def publish_blocks_from_queue(self, start_block_height,
end_block_height) -> Set[int]:
missing_blocks = set()
block_queuing_service = cast(
EthBlockQueuingService,
self.node.block_queuing_service_manager.
get_designated_block_queuing_service())
if block_queuing_service is None:
return missing_blocks
for block_number in range(start_block_height, end_block_height):
block_hash = block_queuing_service.accepted_block_hash_at_height.contents.get(
block_number)
if block_hash:
self.publish(
EthRawBlock(
block_number, block_hash, FeedSource.BDN_INTERNAL,
self.node._get_block_message_lazy(None, block_hash)))
else:
missing_blocks.add(block_number)
return missing_blocks
def publish(self, raw_message: EthRawBlock) -> None:
logger.trace("attempting to publish message: {} for feed {}",
raw_message, self.name)
if raw_message.source not in self.VALID_SOURCES:
return
if self.subscriber_count() == 0:
return
block_hash = raw_message.block_hash
block_number = raw_message.block_number
if block_number < self.last_block_number - gateway_constants.MAX_BLOCK_BACKLOG_TO_PUBLISH:
# published block is too far behind ignore
return
if block_hash in self.published_blocks:
# already published ignore
return
if raw_message.block is None:
block_queuing_service = cast(
EthBlockQueuingService,
self.node.block_queuing_service_manager.
get_designated_block_queuing_service())
best_accepted_height, _ = block_queuing_service.best_accepted_block
logger.warning(
"{} Feed Failed to recover block for message: {},"
"last_block_published {} last block in queueing service {}",
self.name, raw_message, self.last_block_number,
best_accepted_height)
return
self.published_blocks.add(block_hash)
self.published_blocks_height.add(block_number)
if self.last_block_number and block_number > self.last_block_number + 1:
# try to publish all intermediate blocks first
missing_blocks = self.publish_blocks_from_queue(
self.last_block_number + 1, block_number - 1)
if missing_blocks:
logger.info(
"Attempting to publish to feed block: {}, missing previous blocks {} ",
block_number, missing_blocks)
logger.debug("{} Processing new block message: {}", self.name,
raw_message)
super(EthNewBlockFeed, self).publish(raw_message)
if block_number in self.published_blocks_height and block_number <= self.last_block_number:
# possible fork, try to republish all later blocks
_missing_blocks = self.publish_blocks_from_queue(
block_number + 1, self.last_block_number)
if block_number > self.last_block_number:
self.last_block_number = block_number
class EthTransactionReceiptsFeed(Feed[TransactionReceiptsFeedEntry, Union[EthRawBlock, Dict]]):
NAME = rpc_constants.ETH_TRANSACTION_RECEIPTS_FEED_NAME
FIELDS = [
"receipt",
"receipt.block_hash",
"receipt.block_number",
"receipt.contract_address",
"receipt.cumulative_gas_used",
"receipt.from",
"receipt.gas_used",
"receipt.logs",
"receipt.logs_bloom",
"receipt.status",
"receipt.to",
"receipt.transaction_hash",
"receipt.transaction_index"
]
ALL_FIELDS = ["receipt"]
VALID_SOURCES = {
FeedSource.BLOCKCHAIN_SOCKET, FeedSource.BLOCKCHAIN_RPC, FeedSource.BDN_SOCKET
}
def __init__(self, node: "EthGatewayNode", network_num: int = constants.ALL_NETWORK_NUM, ) -> None:
super().__init__(self.NAME, network_num)
self.node = node
self.last_block_number = 0
self.published_blocks = ExpiringSet(
node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_published_blocks"
)
self.published_blocks_height = ExpiringSet(
node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_published_blocks_height"
)
self.blocks_confirmed_by_new_heads_notification = ExpiringSet(
node.alarm_queue, gateway_constants.MAX_BLOCK_CACHE_TIME_S, name="receipts_feed_newHeads_confirmed_blocks"
)
def serialize(self, raw_message: Union[EthRawBlock, Dict]) -> TransactionReceiptsFeedEntry:
# only receipts are serialized for publishing
if isinstance(raw_message, Dict):
return TransactionReceiptsFeedEntry(raw_message["result"])
else:
raise NotImplementedError
def publish(self, raw_message: Union[EthRawBlock, Dict]) -> None:
logger.trace(
"attempting to publish message: {} for feed {}", raw_message, self.name
)
if raw_message.source not in self.VALID_SOURCES or self.subscriber_count() == 0:
return
if isinstance(raw_message, Dict):
# transaction receipts published via parent publish method
raise NotImplementedError
block_hash = raw_message.block_hash
block_number = raw_message.block_number
block = raw_message.block
# receipts won't be available until NewHeads feed notification
if raw_message.source in [FeedSource.BLOCKCHAIN_RPC, FeedSource.BLOCKCHAIN_SOCKET]:
self.blocks_confirmed_by_new_heads_notification.add(block_hash)
if block_number < self.last_block_number - gateway_constants.MAX_BLOCK_BACKLOG_TO_PUBLISH:
# published block is too far behind, ignore
return
if block_hash in self.published_blocks:
# already published ignore
return
if block is None:
block = cast(InternalEthBlockInfo, self.node.block_queuing_service_manager.get_block_data(block_hash))
if block is None:
return
assert block is not None
if block_hash not in self.blocks_confirmed_by_new_heads_notification:
return
if raw_message.source == FeedSource.BDN_SOCKET:
block = block.to_new_block_msg()
self.published_blocks.add(block_hash)
self.published_blocks_height.add(block_number)
if self.last_block_number and block_number > self.last_block_number + 1:
# try to publish all intermediate blocks first
missing_blocks = self._publish_blocks_from_queue(self.last_block_number + 1, block_number - 1)
if missing_blocks:
logger.info(
"Attempting to publish to feed block: {}, missing previous blocks {} ", block_number, missing_blocks
)
logger.debug("{} Attempting to fetch transaction receipts for block {}", self.name, block_hash)
block_hash_str = block_hash.to_string(True)
for tx in block.txns():
asyncio.create_task(self._publish(tx.hash().to_string(True), block_hash_str))
if block_number in self.published_blocks_height and block_number <= self.last_block_number:
# possible fork, try to republish all later blocks
_missing_blocks = self._publish_blocks_from_queue(block_number + 1, self.last_block_number)
if block_number > self.last_block_number:
self.last_block_number = block_number
async def _publish(
self,
transaction_hash: str,
block_hash: str,
retry_count: int = 0
) -> None:
response = None
try:
response = await self.node.eth_ws_proxy_publisher.call_rpc(
rpc_constants.ETH_GET_TRANSACTION_RECEIPT_RPC_METHOD, [transaction_hash],
)
except RpcError as e:
error_response = e.to_json()
logger.warning(
"Failed to fetch transaction receipt for {} in block {}: {}. Ceasing attempts.",
transaction_hash, block_hash, error_response
)
assert response is not None
if response.result is None:
if retry_count == 0 or retry_count == RETRIES_MAX_ATTEMPTS:
logger.debug(
"Failed to fetch transaction receipt for tx {} in block {}: not found. "
"Attempt: {}. Max attempts: {}.",
transaction_hash, block_hash, retry_count + 1, RETRIES_MAX_ATTEMPTS + 1
)
if retry_count < RETRIES_MAX_ATTEMPTS:
sleep_time = utils.fibonacci(retry_count + 1) * 0.1
await asyncio.sleep(sleep_time)
asyncio.create_task(self._publish(transaction_hash, block_hash, retry_count + 1))
return
response.result = humps.decamelize(response.result)
json_response = response.to_json()
if json_response["result"]["block_hash"] != block_hash:
return
super().publish(json_response)
if retry_count > 0:
logger.debug(
"Succeeded in fetching receipt for tx {} in block {} after {} attempts.",
transaction_hash, block_hash, retry_count
)
def _publish_blocks_from_queue(self, start_block_height, end_block_height) -> Set[int]:
missing_blocks = set()
block_queuing_service = cast(
EthBlockQueuingService,
self.node.block_queuing_service_manager.get_designated_block_queuing_service()
)
if block_queuing_service is None:
return missing_blocks
for block_number in range(start_block_height, end_block_height):
block_hash = block_queuing_service.accepted_block_hash_at_height.contents.get(block_number)
if block_hash:
block = self.node.block_queuing_service_manager.get_block_data(block_hash)
if block is not None:
block_hash_str = block_hash.to_string(True)
for tx in block.txns():
asyncio.create_task(self._publish(tx.hash().to_string(True), block_hash_str))
else:
missing_blocks.add(block_number)
return missing_blocks
class EthPendingTransactionFeed(Feed[EthTransactionFeedEntry,
EthRawTransaction]):
NAME = rpc_constants.ETH_PENDING_TRANSACTION_FEED_NAME
FIELDS = [
"tx_hash",
"tx_contents",
"tx_contents.nonce",
"tx_contents.gas_price",
"tx_contents.gas",
"tx_contents.to",
"tx_contents.value",
"tx_contents.input",
"tx_contents.v",
"tx_contents.r",
"tx_contents.s",
"tx_contents.from",
"local_region",
]
FILTERS = {"value", "from", "to"}
ALL_FIELDS = ["tx_hash", "tx_contents", "local_region"]
published_transactions: ExpiringSet[Sha256Hash]
def __init__(
self,
alarm_queue: AlarmQueue,
network_num: int = constants.ALL_NETWORK_NUM,
) -> None:
super().__init__(self.NAME, network_num=network_num)
# enforce uniqueness, since multiple sources can publish to
# pending transactions (eth ws + remote)
self.published_transactions = ExpiringSet(alarm_queue,
EXPIRATION_TIME_S,
"pendingTxs")
def subscribe(
self, options: Dict[str,
Any]) -> Subscriber[EthTransactionFeedEntry]:
duplicates = options.get("duplicates", None)
if duplicates is not None:
if not isinstance(duplicates, bool):
raise RpcInvalidParams('"duplicates" must be a boolean')
return super().subscribe(options)
def publish(self, raw_message: EthRawTransaction) -> None:
if (raw_message.tx_hash in self.published_transactions
and not self.any_subscribers_want_duplicates()):
return
super().publish(raw_message)
self.published_transactions.add(raw_message.tx_hash)
def serialize(self,
raw_message: EthRawTransaction) -> EthTransactionFeedEntry:
return EthTransactionFeedEntry(raw_message.tx_hash,
raw_message.tx_contents,
raw_message.local_region)
def any_subscribers_want_duplicates(self) -> bool:
for subscriber in self.subscribers.values():
if subscriber.options.get("duplicates", False):
return True
return False
def should_publish_message_to_subscriber(
self,
subscriber: Subscriber[EthTransactionFeedEntry],
raw_message: EthRawTransaction,
serialized_message: EthTransactionFeedEntry,
) -> bool:
if (raw_message.tx_hash in self.published_transactions
and not subscriber.options.get("duplicates", False)):
return False
should_publish = True
if subscriber.filters:
logger_filters.trace(
"checking if should publish to {} with filters {}",
subscriber.subscription_id,
subscriber.filters,
)
contents = serialized_message.tx_contents
state = {
"value":
eth_filter_handlers.reformat_tx_value(contents["value"]),
"to": eth_filter_handlers.reformat_address(contents["to"]),
"from": eth_filter_handlers.reformat_address(contents["from"]),
}
should_publish = subscriber.validator(state)
logger_filters.trace("should publish: {}", should_publish)
return should_publish
class EthPendingTransactionFeed(Feed[EthTransactionFeedEntry,
EthRawTransaction]):
NAME = rpc_constants.ETH_PENDING_TRANSACTION_FEED_NAME
FIELDS = ["tx_hash", "tx_contents"]
FILTERS = {"transaction_value_range_eth", "from", "to"}
published_transactions: ExpiringSet[Sha256Hash]
def __init__(self, alarm_queue: AlarmQueue) -> None:
super().__init__(self.NAME)
# enforce uniqueness, since multiple sources can publish to
# pending transactions (eth ws + remote)
self.published_transactions = ExpiringSet(alarm_queue,
EXPIRATION_TIME_S,
"pendingTxs")
def subscribe(
self, options: Dict[str,
Any]) -> Subscriber[EthTransactionFeedEntry]:
duplicates = options.get("duplicates", None)
if duplicates is not None:
if not isinstance(duplicates, bool):
raise RpcInvalidParams('"duplicates" must be a boolean')
return super().subscribe(options)
def publish(self, raw_message: EthRawTransaction) -> None:
if (raw_message.tx_hash in self.published_transactions
and not self.any_subscribers_want_duplicates()):
return
super().publish(raw_message)
self.published_transactions.add(raw_message.tx_hash)
def serialize(self,
raw_message: EthRawTransaction) -> EthTransactionFeedEntry:
return EthTransactionFeedEntry(raw_message.tx_hash,
raw_message.tx_contents)
def any_subscribers_want_duplicates(self) -> bool:
for subscriber in self.subscribers.values():
if subscriber.options.get("duplicates", False):
return True
return False
def should_publish_message_to_subscriber(
self,
subscriber: Subscriber[EthTransactionFeedEntry],
raw_message: EthRawTransaction,
serialized_message: EthTransactionFeedEntry,
) -> bool:
if (raw_message.tx_hash in self.published_transactions
and not subscriber.options.get("duplicates", False)):
return False
should_publish = True
if subscriber.filters:
logger_filters.trace(
"checking if should publish to {} with filters {}",
subscriber.subscription_id,
subscriber.filters,
)
should_publish = filter_dsl.handle(
subscriber.filters,
eth_filter_handlers.handle_filter,
serialized_message,
)
logger_filters.trace("should publish: {}", should_publish)
return should_publish
def reformat_filters(self, filters: Dict[str, Any]) -> Dict[str, Any]:
return filter_dsl.reformat(filters,
eth_filter_handlers.reformat_filter)
