def __init__(self) -> None:
self.name = None
self.log_level = LogLevel.STATS
self.logger = logging.get_logger(__name__)
self.priority_logger = logging.get_logger(__name__)
self.node = None
self.node_id: Optional[str] = None
def __init__(self) -> None:
super(_TransactionStatisticsService, self).__init__()
self.name = "TransactionInfo"
self.logger = logging.get_logger(LogRecordType.TransactionInfo)
self.priority_logger = logging.get_logger(
LogRecordType.TransactionPropagationInfo)
self.log_percentage_for_hash_by_network_num: Dict[int, float] = \
defaultdict(lambda: constants.TRANSACTIONS_BY_HASH_PERCENTAGE_TO_LOG_STATS_FOR)
self.log_percentage_for_sid_by_network_num: Dict[int, float] = \
defaultdict(lambda: constants.TRANSACTIONS_BY_SID_PERCENTAGE_TO_LOG_STATS_FOR)
def __init__(self, interval=0):
super(NodeInfo,
self).__init__("NodeInfo",
interval=interval,
look_back=0,
reset=False,
logger=logging.get_logger(LogRecordType.NodeInfo))
def test_custom_logger(self):
log_config.setup_logging(log_format=log_config.LogFormat.JSON,
default_log_level=log_config.LogLevel.TRACE,
default_logger_names="",
log_level_overrides={},
enable_fluent_logger=True,
fluentd_host="fluentd",
third_party_loggers=[
logging.LoggerConfig(
"test_logging", "{",
logging.LogLevel.TRACE,
handler_type.HandlerType.Fluent)
])
logger = logging.get_logger("test_logging")
handlers = self._get_handlers(logger)
self.assertEqual(len(handlers), 1)
stream_handlers = [
handler for handler in handlers
if isinstance(handler, StreamHandler)
]
fluentd_handlers = [
handler for handler in handlers
if isinstance(handler, FluentHandler)
]
self.assertEqual(len(stream_handlers), 0)
self.assertEqual(len(fluentd_handlers), 1)
for handler in handlers:
self.assertEqual(handler.level, 0)
fluentd_handler = fluentd_handlers[0]
self.assertIsInstance(fluentd_handler.formatter,
formatters.FluentJSONFormatter)
def test_create_logger_fluentd(self):
log_config.setup_logging(log_format=log_config.LogFormat.JSON,
default_log_level=log_config.LogLevel.TRACE,
default_logger_names="",
log_level_overrides={},
enable_fluent_logger=True,
fluentd_host="fluentd")
logger = logging.get_logger("test_logging")
handlers = self._get_handlers(logger)
self.assertEqual(len(handlers), 2)
stream_handlers = [
handler for handler in handlers
if isinstance(handler, StreamHandler)
]
fluentd_handlers = [
handler for handler in handlers
if isinstance(handler, FluentHandler)
]
self.assertEqual(len(stream_handlers), 1)
self.assertEqual(len(fluentd_handlers), 1)
for handler in handlers:
self.assertEqual(handler.level, 0)
fluentd_handler = fluentd_handlers[0]
stream_handler = stream_handlers[0]
self.assertIsInstance(fluentd_handler.formatter,
formatters.JSONFormatter)
self.assertIsInstance(stream_handler.formatter,
formatters.JSONFormatter)
def __init__(self,
interval: int = constants.NODE_STATS_INTERVAL_S) -> None:
super().__init__(
"NodeStatus",
interval,
reset=True,
stat_logger=logging.get_logger(LogRecordType.NodeStatus, __name__),
)
def __init__(self, interval: int = 0) -> None:
super(NodeInfo, self).__init__(
"NodeInfo",
interval=interval,
look_back=0,
reset=False,
stat_logger=logging.get_logger(LogRecordType.NodeInfo, __name__),
)
def __init__(self, interval: int = 0) -> None:
self.sizer_obj = Sizer()
super(MemoryStatsService, self).__init__(
"MemoryStats",
interval=interval,
look_back=5,
reset=False,
stat_logger=logging.get_logger(LogRecordType.Memory, __name__),
)
def __init__(self,
interval: int = gateway_constants.
GATEWAY_TRANSACTION_FEED_STATS_INTERVAL_S,
look_back: int = gateway_constants.
GATEWAY_TRANSACTION_FEED_STATS_LOOKBACK):
super().__init__("TransactionFeedStats",
interval,
look_back,
reset=True,
stat_logger=logging.get_logger(
LogRecordType.TransactionFeedStats))
def __init__(
self,
interval=gateway_constants.GATEWAY_BDN_PERFORMANCE_STATS_INTERVAL_S,
look_back=gateway_constants.GATEWAY_BDN_PERFORMANCE_STATS_LOOKBACK,
) -> None:
super(_GatewayBdnPerformanceStatsService, self).__init__(
"GatewayBdnPerformanceStats",
interval,
look_back,
reset=True,
stat_logger=logging.get_logger(LogRecordType.BdnPerformanceStats, __name__),
)
def __init__(
self,
interval: int = gateway_constants.ETH_GATEWAY_STATS_INTERVAL,
look_back: int = gateway_constants.ETH_GATEWAY_STATS_LOOKBACK,
):
super(_EthGatewayStatsService, self).__init__(
"EthGatewayStatsService",
interval,
look_back,
reset=True,
stat_logger=logging.get_logger(LogRecordType.TransactionStats, __name__),
)
def test_create_logger(self):
log_config.setup_logging(log_format=log_config.LogFormat.JSON,
default_log_level=log_config.LogLevel.TRACE,
default_logger_names="",
log_level_overrides={})
logger = logging.get_logger("test_logging")
handlers = self._get_handlers(logger)
self.assertEqual(len(handlers), 1)
for handler in handlers:
self.assertIsInstance(handler, StreamHandler)
self.assertIsInstance(handler.formatter, formatters.JSONFormatter)
def test_logging(self):
logger = logging.get_logger("test_logging")
with self.assertLogs() as cm:
logger.warning("TEST {}", 1)
self.assertEqual("WARNING:test_logging:TEST 1", cm.output[0])
with self.assertLogs(level=logging.LogLevel.TRACE) as cm:
logger.trace("TEST {}", 1)
self.assertEqual("TRACE:test_logging:TEST 1", cm.output[0])
with self.assertLogs(level=logging.LogLevel.WARNING) as cm:
logger.warning(log_messages.BDN_RETURNED_NO_PEERS, 1)
self.assertEqual(
"WARNING:test_logging:BDN returned no peers at endpoint: 1",
cm.output[0])
def __init__(
self,
interval: int = gateway_constants.GATEWAY_TRANSACTION_STATS_INTERVAL_S,
look_back: int = gateway_constants.GATEWAY_TRANSACTION_STATS_LOOKBACK,
):
super(_GatewayTransactionStatsService, self).__init__(
"GatewayTransactionStats",
interval,
look_back,
reset=True,
stat_logger=logging.get_logger(LogRecordType.TransactionStats,
__name__),
)
def __init__(
self,
interval: int = eth_constants.ETH_ON_BLOCK_FEED_STATS_INTERVAL_S,
look_back: int = eth_constants.ETH_ON_BLOCK_FEED_STATS_LOOKBACK,
) -> None:
super().__init__(
"EthCallFeedStats",
interval,
look_back,
reset=True,
stat_logger=logging.get_logger(LogRecordType.EthOnBlockFeedStats),
)
self.feed_name = rpc_constants.ETH_ON_BLOCK_FEED_NAME
from bxcommon.utils import memory_utils, crypto
from bxcommon.utils.deprecated import deprecated
from bxcommon.utils.memory_utils import ObjectSize, SizeType
from bxcommon.utils.object_encoder import ObjectEncoder
from bxcommon.utils.object_hash import Sha256Hash
from bxcommon.utils.proxy import task_pool_proxy
from bxcommon.utils.proxy.default_map_proxy import DefaultMapProxy
from bxcommon.utils.proxy.map_proxy import MapProxy
from bxcommon.utils.stats import hooks
from bxcommon.utils.stats.transaction_stat_event_type import TransactionStatEventType
from bxcommon.utils.stats.transaction_statistics_service import tx_stats
from bxutils import logging
from bxutils.logging import log_config
from bxutils.logging.log_record_type import LogRecordType
logger = logging.get_logger(__name__)
logger_memory_cleanup = logging.get_logger(LogRecordType.BlockCleanup,
__name__)
UNKNOWN_TRANSACTION_HASH: Sha256Hash = Sha256Hash(
bytearray(b"\xff" * crypto.SHA256_HASH_LEN))
class ExtensionTransactionService(TransactionService):
def __init__(self, node, network_num) -> None:
super(ExtensionTransactionService, self).__init__(node, network_num)
# Log levels need to be set again to include the loggers created in this conditionally imported class
log_config.lazy_set_log_level(node.opts.log_level_overrides)
self.proxy = tpe.TransactionService(
task_pool_proxy.get_pool_size(),
from cryptography.x509 import Certificate
from bxcommon.network.ip_endpoint import IpEndpoint
from bxcommon.network.network_direction import NetworkDirection
from bxcommon.network.socket_connection_state import SocketConnectionState, SocketConnectionStates
from bxcommon.utils.stats import hooks
from bxutils import logging
from bxutils.logging import LogRecordType
from bxutils.ssl import ssl_certificate_factory
if TYPE_CHECKING:
# pylint: disable=ungrouped-imports,cyclic-import
from bxcommon.connections.abstract_node import AbstractNode
logger = logging.get_logger(__name__)
network_troubleshooting_logger = logging.get_logger(LogRecordType.NetworkTroubleshooting, __name__)
SO_QUICKACK = 12
class AbstractSocketConnectionProtocol(BaseProtocol):
transport: Optional[Transport]
file_no: int
endpoint: IpEndpoint
direction: NetworkDirection
can_send: bool
state: SocketConnectionState
is_ssl: bool
_node: "AbstractNode"
_should_retry: bool
from bxcommon.messages.abstract_message import AbstractMessage
from bxcommon.models.blockchain_network_model import BlockchainNetworkModel
from bxcommon.network.socket_connection import SocketConnection
from bxcommon.services import sdn_http_service
from bxcommon.services.broadcast_service import BroadcastService, BroadcastOptions
from bxcommon.utils import memory_utils, json_utils
from bxcommon.utils.alarm_queue import AlarmQueue
from bxcommon.utils.stats.block_statistics_service import block_stats
from bxcommon.utils.stats.memory_statistics_service import memory_statistics
from bxcommon.utils.stats.node_info_service import node_info_statistics
from bxcommon.utils.stats.throughput_service import throughput_statistics
from bxcommon.utils.stats.transaction_statistics_service import tx_stats
from bxutils import logging
from bxutils.logging import LogRecordType
logger = logging.get_logger(__name__)
memory_logger = logging.get_logger(LogRecordType.BxMemory)
class DisconnectRequest(NamedTuple):
fileno: int
should_retry: bool
class AbstractNode:
__meta__ = ABCMeta
FLUSH_SEND_BUFFERS_INTERVAL = constants.OUTPUT_BUFFER_BATCH_MAX_HOLD_TIME * 2
NODE_TYPE = None
def __init__(self, opts: Namespace):
logger.debug("Initializing node of type: {}", self.NODE_TYPE)
from datetime import datetime
import time
import typing
from bxutils import logging
from bxutils.logging.log_record_type import LogRecordType
from bxcommon.utils.proxy import task_pool_proxy
from bxcommon.services.transaction_service import TransactionService
from bxcommon.services.extension_transaction_service import ExtensionTransactionService
from bxcommon.messages.bloxroute.abstract_cleanup_message import AbstractCleanupMessage
import task_pool_executor as tpe # pyre-ignore for now, figure this out later (stub file or Python wrapper?)
logger = logging.get_logger(LogRecordType.TransactionCleanup)
def contents_cleanup(transaction_service: TransactionService,
block_confirmation_message: AbstractCleanupMessage,
cleanup_tasks
):
start_datetime = datetime.utcnow()
start_time = time.time()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task = cleanup_tasks.borrow_task()
cleanup_task.init(tpe.InputBytes(block_confirmation_message.buf), tx_service.proxy)
task_pool_proxy.run_task(cleanup_task)
short_ids = cleanup_task.short_ids()
total_content_removed = cleanup_task.total_content_removed()
tx_count = cleanup_task.tx_count()
from bxcommon.utils.stats.memory_statistics_service import memory_statistics
from bxcommon.utils.stats.node_info_service import node_info_statistics
from bxcommon.utils.stats.node_statistics_service import node_stats_service
from bxcommon.utils.stats.throughput_service import throughput_statistics
from bxcommon.utils.stats.transaction_statistics_service import tx_stats
from bxcommon.utils.transaction_short_id_buckets import TransactionShortIdBuckets
from bxutils import log_messages
from bxutils import logging
from bxutils.exceptions.connection_authentication_error import \
ConnectionAuthenticationError
from bxutils.logging import LogRecordType, LogLevel
from bxutils.services.node_ssl_service import NodeSSLService
from bxutils.ssl.extensions import extensions_factory
from bxutils.ssl.ssl_certificate_type import SSLCertificateType
logger = logging.get_logger(__name__)
memory_logger = logging.get_logger(LogRecordType.BxMemory, __name__)
performance_troubleshooting_logger = logging.get_logger(
LogRecordType.PerformanceTroubleshooting, __name__)
class DisconnectRequest(NamedTuple):
file_no: int
should_retry: bool
# pylint: disable=too-many-public-methods
class AbstractNode:
__meta__ = ABCMeta
FLUSH_SEND_BUFFERS_INTERVAL = constants.OUTPUT_BUFFER_BATCH_MAX_HOLD_TIME * 2
NODE_TYPE: Optional[NodeType] = None
from dataclasses import dataclass
from abc import ABCMeta, abstractmethod
from collections import deque
from datetime import datetime
from threading import Thread, Lock
from typing import Optional, TypeVar, Generic, Deque, Type, Callable, Dict, Any, TYPE_CHECKING
from bxcommon import constants
from bxutils import log_messages
from bxutils import logging
from bxutils.logging import CustomLogger
from bxutils.logging.log_level import LogLevel
from bxutils.logging.log_record_type import LogRecordType
logger = logging.get_logger(__name__)
task_duration_logger = logging.get_logger(LogRecordType.TaskDuration, __name__)
if TYPE_CHECKING:
# pylint: disable=ungrouped-imports,cyclic-import
from bxcommon.connections.abstract_node import AbstractNode
@dataclass
class StatsIntervalData:
start_time: datetime = dataclasses.field(default_factory=datetime.utcnow)
end_time: Optional[datetime] = None
_closed: bool = False
def close(self):
self.end_time = datetime.utcnow()
from bxcommon.connections.connection_type import ConnectionType
from bxcommon.services.transaction_service import TransactionService
from bxcommon.utils import convert
from bxcommon.utils.object_hash import Sha256Hash
from bxgateway.messages.eth.protocol.new_block_eth_protocol_message import NewBlockEthProtocolMessage
from bxgateway.services.abstract_block_cleanup_service import AbstractBlockCleanupService
from bxutils import logging
from bxutils.logging.log_record_type import LogRecordType
if TYPE_CHECKING:
from bxgateway.connections.eth.eth_gateway_node import EthGatewayNode
logger = logging.get_logger(LogRecordType.BlockCleanup, __name__)
class AbstractEthBlockCleanupService(AbstractBlockCleanupService):
"""
Service for managing block cleanup.
"""
def __init__(self, node: "EthGatewayNode", network_num: int) -> None:
"""
Constructor
:param node: reference to node object
:param network_num: network number
"""
super(AbstractEthBlockCleanupService,
self).__init__(node=node, network_num=network_num)
def __init__(self):
super(_TransactionStatisticsService, self).__init__()
self.name = "TransactionInfo"
self.logger = logging.get_logger(LogRecordType.TransactionInfo)
self.log_percentage_by_network_num: Dict[int, float] = \
defaultdict(lambda: constants.TRANSACTIONS_PERCENTAGE_TO_LOG_STATS_FOR)
import sys
import gc
import time
from datetime import datetime
from typing import Dict, Any, Optional
from types import FrameType
from bxcommon import constants
from bxcommon.utils.stats.node_statistics_service import node_stats_service
from bxutils import logging
from bxutils.logging import LogRecordType
logger = logging.get_logger(LogRecordType.GarbageCollection)
_gc_start: Optional[float] = None
def gc_callback(phase: str, info: Dict[str, Any]):
# pylint: disable=global-statement
global _gc_start
gc_start = _gc_start
if phase == "start" and gc_start is None:
_gc_start = time.time()
elif gc_start is not None:
duration = time.time() - gc_start
_gc_start = None
if node_stats_service.node is not None:
node_stats_service.log_gc_duration(info["generation"], duration)
gen0, gen1, gen2 = gc.get_count()
from bxcommon.rpc.bx_json_rpc_request import BxJsonRpcRequest
from bxcommon.rpc.rpc_errors import RpcInvalidParams, RpcAccountIdError
from bxcommon.rpc.rpc_request_type import RpcRequestType
from bxcommon.test_utils.abstract_test_case import AbstractTestCase
from bxcommon.test_utils.helpers import async_test
from bxcommon.feed.feed import Feed
from bxcommon.feed.feed_manager import FeedManager
from bxcommon.feed import filter_parsing
from bxutils.encoding.json_encoder import Case
from bxutils import logging
from bxgateway.testing import gateway_helpers
from bxgateway.rpc.subscription_rpc_handler import SubscriptionRpcHandler
from bxgateway.testing.mocks.mock_gateway_node import MockGatewayNode
logger = logging.get_logger()
class TestFeed(Feed[str, str]):
def serialize(self, raw_message: str) -> str:
return raw_message
class FilterParsingTest(AbstractTestCase):
def setUp(self) -> None:
self.gateway = MockGatewayNode(gateway_helpers.get_gateway_opts(8000))
self.feed_manager = FeedManager(self.gateway)
self.rpc = SubscriptionRpcHandler(self.gateway, self.feed_manager, Case.SNAKE)
self.feed_service_model = FeedServiceModelBase(
allow_filtering=True,
from bxutils import logging
from bxutils.logging.log_record_type import LogRecordType
from bxcommon.services.transaction_service import TransactionService
from bxcommon.utils import memory_utils
from bxcommon.utils.object_encoder import ObjectEncoder
from bxcommon.utils.object_hash import Sha256Hash
from bxcommon.utils.proxy import task_pool_proxy
from bxcommon.utils.proxy.default_map_proxy import DefaultMapProxy
from bxcommon.utils.proxy.map_proxy import MapProxy
from bxcommon import constants
from bxcommon.utils.stats import hooks
import task_pool_executor as tpe # pyre-ignore for now, figure this out later (stub file or Python wrapper?)
logger_memory_cleanup = logging.get_logger(LogRecordType.BlockCleanup)
class ExtensionTransactionService(TransactionService):
def __init__(self, node, network_num):
super(ExtensionTransactionService, self).__init__(node, network_num)
self.proxy = tpe.TransactionService(
task_pool_proxy.get_pool_size(), node.opts.tx_mem_pool_bucket_size,
self._get_final_tx_confirmations_count())
raw_encoder = ObjectEncoder.raw_encoder()
self._tx_cache_key_to_short_ids = DefaultMapProxy(
self.proxy.tx_hash_to_short_ids(), raw_encoder, raw_encoder)
self._short_id_to_tx_cache_key = MapProxy(
self.proxy.short_id_to_tx_hash(), raw_encoder, raw_encoder)
content_encoder = ObjectEncoder(lambda buf_view: memoryview(buf_view),
lambda buf: tpe.InputBytes(buf))
from typing import List, Set
from bxcommon import constants
from bxcommon.feed.eth import eth_filter_handlers
from bxcommon.feed.eth.eth_raw_transaction import EthRawTransaction
from bxcommon.feed.eth.eth_transaction_feed_entry import EthTransactionFeedEntry
from bxcommon.feed.feed import Feed
from bxcommon.feed.subscriber import Subscriber
from bxutils import logging
from bxutils.logging import LogRecordType
logger = logging.get_logger(__name__)
logger_filters = logging.get_logger(LogRecordType.TransactionFiltering, __name__)
class EthTransactionProxyFeed(Feed[EthTransactionFeedEntry, EthRawTransaction]):
def __init__(
self,
name: str,
fields: List[str],
filters: Set[str],
network_num: int = constants.ALL_NETWORK_NUM,
) -> None:
# pylint: disable=invalid-name
self.NAME = name
self.FIELDS = fields
self.FILTERS = filters
super().__init__(self.NAME, network_num)
from abc import ABCMeta
from asyncio import Task
from typing import Dict, Callable, Optional, Generic, TypeVar
from bxcommon.rpc.provider.abstract_provider import SubscriptionNotification
from bxcommon.constants import LOCALHOST, WS_DEFAULT_PORT
from bxcommon.rpc.provider.abstract_ws_provider import AbstractWsProvider
from bxcommon.rpc.ws.ws_client import WsClient
from bxutils import logging
from bxutils.logging.log_record_type import LogRecordType
logger = logging.get_logger(__name__)
memory_logger = logging.get_logger(LogRecordType.BxMemory, __name__)
msg_handling_logger = logging.get_logger(
LogRecordType.MessageHandlingTroubleshooting, __name__)
T = TypeVar("T")
class AbstractFeedConnection(Generic[T], metaclass=ABCMeta):
def __init__(self,
node: T,
feed_ip: str = LOCALHOST,
feed_port: int = WS_DEFAULT_PORT,
headers: Optional[Dict] = None) -> None:
self.node = node
self.feed_ip = feed_ip
self.feed_port = feed_port
self.ws_client = WsClient(f"ws://{self.feed_ip}:{self.feed_port}/ws",
from abc import abstractmethod, ABC
from typing import Optional, List, Iterable, TypeVar, Generic
from bxcommon.connections.abstract_connection import AbstractConnection
from bxcommon.connections.connection_pool import ConnectionPool
from bxcommon.connections.connection_type import ConnectionType
from bxcommon.messages.abstract_message import AbstractMessage
from bxutils import logging
logger = logging.get_logger(__name__)
class BroadcastOptions:
broadcasting_connection: Optional[AbstractConnection]
prepend_to_queue: bool
connection_types: Iterable[ConnectionType]
def __init__(
self,
broadcast_connection: Optional[AbstractConnection] = None,
prepend_to_queue: bool = False,
connection_types: Optional[Iterable[ConnectionType]] = None
) -> None:
if connection_types is None:
connection_types = []
self.broadcasting_connection = broadcast_connection
self.prepend_to_queue = prepend_to_queue
self.connection_types = connection_types
from dataclasses import dataclass
from collections import defaultdict
from typing import Optional, Union, Dict, Type, Any, TYPE_CHECKING
from prometheus_client import Counter
from bxcommon import constants
from bxcommon.network.network_direction import NetworkDirection
from bxcommon.utils.stats.measurement_type import MeasurementType
from bxcommon.utils.stats.peer_stats import PeerStats
from bxcommon.utils.stats.statistics_service import StatisticsService, StatsIntervalData
from bxutils import logging
from bxutils.logging.log_record_type import LogRecordType
logger = logging.get_logger(LogRecordType.Throughput, __name__)
if TYPE_CHECKING:
# noinspection PyUnresolvedReferences
# pylint: disable=ungrouped-imports,cyclic-import
from bxcommon.connections.abstract_node import AbstractNode
bytes_received = Counter("bytes_received",
"Number of bytes received from all connections")
bytes_sent = Counter("bytes_sent", "Number of bytes sent on all connections")
@dataclass
class ThroughputIntervalData(StatsIntervalData):
total_in: int = 0
total_out: int = 0
