def __init__(self) -> None: self.name = None self.log_level = LogLevel.STATS self.logger = logging.get_logger(__name__) self.priority_logger = logging.get_logger(__name__) self.node = None self.node_id: Optional[str] = None
def __init__(self) -> None: super(_TransactionStatisticsService, self).__init__() self.name = "TransactionInfo" self.logger = logging.get_logger(LogRecordType.TransactionInfo) self.priority_logger = logging.get_logger( LogRecordType.TransactionPropagationInfo) self.log_percentage_for_hash_by_network_num: Dict[int, float] = \ defaultdict(lambda: constants.TRANSACTIONS_BY_HASH_PERCENTAGE_TO_LOG_STATS_FOR) self.log_percentage_for_sid_by_network_num: Dict[int, float] = \ defaultdict(lambda: constants.TRANSACTIONS_BY_SID_PERCENTAGE_TO_LOG_STATS_FOR)
def __init__(self, interval=0): super(NodeInfo, self).__init__("NodeInfo", interval=interval, look_back=0, reset=False, logger=logging.get_logger(LogRecordType.NodeInfo))
def test_custom_logger(self): log_config.setup_logging(log_format=log_config.LogFormat.JSON, default_log_level=log_config.LogLevel.TRACE, default_logger_names="", log_level_overrides={}, enable_fluent_logger=True, fluentd_host="fluentd", third_party_loggers=[ logging.LoggerConfig( "test_logging", "{", logging.LogLevel.TRACE, handler_type.HandlerType.Fluent) ]) logger = logging.get_logger("test_logging") handlers = self._get_handlers(logger) self.assertEqual(len(handlers), 1) stream_handlers = [ handler for handler in handlers if isinstance(handler, StreamHandler) ] fluentd_handlers = [ handler for handler in handlers if isinstance(handler, FluentHandler) ] self.assertEqual(len(stream_handlers), 0) self.assertEqual(len(fluentd_handlers), 1) for handler in handlers: self.assertEqual(handler.level, 0) fluentd_handler = fluentd_handlers[0] self.assertIsInstance(fluentd_handler.formatter, formatters.FluentJSONFormatter)
def test_create_logger_fluentd(self): log_config.setup_logging(log_format=log_config.LogFormat.JSON, default_log_level=log_config.LogLevel.TRACE, default_logger_names="", log_level_overrides={}, enable_fluent_logger=True, fluentd_host="fluentd") logger = logging.get_logger("test_logging") handlers = self._get_handlers(logger) self.assertEqual(len(handlers), 2) stream_handlers = [ handler for handler in handlers if isinstance(handler, StreamHandler) ] fluentd_handlers = [ handler for handler in handlers if isinstance(handler, FluentHandler) ] self.assertEqual(len(stream_handlers), 1) self.assertEqual(len(fluentd_handlers), 1) for handler in handlers: self.assertEqual(handler.level, 0) fluentd_handler = fluentd_handlers[0] stream_handler = stream_handlers[0] self.assertIsInstance(fluentd_handler.formatter, formatters.JSONFormatter) self.assertIsInstance(stream_handler.formatter, formatters.JSONFormatter)
def __init__(self, interval: int = constants.NODE_STATS_INTERVAL_S) -> None: super().__init__( "NodeStatus", interval, reset=True, stat_logger=logging.get_logger(LogRecordType.NodeStatus, __name__), )
def __init__(self, interval: int = 0) -> None: super(NodeInfo, self).__init__( "NodeInfo", interval=interval, look_back=0, reset=False, stat_logger=logging.get_logger(LogRecordType.NodeInfo, __name__), )
def __init__(self, interval: int = 0) -> None: self.sizer_obj = Sizer() super(MemoryStatsService, self).__init__( "MemoryStats", interval=interval, look_back=5, reset=False, stat_logger=logging.get_logger(LogRecordType.Memory, __name__), )
def __init__(self, interval: int = gateway_constants. GATEWAY_TRANSACTION_FEED_STATS_INTERVAL_S, look_back: int = gateway_constants. GATEWAY_TRANSACTION_FEED_STATS_LOOKBACK): super().__init__("TransactionFeedStats", interval, look_back, reset=True, stat_logger=logging.get_logger( LogRecordType.TransactionFeedStats))
def __init__( self, interval=gateway_constants.GATEWAY_BDN_PERFORMANCE_STATS_INTERVAL_S, look_back=gateway_constants.GATEWAY_BDN_PERFORMANCE_STATS_LOOKBACK, ) -> None: super(_GatewayBdnPerformanceStatsService, self).__init__( "GatewayBdnPerformanceStats", interval, look_back, reset=True, stat_logger=logging.get_logger(LogRecordType.BdnPerformanceStats, __name__), )
def __init__( self, interval: int = gateway_constants.ETH_GATEWAY_STATS_INTERVAL, look_back: int = gateway_constants.ETH_GATEWAY_STATS_LOOKBACK, ): super(_EthGatewayStatsService, self).__init__( "EthGatewayStatsService", interval, look_back, reset=True, stat_logger=logging.get_logger(LogRecordType.TransactionStats, __name__), )
def test_create_logger(self): log_config.setup_logging(log_format=log_config.LogFormat.JSON, default_log_level=log_config.LogLevel.TRACE, default_logger_names="", log_level_overrides={}) logger = logging.get_logger("test_logging") handlers = self._get_handlers(logger) self.assertEqual(len(handlers), 1) for handler in handlers: self.assertIsInstance(handler, StreamHandler) self.assertIsInstance(handler.formatter, formatters.JSONFormatter)
def test_logging(self): logger = logging.get_logger("test_logging") with self.assertLogs() as cm: logger.warning("TEST {}", 1) self.assertEqual("WARNING:test_logging:TEST 1", cm.output[0]) with self.assertLogs(level=logging.LogLevel.TRACE) as cm: logger.trace("TEST {}", 1) self.assertEqual("TRACE:test_logging:TEST 1", cm.output[0]) with self.assertLogs(level=logging.LogLevel.WARNING) as cm: logger.warning(log_messages.BDN_RETURNED_NO_PEERS, 1) self.assertEqual( "WARNING:test_logging:BDN returned no peers at endpoint: 1", cm.output[0])
def __init__( self, interval: int = gateway_constants.GATEWAY_TRANSACTION_STATS_INTERVAL_S, look_back: int = gateway_constants.GATEWAY_TRANSACTION_STATS_LOOKBACK, ): super(_GatewayTransactionStatsService, self).__init__( "GatewayTransactionStats", interval, look_back, reset=True, stat_logger=logging.get_logger(LogRecordType.TransactionStats, __name__), )
def __init__( self, interval: int = eth_constants.ETH_ON_BLOCK_FEED_STATS_INTERVAL_S, look_back: int = eth_constants.ETH_ON_BLOCK_FEED_STATS_LOOKBACK, ) -> None: super().__init__( "EthCallFeedStats", interval, look_back, reset=True, stat_logger=logging.get_logger(LogRecordType.EthOnBlockFeedStats), ) self.feed_name = rpc_constants.ETH_ON_BLOCK_FEED_NAME
from bxcommon.utils import memory_utils, crypto from bxcommon.utils.deprecated import deprecated from bxcommon.utils.memory_utils import ObjectSize, SizeType from bxcommon.utils.object_encoder import ObjectEncoder from bxcommon.utils.object_hash import Sha256Hash from bxcommon.utils.proxy import task_pool_proxy from bxcommon.utils.proxy.default_map_proxy import DefaultMapProxy from bxcommon.utils.proxy.map_proxy import MapProxy from bxcommon.utils.stats import hooks from bxcommon.utils.stats.transaction_stat_event_type import TransactionStatEventType from bxcommon.utils.stats.transaction_statistics_service import tx_stats from bxutils import logging from bxutils.logging import log_config from bxutils.logging.log_record_type import LogRecordType logger = logging.get_logger(__name__) logger_memory_cleanup = logging.get_logger(LogRecordType.BlockCleanup, __name__) UNKNOWN_TRANSACTION_HASH: Sha256Hash = Sha256Hash( bytearray(b"\xff" * crypto.SHA256_HASH_LEN)) class ExtensionTransactionService(TransactionService): def __init__(self, node, network_num) -> None: super(ExtensionTransactionService, self).__init__(node, network_num) # Log levels need to be set again to include the loggers created in this conditionally imported class log_config.lazy_set_log_level(node.opts.log_level_overrides) self.proxy = tpe.TransactionService( task_pool_proxy.get_pool_size(),
from cryptography.x509 import Certificate from bxcommon.network.ip_endpoint import IpEndpoint from bxcommon.network.network_direction import NetworkDirection from bxcommon.network.socket_connection_state import SocketConnectionState, SocketConnectionStates from bxcommon.utils.stats import hooks from bxutils import logging from bxutils.logging import LogRecordType from bxutils.ssl import ssl_certificate_factory if TYPE_CHECKING: # pylint: disable=ungrouped-imports,cyclic-import from bxcommon.connections.abstract_node import AbstractNode logger = logging.get_logger(__name__) network_troubleshooting_logger = logging.get_logger(LogRecordType.NetworkTroubleshooting, __name__) SO_QUICKACK = 12 class AbstractSocketConnectionProtocol(BaseProtocol): transport: Optional[Transport] file_no: int endpoint: IpEndpoint direction: NetworkDirection can_send: bool state: SocketConnectionState is_ssl: bool _node: "AbstractNode" _should_retry: bool
from bxcommon.messages.abstract_message import AbstractMessage from bxcommon.models.blockchain_network_model import BlockchainNetworkModel from bxcommon.network.socket_connection import SocketConnection from bxcommon.services import sdn_http_service from bxcommon.services.broadcast_service import BroadcastService, BroadcastOptions from bxcommon.utils import memory_utils, json_utils from bxcommon.utils.alarm_queue import AlarmQueue from bxcommon.utils.stats.block_statistics_service import block_stats from bxcommon.utils.stats.memory_statistics_service import memory_statistics from bxcommon.utils.stats.node_info_service import node_info_statistics from bxcommon.utils.stats.throughput_service import throughput_statistics from bxcommon.utils.stats.transaction_statistics_service import tx_stats from bxutils import logging from bxutils.logging import LogRecordType logger = logging.get_logger(__name__) memory_logger = logging.get_logger(LogRecordType.BxMemory) class DisconnectRequest(NamedTuple): fileno: int should_retry: bool class AbstractNode: __meta__ = ABCMeta FLUSH_SEND_BUFFERS_INTERVAL = constants.OUTPUT_BUFFER_BATCH_MAX_HOLD_TIME * 2 NODE_TYPE = None def __init__(self, opts: Namespace): logger.debug("Initializing node of type: {}", self.NODE_TYPE)
from datetime import datetime import time import typing from bxutils import logging from bxutils.logging.log_record_type import LogRecordType from bxcommon.utils.proxy import task_pool_proxy from bxcommon.services.transaction_service import TransactionService from bxcommon.services.extension_transaction_service import ExtensionTransactionService from bxcommon.messages.bloxroute.abstract_cleanup_message import AbstractCleanupMessage import task_pool_executor as tpe # pyre-ignore for now, figure this out later (stub file or Python wrapper?) logger = logging.get_logger(LogRecordType.TransactionCleanup) def contents_cleanup(transaction_service: TransactionService, block_confirmation_message: AbstractCleanupMessage, cleanup_tasks ): start_datetime = datetime.utcnow() start_time = time.time() tx_service = typing.cast(ExtensionTransactionService, transaction_service) cleanup_task = cleanup_tasks.borrow_task() cleanup_task.init(tpe.InputBytes(block_confirmation_message.buf), tx_service.proxy) task_pool_proxy.run_task(cleanup_task) short_ids = cleanup_task.short_ids() total_content_removed = cleanup_task.total_content_removed() tx_count = cleanup_task.tx_count()
from bxcommon.utils.stats.memory_statistics_service import memory_statistics from bxcommon.utils.stats.node_info_service import node_info_statistics from bxcommon.utils.stats.node_statistics_service import node_stats_service from bxcommon.utils.stats.throughput_service import throughput_statistics from bxcommon.utils.stats.transaction_statistics_service import tx_stats from bxcommon.utils.transaction_short_id_buckets import TransactionShortIdBuckets from bxutils import log_messages from bxutils import logging from bxutils.exceptions.connection_authentication_error import \ ConnectionAuthenticationError from bxutils.logging import LogRecordType, LogLevel from bxutils.services.node_ssl_service import NodeSSLService from bxutils.ssl.extensions import extensions_factory from bxutils.ssl.ssl_certificate_type import SSLCertificateType logger = logging.get_logger(__name__) memory_logger = logging.get_logger(LogRecordType.BxMemory, __name__) performance_troubleshooting_logger = logging.get_logger( LogRecordType.PerformanceTroubleshooting, __name__) class DisconnectRequest(NamedTuple): file_no: int should_retry: bool # pylint: disable=too-many-public-methods class AbstractNode: __meta__ = ABCMeta FLUSH_SEND_BUFFERS_INTERVAL = constants.OUTPUT_BUFFER_BATCH_MAX_HOLD_TIME * 2 NODE_TYPE: Optional[NodeType] = None
from dataclasses import dataclass from abc import ABCMeta, abstractmethod from collections import deque from datetime import datetime from threading import Thread, Lock from typing import Optional, TypeVar, Generic, Deque, Type, Callable, Dict, Any, TYPE_CHECKING from bxcommon import constants from bxutils import log_messages from bxutils import logging from bxutils.logging import CustomLogger from bxutils.logging.log_level import LogLevel from bxutils.logging.log_record_type import LogRecordType logger = logging.get_logger(__name__) task_duration_logger = logging.get_logger(LogRecordType.TaskDuration, __name__) if TYPE_CHECKING: # pylint: disable=ungrouped-imports,cyclic-import from bxcommon.connections.abstract_node import AbstractNode @dataclass class StatsIntervalData: start_time: datetime = dataclasses.field(default_factory=datetime.utcnow) end_time: Optional[datetime] = None _closed: bool = False def close(self): self.end_time = datetime.utcnow()
from bxcommon.connections.connection_type import ConnectionType from bxcommon.services.transaction_service import TransactionService from bxcommon.utils import convert from bxcommon.utils.object_hash import Sha256Hash from bxgateway.messages.eth.protocol.new_block_eth_protocol_message import NewBlockEthProtocolMessage from bxgateway.services.abstract_block_cleanup_service import AbstractBlockCleanupService from bxutils import logging from bxutils.logging.log_record_type import LogRecordType if TYPE_CHECKING: from bxgateway.connections.eth.eth_gateway_node import EthGatewayNode logger = logging.get_logger(LogRecordType.BlockCleanup, __name__) class AbstractEthBlockCleanupService(AbstractBlockCleanupService): """ Service for managing block cleanup. """ def __init__(self, node: "EthGatewayNode", network_num: int) -> None: """ Constructor :param node: reference to node object :param network_num: network number """ super(AbstractEthBlockCleanupService, self).__init__(node=node, network_num=network_num)
def __init__(self): super(_TransactionStatisticsService, self).__init__() self.name = "TransactionInfo" self.logger = logging.get_logger(LogRecordType.TransactionInfo) self.log_percentage_by_network_num: Dict[int, float] = \ defaultdict(lambda: constants.TRANSACTIONS_PERCENTAGE_TO_LOG_STATS_FOR)
import sys import gc import time from datetime import datetime from typing import Dict, Any, Optional from types import FrameType from bxcommon import constants from bxcommon.utils.stats.node_statistics_service import node_stats_service from bxutils import logging from bxutils.logging import LogRecordType logger = logging.get_logger(LogRecordType.GarbageCollection) _gc_start: Optional[float] = None def gc_callback(phase: str, info: Dict[str, Any]): # pylint: disable=global-statement global _gc_start gc_start = _gc_start if phase == "start" and gc_start is None: _gc_start = time.time() elif gc_start is not None: duration = time.time() - gc_start _gc_start = None if node_stats_service.node is not None: node_stats_service.log_gc_duration(info["generation"], duration) gen0, gen1, gen2 = gc.get_count()
from bxcommon.rpc.bx_json_rpc_request import BxJsonRpcRequest from bxcommon.rpc.rpc_errors import RpcInvalidParams, RpcAccountIdError from bxcommon.rpc.rpc_request_type import RpcRequestType from bxcommon.test_utils.abstract_test_case import AbstractTestCase from bxcommon.test_utils.helpers import async_test from bxcommon.feed.feed import Feed from bxcommon.feed.feed_manager import FeedManager from bxcommon.feed import filter_parsing from bxutils.encoding.json_encoder import Case from bxutils import logging from bxgateway.testing import gateway_helpers from bxgateway.rpc.subscription_rpc_handler import SubscriptionRpcHandler from bxgateway.testing.mocks.mock_gateway_node import MockGatewayNode logger = logging.get_logger() class TestFeed(Feed[str, str]): def serialize(self, raw_message: str) -> str: return raw_message class FilterParsingTest(AbstractTestCase): def setUp(self) -> None: self.gateway = MockGatewayNode(gateway_helpers.get_gateway_opts(8000)) self.feed_manager = FeedManager(self.gateway) self.rpc = SubscriptionRpcHandler(self.gateway, self.feed_manager, Case.SNAKE) self.feed_service_model = FeedServiceModelBase( allow_filtering=True,
from bxutils import logging from bxutils.logging.log_record_type import LogRecordType from bxcommon.services.transaction_service import TransactionService from bxcommon.utils import memory_utils from bxcommon.utils.object_encoder import ObjectEncoder from bxcommon.utils.object_hash import Sha256Hash from bxcommon.utils.proxy import task_pool_proxy from bxcommon.utils.proxy.default_map_proxy import DefaultMapProxy from bxcommon.utils.proxy.map_proxy import MapProxy from bxcommon import constants from bxcommon.utils.stats import hooks import task_pool_executor as tpe # pyre-ignore for now, figure this out later (stub file or Python wrapper?) logger_memory_cleanup = logging.get_logger(LogRecordType.BlockCleanup) class ExtensionTransactionService(TransactionService): def __init__(self, node, network_num): super(ExtensionTransactionService, self).__init__(node, network_num) self.proxy = tpe.TransactionService( task_pool_proxy.get_pool_size(), node.opts.tx_mem_pool_bucket_size, self._get_final_tx_confirmations_count()) raw_encoder = ObjectEncoder.raw_encoder() self._tx_cache_key_to_short_ids = DefaultMapProxy( self.proxy.tx_hash_to_short_ids(), raw_encoder, raw_encoder) self._short_id_to_tx_cache_key = MapProxy( self.proxy.short_id_to_tx_hash(), raw_encoder, raw_encoder) content_encoder = ObjectEncoder(lambda buf_view: memoryview(buf_view), lambda buf: tpe.InputBytes(buf))
from typing import List, Set from bxcommon import constants from bxcommon.feed.eth import eth_filter_handlers from bxcommon.feed.eth.eth_raw_transaction import EthRawTransaction from bxcommon.feed.eth.eth_transaction_feed_entry import EthTransactionFeedEntry from bxcommon.feed.feed import Feed from bxcommon.feed.subscriber import Subscriber from bxutils import logging from bxutils.logging import LogRecordType logger = logging.get_logger(__name__) logger_filters = logging.get_logger(LogRecordType.TransactionFiltering, __name__) class EthTransactionProxyFeed(Feed[EthTransactionFeedEntry, EthRawTransaction]): def __init__( self, name: str, fields: List[str], filters: Set[str], network_num: int = constants.ALL_NETWORK_NUM, ) -> None: # pylint: disable=invalid-name self.NAME = name self.FIELDS = fields self.FILTERS = filters super().__init__(self.NAME, network_num)
from abc import ABCMeta from asyncio import Task from typing import Dict, Callable, Optional, Generic, TypeVar from bxcommon.rpc.provider.abstract_provider import SubscriptionNotification from bxcommon.constants import LOCALHOST, WS_DEFAULT_PORT from bxcommon.rpc.provider.abstract_ws_provider import AbstractWsProvider from bxcommon.rpc.ws.ws_client import WsClient from bxutils import logging from bxutils.logging.log_record_type import LogRecordType logger = logging.get_logger(__name__) memory_logger = logging.get_logger(LogRecordType.BxMemory, __name__) msg_handling_logger = logging.get_logger( LogRecordType.MessageHandlingTroubleshooting, __name__) T = TypeVar("T") class AbstractFeedConnection(Generic[T], metaclass=ABCMeta): def __init__(self, node: T, feed_ip: str = LOCALHOST, feed_port: int = WS_DEFAULT_PORT, headers: Optional[Dict] = None) -> None: self.node = node self.feed_ip = feed_ip self.feed_port = feed_port self.ws_client = WsClient(f"ws://{self.feed_ip}:{self.feed_port}/ws",
from abc import abstractmethod, ABC from typing import Optional, List, Iterable, TypeVar, Generic from bxcommon.connections.abstract_connection import AbstractConnection from bxcommon.connections.connection_pool import ConnectionPool from bxcommon.connections.connection_type import ConnectionType from bxcommon.messages.abstract_message import AbstractMessage from bxutils import logging logger = logging.get_logger(__name__) class BroadcastOptions: broadcasting_connection: Optional[AbstractConnection] prepend_to_queue: bool connection_types: Iterable[ConnectionType] def __init__( self, broadcast_connection: Optional[AbstractConnection] = None, prepend_to_queue: bool = False, connection_types: Optional[Iterable[ConnectionType]] = None ) -> None: if connection_types is None: connection_types = [] self.broadcasting_connection = broadcast_connection self.prepend_to_queue = prepend_to_queue self.connection_types = connection_types
from dataclasses import dataclass from collections import defaultdict from typing import Optional, Union, Dict, Type, Any, TYPE_CHECKING from prometheus_client import Counter from bxcommon import constants from bxcommon.network.network_direction import NetworkDirection from bxcommon.utils.stats.measurement_type import MeasurementType from bxcommon.utils.stats.peer_stats import PeerStats from bxcommon.utils.stats.statistics_service import StatisticsService, StatsIntervalData from bxutils import logging from bxutils.logging.log_record_type import LogRecordType logger = logging.get_logger(LogRecordType.Throughput, __name__) if TYPE_CHECKING: # noinspection PyUnresolvedReferences # pylint: disable=ungrouped-imports,cyclic-import from bxcommon.connections.abstract_node import AbstractNode bytes_received = Counter("bytes_received", "Number of bytes received from all connections") bytes_sent = Counter("bytes_sent", "Number of bytes sent on all connections") @dataclass class ThroughputIntervalData(StatsIntervalData): total_in: int = 0 total_out: int = 0