def __init__(self, opts: Namespace):
logger.debug("Initializing node of type: {}", self.NODE_TYPE)
self.set_node_config_opts_from_sdn(opts)
self.opts = opts
self.connection_queue: Deque[Tuple[str, int]] = deque()
self.disconnect_queue: Deque[DisconnectRequest] = deque()
self.outbound_peers = opts.outbound_peers[:]
self.connection_pool = ConnectionPool()
self.should_force_exit = False
self.num_retries_by_ip: Dict[Tuple[str, int], int] = defaultdict(int)
# Handle termination gracefully
signal.signal(signal.SIGTERM, self._kill_node)
signal.signal(signal.SIGINT, self._kill_node)
signal.signal(signal.SIGSEGV, self._kill_node)
# Event handling queue for delayed events
self.alarm_queue = AlarmQueue()
self.init_throughput_logging()
self.init_node_info_logging()
self.init_memory_stats_logging()
self.init_block_stats_logging()
self.init_tx_stats_logging()
# TODO: clean this up alongside outputbuffer holding time
# this is Nagle's algorithm and we need to implement it properly
# flush buffers regularly because of output buffer holding time
self.alarm_queue.register_approx_alarm(
self.FLUSH_SEND_BUFFERS_INTERVAL,
constants.OUTPUT_BUFFER_BATCH_MAX_HOLD_TIME,
self.flush_all_send_buffers)
self.network_num = opts.blockchain_network_num
self.broadcast_service = self.get_broadcast_service()
# converting setting in MB to bytes
self.next_report_mem_usage_bytes = self.opts.dump_detailed_report_at_memory_usage * 1024 * 1024
if opts.dump_removed_short_ids:
os.makedirs(opts.dump_removed_short_ids_path, exist_ok=True)
# each time a network has an update regarding txs, blocks, etc. register in a dict,
# this way can verify if node lost connection to requested relay.
self.last_sync_message_received_by_network: Dict[int, float] = {}
opts.has_fully_updated_tx_service = False
self.alarm_queue.register_alarm(constants.TX_SERVICE_SYNC_PROGRESS_S,
self._sync_tx_services)
self._check_sync_relay_connections_alarm_id = self.alarm_queue.register_alarm(
constants.LAST_MSG_FROM_RELAY_THRESHOLD_S,
self._check_sync_relay_connections)
self._transaction_sync_timeout_alarm_id = self.alarm_queue.register_alarm(
constants.TX_SERVICE_CHECK_NETWORKS_SYNCED_S,
self._transaction_sync_timeout)
def setUp(self):
self.conn_pool1 = ConnectionPool()
self.fileno1 = 1
self.ip1 = "123.123.123.123"
self.port1 = 1000
self.node1 = MockNode(
helpers.get_common_opts(1001, external_ip="128.128.128.128"))
self.node_id1 = str(uuid.uuid1())
self.conn1 = MockConnection(
MockSocketConnection(self.fileno1,
ip_address=self.ip1,
port=self.port1), self.node1)
self.fileno2 = 5
self.ip2 = "234.234.234.234"
self.port2 = 2000
self.node2 = MockNode(
helpers.get_common_opts(1003, external_ip="321.321.321.321"))
self.node_id2 = str(uuid.uuid1())
self.conn2 = MockConnection(
MockSocketConnection(self.fileno2,
ip_address=self.ip2,
port=self.port2), self.node2)
self.fileno3 = 6
self.ip3 = "234.234.234.234"
self.port3 = 3000
self.node3 = MockNode(
helpers.get_common_opts(1003, external_ip="213.213.213.213."))
self.node_id3 = str(uuid.uuid1())
self.conn3 = MockConnection(
MockSocketConnection(self.fileno3,
ip_address=self.ip3,
port=self.port3), self.node3)
def setUp(self):
self.node = MockNode(helpers.get_common_opts(8888))
# noinspection PyTypeChecker
connection = helpers.create_connection(MockConnection,
self.node,
port=9999)
self.node.connection_pool = ConnectionPool()
self.node.connection_pool.add(1, connection.peer_ip,
connection.peer_port, connection)
node_info_statistics.set_node(self.node)
def setUp(self):
self.node = MockGatewayNode(gateway_helpers.get_gateway_opts(8000))
self.node.connection_pool = ConnectionPool()
self.blockchain_sync_service = BlockchainSyncService(
self.node, {BtcMessageType.GET_HEADERS: BtcMessageType.HEADERS})
self.blockchain_connection = MockConnection(1, (LOCALHOST, 8001),
self.node)
self.blockchain_connection.state |= ConnectionState.ESTABLISHED
self.node.node_conn = self.blockchain_connection
self.alarms = []
def test_get_by_connection_types_performance(self):
log_config.set_level([
"bxcommon.connections.abstract_node",
"bxcommon.services.transaction_service"
], LogLevel.INFO)
conn_pool = ConnectionPool()
self.conn1.CONNECTION_TYPE = ConnectionType.EXTERNAL_GATEWAY
self.conn2.CONNECTION_TYPE = ConnectionType.RELAY_BLOCK
self.conn3.CONNECTION_TYPE = ConnectionType.RELAY_ALL
number_of_iteration = 100
for i in range(40):
ip = f"{i}.{i}.{i}.{i}"
node = MockNode(helpers.get_common_opts(i, external_ip=ip))
conn = MockConnection(
MockSocketConnection(i, ip_address=ip, port=i), node)
if i % 7 == 0:
conn.CONNECTION_TYPE = ConnectionType.RELAY_BLOCK
elif i % 5 == 0:
conn.CONNECTION_TYPE = ConnectionType.RELAY_TRANSACTION
elif i % 3 == 0:
conn.CONNECTION_TYPE = ConnectionType.INTERNAL_GATEWAY
else:
conn.CONNECTION_TYPE = ConnectionType.EXTERNAL_GATEWAY
conn_pool.add(i, ip, i, conn)
timeit_get_by_connections_types_one_type = timeit.timeit(
lambda: conn_pool.get_by_connection_types([ConnectionType.GATEWAY]
),
number=number_of_iteration)
timeit_get_by_connections_types_two_types = timeit.timeit(
lambda: conn_pool.get_by_connection_types(
[ConnectionType.GATEWAY, ConnectionType.RELAY_TRANSACTION]),
number=number_of_iteration)
print(
f"\ntimeit_get_by_connections_types_one_type # 2: {timeit_get_by_connections_types_one_type * 1000 / number_of_iteration:.4f}ms, "
f"#connections: {len(list(conn_pool.get_by_connection_types([ConnectionType.GATEWAY])))}"
f"\ntimeit_get_by_connections_types_two_types # 2: {timeit_get_by_connections_types_two_types * 1000 / number_of_iteration:.4f}ms, "
f"#connections: {len(list(conn_pool.get_by_connection_types([ConnectionType.GATEWAY, ConnectionType.RELAY_TRANSACTION])))}"
)
print("*****")
for c in conn_pool.get_by_connection_types(
[ConnectionType.GATEWAY, ConnectionType.RELAY_TRANSACTION]):
print(f"connection: {c}, connection type: {c.CONNECTION_TYPE}")
def __init__(self,
opts: CommonOpts,
node_ssl_service: NodeSSLService,
connection_pool: Optional[ConnectionPool] = None):
self.node_ssl_service = node_ssl_service
logger.debug("Initializing node of type: {}", self.NODE_TYPE)
self.server_endpoints = [
IpEndpoint(constants.LISTEN_ON_IP_ADDRESS, opts.external_port),
# TODO: remove this after v1 is no longer supported
IpEndpoint(constants.LISTEN_ON_IP_ADDRESS, opts.non_ssl_port)
]
self.set_node_config_opts_from_sdn(opts)
self.opts: CommonOpts = opts
self.pending_connection_requests: Set[ConnectionPeerInfo] = set()
self.pending_connection_attempts: Set[ConnectionPeerInfo] = set()
self.outbound_peers: Set[OutboundPeerModel] = opts.outbound_peers.copy(
)
if connection_pool is not None:
self.connection_pool = connection_pool
else:
self.connection_pool = ConnectionPool()
self.should_force_exit = False
self.should_restart_on_high_memory = False
self.num_retries_by_ip: Dict[Tuple[str, int], int] = defaultdict(int)
# Event handling queue for delayed events
self.alarm_queue = AlarmQueue()
self.init_node_status_logging()
self.init_throughput_logging()
self.init_node_info_logging()
self.init_memory_stats_logging()
self.init_block_stats_logging()
self.init_tx_stats_logging()
# TODO: clean this up alongside outputbuffer holding time
# this is Nagle's algorithm and we need to implement it properly
# flush buffers regularly because of output buffer holding time
self.alarm_queue.register_approx_alarm(
self.FLUSH_SEND_BUFFERS_INTERVAL,
constants.OUTPUT_BUFFER_BATCH_MAX_HOLD_TIME,
self.flush_all_send_buffers)
self.network_num = opts.blockchain_network_num
self.broadcast_service = self.get_broadcast_service()
# converting setting in MB to bytes
self.next_report_mem_usage_bytes = self.opts.dump_detailed_report_at_memory_usage * 1024 * 1024
if opts.dump_removed_short_ids:
os.makedirs(opts.dump_removed_short_ids_path, exist_ok=True)
# each time a network has an update regarding txs, blocks, etc. register in a dict,
# this way can verify if node lost connection to requested relay.
self.last_sync_message_received_by_network: Dict[int, float] = {}
self.start_sync_time: Optional[float] = None
self.sync_metrics: Dict[int, Counter] = defaultdict(Counter)
self.sync_short_id_buckets: Dict[
int,
TransactionShortIdBuckets] = defaultdict(TransactionShortIdBuckets)
opts.has_fully_updated_tx_service = False
self.check_sync_relay_connections_alarm_id: Optional[AlarmId] = None
self.transaction_sync_timeout_alarm_id: Optional[AlarmId] = None
self.requester = ThreadedRequestService(
# pyre-fixme[16]: `Optional` has no attribute `name`.
self.NODE_TYPE.name.lower(),
self.alarm_queue,
constants.THREADED_HTTP_POOL_SLEEP_INTERVAL_S)
self._last_responsiveness_check_log_time = time.time()
self._last_responsiveness_check_details = {}
self.gc_logging_enabled = False
self.serialized_message_cache = SerializedMessageCache(
self.alarm_queue)
self.alarm_queue.register_alarm(
constants.RESPONSIVENESS_CHECK_INTERVAL_S,
self._responsiveness_check_log)
def setUp(self) -> None:
self.node = MockNode(helpers.get_common_opts(8000))
self.connection_pool = ConnectionPool()
self.sut = TestBroadcastService(self.connection_pool)
def setUp(self):
self.conn_pool = ConnectionPool()
self.source_version = "v1.0.0"
self.ip_address = "0.0.0.0"
self.continent = "NA"
self.country = "United States"
self.account_id = None
self.fileno1 = 1
self.ip1 = "123.123.123.123"
self.port1 = 1000
self.node1 = MockNode(
helpers.get_common_opts(1001, external_ip="128.128.128.128"))
self.node_id1 = str(uuid.uuid1())
self.conn1 = MockConnection(
MockSocketConnection(self.fileno1,
self.node1,
ip_address=self.ip1,
port=self.port1), self.node1)
self.conn1.CONNECTION_TYPE = ConnectionType.RELAY_BLOCK
self.fileno2 = 5
self.ip2 = "234.234.234.234"
self.port2 = 2000
self.node2 = MockNode(
helpers.get_common_opts(1003, external_ip="321.321.321.321"))
self.node_id2 = str(uuid.uuid1())
self.conn2 = MockConnection(
MockSocketConnection(self.fileno2,
self.node2,
ip_address=self.ip2,
port=self.port2), self.node2)
self.conn2.CONNECTION_TYPE = ConnectionType.RELAY_TRANSACTION
self.fileno3 = 6
self.ip3 = "234.234.234.234"
self.port3 = 3000
self.node3 = MockNode(
helpers.get_common_opts(1003, external_ip="213.213.213.213"))
self.node_id3 = str(uuid.uuid1())
self.conn3 = MockConnection(
MockSocketConnection(self.fileno3,
self.node3,
ip_address=self.ip3,
port=self.port3), self.node3)
self.conn3.CONNECTION_TYPE = ConnectionType.BLOCKCHAIN_NODE
self.fileno4 = 8
self.ip4 = "111.222.111.222"
self.port4 = 3000
self.node4 = MockNode(
helpers.get_common_opts(1003, external_ip="101.101.101.101"))
self.node_id4 = str(uuid.uuid1())
self.conn4 = MockConnection(
MockSocketConnection(self.fileno4,
self.node4,
ip_address=self.ip4,
port=self.port4), self.node4)
self.conn4.CONNECTION_TYPE = ConnectionType.REMOTE_BLOCKCHAIN_NODE
self.quota_level = 0
initialize(False, self.source_version, self.ip_address, self.continent,
self.country, False, self.account_id, self.quota_level)
path = config.get_data_file(STATUS_FILE_NAME)
self.addCleanup(os.remove, path)