async def initialize_address_manager(self):
mkdir(self.peer_db_path.parent)
self.connection = await aiosqlite.connect(self.peer_db_path)
self.address_manager_store = await AddressManagerStore.create(self.connection)
if not await self.address_manager_store.is_empty():
self.address_manager = await self.address_manager_store.deserialize()
else:
await self.address_manager_store.clear()
self.address_manager = AddressManager()
self.server.set_received_message_callback(self.update_peer_timestamp_on_message)
async def initialize_address_manager(self):
mkdir(self.peer_db_path.parent)
self.connection = await aiosqlite.connect(self.peer_db_path)
self.address_manager_store = await AddressManagerStore.create(
self.connection)
if not await self.address_manager_store.is_empty():
self.address_manager = await self.address_manager_store.deserialize(
)
else:
await self.address_manager_store.clear()
self.address_manager = AddressManager()
class FullNodeDiscovery:
def __init__(
self,
server: ChiaServer,
root_path: Path,
target_outbound_count: int,
peer_db_path: str,
introducer_info: Optional[Dict],
peer_connect_interval: int,
log,
):
self.server: ChiaServer = server
self.message_queue: asyncio.Queue = asyncio.Queue()
self.is_closed = False
self.target_outbound_count = target_outbound_count
self.peer_db_path = path_from_root(root_path, peer_db_path)
if introducer_info is not None:
self.introducer_info: Optional[PeerInfo] = PeerInfo(
introducer_info["host"],
introducer_info["port"],
)
else:
self.introducer_info = None
self.peer_connect_interval = peer_connect_interval
self.log = log
self.relay_queue = None
self.address_manager = None
self.connection_time_pretest: Dict = {}
async def initialize_address_manager(self):
mkdir(self.peer_db_path.parent)
self.connection = await aiosqlite.connect(self.peer_db_path)
self.address_manager_store = await AddressManagerStore.create(
self.connection)
if not await self.address_manager_store.is_empty():
self.address_manager = await self.address_manager_store.deserialize(
)
else:
await self.address_manager_store.clear()
self.address_manager = AddressManager()
self.server.set_received_message_callback(
self.update_peer_timestamp_on_message)
async def start_tasks(self):
random = Random()
self.connect_peers_task = asyncio.create_task(
self._connect_to_peers(random))
self.serialize_task = asyncio.create_task(
self._periodically_serialize(random))
self.cleanup_task = asyncio.create_task(self._periodically_cleanup())
async def _close_common(self):
self.is_closed = True
self.connect_peers_task.cancel()
self.serialize_task.cancel()
self.cleanup_task.cancel()
await self.connection.close()
def add_message(self, message, data):
self.message_queue.put_nowait((message, data))
async def on_connect(self, peer: ws.WSChiaConnection):
if (peer.is_outbound is False and peer.peer_server_port is not None
and peer.connection_type is NodeType.FULL_NODE
and self.server._local_type is NodeType.FULL_NODE
and self.address_manager is not None):
timestamped_peer_info = TimestampedPeerInfo(
peer.peer_host,
peer.peer_server_port,
uint64(int(time.time())),
)
await self.address_manager.add_to_new_table(
[timestamped_peer_info], peer.get_peer_info(), 0)
if self.relay_queue is not None:
self.relay_queue.put_nowait((timestamped_peer_info, 1))
if (peer.is_outbound and peer.peer_server_port is not None
and peer.connection_type is NodeType.FULL_NODE
and self.server._local_type is NodeType.FULL_NODE
and self.address_manager is not None):
msg = Message("request_peers", full_node_protocol.RequestPeers())
await peer.send_message(msg)
# Updates timestamps each time we receive a message for outbound connections.
async def update_peer_timestamp_on_message(self,
peer: ws.WSChiaConnection):
if (peer.is_outbound and peer.peer_server_port is not None
and peer.connection_type is NodeType.FULL_NODE
and self.server._local_type is NodeType.FULL_NODE
and self.address_manager is not None):
peer_info = peer.get_peer_info()
if peer_info.host not in self.connection_time_pretest:
self.connection_time_pretest[peer_info.host] = time.time()
if time.time() - self.connection_time_pretest[
peer_info.host] > 600:
self.connection_time_pretest[peer_info.host] = time.time()
await self.address_manager.connect(peer_info)
def _num_needed_peers(self) -> int:
diff = self.target_outbound_count
outgoing = self.server.get_outgoing_connections()
diff -= len(outgoing)
return diff if diff >= 0 else 0
"""
Uses the Poisson distribution to determine the next time
when we'll initiate a feeler connection.
(https://en.wikipedia.org/wiki/Poisson_distribution)
"""
def _poisson_next_send(self, now, avg_interval_seconds, random):
return now + (math.log(
random.randrange(1 << 48) * -0.0000000000000035527136788 + 1) *
avg_interval_seconds * -1000000.0 + 0.5)
async def _introducer_client(self):
if self.introducer_info is None:
return
async def on_connect(peer: ws.WSChiaConnection):
msg = Message("request_peers", introducer_protocol.RequestPeers())
await peer.send_message(msg)
await self.server.start_client(self.introducer_info, on_connect)
async def _connect_to_peers(self, random):
next_feeler = self._poisson_next_send(time.time() * 1000 * 1000, 240,
random)
empty_tables = False
local_peerinfo: Optional[PeerInfo] = await self.server.get_peer_info()
last_timestamp_local_info: uint64 = uint64(int(time.time()))
while not self.is_closed:
try:
# We don't know any address, connect to the introducer to get some.
size = await self.address_manager.size()
if size == 0 or empty_tables:
await self._introducer_client()
try:
await asyncio.sleep(min(5, self.peer_connect_interval))
except asyncio.CancelledError:
return
empty_tables = False
continue
# Only connect out to one peer per network group (/16 for IPv4).
groups = []
full_node_connected = self.server.get_full_node_connections()
connected = [c.get_peer_info() for c in full_node_connected]
connected = [c for c in connected if c is not None]
for conn in full_node_connected:
peer = conn.get_peer_info()
if peer is None:
continue
group = peer.get_group()
if group not in groups:
groups.append(group)
# Feeler Connections
#
# Design goals:
# * Increase the number of connectable addresses in the tried table.
#
# Method:
# * Choose a random address from new and attempt to connect to it if we can connect
# successfully it is added to tried.
# * Start attempting feeler connections only after node finishes making outbound
# connections.
# * Only make a feeler connection once every few minutes.
is_feeler = False
has_collision = False
if self._num_needed_peers() == 0:
if time.time() * 1000 * 1000 > next_feeler:
next_feeler = self._poisson_next_send(
time.time() * 1000 * 1000, 240, random)
is_feeler = True
await self.address_manager.resolve_tried_collisions()
tries = 0
now = time.time()
got_peer = False
addr: Optional[PeerInfo] = None
max_tries = 50
if len(groups) < 3:
max_tries = 10
elif len(groups) <= 5:
max_tries = 25
while not got_peer and not self.is_closed:
sleep_interval = 1 + len(groups) * 0.5
sleep_interval = min(sleep_interval,
self.peer_connect_interval)
try:
await asyncio.sleep(sleep_interval)
except asyncio.CancelledError:
return
tries += 1
if tries > max_tries:
addr = None
empty_tables = True
break
info: Optional[
ExtendedPeerInfo] = await self.address_manager.select_tried_collision(
)
if info is None:
info = await self.address_manager.select_peer(is_feeler
)
else:
has_collision = True
if info is None:
if not is_feeler:
empty_tables = True
break
# Require outbound connections, other than feelers,
# to be to distinct network groups.
addr = info.peer_info
if has_collision:
break
if addr is not None and not addr.is_valid():
addr = None
continue
if not is_feeler and addr.get_group() in groups:
addr = None
continue
if addr in connected:
addr = None
continue
# only consider very recently tried nodes after 30 failed attempts
if now - info.last_try < 600 and tries < 30:
continue
if time.time(
) - last_timestamp_local_info > 1800 or local_peerinfo is None:
local_peerinfo = await self.server.get_peer_info()
last_timestamp_local_info = uint64(int(time.time()))
if local_peerinfo is not None and addr == local_peerinfo:
continue
got_peer = True
disconnect_after_handshake = is_feeler
if self._num_needed_peers() == 0:
disconnect_after_handshake = True
empty_tables = False
initiate_connection = self._num_needed_peers(
) > 0 or has_collision or is_feeler
connected = False
if addr is not None and initiate_connection:
try:
connected = await self.server.start_client(
addr,
is_feeler=disconnect_after_handshake,
on_connect=self.server.on_connect,
)
except Exception as e:
self.log.error(
f"Exception in create outbound connections: {e}")
self.log.error(f"Traceback: {traceback.format_exc()}")
if self.server.is_duplicate_or_self_connection(addr):
# Mark it as a softer attempt, without counting the failures.
await self.address_manager.attempt(addr, False)
else:
if connected is True:
await self.address_manager.mark_good(addr)
await self.address_manager.connect(addr)
else:
await self.address_manager.attempt(addr, True)
sleep_interval = 1 + len(groups) * 0.5
sleep_interval = min(sleep_interval,
self.peer_connect_interval)
await asyncio.sleep(sleep_interval)
except Exception as e:
self.log.error(
f"Exception in create outbound connections: {e}")
self.log.error(f"Traceback: {traceback.format_exc()}")
async def _periodically_serialize(self, random: Random):
while not self.is_closed:
if self.address_manager is None:
await asyncio.sleep(10)
continue
serialize_interval = random.randint(15 * 60, 30 * 60)
await asyncio.sleep(serialize_interval)
async with self.address_manager.lock:
await self.address_manager_store.serialize(self.address_manager
)
async def _periodically_cleanup(self):
while not self.is_closed:
# Removes entries with timestamp worse than 14 days ago
# and with a high number of failed attempts.
# Most likely, the peer left the network,
# so we can save space in the peer tables.
cleanup_interval = 1800
max_timestamp_difference = 14 * 3600 * 24
max_consecutive_failures = 10
await asyncio.sleep(cleanup_interval)
# Perform the cleanup only if we have at least 3 connections.
full_node_connected = self.server.get_full_node_connections()
connected = [c.get_peer_info() for c in full_node_connected]
connected = [c for c in connected if c is not None]
if len(connected) >= 3:
async with self.address_manager.lock:
self.address_manager.cleanup(max_timestamp_difference,
max_consecutive_failures)
async def _respond_peers_common(self, request, peer_src, is_full_node):
# Check if we got the peers from a full node or from the introducer.
peers_adjusted_timestamp = []
for peer in request.peer_list:
if peer.timestamp < 100000000 or peer.timestamp > time.time(
) + 10 * 60:
# Invalid timestamp, predefine a bad one.
current_peer = TimestampedPeerInfo(
peer.host,
peer.port,
uint64(int(time.time() - 5 * 24 * 60 * 60)),
)
else:
current_peer = peer
if not is_full_node:
current_peer = TimestampedPeerInfo(
peer.host,
peer.port,
uint64(0),
)
peers_adjusted_timestamp.append(current_peer)
if is_full_node:
await self.address_manager.add_to_new_table(
peers_adjusted_timestamp, peer_src, 2 * 60 * 60)
else:
await self.address_manager.add_to_new_table(
peers_adjusted_timestamp, None, 0)
async def deserialize(self) -> AddressManager:
address_manager = AddressManager()
metadata = await self.get_metadata()
nodes = await self.get_nodes()
new_table_entries = await self.get_new_table()
address_manager.clear()
address_manager.key = int(metadata["key"])
address_manager.new_count = int(metadata["new_count"])
address_manager.tried_count = int(metadata["tried_count"])
new_table_nodes = [(node_id, info) for node_id, info in nodes
if node_id < address_manager.new_count]
for n, info in new_table_nodes:
address_manager.map_addr[info.peer_info.host] = n
address_manager.map_info[n] = info
info.random_pos = len(address_manager.random_pos)
address_manager.random_pos.append(n)
address_manager.id_count = len(new_table_nodes)
tried_table_nodes = [(node_id, info) for node_id, info in nodes
if node_id >= address_manager.new_count]
lost_count = 0
for node_id, info in tried_table_nodes:
tried_bucket = info.get_tried_bucket(address_manager.key)
tried_bucket_pos = info.get_bucket_position(
address_manager.key, False, tried_bucket)
if address_manager.tried_matrix[tried_bucket][
tried_bucket_pos] == -1:
info.random_pos = len(address_manager.random_pos)
info.is_tried = True
id_count = address_manager.id_count
address_manager.random_pos.append(id_count)
address_manager.map_info[id_count] = info
address_manager.map_addr[info.peer_info.host] = id_count
address_manager.tried_matrix[tried_bucket][
tried_bucket_pos] = id_count
address_manager.id_count += 1
else:
lost_count += 1
address_manager.tried_count -= lost_count
for node_id, bucket in new_table_entries:
if node_id >= 0 and node_id < address_manager.new_count:
info = address_manager.map_info[node_id]
bucket_pos = info.get_bucket_position(address_manager.key,
True, bucket)
if (address_manager.new_matrix[bucket][bucket_pos] == -1
and info.ref_count < NEW_BUCKETS_PER_ADDRESS):
info.ref_count += 1
address_manager.new_matrix[bucket][bucket_pos] = node_id
for node_id, info in list(address_manager.map_info.items()):
if not info.is_tried and info.ref_count == 0:
address_manager.delete_new_entry_(node_id)
return address_manager