async def _respond_peers_common(self, request, peer_src, is_full_node):
# Check if we got the peers from a full node or from the introducer.
peers_adjusted_timestamp = []
for peer in request.peer_list:
if peer.timestamp < 100000000 or peer.timestamp > time.time(
) + 10 * 60:
# Invalid timestamp, predefine a bad one.
current_peer = TimestampedPeerInfo(
peer.host,
peer.port,
uint64(int(time.time() - 5 * 24 * 60 * 60)),
)
else:
current_peer = peer
if not is_full_node:
current_peer = TimestampedPeerInfo(
peer.host,
peer.port,
uint64(0),
)
peers_adjusted_timestamp.append(current_peer)
if is_full_node:
await self.address_manager.add_to_new_table(
peers_adjusted_timestamp, peer_src, 2 * 60 * 60)
else:
await self.address_manager.add_to_new_table(
peers_adjusted_timestamp, None, 0)
async def test_addrman_new_bucket(self):
t_peer1 = TimestampedPeerInfo("250.1.2.1", 8444, 0)
source1 = PeerInfo("250.1.2.1", 8444)
t_peer2 = TimestampedPeerInfo("250.1.2.1", 9999, 0)
peer_info1 = ExtendedPeerInfo(t_peer1, source1)
# Test: Make sure key actually randomizes bucket placement. A fail on
# this test could be a security issue.
key1 = 2**256 - 1
key2 = 2**128 - 1
bucket1 = peer_info1.get_new_bucket(key1)
bucket2 = peer_info1.get_new_bucket(key2)
assert bucket1 != bucket2
# Test: Ports should not affect bucket placement in the addr
peer_info2 = ExtendedPeerInfo(t_peer2, source1)
assert peer_info1.get_new_bucket(key1) == peer_info2.get_new_bucket(
key1)
# Test: IP addresses in the same group (\16 prefix for IPv4) should
# always map to the same bucket.
buckets = []
for i in range(255):
peer = PeerInfo("250.1.1." + str(i), 8444)
t_peer = TimestampedPeerInfo("250.1.1." + str(i), 8444, 0)
extended_peer_info = ExtendedPeerInfo(t_peer, peer)
bucket = extended_peer_info.get_new_bucket(key1)
if bucket not in buckets:
buckets.append(bucket)
assert len(buckets) == 1
# Test: IP addresses in the same source groups should map to no more
# than 64 buckets.
buckets = []
for i in range(4 * 255):
src = PeerInfo("251.4.1.1", 8444)
peer = PeerInfo(
str(250 + i // 255) + "." + str(i % 256) + ".1.1", 8444)
t_peer = TimestampedPeerInfo(
str(250 + i // 255) + "." + str(i % 256) + ".1.1", 8444, 0)
extended_peer_info = ExtendedPeerInfo(t_peer, src)
bucket = extended_peer_info.get_new_bucket(key1)
if bucket not in buckets:
buckets.append(bucket)
assert len(buckets) <= 64
# Test: IP addresses in the different source groups should map to more
# than 64 buckets.
buckets = []
for i in range(255):
src = PeerInfo("250." + str(i) + ".1.1", 8444)
peer = PeerInfo("250.1.1.1", 8444)
t_peer = TimestampedPeerInfo("250.1.1.1", 8444, 0)
extended_peer_info = ExtendedPeerInfo(t_peer, src)
bucket = extended_peer_info.get_new_bucket(key1)
if bucket not in buckets:
buckets.append(bucket)
assert len(buckets) > 64
async def test_cleanup(self):
addrman = AddressManagerTest()
peer1 = TimestampedPeerInfo("250.250.2.1", 8444, 100000)
peer2 = TimestampedPeerInfo("250.250.2.2", 9999, time.time())
source = PeerInfo("252.5.1.1", 8333)
assert await addrman.add_to_new_table([peer1], source)
assert await addrman.add_to_new_table([peer2], source)
await addrman.mark_good(PeerInfo("250.250.2.2", 9999))
assert await addrman.size() == 2
for _ in range(5):
await addrman.attempt(peer1, True, time.time() - 61)
addrman.cleanup(7 * 3600 * 24, 5)
assert await addrman.size() == 1
def from_string(cls, peer_str: str):
blobs = peer_str.split(" ")
assert len(blobs) == 4
peer_info = TimestampedPeerInfo(blobs[0], uint16(int(blobs[1])),
uint64(0))
src_peer = PeerInfo(blobs[2], uint16(int(blobs[3])))
return cls(peer_info, src_peer)
async def _periodically_self_advertise(self):
while not self.is_closed:
try:
try:
await asyncio.sleep(24 * 3600)
except asyncio.CancelledError:
return
# Clean up known nodes for neighbours every 24 hours.
async with self.lock:
for neighbour in list(self.neighbour_known_peers.keys()):
self.neighbour_known_peers[neighbour].clear()
# Self advertise every 24 hours.
peer = await self.server.get_peer_info()
if peer is None:
continue
timestamped_peer = [
TimestampedPeerInfo(
peer.host,
peer.port,
uint64(int(time.time())),
)
]
msg = Message(
"respond_peers",
full_node_protocol.RespondPeers(timestamped_peer),
)
await self.server.send_to_all([msg], NodeType.FULL_NODE)
except Exception as e:
self.log.error(f"Exception in self advertise: {e}")
self.log.error(f"Traceback: {traceback.format_exc()}")
async def on_connect(self, peer: ws.WSChiaConnection):
if (
peer.is_outbound is False
and peer.peer_server_port is not None
and peer.connection_type is NodeType.FULL_NODE
and self.server._local_type is NodeType.FULL_NODE
and self.address_manager is not None
):
timestamped_peer_info = TimestampedPeerInfo(
peer.peer_host,
peer.peer_server_port,
uint64(int(time.time())),
)
await self.address_manager.add_to_new_table([timestamped_peer_info], peer.get_peer_info(), 0)
if self.relay_queue is not None:
self.relay_queue.put_nowait((timestamped_peer_info, 1))
if (
peer.is_outbound
and peer.peer_server_port is not None
and peer.connection_type is NodeType.FULL_NODE
and self.server._local_type is NodeType.FULL_NODE
and self.address_manager is not None
):
msg = make_msg(ProtocolMessageTypes.request_peers, full_node_protocol.RequestPeers())
await peer.send_message(msg)
async def _periodically_self_advertise(self):
while not self.is_closed:
try:
await asyncio.sleep(24 * 3600)
# Clean up known nodes for neighbours every 24 hours.
async with self.lock:
for neighbour in list(self.neighbour_known_peers.keys()):
self.neighbour_known_peers[neighbour].clear()
# Self advertise every 24 hours.
peer = await self.global_connections.get_local_peerinfo()
if peer is None:
continue
timestamped_peer = [
TimestampedPeerInfo(
peer.host,
peer.port,
uint64(int(time.time())),
)
]
outbound_message = OutboundMessage(
NodeType.FULL_NODE,
Message(
"respond_peers_full_node",
full_node_protocol.RespondPeers(timestamped_peer),
),
Delivery.BROADCAST,
)
if self.server is not None:
self.server.push_message(outbound_message)
except Exception as e:
self.log.error(f"Exception in self advertise: {e}")
self.log.error(f"Traceback: {traceback.format_exc()}")
async def _process_messages(self):
connection_time_pretest: Dict = {}
while not self.is_closed:
try:
message, peer_info = await self.message_queue.get()
if peer_info is None or not peer_info.port:
continue
if message == "make_tried":
await self.address_manager.mark_good(peer_info, True)
await self.address_manager.connect(peer_info)
elif message == "mark_attempted":
await self.address_manager.attempt(peer_info, True)
elif message == "mark_attempted_soft":
await self.address_manager.attempt(peer_info, False)
elif message == "update_connection_time":
if peer_info.host not in connection_time_pretest:
connection_time_pretest[peer_info.host] = time.time()
if time.time() - connection_time_pretest[
peer_info.host] > 60:
await self.address_manager.connect(peer_info)
connection_time_pretest[peer_info.host] = time.time()
elif message == "new_inbound_connection":
timestamped_peer_info = TimestampedPeerInfo(
peer_info.host,
peer_info.port,
uint64(int(time.time())),
)
await self.address_manager.add_to_new_table(
[timestamped_peer_info], peer_info, 0)
# await self.address_manager.mark_good(peer_info, True)
if self.relay_queue is not None:
self.relay_queue.put_nowait((timestamped_peer_info, 1))
except Exception as e:
self.log.error(f"Exception in process message: {e}")
async def request_peers_introducer(
self,
request: RequestPeersIntroducer,
peer: WSChiaConnection,
) -> Optional[Message]:
max_peers = self.introducer.max_peers_to_send
if self.introducer.server is None or self.introducer.server.introducer_peers is None:
return None
rawpeers = self.introducer.server.introducer_peers.get_peers(
max_peers * 5, True, self.introducer.recent_peer_threshold)
peers = []
for r_peer in rawpeers:
if r_peer.get_hash() not in self.introducer.vetted:
continue
if self.introducer.vetted[r_peer.get_hash()]:
if r_peer.host == peer.peer_host and r_peer.port == peer.peer_server_port:
continue
peer_without_timestamp = TimestampedPeerInfo(
r_peer.host,
r_peer.port,
uint64(0),
)
peers.append(peer_without_timestamp)
if len(peers) >= max_peers:
break
self.introducer.log.info(f"Sending vetted {peers}")
msg = make_msg(ProtocolMessageTypes.respond_peers_introducer,
RespondPeersIntroducer(peers))
return msg
async def request_peers_with_peer_info(
self,
request: RequestPeers,
peer_info: PeerInfo,
) -> AsyncGenerator[OutboundMessage, None]:
max_peers = self.max_peers_to_send
if self.global_connections.introducer_peers is None:
return
rawpeers = self.global_connections.introducer_peers.get_peers(
max_peers * 5, True, self.recent_peer_threshold
)
peers = []
for peer in rawpeers:
if peer.get_hash() not in self.vetted:
continue
if self.vetted[peer.get_hash()]:
if peer.host == peer_info.host and peer.port == peer_info.port:
continue
peer_without_timestamp = TimestampedPeerInfo(
peer.host,
peer.port,
uint64(0),
)
peers.append(peer_without_timestamp)
if len(peers) >= max_peers:
break
log.info(f"Sending vetted {peers}")
msg = Message("respond_peers", RespondPeers(peers))
yield OutboundMessage(NodeType.FULL_NODE, msg, Delivery.RESPOND)
yield OutboundMessage(NodeType.WALLET, msg, Delivery.RESPOND)
async def test_addrman_tried_bucket(self):
peer1 = PeerInfo("250.1.1.1", 8444)
t_peer1 = TimestampedPeerInfo("250.1.1.1", 8444, 0)
peer2 = PeerInfo("250.1.1.1", 9999)
t_peer2 = TimestampedPeerInfo("250.1.1.1", 9999, 0)
source1 = PeerInfo("250.1.1.1", 8444)
peer_info1 = ExtendedPeerInfo(t_peer1, source1)
# Test: Make sure key actually randomizes bucket placement. A fail on
# this test could be a security issue.
key1 = 2**256 - 1
key2 = 2**128 - 1
bucket1 = peer_info1.get_tried_bucket(key1)
bucket2 = peer_info1.get_tried_bucket(key2)
assert bucket1 != bucket2
# Test: Two addresses with same IP but different ports can map to
# different buckets because they have different keys.
peer_info2 = ExtendedPeerInfo(t_peer2, source1)
assert peer1.get_key() != peer2.get_key()
assert peer_info1.get_tried_bucket(
key1) != peer_info2.get_tried_bucket(key1)
# Test: IP addresses in the same group (\16 prefix for IPv4) should
# never get more than 8 buckets
buckets = []
for i in range(255):
peer = PeerInfo("250.1.1." + str(i), 8444)
t_peer = TimestampedPeerInfo("250.1.1." + str(i), 8444, 0)
extended_peer_info = ExtendedPeerInfo(t_peer, peer)
bucket = extended_peer_info.get_tried_bucket(key1)
if bucket not in buckets:
buckets.append(bucket)
assert len(buckets) == 8
# Test: IP addresses in the different groups should map to more than
# 8 buckets.
buckets = []
for i in range(255):
peer = PeerInfo("250." + str(i) + ".1.1", 8444)
t_peer = TimestampedPeerInfo("250." + str(i) + ".1.1", 8444, 0)
extended_peer_info = ExtendedPeerInfo(t_peer, peer)
bucket = extended_peer_info.get_tried_bucket(key1)
if bucket not in buckets:
buckets.append(bucket)
assert len(buckets) > 8
async def _respond_peers_common(self, request, peer_src, is_full_node):
# Check if we got the peers from a full node or from the introducer.
peers_adjusted_timestamp = []
is_misbehaving = False
if len(request.peer_list) > MAX_PEERS_RECEIVED_PER_REQUEST:
is_misbehaving = True
if is_full_node:
if peer_src is None:
return
async with self.lock:
if peer_src.host not in self.received_count_from_peers:
self.received_count_from_peers[peer_src.host] = 0
self.received_count_from_peers[peer_src.host] += len(
request.peer_list)
if self.received_count_from_peers[
peer_src.host] > MAX_TOTAL_PEERS_RECEIVED:
is_misbehaving = True
if is_misbehaving:
return
for peer in request.peer_list:
if peer.timestamp < 100000000 or peer.timestamp > time.time(
) + 10 * 60:
# Invalid timestamp, predefine a bad one.
current_peer = TimestampedPeerInfo(
peer.host,
peer.port,
uint64(int(time.time() - 5 * 24 * 60 * 60)),
)
else:
current_peer = peer
if not is_full_node:
current_peer = TimestampedPeerInfo(
peer.host,
peer.port,
uint64(0),
)
peers_adjusted_timestamp.append(current_peer)
if is_full_node:
await self.address_manager.add_to_new_table(
peers_adjusted_timestamp, peer_src, 2 * 60 * 60)
else:
await self.address_manager.add_to_new_table(
peers_adjusted_timestamp, None, 0)
async def test_serialization(self):
addrman = AddressManagerTest()
now = int(math.floor(time.time()))
t_peer1 = TimestampedPeerInfo("250.7.1.1", 8333, now - 10000)
t_peer2 = TimestampedPeerInfo("250.7.2.2", 9999, now - 20000)
t_peer3 = TimestampedPeerInfo("250.7.3.3", 9999, now - 30000)
source = PeerInfo("252.5.1.1", 8333)
await addrman.add_to_new_table([t_peer1, t_peer2, t_peer3], source)
await addrman.mark_good(PeerInfo("250.7.1.1", 8333))
db_filename = Path("peer_table.db")
if db_filename.exists():
db_filename.unlink()
connection = await aiosqlite.connect(db_filename)
address_manager_store = await AddressManagerStore.create(connection)
await address_manager_store.serialize(addrman)
addrman2 = await address_manager_store.deserialize()
retrieved_peers = []
for _ in range(50):
peer = await addrman2.select_peer()
if peer not in retrieved_peers:
retrieved_peers.append(peer)
if len(retrieved_peers) == 3:
break
assert len(retrieved_peers) == 3
wanted_peers = [
ExtendedPeerInfo(t_peer1, source),
ExtendedPeerInfo(t_peer2, source),
ExtendedPeerInfo(t_peer3, source),
]
recovered = 0
for target_peer in wanted_peers:
for current_peer in retrieved_peers:
if (
current_peer.peer_info == target_peer.peer_info
and current_peer.src == target_peer.src
and current_peer.timestamp == target_peer.timestamp
):
recovered += 1
assert recovered == 3
await connection.close()
db_filename.unlink()
async def test_addrman_create(self):
addrman = AddressManagerTest()
assert await addrman.size() == 0
peer1 = PeerInfo("250.1.2.1", 8444)
t_peer = TimestampedPeerInfo("250.1.2.1", 8444, 0)
info, node_id = addrman.create_(t_peer, peer1)
assert info.peer_info == peer1
info, _ = addrman.find_(peer1)
assert info.peer_info == peer1
async def add_peer_info(self, peers, peer_src=None):
timestamped_peers = [
TimestampedPeerInfo(
peer.host,
peer.port,
0,
) for peer in peers
]
added = await self.add_to_new_table(timestamped_peers, peer_src)
return added
def get_peers(self,
max_peers: int = 0,
randomize: bool = False,
recent_threshold=9999999) -> List[TimestampedPeerInfo]:
target_peers = [
TimestampedPeerInfo(peer.host, uint16(peer.port), uint64(0))
for peer in self._peers if time.time() -
self.time_added[peer.get_hash()] < recent_threshold
]
if not max_peers or max_peers > len(target_peers):
max_peers = len(target_peers)
if randomize:
random.shuffle(target_peers)
return target_peers[:max_peers]
async def test_addrman_delete(self):
addrman = AddressManagerTest()
assert await addrman.size() == 0
peer1 = PeerInfo("250.1.2.1", 8444)
t_peer = TimestampedPeerInfo("250.1.2.1", 8444, 0)
info, node_id = addrman.create_(t_peer, peer1)
# Test: Delete should actually delete the addr.
assert await addrman.size() == 1
addrman.delete_new_entry_(node_id)
assert await addrman.size() == 0
info2, _ = addrman.find_(peer1)
assert info2 is None
def get_peers_(self) -> List[TimestampedPeerInfo]:
addr: List[TimestampedPeerInfo] = []
num_nodes = math.ceil(23 * len(self.random_pos) / 100)
if num_nodes > 1000:
num_nodes = 1000
for n in range(len(self.random_pos)):
if len(addr) >= num_nodes:
return addr
rand_pos = randrange(len(self.random_pos) - n) + n
self.swap_random_(n, rand_pos)
info = self.map_info[self.random_pos[n]]
if not info.is_terrible():
cur_peer_info = TimestampedPeerInfo(
info.peer_info.host,
uint16(info.peer_info.port),
uint64(info.timestamp),
)
addr.append(cur_peer_info)
return addr
async def test_addrman_get_peers(self):
addrman = AddressManagerTest()
assert await addrman.size() == 0
peers1 = await addrman.get_peers()
assert len(peers1) == 0
peer1 = TimestampedPeerInfo("250.250.2.1", 8444, time.time())
peer2 = TimestampedPeerInfo("250.250.2.2", 9999, time.time())
peer3 = TimestampedPeerInfo("251.252.2.3", 8444, time.time())
peer4 = TimestampedPeerInfo("251.252.2.4", 8444, time.time())
peer5 = TimestampedPeerInfo("251.252.2.5", 8444, time.time())
source1 = PeerInfo("250.1.2.1", 8444)
source2 = PeerInfo("250.2.3.3", 8444)
# Test: Ensure GetPeers works with new addresses.
assert await addrman.add_to_new_table([peer1], source1)
assert await addrman.add_to_new_table([peer2], source2)
assert await addrman.add_to_new_table([peer3], source1)
assert await addrman.add_to_new_table([peer4], source1)
assert await addrman.add_to_new_table([peer5], source1)
# GetPeers returns 23% of addresses, 23% of 5 is 2 rounded up.
peers2 = await addrman.get_peers()
assert len(peers2) == 2
# Test: Ensure GetPeers works with new and tried addresses.
await addrman.mark_good(peer1)
await addrman.mark_good(peer2)
peers3 = await addrman.get_peers()
assert len(peers3) == 2
# Test: Ensure GetPeers still returns 23% when addrman has many addrs.
for i in range(1, 8 * 256):
octet1 = i % 256
octet2 = i >> 8 % 256
peer = TimestampedPeerInfo(
str(octet1) + "." + str(octet2) + ".1.23", 8444, time.time())
await addrman.add_to_new_table([peer])
if i % 8 == 0:
await addrman.mark_good(peer)
peers4 = await addrman.get_peers()
percent = await addrman.size()
percent = math.ceil(percent * 23 / 100)
assert len(peers4) == percent
