async def _make_protocol(self, other_peer, node_id, address, udp_port, tcp_port):
proto = KademliaProtocol(
self.loop, PeerManager(self.loop), node_id, address, udp_port, tcp_port
)
await self.loop.create_datagram_endpoint(lambda: proto, (address, 4444))
proto.start()
return proto, other_peer.peer_manager.get_kademlia_peer(node_id, address, udp_port=udp_port)
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: typing.Optional[float] = 5.0):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
async def test_add_peer_after_handle_request(self):
with dht_mocks.mock_network_loop(self.loop):
node_id1 = constants.generate_id()
node_id2 = constants.generate_id()
node_id3 = constants.generate_id()
node_id4 = constants.generate_id()
peer1 = KademliaProtocol(
self.loop, PeerManager(self.loop), node_id1, '1.2.3.4', 4444, 3333
)
await self.loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
peer1.start()
peer2, peer_2_from_peer_1 = await self._make_protocol(peer1, node_id2, '1.2.3.5', 4444, 3333)
peer3, peer_3_from_peer_1 = await self._make_protocol(peer1, node_id3, '1.2.3.6', 4444, 3333)
peer4, peer_4_from_peer_1 = await self._make_protocol(peer1, node_id4, '1.2.3.7', 4444, 3333)
# peers who reply should be added
await peer1.get_rpc_peer(peer_2_from_peer_1).ping()
await asyncio.sleep(0.5)
self.assertListEqual([peer_2_from_peer_1], peer1.routing_table.get_peers())
peer1.routing_table.remove_peer(peer_2_from_peer_1)
# peers not known by be good/bad should be enqueued to maybe-ping
peer1_from_peer3 = peer3.get_rpc_peer(peer3.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
pong = await peer1_from_peer3.ping()
self.assertEqual(b'pong', pong)
self.assertEqual(1, len(peer1.ping_queue._pending_contacts))
peer1.ping_queue._pending_contacts.clear()
# peers who are already good should be added
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
peer1.peer_manager.report_last_replied('1.2.3.7', 4444)
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
pong = await peer1_from_peer4.ping()
self.assertEqual(b'pong', pong)
await asyncio.sleep(0.5)
self.assertEqual(1, len(peer1.routing_table.get_peers()))
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
peer1.routing_table.buckets[0].peers.clear()
# peers who are known to be bad recently should not be added or maybe-pinged
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
peer1.peer_manager.report_failure('1.2.3.7', 4444)
peer1.peer_manager.report_failure('1.2.3.7', 4444)
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
pong = await peer1_from_peer4.ping()
self.assertEqual(b'pong', pong)
self.assertEqual(0, len(peer1.routing_table.get_peers()))
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
for p in [peer1, peer2, peer3, peer4]:
p.stop()
p.disconnect()
def __init__(self,
loop: asyncio.BaseEventLoop,
peer_manager: 'PeerManager',
node_id: bytes,
udp_port: int,
internal_udp_port: int,
peer_port: int,
external_ip: str,
rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.
split_buckets_under_index):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id,
external_ip, udp_port, peer_port,
rpc_timeout,
split_buckets_under_index)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
async def test_store_to_peer(self):
loop = asyncio.get_event_loop()
with dht_mocks.mock_network_loop(loop):
node_id1 = constants.generate_id()
peer1 = KademliaProtocol(loop, PeerManager(loop), node_id1,
'1.2.3.4', 4444, 3333)
peer2 = KademliaProtocol(loop, PeerManager(loop),
constants.generate_id(), '1.2.3.5', 4444,
3333)
await loop.create_datagram_endpoint(lambda: peer1,
('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2,
('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1,
'1.2.3.4',
udp_port=4444)
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port)
peer2_from_peer1.update_tcp_port(3333)
peer3 = peer1.peer_manager.get_kademlia_peer(
constants.generate_id(), '1.2.3.6', udp_port=4444)
store_result = await peer2.store_to_peer(b'2' * 48, peer)
self.assertEqual(store_result[0], peer.node_id)
self.assertEqual(True, store_result[1])
self.assertEqual(True,
peer1.data_store.has_peers_for_blob(b'2' * 48))
self.assertEqual(False,
peer1.data_store.has_peers_for_blob(b'3' * 48))
self.assertListEqual([peer2_from_peer1],
peer1.data_store.get_storing_contacts())
peer1.data_store.completed_blobs.add(
binascii.hexlify(b'2' * 48).decode())
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
self.assertEqual(len(find_value_response[b'contacts']), 0)
self.assertSetEqual(
{b'2' * 48, b'token', b'protocolVersion', b'contacts'},
set(find_value_response.keys()))
self.assertEqual(2, len(find_value_response[b'2' * 48]))
self.assertEqual(find_value_response[b'2' * 48][0],
peer2_from_peer1.compact_address_tcp())
self.assertDictEqual(bdecode(bencode(find_value_response)),
find_value_response)
peer1.stop()
peer2.stop()
peer1.disconnect()
peer2.disconnect()
async def test_update_token(self):
loop = asyncio.get_event_loop()
with dht_mocks.mock_network_loop(loop):
node_id1 = constants.generate_id()
peer1 = KademliaProtocol(loop, PeerManager(loop), node_id1,
'1.2.3.4', 4444, 3333)
peer2 = KademliaProtocol(loop, PeerManager(loop),
constants.generate_id(), '1.2.3.5', 4444,
3333)
await loop.create_datagram_endpoint(lambda: peer1,
('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2,
('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1,
'1.2.3.4',
udp_port=4444)
self.assertEqual(None,
peer2.peer_manager.get_node_token(peer.node_id))
await peer2.get_rpc_peer(peer).find_value(b'1' * 48)
self.assertNotEqual(
None, peer2.peer_manager.get_node_token(peer.node_id))
peer1.stop()
peer2.stop()
peer1.disconnect()
peer2.disconnect()
async def test_ping(self):
loop = asyncio.get_event_loop()
with dht_mocks.mock_network_loop(loop):
node_id1 = constants.generate_id()
peer1 = KademliaProtocol(loop, PeerManager(loop), node_id1,
'1.2.3.4', 4444, 3333)
peer2 = KademliaProtocol(loop, PeerManager(loop),
constants.generate_id(), '1.2.3.5', 4444,
3333)
await loop.create_datagram_endpoint(lambda: peer1,
('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2,
('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1,
'1.2.3.4',
udp_port=4444)
result = await peer2.get_rpc_peer(peer).ping()
self.assertEqual(result, b'pong')
peer1.stop()
peer2.stop()
peer1.disconnect()
peer2.disconnect()
class Node:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: typing.Optional[float] = 5.0):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
async def refresh_node(self):
while True:
# remove peers with expired blob announcements from the datastore
self.protocol.data_store.removed_expired_peers()
total_peers: typing.List['KademliaPeer'] = []
# add all peers in the routing table
total_peers.extend(self.protocol.routing_table.get_peers())
# add all the peers who have announed blobs to us
total_peers.extend(self.protocol.data_store.get_storing_contacts())
# get ids falling in the midpoint of each bucket that hasn't been recently updated
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
# populate/split the buckets further
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
if buckets_with_contacts <= 3:
for i in range(buckets_with_contacts):
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
if self.protocol.routing_table.get_peers():
# if we have node ids to look up, perform the iterative search until we have k results
while node_ids:
peers = await self.peer_search(node_ids.pop())
total_peers.extend(peers)
else:
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
await fut
continue
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
if to_ping:
await self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
await fut
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
announced_to_node_ids = []
while not announced_to_node_ids:
hash_value = binascii.unhexlify(blob_hash.encode())
assert len(hash_value) == constants.hash_length
peers = await self.peer_search(hash_value)
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.debug("Store to %i peers", len(peers))
for peer in peers:
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
)
announced_to_node_ids.extend([node_id for node_id, contacted in stored_to_tup if contacted])
log.info("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
len(announced_to_node_ids), len(peers))
return announced_to_node_ids
def stop(self) -> None:
if self.joined.is_set():
self.joined.clear()
if self._join_task:
self._join_task.cancel()
if self._refresh_task and not (self._refresh_task.done() or self._refresh_task.cancelled()):
self._refresh_task.cancel()
if self.protocol and self.protocol.ping_queue.running:
self.protocol.ping_queue.stop()
if self.listening_port is not None:
self.listening_port.close()
self._join_task = None
self.listening_port = None
log.info("Stopped DHT node")
async def start_listening(self, interface: str = '') -> None:
if not self.listening_port:
self.listening_port, _ = await self.loop.create_datagram_endpoint(
lambda: self.protocol, (interface, self.internal_udp_port)
)
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
else:
log.warning("Already bound to port %s", self.listening_port)
async def join_network(self, interface: typing.Optional[str] = '',
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
if not self.listening_port:
await self.start_listening(interface)
self.protocol.ping_queue.start()
self._refresh_task = self.loop.create_task(self.refresh_node())
# resolve the known node urls
known_node_addresses = []
url_to_addr = {}
if known_node_urls:
for host, port in known_node_urls:
address = await resolve_host(host, port, proto='udp')
if (address, port) not in known_node_addresses:
known_node_addresses.append((address, port))
url_to_addr[address] = host
if known_node_addresses:
while not self.protocol.routing_table.get_peers():
success = False
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
try:
await peer.ping()
success = True
except asyncio.TimeoutError:
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
round(self.protocol.rpc_timeout, 2))
if success:
break
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
# in the routing table with good peers who are near us
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
try:
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
self.joined.set()
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
self._join_task = self.loop.create_task(
self.join_network(
interface=interface, known_node_urls=known_node_urls
)
)
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
bottom_out_limit: int = constants.bottom_out_limit,
max_results: int = constants.k) -> IterativeNodeFinder:
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
bottom_out_limit: int = 40,
max_results: int = -1) -> IterativeValueFinder:
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
@contextlib.asynccontextmanager
async def stream_peer_search_junction(self, hash_queue: asyncio.Queue, bottom_out_limit=20,
max_results=-1) -> AsyncGeneratorJunction:
peer_generator = AsyncGeneratorJunction(self.loop)
async def _add_hashes_from_queue():
while True:
try:
blob_hash = await hash_queue.get()
except asyncio.CancelledError:
break
peer_generator.add_generator(
self.get_iterative_value_finder(
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
max_results=max_results
)
)
add_hashes_task = self.loop.create_task(_add_hashes_from_queue())
try:
async with peer_generator as junction:
yield junction
await peer_generator.finished.wait()
except asyncio.CancelledError:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
raise
finally:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
def peer_search_junction(self, node_id: bytes, max_results=constants.k*2,
bottom_out_limit=20) -> AsyncGeneratorJunction:
peer_generator = AsyncGeneratorJunction(self.loop)
peer_generator.add_generator(
self.get_iterative_node_finder(
node_id, bottom_out_limit=bottom_out_limit, max_results=max_results
)
)
return peer_generator
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
bottom_out_limit=20) -> typing.List['KademliaPeer']:
accumulated: typing.List['KademliaPeer'] = []
async with self.peer_search_junction(node_id, max_results=max_results,
bottom_out_limit=bottom_out_limit) as junction:
async for peers in junction:
accumulated.extend(peers)
distance = Distance(node_id)
accumulated.sort(key=lambda peer: distance(peer.node_id))
return accumulated[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
try:
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
async for peers in search_junction:
if peers:
result_queue.put_nowait([
peer for peer in peers
if not (
peer.address == self.protocol.external_ip
and peer.tcp_port == self.protocol.peer_port
)
])
except asyncio.CancelledError:
return
def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue()
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
class Node:
def __init__(self,
loop: asyncio.BaseEventLoop,
peer_manager: 'PeerManager',
node_id: bytes,
udp_port: int,
internal_udp_port: int,
peer_port: int,
external_ip: str,
rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.
split_buckets_under_index):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id,
external_ip, udp_port, peer_port,
rpc_timeout,
split_buckets_under_index)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
async def refresh_node(self, force_once=False):
while True:
# remove peers with expired blob announcements from the datastore
self.protocol.data_store.removed_expired_peers()
total_peers: typing.List['KademliaPeer'] = []
# add all peers in the routing table
total_peers.extend(self.protocol.routing_table.get_peers())
# add all the peers who have announed blobs to us
total_peers.extend(self.protocol.data_store.get_storing_contacts())
# get ids falling in the midpoint of each bucket that hasn't been recently updated
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
# populate/split the buckets further
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts(
)
if buckets_with_contacts <= 3:
for i in range(buckets_with_contacts):
node_ids.append(
self.protocol.routing_table.random_id_in_bucket_range(
i))
node_ids.append(
self.protocol.routing_table.random_id_in_bucket_range(
i))
if self.protocol.routing_table.get_peers():
# if we have node ids to look up, perform the iterative search until we have k results
while node_ids:
peers = await self.peer_search(node_ids.pop())
total_peers.extend(peers)
else:
if force_once:
break
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval // 4,
fut.set_result, None)
await fut
continue
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
to_ping = [
peer for peer in set(total_peers)
if self.protocol.peer_manager.peer_is_good(peer) is not True
]
if to_ping:
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
if force_once:
break
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval, fut.set_result,
None)
await fut
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
hash_value = binascii.unhexlify(blob_hash.encode())
assert len(hash_value) == constants.hash_length
peers = await self.peer_search(hash_value)
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.debug("Store to %i peers", len(peers))
for peer in peers:
log.debug("store to %s %s %s", peer.address, peer.udp_port,
peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers),
loop=self.loop)
stored_to = [
node_id for node_id, contacted in stored_to_tup if contacted
]
if stored_to:
log.info("Stored %s to %i of %i attempted peers",
binascii.hexlify(hash_value).decode()[:8], len(stored_to),
len(peers))
else:
log.warning("Failed announcing %s, stored to 0 peers",
blob_hash[:8])
return stored_to
def stop(self) -> None:
if self.joined.is_set():
self.joined.clear()
if self._join_task:
self._join_task.cancel()
if self._refresh_task and not (self._refresh_task.done()
or self._refresh_task.cancelled()):
self._refresh_task.cancel()
if self.protocol and self.protocol.ping_queue.running:
self.protocol.ping_queue.stop()
self.protocol.stop()
if self.listening_port is not None:
self.listening_port.close()
self._join_task = None
self.listening_port = None
log.info("Stopped DHT node")
async def start_listening(self, interface: str = '') -> None:
if not self.listening_port:
self.listening_port, _ = await self.loop.create_datagram_endpoint(
lambda: self.protocol, (interface, self.internal_udp_port))
log.info("DHT node listening on UDP %s:%i", interface,
self.internal_udp_port)
self.protocol.start()
else:
log.warning("Already bound to port %s", self.listening_port)
async def join_network(self,
interface: typing.Optional[str] = '',
known_node_urls: typing.Optional[typing.List[
typing.Tuple[str, int]]] = None):
if not self.listening_port:
await self.start_listening(interface)
self.protocol.ping_queue.start()
self._refresh_task = self.loop.create_task(self.refresh_node())
# resolve the known node urls
known_node_addresses = []
url_to_addr = {}
if known_node_urls:
for host, port in known_node_urls:
address = await resolve_host(host, port, proto='udp')
if (address, port) not in known_node_addresses and\
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
known_node_addresses.append((address, port))
url_to_addr[address] = host
if known_node_addresses:
peers = [
KademliaPeer(self.loop, address, udp_port=port)
for (address, port) in known_node_addresses
]
while True:
if not self.protocol.routing_table.get_peers():
if self.joined.is_set():
self.joined.clear()
self.protocol.peer_manager.reset()
self.protocol.ping_queue.enqueue_maybe_ping(*peers,
delay=0.0)
peers.extend(await self.peer_search(self.protocol.node_id,
shortlist=peers,
count=32))
if self.protocol.routing_table.get_peers():
self.joined.set()
log.info(
"Joined DHT, %i peers known in %i buckets",
len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts(
))
else:
continue
await asyncio.sleep(1, loop=self.loop)
log.info("Joined DHT, %i peers known in %i buckets",
len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
self.joined.set()
def start(self, interface: str,
known_node_urls: typing.List[typing.Tuple[str, int]]):
self._join_task = self.loop.create_task(
self.join_network(interface=interface,
known_node_urls=known_node_urls))
def get_iterative_node_finder(
self,
key: bytes,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = constants.bottom_out_limit,
max_results: int = constants.k) -> IterativeNodeFinder:
return IterativeNodeFinder(self.loop, self.protocol.peer_manager,
self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None,
shortlist)
def get_iterative_value_finder(
self,
key: bytes,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = 40,
max_results: int = -1) -> IterativeValueFinder:
return IterativeValueFinder(self.loop, self.protocol.peer_manager,
self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None,
shortlist)
async def peer_search(
self,
node_id: bytes,
count=constants.k,
max_results=constants.k * 2,
bottom_out_limit=20,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None
) -> typing.List['KademliaPeer']:
peers = []
async for iteration_peers in self.get_iterative_node_finder(
node_id,
shortlist=shortlist,
bottom_out_limit=bottom_out_limit,
max_results=max_results):
peers.extend(iteration_peers)
distance = Distance(node_id)
peers.sort(key=lambda peer: distance(peer.node_id))
return peers[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
tasks = []
try:
while True:
blob_hash = await search_queue.get()
tasks.append(
self.loop.create_task(
self._value_producer(blob_hash, result_queue)))
finally:
for task in tasks:
task.cancel()
async def _value_producer(self, blob_hash: str,
result_queue: asyncio.Queue):
async for results in self.get_iterative_value_finder(
binascii.unhexlify(blob_hash.encode())):
result_queue.put_nowait(results)
def accumulate_peers(
self,
search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None
) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue(loop=self.loop)
return q, self.loop.create_task(
self._accumulate_search_junction(search_queue, q))