async def main(host: str, port: int, db_file_path: str,
bootstrap_node: Optional[str], prometheus_port: int):
loop = asyncio.get_event_loop()
conf = Config()
storage = SQLiteStorage(conf, db_file_path, loop, loop.time)
if bootstrap_node:
nodes = bootstrap_node.split(':')
nodes = [(nodes[0], int(nodes[1]))]
else:
nodes = conf.known_dht_nodes
await storage.open()
node = Node(loop,
PeerManager(loop),
generate_id(),
port,
port,
3333,
None,
storage=storage)
if prometheus_port > 0:
metrics = SimpleMetrics(prometheus_port, node)
await metrics.start()
node.start(host, nodes)
while True:
await asyncio.sleep(10)
PEERS.labels('main').set(len(node.protocol.routing_table.get_peers()))
BLOBS_STORED.labels('main').set(
len(node.protocol.data_store.get_storing_contacts()))
log.info(
"Known peers: %d. Storing contact information for %d blobs from %d peers.",
len(node.protocol.routing_table.get_peers()),
len(node.protocol.data_store),
len(node.protocol.data_store.get_storing_contacts()))
class DHTComponent(Component):
component_name = DHT_COMPONENT
depends_on = [UPNP_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.dht_node: typing.Optional[Node] = None
self.external_udp_port = None
self.external_peer_port = None
@property
def component(self) -> typing.Optional[Node]:
return self.dht_node
async def get_status(self):
return {
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
}
def get_node_id(self):
node_id_filename = os.path.join(self.conf.data_dir, "node_id")
if os.path.isfile(node_id_filename):
with open(node_id_filename, "r") as node_id_file:
return base58.b58decode(str(node_id_file.read()).strip())
node_id = utils.generate_id()
with open(node_id_filename, "w") as node_id_file:
node_id_file.write(base58.b58encode(node_id).decode())
return node_id
async def start(self):
log.info("start the dht")
upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
self.external_udp_port = upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
external_ip = upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = await utils.get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
self.dht_node = Node(
self.component_manager.loop,
self.component_manager.peer_manager,
node_id=self.get_node_id(),
internal_udp_port=self.conf.udp_port,
udp_port=self.external_udp_port,
external_ip=external_ip,
peer_port=self.external_peer_port,
rpc_timeout=self.conf.node_rpc_timeout,
split_buckets_under_index=self.conf.split_buckets_under_index
)
self.dht_node.start(
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
)
log.info("Started the dht")
async def stop(self):
self.dht_node.stop()
async def start(self):
log.info("start the dht")
upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = upnp_component.upnp_redirects.get(
"TCP", self.conf.tcp_port)
self.external_udp_port = upnp_component.upnp_redirects.get(
"UDP", self.conf.udp_port)
external_ip = upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = await utils.get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
self.dht_node = Node(
self.component_manager.loop,
self.component_manager.peer_manager,
node_id=self.get_node_id(),
internal_udp_port=self.conf.udp_port,
udp_port=self.external_udp_port,
external_ip=external_ip,
peer_port=self.external_peer_port,
rpc_timeout=self.conf.node_rpc_timeout,
split_buckets_under_index=self.conf.split_buckets_under_index)
self.dht_node.start(interface=self.conf.network_interface,
known_node_urls=self.conf.known_dht_nodes)
log.info("Started the dht")
async def setup_node(self, peer_addresses, address, node_id):
self.nodes: typing.Dict[int, Node] = {}
self.advance = dht_mocks.get_time_accelerator(self.loop, self.loop.time())
self.conf = Config()
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
await self.storage.open()
self.peer_manager = PeerManager(self.loop)
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address)
await self.node.start_listening(address)
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
for node_id, address in peer_addresses:
await self.add_peer(node_id, address)
self.node.joined.set()
self.node._refresh_task = self.loop.create_task(self.node.refresh_node())
async def setup_network(self, size: int, start_port=40000, seed_nodes=1):
for i in range(size):
node_port = start_port + i
node = Node(self.loop, PeerManager(self.loop), node_id=constants.generate_id(i),
udp_port=node_port, internal_udp_port=node_port,
peer_port=3333, external_ip='127.0.0.1')
self.nodes.append(node)
self.known_node_addresses.append(('127.0.0.1', node_port))
await node.start_listening('127.0.0.1')
self.addCleanup(node.stop)
for node in self.nodes:
node.protocol.rpc_timeout = .5
node.protocol.ping_queue._default_delay = .5
node.start('127.0.0.1', self.known_node_addresses[:seed_nodes])
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
async def add_peer(self, node_id, address, add_to_routing_table=True):
#print('add', node_id.hex()[:8], address)
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
await n.start_listening(address)
self.nodes.update({len(self.nodes): n})
if add_to_routing_table:
self.add_peer_to_routing_table(self.node, n)
async def add_peer(self, node_id, address, add_to_routing_table=True):
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333,
address)
await n.start_listening(address)
self.nodes.update({len(self.nodes): n})
if add_to_routing_table:
self.node.protocol.add_peer(
make_kademlia_peer(n.protocol.node_id, n.protocol.external_ip,
n.protocol.udp_port))
async def test_cant_add_peer_without_a_node_id_gracefully(self):
loop = asyncio.get_event_loop()
node = Node(loop, PeerManager(loop), constants.generate_id(), 4444,
4444, 3333, '1.2.3.4')
bad_peer = make_kademlia_peer(None, '1.2.3.4', 5555)
with self.assertLogs(level='WARNING') as logged:
self.assertFalse(await node.protocol._add_peer(bad_peer))
self.assertEqual(1, len(logged.output))
self.assertTrue(logged.output[0].endswith(
'Tried adding a peer with no node id!'))
async def main(host: str, port: int, db_file_path: str,
bootstrap_node: Optional[str], prometheus_port: int,
export: bool):
loop = asyncio.get_event_loop()
conf = Config()
if not db_file_path.startswith(':memory:'):
node_id_file_path = db_file_path + 'node_id'
if os.path.exists(node_id_file_path):
with open(node_id_file_path, 'rb') as node_id_file:
node_id = node_id_file.read()
else:
with open(node_id_file_path, 'wb') as node_id_file:
node_id = generate_id()
node_id_file.write(node_id)
storage = SQLiteStorage(conf, db_file_path, loop, loop.time)
if bootstrap_node:
nodes = bootstrap_node.split(':')
nodes = [(nodes[0], int(nodes[1]))]
else:
nodes = conf.known_dht_nodes
await storage.open()
node = Node(loop,
PeerManager(loop),
node_id,
port,
port,
3333,
None,
storage=storage)
if prometheus_port > 0:
metrics = SimpleMetrics(prometheus_port, node if export else None)
await metrics.start()
node.start(host, nodes)
log.info("Peer with id %s started", node_id.hex())
while True:
await asyncio.sleep(10)
log.info(
"Known peers: %d. Storing contact information for %d blobs from %d peers.",
len(node.protocol.routing_table.get_peers()),
len(node.protocol.data_store),
len(node.protocol.data_store.get_storing_contacts()))
async def setup_node(self, peer_addresses, address, node_id):
self.nodes: typing.Dict[int, Node] = {}
self.advance = dht_mocks.get_time_accelerator(self.loop)
self.instant_advance = dht_mocks.get_time_accelerator(self.loop)
self.conf = Config()
self.peer_manager = PeerManager(self.loop)
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address)
await self.node.start_listening(address)
await asyncio.gather(*[self.add_peer(node_id, address) for node_id, address in peer_addresses])
for first_peer in self.nodes.values():
for second_peer in self.nodes.values():
if first_peer == second_peer:
continue
self.add_peer_to_routing_table(first_peer, second_peer)
self.add_peer_to_routing_table(second_peer, first_peer)
await self.advance(0.1) # just to make pings go through
self.node.joined.set()
self.node._refresh_task = self.loop.create_task(self.node.refresh_node())
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
await self.storage.open()
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
async def create_node(self, node_id, port, external_ip='127.0.0.1'):
storage = SQLiteStorage(Config(), ":memory:", self.loop,
self.loop.time)
await storage.open()
node = Node(self.loop,
PeerManager(self.loop),
node_id=node_id,
udp_port=port,
internal_udp_port=port,
peer_port=3333,
external_ip=external_ip,
storage=storage)
self.addCleanup(node.stop)
node.protocol.rpc_timeout = .5
node.protocol.ping_queue._default_delay = .5
return node
async def test_split_buckets(self):
loop = asyncio.get_event_loop()
peer_addresses = [
(constants.generate_id(1), '1.2.3.1'),
]
for i in range(2, 200):
peer_addresses.append((constants.generate_id(i), f'1.2.3.{i}'))
with dht_mocks.mock_network_loop(loop):
nodes = {
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333,
address)
for i, (node_id, address) in enumerate(peer_addresses)
}
node_1 = nodes[0]
for i in range(1, len(peer_addresses)):
node = nodes[i]
peer = node_1.protocol.peer_manager.get_kademlia_peer(
node.protocol.node_id,
node.protocol.external_ip,
udp_port=node.protocol.udp_port)
# set all of the peers to good (as to not attempt pinging stale ones during split)
node_1.protocol.peer_manager.report_last_replied(
peer.address, peer.udp_port)
node_1.protocol.peer_manager.report_last_replied(
peer.address, peer.udp_port)
await node_1.protocol._add_peer(peer)
# check that bucket 0 is always the one covering the local node id
self.assertEqual(
True,
node_1.protocol.routing_table.buckets[0].key_in_range(
node_1.protocol.node_id))
self.assertEqual(40,
len(node_1.protocol.routing_table.get_peers()))
self.assertEqual(len(expected_ranges),
len(node_1.protocol.routing_table.buckets))
covered = 0
for (expected_min, expected_max), bucket in zip(
expected_ranges, node_1.protocol.routing_table.buckets):
self.assertEqual(expected_min, bucket.range_min)
self.assertEqual(expected_max, bucket.range_max)
covered += bucket.range_max - bucket.range_min
self.assertEqual(2**384, covered)
for node in nodes.values():
node.stop()
async def test_fill_one_bucket(self):
loop = asyncio.get_event_loop()
peer_addresses = [
(constants.generate_id(1), '1.2.3.1'),
(constants.generate_id(2), '1.2.3.2'),
(constants.generate_id(3), '1.2.3.3'),
(constants.generate_id(4), '1.2.3.4'),
(constants.generate_id(5), '1.2.3.5'),
(constants.generate_id(6), '1.2.3.6'),
(constants.generate_id(7), '1.2.3.7'),
(constants.generate_id(8), '1.2.3.8'),
(constants.generate_id(9), '1.2.3.9'),
]
with dht_mocks.mock_network_loop(loop):
nodes = {
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333,
address)
for i, (node_id, address) in enumerate(peer_addresses)
}
node_1 = nodes[0]
contact_cnt = 0
for i in range(1, len(peer_addresses)):
self.assertEqual(
len(node_1.protocol.routing_table.get_peers()),
contact_cnt)
node = nodes[i]
peer = node_1.protocol.peer_manager.get_kademlia_peer(
node.protocol.node_id,
node.protocol.external_ip,
udp_port=node.protocol.udp_port)
added = await node_1.protocol._add_peer(peer)
self.assertEqual(True, added)
contact_cnt += 1
self.assertEqual(len(node_1.protocol.routing_table.get_peers()), 8)
self.assertEqual(
node_1.protocol.routing_table.buckets_with_contacts(), 1)
for node in nodes.values():
node.protocol.stop()
async def test_ping_queue_discover(self):
loop = asyncio.get_event_loop()
loop.set_debug(False)
peer_addresses = [
(constants.generate_id(1), '1.2.3.1'),
(constants.generate_id(2), '1.2.3.2'),
(constants.generate_id(3), '1.2.3.3'),
(constants.generate_id(4), '1.2.3.4'),
(constants.generate_id(5), '1.2.3.5'),
(constants.generate_id(6), '1.2.3.6'),
(constants.generate_id(7), '1.2.3.7'),
(constants.generate_id(8), '1.2.3.8'),
(constants.generate_id(9), '1.2.3.9'),
]
with dht_mocks.mock_network_loop(loop):
advance = dht_mocks.get_time_accelerator(loop)
# start the nodes
nodes: typing.Dict[int, Node] = {
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333,
address)
for i, (node_id, address) in enumerate(peer_addresses)
}
for i, n in nodes.items():
n.start(peer_addresses[i][1], [])
await advance(1)
node_1 = nodes[0]
# ping 8 nodes from node_1, this will result in a delayed return ping
futs = []
for i in range(1, len(peer_addresses)):
node = nodes[i]
assert node.protocol.node_id != node_1.protocol.node_id
peer = make_kademlia_peer(node.protocol.node_id,
node.protocol.external_ip,
udp_port=node.protocol.udp_port)
futs.append(node_1.protocol.get_rpc_peer(peer).ping())
await advance(3)
replies = await asyncio.gather(*tuple(futs))
self.assertTrue(all(map(lambda reply: reply == b"pong", replies)))
# run for long enough for the delayed pings to have been sent by node 1
await advance(1000)
# verify all of the previously pinged peers have node_1 in their routing tables
for n in nodes.values():
peers = n.protocol.routing_table.get_peers()
if n is node_1:
self.assertEqual(8, len(peers))
# TODO: figure out why this breaks
# else:
# self.assertEqual(1, len(peers))
# self.assertEqual((peers[0].node_id, peers[0].address, peers[0].udp_port),
# (node_1.protocol.node_id, node_1.protocol.external_ip, node_1.protocol.udp_port))
# run long enough for the refresh loop to run
await advance(3600)
# verify all the nodes know about each other
for n in nodes.values():
if n is node_1:
continue
peers = n.protocol.routing_table.get_peers()
self.assertEqual(8, len(peers))
self.assertSetEqual(
{
n_id[0]
for n_id in peer_addresses
if n_id[0] != n.protocol.node_id
}, {c.node_id
for c in peers})
self.assertSetEqual(
{
n_addr[1]
for n_addr in peer_addresses
if n_addr[1] != n.protocol.external_ip
}, {c.address
for c in peers})
# teardown
for n in nodes.values():
n.stop()
async def test_losing_connection(self):
async def wait_for(check_ok, insist, timeout=20):
start = time.time()
while time.time() - start < timeout:
if check_ok():
break
await asyncio.sleep(0)
else:
insist()
loop = self.loop
loop.set_debug(False)
peer_addresses = [('1.2.3.4', 40000 + i) for i in range(10)]
node_ids = [constants.generate_id(i) for i in range(10)]
nodes = [
Node(loop,
PeerManager(loop),
node_id,
udp_port,
udp_port,
3333,
address,
storage=SQLiteStorage(Config(), ":memory:", self.loop,
self.loop.time))
for node_id, (address, udp_port) in zip(node_ids, peer_addresses)
]
dht_network = {
peer_addresses[i]: node.protocol
for i, node in enumerate(nodes)
}
num_seeds = 3
with dht_mocks.mock_network_loop(loop, dht_network):
for i, n in enumerate(nodes):
await n._storage.open()
self.addCleanup(n.stop)
n.start(peer_addresses[i][0], peer_addresses[:num_seeds])
await asyncio.gather(*[n.joined.wait() for n in nodes])
node = nodes[-1]
advance = dht_mocks.get_time_accelerator(loop)
await advance(500)
# Join the network, assert that at least the known peers are in RT
self.assertTrue(node.joined.is_set())
self.assertTrue(
len(node.protocol.routing_table.get_peers()) >= num_seeds)
# Refresh, so that the peers are persisted
self.assertFalse(
len(await node._storage.get_persisted_kademlia_peers()) >
num_seeds)
await advance(4000)
self.assertTrue(
len(await node._storage.get_persisted_kademlia_peers()) >
num_seeds)
# We lost internet connection - all the peers stop responding
dht_network.pop(
(node.protocol.external_ip, node.protocol.udp_port))
# The peers are cleared on refresh from RT and storage
await advance(4000)
self.assertListEqual([], await
node._storage.get_persisted_kademlia_peers())
await wait_for(
lambda: len(node.protocol.routing_table.get_peers()) == 0,
lambda: self.assertListEqual(
node.protocol.routing_table.get_peers(), []))
# Reconnect
dht_network[(node.protocol.external_ip,
node.protocol.udp_port)] = node.protocol
# Check that node reconnects at least to them
await advance(1000)
await wait_for(
lambda: len(node.protocol.routing_table.get_peers()) >=
num_seeds, lambda: self.assertGreaterEqual(
len(node.protocol.routing_table.get_peers()), num_seeds))
class TestBlobAnnouncer(AsyncioTestCase):
async def setup_node(self, peer_addresses, address, node_id):
self.nodes: typing.Dict[int, Node] = {}
self.advance = dht_mocks.get_time_accelerator(self.loop,
self.loop.time())
self.conf = Config()
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop,
self.loop.time)
await self.storage.open()
self.peer_manager = PeerManager(self.loop)
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444,
3333, address)
await self.node.start_listening(address)
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
for node_id, address in peer_addresses:
await self.add_peer(node_id, address)
self.node.joined.set()
self.node._refresh_task = self.loop.create_task(
self.node.refresh_node())
async def add_peer(self, node_id, address, add_to_routing_table=True):
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333,
address)
await n.start_listening(address)
self.nodes.update({len(self.nodes): n})
if add_to_routing_table:
self.node.protocol.add_peer(
self.peer_manager.get_kademlia_peer(n.protocol.node_id,
n.protocol.external_ip,
n.protocol.udp_port))
@contextlib.asynccontextmanager
async def _test_network_context(self, peer_addresses=None):
self.peer_addresses = peer_addresses or [
(constants.generate_id(2), '1.2.3.2'),
(constants.generate_id(3), '1.2.3.3'),
(constants.generate_id(4), '1.2.3.4'),
(constants.generate_id(5), '1.2.3.5'),
(constants.generate_id(6), '1.2.3.6'),
(constants.generate_id(7), '1.2.3.7'),
(constants.generate_id(8), '1.2.3.8'),
(constants.generate_id(9), '1.2.3.9'),
]
try:
with dht_mocks.mock_network_loop(self.loop):
await self.setup_node(self.peer_addresses, '1.2.3.1',
constants.generate_id(1))
yield
finally:
self.blob_announcer.stop()
self.node.stop()
for n in self.nodes.values():
n.stop()
async def chain_peer(self, node_id, address):
previous_last_node = self.nodes[len(self.nodes) - 1]
await self.add_peer(node_id, address, False)
last_node = self.nodes[len(self.nodes) - 1]
peer = last_node.protocol.get_rpc_peer(
last_node.protocol.peer_manager.get_kademlia_peer(
previous_last_node.protocol.node_id,
previous_last_node.protocol.external_ip,
previous_last_node.protocol.udp_port))
await peer.ping()
return peer
async def test_announce_blobs(self):
blob1 = binascii.hexlify(b'1' * 48).decode()
blob2 = binascii.hexlify(b'2' * 48).decode()
async with self._test_network_context():
await self.storage.add_blobs((blob1, 1024), (blob2, 1024),
finished=True)
await self.storage.db.execute(
"update blob set next_announce_time=0, should_announce=1 where blob_hash in (?, ?)",
(blob1, blob2))
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(2, len(to_announce))
self.blob_announcer.start(
batch_size=1) # so it covers batching logic
# takes 60 seconds to start, but we advance 120 to ensure it processed all batches
await self.advance(60.0 * 2)
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(0, len(to_announce))
self.blob_announcer.stop()
# test that we can route from a poorly connected peer all the way to the announced blob
await self.chain_peer(constants.generate_id(10), '1.2.3.10')
await self.chain_peer(constants.generate_id(11), '1.2.3.11')
await self.chain_peer(constants.generate_id(12), '1.2.3.12')
await self.chain_peer(constants.generate_id(13), '1.2.3.13')
await self.chain_peer(constants.generate_id(14), '1.2.3.14')
await self.advance(61.0)
last = self.nodes[len(self.nodes) - 1]
search_q, peer_q = asyncio.Queue(loop=self.loop), asyncio.Queue(
loop=self.loop)
search_q.put_nowait(blob1)
_, task = last.accumulate_peers(search_q, peer_q)
found_peers = await peer_q.get()
task.cancel()
self.assertEqual(1, len(found_peers))
self.assertEqual(self.node.protocol.node_id,
found_peers[0].node_id)
self.assertEqual(self.node.protocol.external_ip,
found_peers[0].address)
self.assertEqual(self.node.protocol.peer_port,
found_peers[0].tcp_port)
async def test_popular_blob(self):
peer_count = 150
addresses = [
(constants.generate_id(i + 1),
socket.inet_ntoa(int(i + 1).to_bytes(length=4, byteorder='big')))
for i in range(peer_count)
]
blob_hash = b'1' * 48
async with self._test_network_context(peer_addresses=addresses):
total_seen = set()
announced_to = self.nodes[0]
for i in range(1, peer_count):
node = self.nodes[i]
kad_peer = announced_to.protocol.peer_manager.get_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip,
node.protocol.udp_port)
await announced_to.protocol._add_peer(kad_peer)
peer = node.protocol.get_rpc_peer(
node.protocol.peer_manager.get_kademlia_peer(
announced_to.protocol.node_id,
announced_to.protocol.external_ip,
announced_to.protocol.udp_port))
response = await peer.store(blob_hash)
self.assertEqual(response, b'OK')
peers_for_blob = await peer.find_value(blob_hash, 0)
if i == 1:
self.assertTrue(blob_hash not in peers_for_blob)
self.assertEqual(peers_for_blob[b'p'], 0)
else:
self.assertEqual(len(peers_for_blob[blob_hash]),
min(i - 1, constants.k))
self.assertEqual(
len(
announced_to.protocol.data_store.
get_peers_for_blob(blob_hash)), i)
if i - 1 > constants.k:
self.assertEqual(len(peers_for_blob[b'contacts']),
constants.k)
self.assertEqual(peers_for_blob[b'p'],
((i - 1) // (constants.k + 1)) + 1)
seen = set(peers_for_blob[blob_hash])
self.assertEqual(len(seen), constants.k)
self.assertEqual(len(peers_for_blob[blob_hash]), len(seen))
for pg in range(1, peers_for_blob[b'p']):
page_x = await peer.find_value(blob_hash, pg)
self.assertNotIn(b'contacts', page_x)
page_x_set = set(page_x[blob_hash])
self.assertEqual(len(page_x[blob_hash]),
len(page_x_set))
self.assertTrue(len(page_x_set) > 0)
self.assertSetEqual(seen.intersection(page_x_set),
set())
seen.intersection_update(page_x_set)
total_seen.update(page_x_set)
else:
self.assertEqual(len(peers_for_blob[b'contacts']), i - 1)
self.assertEqual(len(total_seen), peer_count - 2)
import asyncio
from aiohttp import web
from lbry.blob_exchange.serialization import BlobRequest, BlobResponse
from lbry.dht.constants import generate_id
from lbry.dht.node import Node
from lbry.dht.peer import make_kademlia_peer, PeerManager
from lbry.extras.daemon.storage import SQLiteStorage
loop = asyncio.get_event_loop()
NODE = Node(loop,
PeerManager(loop),
generate_id(),
60600,
60600,
3333,
None,
storage=SQLiteStorage(None, ":memory:", loop, loop.time))
async def check_p2p(ip, port):
writer = None
try:
reader, writer = await asyncio.open_connection(ip, port)
writer.write(
BlobRequest.make_request_for_blob_hash('0' * 96).serialize())
return BlobResponse.deserialize(
await
reader.readuntil(b'}')).get_address_response().lbrycrd_address
except OSError:
return None
class TestBlobAnnouncer(AsyncioTestCase):
async def setup_node(self, peer_addresses, address, node_id):
self.nodes: typing.Dict[int, Node] = {}
self.advance = dht_mocks.get_time_accelerator(self.loop)
self.instant_advance = dht_mocks.get_time_accelerator(self.loop)
self.conf = Config()
self.peer_manager = PeerManager(self.loop)
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address)
await self.node.start_listening(address)
await asyncio.gather(*[self.add_peer(node_id, address) for node_id, address in peer_addresses])
for first_peer in self.nodes.values():
for second_peer in self.nodes.values():
if first_peer == second_peer:
continue
self.add_peer_to_routing_table(first_peer, second_peer)
self.add_peer_to_routing_table(second_peer, first_peer)
await self.advance(0.1) # just to make pings go through
self.node.joined.set()
self.node._refresh_task = self.loop.create_task(self.node.refresh_node())
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
await self.storage.open()
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
async def add_peer(self, node_id, address, add_to_routing_table=True):
#print('add', node_id.hex()[:8], address)
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
await n.start_listening(address)
self.nodes.update({len(self.nodes): n})
if add_to_routing_table:
self.add_peer_to_routing_table(self.node, n)
def add_peer_to_routing_table(self, adder, being_added):
adder.protocol.add_peer(
make_kademlia_peer(
being_added.protocol.node_id, being_added.protocol.external_ip, being_added.protocol.udp_port
)
)
@contextlib.asynccontextmanager
async def _test_network_context(self, peer_count=200):
self.peer_addresses = [
(constants.generate_id(i), socket.inet_ntoa(int(i + 0x01000001).to_bytes(length=4, byteorder='big')))
for i in range(1, peer_count + 1)
]
try:
with dht_mocks.mock_network_loop(self.loop):
await self.setup_node(self.peer_addresses, '1.2.3.1', constants.generate_id(1000))
yield
finally:
self.blob_announcer.stop()
self.node.stop()
for n in self.nodes.values():
n.stop()
async def chain_peer(self, node_id, address):
previous_last_node = self.nodes[len(self.nodes) - 1]
await self.add_peer(node_id, address, False)
last_node = self.nodes[len(self.nodes) - 1]
peer = last_node.protocol.get_rpc_peer(
make_kademlia_peer(
previous_last_node.protocol.node_id, previous_last_node.protocol.external_ip,
previous_last_node.protocol.udp_port
)
)
await peer.ping()
return last_node
async def test_announce_blobs(self):
blob1 = binascii.hexlify(b'1' * 48).decode()
blob2 = binascii.hexlify(b'2' * 48).decode()
async with self._test_network_context(peer_count=100):
await self.storage.add_blobs((blob1, 1024, 0, True), (blob2, 1024, 0, True), finished=True)
await self.storage.add_blobs(
*((constants.generate_id(value).hex(), 1024, 0, True) for value in range(1000, 1090)),
finished=True)
await self.storage.db.execute("update blob set next_announce_time=0, should_announce=1")
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(92, len(to_announce))
self.blob_announcer.start(batch_size=10) # so it covers batching logic
# takes 60 seconds to start, but we advance 120 to ensure it processed all batches
ongoing_announcements = asyncio.ensure_future(self.blob_announcer.wait())
await self.instant_advance(60.0)
await ongoing_announcements
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(0, len(to_announce))
self.blob_announcer.stop()
# as routing table pollution will cause some peers to be hard to reach, we add a tolerance for CI
tolerance = 0.8 # at least 80% of the announcements are within the top K
for blob in await self.storage.get_all_blob_hashes():
distance = Distance(bytes.fromhex(blob))
candidates = list(self.nodes.values())
candidates.sort(key=lambda sorting_node: distance(sorting_node.protocol.node_id))
has_it = 0
for index, node in enumerate(candidates[:constants.K], start=1):
if node.protocol.data_store.get_peers_for_blob(bytes.fromhex(blob)):
has_it += 1
else:
logging.warning("blob %s wasnt found between the best K (%s)", blob[:8], node.protocol.node_id.hex()[:8])
self.assertGreaterEqual(has_it, int(tolerance * constants.K))
# test that we can route from a poorly connected peer all the way to the announced blob
current = len(self.nodes)
await self.chain_peer(constants.generate_id(current + 1), '1.2.3.10')
await self.chain_peer(constants.generate_id(current + 2), '1.2.3.11')
await self.chain_peer(constants.generate_id(current + 3), '1.2.3.12')
await self.chain_peer(constants.generate_id(current + 4), '1.2.3.13')
last = await self.chain_peer(constants.generate_id(current + 5), '1.2.3.14')
search_q, peer_q = asyncio.Queue(loop=self.loop), asyncio.Queue(loop=self.loop)
search_q.put_nowait(blob1)
_, task = last.accumulate_peers(search_q, peer_q)
found_peers = await asyncio.wait_for(peer_q.get(), 1.0)
task.cancel()
self.assertEqual(1, len(found_peers))
self.assertEqual(self.node.protocol.node_id, found_peers[0].node_id)
self.assertEqual(self.node.protocol.external_ip, found_peers[0].address)
self.assertEqual(self.node.protocol.peer_port, found_peers[0].tcp_port)
async def test_popular_blob(self):
peer_count = 150
blob_hash = constants.generate_id(99999)
async with self._test_network_context(peer_count=peer_count):
total_seen = set()
announced_to = self.nodes.pop(0)
for i, node in enumerate(self.nodes.values()):
self.add_peer_to_routing_table(announced_to, node)
peer = node.protocol.get_rpc_peer(
make_kademlia_peer(
announced_to.protocol.node_id,
announced_to.protocol.external_ip,
announced_to.protocol.udp_port
)
)
response = await peer.store(blob_hash)
self.assertEqual(response, b'OK')
peers_for_blob = await peer.find_value(blob_hash, 0)
if i == 0:
self.assertNotIn(blob_hash, peers_for_blob)
self.assertEqual(peers_for_blob[b'p'], 0)
else:
self.assertEqual(len(peers_for_blob[blob_hash]), min(i, constants.K))
self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i + 1)
if i - 1 > constants.K:
self.assertEqual(len(peers_for_blob[b'contacts']), constants.K)
self.assertEqual(peers_for_blob[b'p'], (i // (constants.K + 1)) + 1)
seen = set(peers_for_blob[blob_hash])
self.assertEqual(len(seen), constants.K)
self.assertEqual(len(peers_for_blob[blob_hash]), len(seen))
for pg in range(1, peers_for_blob[b'p']):
page_x = await peer.find_value(blob_hash, pg)
self.assertNotIn(b'contacts', page_x)
page_x_set = set(page_x[blob_hash])
self.assertEqual(len(page_x[blob_hash]), len(page_x_set))
self.assertGreater(len(page_x_set), 0)
self.assertSetEqual(seen.intersection(page_x_set), set())
seen.intersection_update(page_x_set)
total_seen.update(page_x_set)
else:
self.assertEqual(len(peers_for_blob[b'contacts']), 8) # we always add 8 on first page
self.assertEqual(len(total_seen), peer_count - 2)
