async def test_update_token(self):
loop = asyncio.get_event_loop()
with dht_mocks.mock_network_loop(loop):
node_id1 = constants.generate_id()
peer1 = KademliaProtocol(loop, PeerManager(loop), node_id1,
'1.2.3.4', 4444, 3333)
peer2 = KademliaProtocol(loop, PeerManager(loop),
constants.generate_id(), '1.2.3.5', 4444,
3333)
await loop.create_datagram_endpoint(lambda: peer1,
('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2,
('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1,
'1.2.3.4',
udp_port=4444)
self.assertEqual(None,
peer2.peer_manager.get_node_token(peer.node_id))
await peer2.get_rpc_peer(peer).find_value(b'1' * 48)
self.assertNotEqual(
None, peer2.peer_manager.get_node_token(peer.node_id))
peer1.stop()
peer2.stop()
peer1.disconnect()
peer2.disconnect()
class PeerTest(AsyncioTestCase):
def setUp(self):
self.loop = asyncio.get_event_loop()
self.peer_manager = PeerManager(self.loop)
self.node_ids = [generate_id(), generate_id(), generate_id()]
self.first_contact = self.peer_manager.get_kademlia_peer(
self.node_ids[1], '127.0.0.1', udp_port=1000)
self.second_contact = self.peer_manager.get_kademlia_peer(
self.node_ids[0], '192.168.0.1', udp_port=1000)
def test_make_contact_error_cases(self):
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer,
self.node_ids[1], '192.168.1.20', 100000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer,
self.node_ids[1], '192.168.1.20.1', 1000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer,
self.node_ids[1], 'this is not an ip', 1000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer,
self.node_ids[1], '192.168.1.20', -1000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer,
b'not valid node id', '192.168.1.20', 1000)
def test_boolean(self):
self.assertNotEqual(self.first_contact, self.second_contact)
self.assertEqual(
self.second_contact,
self.peer_manager.get_kademlia_peer(self.node_ids[0],
'192.168.0.1',
udp_port=1000))
def test_compact_ip(self):
self.assertEqual(self.first_contact.compact_ip(), b'\x7f\x00\x00\x01')
self.assertEqual(self.second_contact.compact_ip(), b'\xc0\xa8\x00\x01')
def setUp(self):
self.loop = asyncio.get_event_loop()
self.peer_manager = PeerManager(self.loop)
self.node_ids = [generate_id(), generate_id(), generate_id()]
self.first_contact = self.peer_manager.get_kademlia_peer(
self.node_ids[1], '127.0.0.1', udp_port=1000)
self.second_contact = self.peer_manager.get_kademlia_peer(
self.node_ids[0], '192.168.0.1', udp_port=1000)
async def setup_node(self, peer_addresses, address, node_id):
self.nodes: typing.Dict[int, Node] = {}
self.advance = dht_mocks.get_time_accelerator(self.loop, self.loop.time())
self.conf = Config()
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
await self.storage.open()
self.peer_manager = PeerManager(self.loop)
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address)
await self.node.start_listening(address)
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
for node_id, address in peer_addresses:
await self.add_peer(node_id, address)
self.node.joined.set()
async def test_store_to_peer(self):
loop = asyncio.get_event_loop()
with dht_mocks.mock_network_loop(loop):
node_id1 = constants.generate_id()
peer1 = KademliaProtocol(loop, PeerManager(loop), node_id1,
'1.2.3.4', 4444, 3333)
peer2 = KademliaProtocol(loop, PeerManager(loop),
constants.generate_id(), '1.2.3.5', 4444,
3333)
await loop.create_datagram_endpoint(lambda: peer1,
('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2,
('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1,
'1.2.3.4',
udp_port=4444)
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port)
peer2_from_peer1.update_tcp_port(3333)
peer3 = peer1.peer_manager.get_kademlia_peer(
constants.generate_id(), '1.2.3.6', udp_port=4444)
store_result = await peer2.store_to_peer(b'2' * 48, peer)
self.assertEqual(store_result[0], peer.node_id)
self.assertEqual(True, store_result[1])
self.assertEqual(True,
peer1.data_store.has_peers_for_blob(b'2' * 48))
self.assertEqual(False,
peer1.data_store.has_peers_for_blob(b'3' * 48))
self.assertListEqual([peer2_from_peer1],
peer1.data_store.get_storing_contacts())
peer1.data_store.completed_blobs.add(
binascii.hexlify(b'2' * 48).decode())
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
self.assertEqual(len(find_value_response[b'contacts']), 0)
self.assertSetEqual(
{b'2' * 48, b'token', b'protocolVersion', b'contacts'},
set(find_value_response.keys()))
self.assertEqual(2, len(find_value_response[b'2' * 48]))
self.assertEqual(find_value_response[b'2' * 48][0],
peer2_from_peer1.compact_address_tcp())
self.assertDictEqual(bdecode(bencode(find_value_response)),
find_value_response)
peer1.stop()
peer2.stop()
peer1.disconnect()
peer2.disconnect()
async def _make_protocol(self, other_peer, node_id, address, udp_port, tcp_port):
proto = KademliaProtocol(
self.loop, PeerManager(self.loop), node_id, address, udp_port, tcp_port
)
await self.loop.create_datagram_endpoint(lambda: proto, (address, 4444))
proto.start()
return proto, other_peer.peer_manager.get_kademlia_peer(node_id, address, udp_port=udp_port)
async def test_split_buckets(self):
loop = asyncio.get_event_loop()
peer_addresses = [
(constants.generate_id(1), '1.2.3.1'),
]
for i in range(2, 200):
peer_addresses.append((constants.generate_id(i), f'1.2.3.{i}'))
with dht_mocks.mock_network_loop(loop):
nodes = {
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address)
for i, (node_id, address) in enumerate(peer_addresses)
}
node_1 = nodes[0]
for i in range(1, len(peer_addresses)):
node = nodes[i]
peer = node_1.protocol.peer_manager.get_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip,
udp_port=node.protocol.udp_port
)
# set all of the peers to good (as to not attempt pinging stale ones during split)
node_1.protocol.peer_manager.report_last_replied(peer.address, peer.udp_port)
node_1.protocol.peer_manager.report_last_replied(peer.address, peer.udp_port)
await node_1.protocol._add_peer(peer)
# check that bucket 0 is always the one covering the local node id
self.assertEqual(True, node_1.protocol.routing_table.buckets[0].key_in_range(node_1.protocol.node_id))
self.assertEqual(40, len(node_1.protocol.routing_table.get_peers()))
self.assertEqual(len(expected_ranges), len(node_1.protocol.routing_table.buckets))
covered = 0
for (expected_min, expected_max), bucket in zip(expected_ranges, node_1.protocol.routing_table.buckets):
self.assertEqual(expected_min, bucket.range_min)
self.assertEqual(expected_max, bucket.range_max)
covered += bucket.range_max - bucket.range_min
self.assertEqual(2**384, covered)
for node in nodes.values():
node.stop()
def __init__(self,
conf: Config,
analytics_manager=None,
skip_components=None,
peer_manager=None,
**override_components):
self.conf = conf
self.skip_components = skip_components or []
self.loop = asyncio.get_event_loop()
self.analytics_manager = analytics_manager
self.component_classes = {}
self.components = set()
self.started = asyncio.Event(loop=self.loop)
self.peer_manager = peer_manager or PeerManager(
asyncio.get_event_loop_policy().get_event_loop())
for component_name, component_class in self.default_component_classes.items(
):
if component_name in override_components:
component_class = override_components.pop(component_name)
if component_name not in self.skip_components:
self.component_classes[component_name] = component_class
if override_components:
raise SyntaxError("unexpected components: %s" %
override_components)
for component_class in self.component_classes.values():
self.components.add(component_class(self))
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
self.client_dir = tempfile.mkdtemp()
self.server_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.client_dir)
self.addCleanup(shutil.rmtree, self.server_dir)
self.server_storage = SQLiteStorage(
Config(), os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir,
self.server_storage)
self.server = BlobServer(self.loop, self.server_blob_manager,
'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_storage = SQLiteStorage(
Config(), os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir,
self.client_storage)
self.client_peer_manager = PeerManager(self.loop)
self.server_from_client = KademliaPeer(self.loop,
"127.0.0.1",
b'1' * 48,
tcp_port=33333)
await self.client_storage.open()
await self.server_storage.open()
await self.client_blob_manager.setup()
await self.server_blob_manager.setup()
self.server.start_server(33333, '127.0.0.1')
await self.server.started_listening.wait()
async def test_fill_one_bucket(self):
loop = asyncio.get_event_loop()
peer_addresses = [
(constants.generate_id(1), '1.2.3.1'),
(constants.generate_id(2), '1.2.3.2'),
(constants.generate_id(3), '1.2.3.3'),
(constants.generate_id(4), '1.2.3.4'),
(constants.generate_id(5), '1.2.3.5'),
(constants.generate_id(6), '1.2.3.6'),
(constants.generate_id(7), '1.2.3.7'),
(constants.generate_id(8), '1.2.3.8'),
(constants.generate_id(9), '1.2.3.9'),
]
with dht_mocks.mock_network_loop(loop):
nodes = {
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address)
for i, (node_id, address) in enumerate(peer_addresses)
}
node_1 = nodes[0]
contact_cnt = 0
for i in range(1, len(peer_addresses)):
self.assertEqual(len(node_1.protocol.routing_table.get_peers()), contact_cnt)
node = nodes[i]
peer = node_1.protocol.peer_manager.get_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip,
udp_port=node.protocol.udp_port
)
added = await node_1.protocol._add_peer(peer)
self.assertEqual(True, added)
contact_cnt += 1
self.assertEqual(len(node_1.protocol.routing_table.get_peers()), 8)
self.assertEqual(node_1.protocol.routing_table.buckets_with_contacts(), 1)
for node in nodes.values():
node.protocol.stop()
async def test_add_peer_after_handle_request(self):
with dht_mocks.mock_network_loop(self.loop):
node_id1 = constants.generate_id()
node_id2 = constants.generate_id()
node_id3 = constants.generate_id()
node_id4 = constants.generate_id()
peer1 = KademliaProtocol(
self.loop, PeerManager(self.loop), node_id1, '1.2.3.4', 4444, 3333
)
await self.loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
peer1.start()
peer2, peer_2_from_peer_1 = await self._make_protocol(peer1, node_id2, '1.2.3.5', 4444, 3333)
peer3, peer_3_from_peer_1 = await self._make_protocol(peer1, node_id3, '1.2.3.6', 4444, 3333)
peer4, peer_4_from_peer_1 = await self._make_protocol(peer1, node_id4, '1.2.3.7', 4444, 3333)
# peers who reply should be added
await peer1.get_rpc_peer(peer_2_from_peer_1).ping()
await asyncio.sleep(0.5)
self.assertListEqual([peer_2_from_peer_1], peer1.routing_table.get_peers())
peer1.routing_table.remove_peer(peer_2_from_peer_1)
# peers not known by be good/bad should be enqueued to maybe-ping
peer1_from_peer3 = peer3.get_rpc_peer(peer3.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
pong = await peer1_from_peer3.ping()
self.assertEqual(b'pong', pong)
self.assertEqual(1, len(peer1.ping_queue._pending_contacts))
peer1.ping_queue._pending_contacts.clear()
# peers who are already good should be added
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
peer1.peer_manager.report_last_replied('1.2.3.7', 4444)
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
pong = await peer1_from_peer4.ping()
self.assertEqual(b'pong', pong)
await asyncio.sleep(0.5)
self.assertEqual(1, len(peer1.routing_table.get_peers()))
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
peer1.routing_table.buckets[0].peers.clear()
# peers who are known to be bad recently should not be added or maybe-pinged
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
peer1.peer_manager.report_failure('1.2.3.7', 4444)
peer1.peer_manager.report_failure('1.2.3.7', 4444)
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
pong = await peer1_from_peer4.ping()
self.assertEqual(b'pong', pong)
self.assertEqual(0, len(peer1.routing_table.get_peers()))
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
for p in [peer1, peer2, peer3, peer4]:
p.stop()
p.disconnect()
async def add_peer(self, node_id, address, add_to_routing_table=True):
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
await n.start_listening(address)
self.nodes.update({len(self.nodes): n})
if add_to_routing_table:
await self.node.protocol.add_peer(
self.peer_manager.get_kademlia_peer(
n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port
)
)
async def test_ping(self):
loop = asyncio.get_event_loop()
with dht_mocks.mock_network_loop(loop):
node_id1 = constants.generate_id()
peer1 = KademliaProtocol(
loop, PeerManager(loop), node_id1, '1.2.3.4', 4444, 3333
)
peer2 = KademliaProtocol(
loop, PeerManager(loop), constants.generate_id(), '1.2.3.5', 4444, 3333
)
await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
result = await peer2.get_rpc_peer(peer).ping()
self.assertEqual(result, b'pong')
peer1.stop()
peer2.stop()
peer1.disconnect()
peer2.disconnect()
async def setup_network(self, size: int, start_port=40000, seed_nodes=1):
for i in range(size):
node_port = start_port + i
node = Node(self.loop, PeerManager(self.loop), node_id=constants.generate_id(i),
udp_port=node_port, internal_udp_port=node_port,
peer_port=3333, external_ip='127.0.0.1')
self.nodes.append(node)
self.known_node_addresses.append(('127.0.0.1', node_port))
await node.start_listening('127.0.0.1')
self.addCleanup(node.stop)
for node in self.nodes:
node.protocol.rpc_timeout = .5
node.protocol.ping_queue._default_delay = .5
node.start('127.0.0.1', self.known_node_addresses[:seed_nodes])
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
class DataStoreTests(TestCase):
def setUp(self):
self.loop = mock.Mock(spec=asyncio.BaseEventLoop)
self.loop.time = lambda: 0.0
self.peer_manager = PeerManager(self.loop)
self.data_store = DictDataStore(self.loop, self.peer_manager)
def _test_add_peer_to_blob(self,
blob=b'2' * 48,
node_id=b'1' * 48,
address='1.2.3.4',
tcp_port=3333,
udp_port=4444):
peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
peer.update_tcp_port(tcp_port)
before = self.data_store.get_peers_for_blob(blob)
self.data_store.add_peer_to_blob(peer, blob)
self.assertListEqual(before + [peer],
self.data_store.get_peers_for_blob(blob))
return peer
def test_refresh_peer_to_blob(self):
blob = b'f' * 48
self.assertListEqual([], self.data_store.get_peers_for_blob(blob))
peer = self._test_add_peer_to_blob(blob=blob,
node_id=b'a' * 48,
address='1.2.3.4')
self.assertTrue(self.data_store.has_peers_for_blob(blob))
self.assertEqual(len(self.data_store.get_peers_for_blob(blob)), 1)
self.assertEqual(self.data_store._data_store[blob][0][1], 0)
self.loop.time = lambda: 100.0
self.assertEqual(self.data_store._data_store[blob][0][1], 0)
self.data_store.add_peer_to_blob(peer, blob)
self.assertEqual(self.data_store._data_store[blob][0][1], 100)
def test_add_peer_to_blob(self, blob=b'f' * 48, peers=None):
peers = peers or [
(b'a' * 48, '1.2.3.4'),
(b'b' * 48, '1.2.3.5'),
(b'c' * 48, '1.2.3.6'),
]
self.assertListEqual([], self.data_store.get_peers_for_blob(blob))
peer_objects = []
for (node_id, address) in peers:
peer_objects.append(
self._test_add_peer_to_blob(blob=blob,
node_id=node_id,
address=address))
self.assertTrue(self.data_store.has_peers_for_blob(blob))
self.assertEqual(len(self.data_store.get_peers_for_blob(blob)),
len(peers))
return peer_objects
def test_get_storing_contacts(self,
peers=None,
blob1=b'd' * 48,
blob2=b'e' * 48):
peers = peers or [
(b'a' * 48, '1.2.3.4'),
(b'b' * 48, '1.2.3.5'),
(b'c' * 48, '1.2.3.6'),
]
peer_objs1 = self.test_add_peer_to_blob(blob=blob1, peers=peers)
self.assertEqual(len(peers), len(peer_objs1))
self.assertEqual(len(peers),
len(self.data_store.get_storing_contacts()))
peer_objs2 = self.test_add_peer_to_blob(blob=blob2, peers=peers)
self.assertEqual(len(peers), len(peer_objs2))
self.assertEqual(len(peers),
len(self.data_store.get_storing_contacts()))
for o1, o2 in zip(peer_objs1, peer_objs2):
self.assertIs(o1, o2)
def test_remove_expired_peers(self):
peers = [
(b'a' * 48, '1.2.3.4'),
(b'b' * 48, '1.2.3.5'),
(b'c' * 48, '1.2.3.6'),
]
blob1 = b'd' * 48
blob2 = b'e' * 48
self.data_store.removed_expired_peers() # nothing should happen
self.test_get_storing_contacts(peers, blob1, blob2)
self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)),
len(peers))
self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)),
len(peers))
self.assertEqual(len(self.data_store.get_storing_contacts()),
len(peers))
# expire the first peer from blob1
first = self.data_store._data_store[blob1][0][0]
self.data_store._data_store[blob1][0] = (first, -86401)
self.assertEqual(len(self.data_store.get_storing_contacts()),
len(peers))
self.data_store.removed_expired_peers()
self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)),
len(peers) - 1)
self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)),
len(peers))
self.assertEqual(len(self.data_store.get_storing_contacts()),
len(peers))
# expire the first peer from blob2
first = self.data_store._data_store[blob2][0][0]
self.data_store._data_store[blob2][0] = (first, -86401)
self.data_store.removed_expired_peers()
self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)),
len(peers) - 1)
self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)),
len(peers) - 1)
self.assertEqual(len(self.data_store.get_storing_contacts()),
len(peers) - 1)
# expire the second and third peers from blob1
first = self.data_store._data_store[blob2][0][0]
self.data_store._data_store[blob1][0] = (first, -86401)
second = self.data_store._data_store[blob2][1][0]
self.data_store._data_store[blob1][1] = (second, -86401)
self.data_store.removed_expired_peers()
self.assertEqual(len(self.data_store.get_peers_for_blob(blob1)), 0)
self.assertEqual(len(self.data_store.get_peers_for_blob(blob2)),
len(peers) - 1)
self.assertEqual(len(self.data_store.get_storing_contacts()),
len(peers) - 1)
async def test_ping_queue_discover(self):
loop = asyncio.get_event_loop()
loop.set_debug(False)
peer_addresses = [
(constants.generate_id(1), '1.2.3.1'),
(constants.generate_id(2), '1.2.3.2'),
(constants.generate_id(3), '1.2.3.3'),
(constants.generate_id(4), '1.2.3.4'),
(constants.generate_id(5), '1.2.3.5'),
(constants.generate_id(6), '1.2.3.6'),
(constants.generate_id(7), '1.2.3.7'),
(constants.generate_id(8), '1.2.3.8'),
(constants.generate_id(9), '1.2.3.9'),
]
with dht_mocks.mock_network_loop(loop):
advance = dht_mocks.get_time_accelerator(loop, loop.time())
# start the nodes
nodes: typing.Dict[int, Node] = {
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333,
address)
for i, (node_id, address) in enumerate(peer_addresses)
}
for i, n in nodes.items():
n.start(peer_addresses[i][1], [])
await advance(1)
node_1 = nodes[0]
# ping 8 nodes from node_1, this will result in a delayed return ping
futs = []
for i in range(1, len(peer_addresses)):
node = nodes[i]
assert node.protocol.node_id != node_1.protocol.node_id
peer = node_1.protocol.peer_manager.get_kademlia_peer(
node.protocol.node_id,
node.protocol.external_ip,
udp_port=node.protocol.udp_port)
futs.append(node_1.protocol.get_rpc_peer(peer).ping())
await advance(3)
replies = await asyncio.gather(*tuple(futs))
self.assertTrue(all(map(lambda reply: reply == b"pong", replies)))
# run for long enough for the delayed pings to have been sent by node 1
await advance(1000)
# verify all of the previously pinged peers have node_1 in their routing tables
for n in nodes.values():
peers = n.protocol.routing_table.get_peers()
if n is node_1:
self.assertEqual(8, len(peers))
else:
self.assertEqual(1, len(peers))
self.assertEqual(
(peers[0].node_id, peers[0].address,
peers[0].udp_port),
(node_1.protocol.node_id, node_1.protocol.external_ip,
node_1.protocol.udp_port))
# run long enough for the refresh loop to run
await advance(3600)
# verify all the nodes know about each other
for n in nodes.values():
if n is node_1:
continue
peers = n.protocol.routing_table.get_peers()
self.assertEqual(8, len(peers))
self.assertSetEqual(
{
n_id[0]
for n_id in peer_addresses
if n_id[0] != n.protocol.node_id
}, {c.node_id
for c in peers})
self.assertSetEqual(
{
n_addr[1]
for n_addr in peer_addresses
if n_addr[1] != n.protocol.external_ip
}, {c.address
for c in peers})
# teardown
for n in nodes.values():
n.stop()
class TestBlobAnnouncer(AsyncioTestCase):
async def setup_node(self, peer_addresses, address, node_id):
self.nodes: typing.Dict[int, Node] = {}
self.advance = dht_mocks.get_time_accelerator(self.loop, self.loop.time())
self.conf = Config()
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
await self.storage.open()
self.peer_manager = PeerManager(self.loop)
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address)
await self.node.start_listening(address)
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
for node_id, address in peer_addresses:
await self.add_peer(node_id, address)
self.node.joined.set()
async def add_peer(self, node_id, address, add_to_routing_table=True):
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
await n.start_listening(address)
self.nodes.update({len(self.nodes): n})
if add_to_routing_table:
await self.node.protocol.add_peer(
self.peer_manager.get_kademlia_peer(
n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port
)
)
@contextlib.asynccontextmanager
async def _test_network_context(self, peer_addresses=None):
self.peer_addresses = peer_addresses or [
(constants.generate_id(2), '1.2.3.2'),
(constants.generate_id(3), '1.2.3.3'),
(constants.generate_id(4), '1.2.3.4'),
(constants.generate_id(5), '1.2.3.5'),
(constants.generate_id(6), '1.2.3.6'),
(constants.generate_id(7), '1.2.3.7'),
(constants.generate_id(8), '1.2.3.8'),
(constants.generate_id(9), '1.2.3.9'),
]
try:
with dht_mocks.mock_network_loop(self.loop):
await self.setup_node(self.peer_addresses, '1.2.3.1', constants.generate_id(1))
yield
finally:
self.blob_announcer.stop()
self.node.stop()
for n in self.nodes.values():
n.stop()
async def chain_peer(self, node_id, address):
previous_last_node = self.nodes[len(self.nodes) - 1]
await self.add_peer(node_id, address, False)
last_node = self.nodes[len(self.nodes) - 1]
peer = last_node.protocol.get_rpc_peer(
last_node.protocol.peer_manager.get_kademlia_peer(
previous_last_node.protocol.node_id, previous_last_node.protocol.external_ip,
previous_last_node.protocol.udp_port
)
)
await peer.ping()
return peer
async def test_announce_blobs(self):
blob1 = binascii.hexlify(b'1' * 48).decode()
blob2 = binascii.hexlify(b'2' * 48).decode()
async with self._test_network_context():
await self.storage.add_completed_blob(blob1, 1024)
await self.storage.add_completed_blob(blob2, 1024)
await self.storage.db.execute(
"update blob set next_announce_time=0, should_announce=1 where blob_hash in (?, ?)",
(blob1, blob2)
)
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(2, len(to_announce))
self.blob_announcer.start()
await self.advance(61.0)
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(0, len(to_announce))
self.blob_announcer.stop()
# test that we can route from a poorly connected peer all the way to the announced blob
await self.chain_peer(constants.generate_id(10), '1.2.3.10')
await self.chain_peer(constants.generate_id(11), '1.2.3.11')
await self.chain_peer(constants.generate_id(12), '1.2.3.12')
await self.chain_peer(constants.generate_id(13), '1.2.3.13')
await self.chain_peer(constants.generate_id(14), '1.2.3.14')
last = self.nodes[len(self.nodes) - 1]
search_q, peer_q = asyncio.Queue(loop=self.loop), asyncio.Queue(loop=self.loop)
search_q.put_nowait(blob1)
_, task = last.accumulate_peers(search_q, peer_q)
found_peers = await peer_q.get()
task.cancel()
self.assertEqual(1, len(found_peers))
self.assertEqual(self.node.protocol.node_id, found_peers[0].node_id)
self.assertEqual(self.node.protocol.external_ip, found_peers[0].address)
self.assertEqual(self.node.protocol.peer_port, found_peers[0].tcp_port)
class TestKBucket(AsyncioTestCase):
def setUp(self):
self.loop = asyncio.get_event_loop()
self.address_generator = address_generator()
self.peer_manager = PeerManager(self.loop)
self.kbucket = KBucket(self.peer_manager, 0, 2**constants.hash_bits,
generate_id())
def test_add_peer(self):
# Test if contacts can be added to empty list
# Add k contacts to bucket
for i in range(constants.k):
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertTrue(self.kbucket.add_peer(peer))
self.assertEqual(peer, self.kbucket.peers[i])
# Test if contact is not added to full list
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertFalse(self.kbucket.add_peer(peer))
# Test if an existing contact is updated correctly if added again
existing_peer = self.kbucket.peers[0]
self.assertTrue(self.kbucket.add_peer(existing_peer))
self.assertEqual(existing_peer, self.kbucket.peers[-1])
# def testGetContacts(self):
# # try and get 2 contacts from empty list
# result = self.kbucket.getContacts(2)
# self.assertFalse(len(result) != 0, "Returned list should be empty; returned list length: %d" %
# (len(result)))
#
# # Add k-2 contacts
# node_ids = []
# if constants.k >= 2:
# for i in range(constants.k-2):
# node_ids.append(generate_id())
# tmpContact = self.contact_manager.make_contact(node_ids[-1], next(self.address_generator), 4444, 0,
# None)
# self.kbucket.addContact(tmpContact)
# else:
# # add k contacts
# for i in range(constants.k):
# node_ids.append(generate_id())
# tmpContact = self.contact_manager.make_contact(node_ids[-1], next(self.address_generator), 4444, 0,
# None)
# self.kbucket.addContact(tmpContact)
#
# # try to get too many contacts
# # requested count greater than bucket size; should return at most k contacts
# contacts = self.kbucket.getContacts(constants.k+3)
# self.assertTrue(len(contacts) <= constants.k,
# 'Returned list should not have more than k entries!')
#
# # verify returned contacts in list
# for node_id, i in zip(node_ids, range(constants.k-2)):
# self.assertFalse(self.kbucket._contacts[i].id != node_id,
# "Contact in position %s not same as added contact" % (str(i)))
#
# # try to get too many contacts
# # requested count one greater than number of contacts
# if constants.k >= 2:
# result = self.kbucket.getContacts(constants.k-1)
# self.assertFalse(len(result) != constants.k-2,
# "Too many contacts in returned list %s - should be %s" %
# (len(result), constants.k-2))
# else:
# result = self.kbucket.getContacts(constants.k-1)
# # if the count is <= 0, it should return all of it's contats
# self.assertFalse(len(result) != constants.k,
# "Too many contacts in returned list %s - should be %s" %
# (len(result), constants.k-2))
# result = self.kbucket.getContacts(constants.k-3)
# self.assertFalse(len(result) != constants.k-3,
# "Too many contacts in returned list %s - should be %s" %
# (len(result), constants.k-3))
def test_remove_peer(self):
# try remove contact from empty list
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertRaises(ValueError, self.kbucket.remove_peer, peer)
added = []
# Add couple contacts
for i in range(constants.k - 2):
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertTrue(self.kbucket.add_peer(peer))
added.append(peer)
while added:
peer = added.pop()
self.assertIn(peer, self.kbucket.peers)
self.kbucket.remove_peer(peer)
self.assertNotIn(peer, self.kbucket.peers)
def setUp(self):
self.loop = asyncio.get_event_loop()
self.address_generator = address_generator()
self.peer_manager = PeerManager(self.loop)
self.kbucket = KBucket(self.peer_manager, 0, 2**constants.hash_bits,
generate_id())
def setUp(self):
self.loop = mock.Mock(spec=asyncio.BaseEventLoop)
self.loop.time = lambda: 0.0
self.peer_manager = PeerManager(self.loop)
self.data_store = DictDataStore(self.loop, self.peer_manager)