class NodeRPC(AuthJSONRPCServer): def __init__(self, lbryid, seeds, node_port, rpc_port): AuthJSONRPCServer.__init__(self, False) self.root = None self.port = None self.seeds = seeds self.node_port = node_port self.rpc_port = rpc_port if lbryid: lbryid = lbryid.decode('hex') else: lbryid = generate_id() self.node_id = lbryid self.external_ip = get_external_ip_and_setup_upnp() self.node_port = node_port @defer.inlineCallbacks def setup(self): self.node = Node(node_id=self.node_id, udpPort=self.node_port, externalIP=self.external_ip) hosts = [] for hostname, hostport in self.seeds: host_ip = yield reactor.resolve(hostname) hosts.append((host_ip, hostport)) log.info("connecting to dht") yield self.node.joinNetwork(tuple(hosts)) log.info("connected to dht") if not self.announced_startup: self.announced_startup = True self.start_api() log.info("lbry id: %s (%i bytes)", self.node.node_id.encode('hex'), len(self.node.node_id)) def start_api(self): root = resource.Resource() root.putChild('', self) self.port = reactor.listenTCP(self.rpc_port, Site(root), interface='localhost') log.info("started jsonrpc server") @defer.inlineCallbacks def jsonrpc_node_id_set(self, node_id): old_id = self.node.node_id self.node.stop() del self.node self.node_id = node_id.decode('hex') yield self.setup() msg = "changed dht id from %s to %s" % ( old_id.encode('hex'), self.node.node_id.encode('hex')) defer.returnValue(msg) def jsonrpc_node_id_get(self): return self._render_response(self.node.node_id.encode('hex')) @defer.inlineCallbacks def jsonrpc_peer_find(self, node_id): node_id = node_id.decode('hex') contact = yield self.node.findContact(node_id) result = None if contact: result = (contact.address, contact.port) defer.returnValue(result) @defer.inlineCallbacks def jsonrpc_peer_list_for_blob(self, blob_hash): peers = yield self.node.getPeersForBlob(blob_hash.decode('hex')) defer.returnValue(peers) @defer.inlineCallbacks def jsonrpc_ping(self, node_id): contact_host = yield self.jsonrpc_peer_find(node_id=node_id) if not contact_host: defer.returnValue("failed to find node") contact_ip, contact_port = contact_host contact = Contact(node_id.decode('hex'), contact_ip, contact_port, self.node._protocol) try: result = yield contact.ping() except TimeoutError: self.node.removeContact(contact.id) self.node._dataStore.removePeer(contact.id) result = {'error': 'timeout'} defer.returnValue(result) def get_routing_table(self): result = {} data_store = deepcopy(self.node._dataStore._dict) datastore_len = len(data_store) hosts = {} missing_contacts = [] if datastore_len: for k, v in data_store.iteritems(): for value, lastPublished, originallyPublished, originalPublisherID in v: try: contact = self.node._routingTable.getContact( originalPublisherID) except ValueError: if originalPublisherID.encode( 'hex') not in missing_contacts: missing_contacts.append( originalPublisherID.encode('hex')) continue if contact in hosts: blobs = hosts[contact] else: blobs = [] blobs.append(k.encode('hex')) hosts[contact] = blobs contact_set = [] blob_hashes = [] result['buckets'] = {} for i in range(len(self.node._routingTable._buckets)): for contact in self.node._routingTable._buckets[i]._contacts: contacts = result['buckets'].get(i, []) if contact in hosts: blobs = hosts[contact] del hosts[contact] else: blobs = [] host = { "address": contact.address, "id": contact.id.encode("hex"), "blobs": blobs, } for blob_hash in blobs: if blob_hash not in blob_hashes: blob_hashes.append(blob_hash) contacts.append(host) result['buckets'][i] = contacts contact_set.append(contact.id.encode("hex")) if hosts: result['datastore extra'] = [{ "id": host.id.encode('hex'), "blobs": hosts[host], } for host in hosts] result['missing contacts'] = missing_contacts result['contacts'] = contact_set result['blob hashes'] = blob_hashes result['node id'] = self.node_id.encode('hex') return result def jsonrpc_routing_table_get(self): return self._render_response(self.get_routing_table())
class KademliaProtocolTest(unittest.TestCase): """ Test case for the Protocol class """ udpPort = 9182 def setUp(self): self._reactor = Clock() self.node = Node(node_id=b'1' * 48, udpPort=self.udpPort, externalIP="127.0.0.1", listenUDP=listenUDP, resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) self.remote_node = Node(node_id=b'2' * 48, udpPort=self.udpPort, externalIP="127.0.0.2", listenUDP=listenUDP, resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) self.remote_contact = self.node.contact_manager.make_contact(b'2' * 48, '127.0.0.2', 9182, self.node._protocol) self.us_from_them = self.remote_node.contact_manager.make_contact(b'1' * 48, '127.0.0.1', 9182, self.remote_node._protocol) self.node.start_listening() self.remote_node.start_listening() @defer.inlineCallbacks def tearDown(self): yield self.node.stop() yield self.remote_node.stop() del self._reactor @defer.inlineCallbacks def testReactor(self): """ Tests if the reactor can start/stop the protocol correctly """ d = defer.Deferred() self._reactor.callLater(1, d.callback, True) self._reactor.advance(1) result = yield d self.assertTrue(result) @defer.inlineCallbacks def testRPCTimeout(self): """ Tests if a RPC message sent to a dead remote node times out correctly """ yield self.remote_node.stop() self._reactor.pump([1 for _ in range(10)]) self.node.addContact(self.remote_contact) @rpcmethod def fake_ping(*args, **kwargs): time.sleep(lbrynet.dht.constants.rpcTimeout + 1) return 'pong' real_ping = self.node.ping real_timeout = lbrynet.dht.constants.rpcTimeout real_attempts = lbrynet.dht.constants.rpcAttempts lbrynet.dht.constants.rpcAttempts = 1 lbrynet.dht.constants.rpcTimeout = 1 self.node.ping = fake_ping # Make sure the contact was added self.assertFalse(self.remote_contact not in self.node.contacts, 'Contact not added to fake node (error in test code)') self.node.start_listening() # Run the PING RPC (which should raise a timeout error) df = self.remote_contact.ping() def check_timeout(err): self.assertEqual(err.type, TimeoutError) df.addErrback(check_timeout) def reset_values(): self.node.ping = real_ping lbrynet.dht.constants.rpcTimeout = real_timeout lbrynet.dht.constants.rpcAttempts = real_attempts # See if the contact was removed due to the timeout def check_removed_contact(): self.assertFalse(self.remote_contact in self.node.contacts, 'Contact was not removed after RPC timeout; check exception types.') df.addCallback(lambda _: reset_values()) # Stop the reactor if a result arrives (timeout or not) df.addCallback(lambda _: check_removed_contact()) self._reactor.pump([1 for _ in range(20)]) @defer.inlineCallbacks def testRPCRequest(self): """ Tests if a valid RPC request is executed and responded to correctly """ yield self.node.addContact(self.remote_contact) self.error = None def handleError(f): self.error = 'An RPC error occurred: %s' % f.getErrorMessage() def handleResult(result): expectedResult = b'pong' if result != expectedResult: self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' \ % (expectedResult, result) # Simulate the RPC df = self.remote_contact.ping() df.addCallback(handleResult) df.addErrback(handleError) self._reactor.advance(2) yield df self.assertFalse(self.error, self.error) # The list of sent RPC messages should be empty at this stage self.assertEqual(len(self.node._protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, ' 'but the transaction is already done!') def testRPCAccess(self): """ Tests invalid RPC requests Verifies that a RPC request for an existing but unpublished method is denied, and that the associated (remote) exception gets raised locally """ self.assertRaises(AttributeError, getattr, self.remote_contact, "not_a_rpc_function") def testRPCRequestArgs(self): """ Tests if an RPC requiring arguments is executed correctly """ self.node.addContact(self.remote_contact) self.error = None def handleError(f): self.error = 'An RPC error occurred: %s' % f.getErrorMessage() def handleResult(result): expectedResult = b'pong' if result != expectedResult: self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' % \ (expectedResult, result) # Publish the "local" node on the network self.node.start_listening() # Simulate the RPC df = self.remote_contact.ping() df.addCallback(handleResult) df.addErrback(handleError) self._reactor.pump([1 for _ in range(10)]) self.assertFalse(self.error, self.error) # The list of sent RPC messages should be empty at this stage self.assertEqual(len(self.node._protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, ' 'but the transaction is already done!') @defer.inlineCallbacks def testDetectProtocolVersion(self): original_findvalue = self.remote_node.findValue fake_blob = unhexlify("AB" * 48) @rpcmethod def findValue(contact, key): result = original_findvalue(contact, key) result.pop(b'protocolVersion') return result self.remote_node.findValue = findValue d = self.remote_contact.findValue(fake_blob) self._reactor.advance(3) find_value_response = yield d self.assertEqual(self.remote_contact.protocolVersion, 0) self.assertNotIn('protocolVersion', find_value_response) self.remote_node.findValue = original_findvalue d = self.remote_contact.findValue(fake_blob) self._reactor.advance(3) find_value_response = yield d self.assertEqual(self.remote_contact.protocolVersion, 1) self.assertNotIn('protocolVersion', find_value_response) self.remote_node.findValue = findValue d = self.remote_contact.findValue(fake_blob) self._reactor.advance(3) find_value_response = yield d self.assertEqual(self.remote_contact.protocolVersion, 0) self.assertNotIn('protocolVersion', find_value_response) @defer.inlineCallbacks def testStoreToPre_0_20_0_Node(self): def _dont_migrate(contact, method, *args): return args, {} self.remote_node._protocol._migrate_incoming_rpc_args = _dont_migrate original_findvalue = self.remote_node.findValue original_store = self.remote_node.store @rpcmethod def findValue(contact, key): result = original_findvalue(contact, key) if b'protocolVersion' in result: result.pop(b'protocolVersion') return result @rpcmethod def store(contact, key, value, originalPublisherID=None, self_store=False, **kwargs): self.assertEqual(len(key), 48) self.assertSetEqual(set(value.keys()), {b'token', b'lbryid', b'port'}) self.assertFalse(self_store) self.assertDictEqual(kwargs, {}) return original_store( # pylint: disable=too-many-function-args contact, key, value[b'token'], value[b'port'], originalPublisherID, 0 ) self.remote_node.findValue = findValue self.remote_node.store = store fake_blob = unhexlify("AB" * 48) d = self.remote_contact.findValue(fake_blob) self._reactor.advance(3) find_value_response = yield d self.assertEqual(self.remote_contact.protocolVersion, 0) self.assertNotIn(b'protocolVersion', find_value_response) token = find_value_response[b'token'] d = self.remote_contact.store(fake_blob, token, 3333, self.node.node_id, 0) self._reactor.advance(3) response = yield d self.assertEqual(response, b'OK') self.assertEqual(self.remote_contact.protocolVersion, 0) self.assertTrue(self.remote_node._dataStore.hasPeersForBlob(fake_blob)) self.assertEqual(len(self.remote_node._dataStore.getStoringContacts()), 1) @defer.inlineCallbacks def testStoreFromPre_0_20_0_Node(self): def _dont_migrate(contact, method, *args): return args self.remote_node._protocol._migrate_outgoing_rpc_args = _dont_migrate us_from_them = self.remote_node.contact_manager.make_contact(b'1' * 48, '127.0.0.1', self.udpPort, self.remote_node._protocol) fake_blob = unhexlify("AB" * 48) d = us_from_them.findValue(fake_blob) self._reactor.advance(3) find_value_response = yield d self.assertEqual(self.remote_contact.protocolVersion, 0) self.assertNotIn(b'protocolVersion', find_value_response) token = find_value_response[b'token'] us_from_them.update_protocol_version(0) d = self.remote_node._protocol.sendRPC( us_from_them, b"store", (fake_blob, {b'lbryid': self.remote_node.node_id, b'token': token, b'port': 3333}) ) self._reactor.advance(3) response = yield d self.assertEqual(response, b'OK') self.assertEqual(self.remote_contact.protocolVersion, 0) self.assertTrue(self.node._dataStore.hasPeersForBlob(fake_blob)) self.assertEqual(len(self.node._dataStore.getStoringContacts()), 1) self.assertIs(self.node._dataStore.getStoringContacts()[0], self.remote_contact) @defer.inlineCallbacks def test_find_node(self): self.node.addContact(self.node.contact_manager.make_contact( self.remote_contact.id, self.remote_contact.address, self.remote_contact.port, self.node._protocol) ) result = self.node.findContact(b'0'*48) for _ in range(6): self._reactor.advance(1) self.assertIsNone((yield result)) result = self.node.findContact(self.remote_contact.id) for _ in range(6): self._reactor.advance(1) self.assertEqual((yield result).id, self.remote_contact.id)