class NodeRPC(AuthJSONRPCServer):
def __init__(self, lbryid, seeds, node_port, rpc_port):
AuthJSONRPCServer.__init__(self, False)
self.root = None
self.port = None
self.seeds = seeds
self.node_port = node_port
self.rpc_port = rpc_port
if lbryid:
lbryid = lbryid.decode('hex')
else:
lbryid = generate_id()
self.node_id = lbryid
self.external_ip = get_external_ip_and_setup_upnp()
self.node_port = node_port
@defer.inlineCallbacks
def setup(self):
self.node = Node(node_id=self.node_id,
udpPort=self.node_port,
externalIP=self.external_ip)
hosts = []
for hostname, hostport in self.seeds:
host_ip = yield reactor.resolve(hostname)
hosts.append((host_ip, hostport))
log.info("connecting to dht")
yield self.node.joinNetwork(tuple(hosts))
log.info("connected to dht")
if not self.announced_startup:
self.announced_startup = True
self.start_api()
log.info("lbry id: %s (%i bytes)", self.node.node_id.encode('hex'),
len(self.node.node_id))
def start_api(self):
root = resource.Resource()
root.putChild('', self)
self.port = reactor.listenTCP(self.rpc_port,
Site(root),
interface='localhost')
log.info("started jsonrpc server")
@defer.inlineCallbacks
def jsonrpc_node_id_set(self, node_id):
old_id = self.node.node_id
self.node.stop()
del self.node
self.node_id = node_id.decode('hex')
yield self.setup()
msg = "changed dht id from %s to %s" % (
old_id.encode('hex'), self.node.node_id.encode('hex'))
defer.returnValue(msg)
def jsonrpc_node_id_get(self):
return self._render_response(self.node.node_id.encode('hex'))
@defer.inlineCallbacks
def jsonrpc_peer_find(self, node_id):
node_id = node_id.decode('hex')
contact = yield self.node.findContact(node_id)
result = None
if contact:
result = (contact.address, contact.port)
defer.returnValue(result)
@defer.inlineCallbacks
def jsonrpc_peer_list_for_blob(self, blob_hash):
peers = yield self.node.getPeersForBlob(blob_hash.decode('hex'))
defer.returnValue(peers)
@defer.inlineCallbacks
def jsonrpc_ping(self, node_id):
contact_host = yield self.jsonrpc_peer_find(node_id=node_id)
if not contact_host:
defer.returnValue("failed to find node")
contact_ip, contact_port = contact_host
contact = Contact(node_id.decode('hex'), contact_ip, contact_port,
self.node._protocol)
try:
result = yield contact.ping()
except TimeoutError:
self.node.removeContact(contact.id)
self.node._dataStore.removePeer(contact.id)
result = {'error': 'timeout'}
defer.returnValue(result)
def get_routing_table(self):
result = {}
data_store = deepcopy(self.node._dataStore._dict)
datastore_len = len(data_store)
hosts = {}
missing_contacts = []
if datastore_len:
for k, v in data_store.iteritems():
for value, lastPublished, originallyPublished, originalPublisherID in v:
try:
contact = self.node._routingTable.getContact(
originalPublisherID)
except ValueError:
if originalPublisherID.encode(
'hex') not in missing_contacts:
missing_contacts.append(
originalPublisherID.encode('hex'))
continue
if contact in hosts:
blobs = hosts[contact]
else:
blobs = []
blobs.append(k.encode('hex'))
hosts[contact] = blobs
contact_set = []
blob_hashes = []
result['buckets'] = {}
for i in range(len(self.node._routingTable._buckets)):
for contact in self.node._routingTable._buckets[i]._contacts:
contacts = result['buckets'].get(i, [])
if contact in hosts:
blobs = hosts[contact]
del hosts[contact]
else:
blobs = []
host = {
"address": contact.address,
"id": contact.id.encode("hex"),
"blobs": blobs,
}
for blob_hash in blobs:
if blob_hash not in blob_hashes:
blob_hashes.append(blob_hash)
contacts.append(host)
result['buckets'][i] = contacts
contact_set.append(contact.id.encode("hex"))
if hosts:
result['datastore extra'] = [{
"id": host.id.encode('hex'),
"blobs": hosts[host],
} for host in hosts]
result['missing contacts'] = missing_contacts
result['contacts'] = contact_set
result['blob hashes'] = blob_hashes
result['node id'] = self.node_id.encode('hex')
return result
def jsonrpc_routing_table_get(self):
return self._render_response(self.get_routing_table())
class KademliaProtocolTest(unittest.TestCase):
""" Test case for the Protocol class """
udpPort = 9182
def setUp(self):
self._reactor = Clock()
self.node = Node(node_id=b'1' * 48, udpPort=self.udpPort, externalIP="127.0.0.1", listenUDP=listenUDP,
resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater)
self.remote_node = Node(node_id=b'2' * 48, udpPort=self.udpPort, externalIP="127.0.0.2", listenUDP=listenUDP,
resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater)
self.remote_contact = self.node.contact_manager.make_contact(b'2' * 48, '127.0.0.2', 9182, self.node._protocol)
self.us_from_them = self.remote_node.contact_manager.make_contact(b'1' * 48, '127.0.0.1', 9182,
self.remote_node._protocol)
self.node.start_listening()
self.remote_node.start_listening()
@defer.inlineCallbacks
def tearDown(self):
yield self.node.stop()
yield self.remote_node.stop()
del self._reactor
@defer.inlineCallbacks
def testReactor(self):
""" Tests if the reactor can start/stop the protocol correctly """
d = defer.Deferred()
self._reactor.callLater(1, d.callback, True)
self._reactor.advance(1)
result = yield d
self.assertTrue(result)
@defer.inlineCallbacks
def testRPCTimeout(self):
""" Tests if a RPC message sent to a dead remote node times out correctly """
yield self.remote_node.stop()
self._reactor.pump([1 for _ in range(10)])
self.node.addContact(self.remote_contact)
@rpcmethod
def fake_ping(*args, **kwargs):
time.sleep(lbrynet.dht.constants.rpcTimeout + 1)
return 'pong'
real_ping = self.node.ping
real_timeout = lbrynet.dht.constants.rpcTimeout
real_attempts = lbrynet.dht.constants.rpcAttempts
lbrynet.dht.constants.rpcAttempts = 1
lbrynet.dht.constants.rpcTimeout = 1
self.node.ping = fake_ping
# Make sure the contact was added
self.assertFalse(self.remote_contact not in self.node.contacts,
'Contact not added to fake node (error in test code)')
self.node.start_listening()
# Run the PING RPC (which should raise a timeout error)
df = self.remote_contact.ping()
def check_timeout(err):
self.assertEqual(err.type, TimeoutError)
df.addErrback(check_timeout)
def reset_values():
self.node.ping = real_ping
lbrynet.dht.constants.rpcTimeout = real_timeout
lbrynet.dht.constants.rpcAttempts = real_attempts
# See if the contact was removed due to the timeout
def check_removed_contact():
self.assertFalse(self.remote_contact in self.node.contacts,
'Contact was not removed after RPC timeout; check exception types.')
df.addCallback(lambda _: reset_values())
# Stop the reactor if a result arrives (timeout or not)
df.addCallback(lambda _: check_removed_contact())
self._reactor.pump([1 for _ in range(20)])
@defer.inlineCallbacks
def testRPCRequest(self):
""" Tests if a valid RPC request is executed and responded to correctly """
yield self.node.addContact(self.remote_contact)
self.error = None
def handleError(f):
self.error = 'An RPC error occurred: %s' % f.getErrorMessage()
def handleResult(result):
expectedResult = b'pong'
if result != expectedResult:
self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' \
% (expectedResult, result)
# Simulate the RPC
df = self.remote_contact.ping()
df.addCallback(handleResult)
df.addErrback(handleError)
self._reactor.advance(2)
yield df
self.assertFalse(self.error, self.error)
# The list of sent RPC messages should be empty at this stage
self.assertEqual(len(self.node._protocol._sentMessages), 0,
'The protocol is still waiting for a RPC result, '
'but the transaction is already done!')
def testRPCAccess(self):
""" Tests invalid RPC requests
Verifies that a RPC request for an existing but unpublished
method is denied, and that the associated (remote) exception gets
raised locally """
self.assertRaises(AttributeError, getattr, self.remote_contact, "not_a_rpc_function")
def testRPCRequestArgs(self):
""" Tests if an RPC requiring arguments is executed correctly """
self.node.addContact(self.remote_contact)
self.error = None
def handleError(f):
self.error = 'An RPC error occurred: %s' % f.getErrorMessage()
def handleResult(result):
expectedResult = b'pong'
if result != expectedResult:
self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' % \
(expectedResult, result)
# Publish the "local" node on the network
self.node.start_listening()
# Simulate the RPC
df = self.remote_contact.ping()
df.addCallback(handleResult)
df.addErrback(handleError)
self._reactor.pump([1 for _ in range(10)])
self.assertFalse(self.error, self.error)
# The list of sent RPC messages should be empty at this stage
self.assertEqual(len(self.node._protocol._sentMessages), 0,
'The protocol is still waiting for a RPC result, '
'but the transaction is already done!')
@defer.inlineCallbacks
def testDetectProtocolVersion(self):
original_findvalue = self.remote_node.findValue
fake_blob = unhexlify("AB" * 48)
@rpcmethod
def findValue(contact, key):
result = original_findvalue(contact, key)
result.pop(b'protocolVersion')
return result
self.remote_node.findValue = findValue
d = self.remote_contact.findValue(fake_blob)
self._reactor.advance(3)
find_value_response = yield d
self.assertEqual(self.remote_contact.protocolVersion, 0)
self.assertNotIn('protocolVersion', find_value_response)
self.remote_node.findValue = original_findvalue
d = self.remote_contact.findValue(fake_blob)
self._reactor.advance(3)
find_value_response = yield d
self.assertEqual(self.remote_contact.protocolVersion, 1)
self.assertNotIn('protocolVersion', find_value_response)
self.remote_node.findValue = findValue
d = self.remote_contact.findValue(fake_blob)
self._reactor.advance(3)
find_value_response = yield d
self.assertEqual(self.remote_contact.protocolVersion, 0)
self.assertNotIn('protocolVersion', find_value_response)
@defer.inlineCallbacks
def testStoreToPre_0_20_0_Node(self):
def _dont_migrate(contact, method, *args):
return args, {}
self.remote_node._protocol._migrate_incoming_rpc_args = _dont_migrate
original_findvalue = self.remote_node.findValue
original_store = self.remote_node.store
@rpcmethod
def findValue(contact, key):
result = original_findvalue(contact, key)
if b'protocolVersion' in result:
result.pop(b'protocolVersion')
return result
@rpcmethod
def store(contact, key, value, originalPublisherID=None, self_store=False, **kwargs):
self.assertEqual(len(key), 48)
self.assertSetEqual(set(value.keys()), {b'token', b'lbryid', b'port'})
self.assertFalse(self_store)
self.assertDictEqual(kwargs, {})
return original_store( # pylint: disable=too-many-function-args
contact, key, value[b'token'], value[b'port'], originalPublisherID, 0
)
self.remote_node.findValue = findValue
self.remote_node.store = store
fake_blob = unhexlify("AB" * 48)
d = self.remote_contact.findValue(fake_blob)
self._reactor.advance(3)
find_value_response = yield d
self.assertEqual(self.remote_contact.protocolVersion, 0)
self.assertNotIn(b'protocolVersion', find_value_response)
token = find_value_response[b'token']
d = self.remote_contact.store(fake_blob, token, 3333, self.node.node_id, 0)
self._reactor.advance(3)
response = yield d
self.assertEqual(response, b'OK')
self.assertEqual(self.remote_contact.protocolVersion, 0)
self.assertTrue(self.remote_node._dataStore.hasPeersForBlob(fake_blob))
self.assertEqual(len(self.remote_node._dataStore.getStoringContacts()), 1)
@defer.inlineCallbacks
def testStoreFromPre_0_20_0_Node(self):
def _dont_migrate(contact, method, *args):
return args
self.remote_node._protocol._migrate_outgoing_rpc_args = _dont_migrate
us_from_them = self.remote_node.contact_manager.make_contact(b'1' * 48, '127.0.0.1', self.udpPort,
self.remote_node._protocol)
fake_blob = unhexlify("AB" * 48)
d = us_from_them.findValue(fake_blob)
self._reactor.advance(3)
find_value_response = yield d
self.assertEqual(self.remote_contact.protocolVersion, 0)
self.assertNotIn(b'protocolVersion', find_value_response)
token = find_value_response[b'token']
us_from_them.update_protocol_version(0)
d = self.remote_node._protocol.sendRPC(
us_from_them, b"store", (fake_blob, {b'lbryid': self.remote_node.node_id, b'token': token, b'port': 3333})
)
self._reactor.advance(3)
response = yield d
self.assertEqual(response, b'OK')
self.assertEqual(self.remote_contact.protocolVersion, 0)
self.assertTrue(self.node._dataStore.hasPeersForBlob(fake_blob))
self.assertEqual(len(self.node._dataStore.getStoringContacts()), 1)
self.assertIs(self.node._dataStore.getStoringContacts()[0], self.remote_contact)
@defer.inlineCallbacks
def test_find_node(self):
self.node.addContact(self.node.contact_manager.make_contact(
self.remote_contact.id, self.remote_contact.address, self.remote_contact.port, self.node._protocol)
)
result = self.node.findContact(b'0'*48)
for _ in range(6):
self._reactor.advance(1)
self.assertIsNone((yield result))
result = self.node.findContact(self.remote_contact.id)
for _ in range(6):
self._reactor.advance(1)
self.assertEqual((yield result).id, self.remote_contact.id)
