class KademliaProtocolTest(unittest.TestCase):
def setUp(self):
self.version = PROTOCOL_VERSION
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
signed_pubkey = self.signing_key.sign(str(verify_key))
h = nacl.hash.sha512(signed_pubkey)
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, signed_pubkey, True)
self.db = datastore.Database(filepath="test.db")
self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db)
self.wire_protocol = OpenBazaarProtocol(self.own_addr, "Full Cone")
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol)
self.handler.connection = self.con
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
def tearDown(self):
if self.con.state != connection.State.SHUTDOWN:
self.con.shutdown()
self.wire_protocol.shutdown()
os.remove("test.db")
def test_invalid_datagram(self):
self.assertFalse(self.handler.receive_message("hi"))
self.assertFalse(self.handler.receive_message("hihihihihihihihihihihihihihihihihihihihih"))
def test_rpc_ping(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("PING")
m.protoVer = self.version
m.testnet = False
data = m.SerializeToString()
m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_store(self):
self._connecting_to_connected()
self.protocol.router.addContact(self.protocol.sourceNode)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
data = m.SerializeToString()
del m.arguments[-4:]
m.arguments.append("True")
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
def test_bad_rpc_store(self):
r = self.protocol.rpc_store(self.node, 'testkeyword', 'kw', 'val', 10)
self.assertEqual(r, ['False'])
def test_rpc_delete(self):
self._connecting_to_connected()
self.protocol.router.addContact(self.protocol.sourceNode)
# Set a keyword to store
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
data = m.SerializeToString()
del m.arguments[-4:]
m.arguments.append("True")
expected_message1 = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
# Test bad signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key", "Bad Signature"])
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("False")
expected_message2 = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list
)
self.assertEqual(sent_packets[0].payload, expected_message1)
self.assertEqual(sent_packets[1].payload, expected_message2)
self.proto_mock.send_datagram.call_args_list = []
# Test good signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key", self.signing_key.sign("Key")[:64]])
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("True")
expected_message3 = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
self.assertEqual(sent_packet.payload, expected_message3)
self.assertTrue(self.storage.getSpecific(digest("Keyword"), "Key") is None)
def test_rpc_stun(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STUN")
m.protoVer = self.version
m.testnet = False
data = m.SerializeToString()
m.arguments.extend([self.public_ip, str(self.port)])
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_node(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_NODE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("nodetofind"))
data = m.SerializeToString()
del m.arguments[-1]
m.arguments.extend([node2.getProto().SerializeToString(), node1.getProto().SerializeToString(),
node3.getProto().SerializeToString()])
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_value(self):
self._connecting_to_connected()
self.protocol.router.addContact(self.protocol.sourceNode)
# Set a value to find
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.protoVer = self.version
m.arguments.extend([digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
data = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
# Send the find_value rpc
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("Keyword"))
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
value = objects.Value()
value.valueKey = "Key"
value.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
value.ttl = 10
m.arguments.append("value")
m.arguments.append(value.SerializeToString())
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list
)
received_message = sent_packets[1].payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 3)
def test_rpc_find_without_value(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("Keyword"))
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
m.arguments.extend([node3.getProto().SerializeToString(), node1.getProto().SerializeToString(),
node2.getProto().SerializeToString()])
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
m = message.Message()
m.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_callPing(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callPing(n)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.PING)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
def test_callStore(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callStore(n, digest("Keyword"), digest("Key"),
self.protocol.sourceNode.getProto().SerializeToString(), 10)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.STORE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], self.protocol.sourceNode.getProto().SerializeToString())
def test_callFindValue(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
keyword = Node(digest("Keyword"))
self.protocol.callFindValue(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.FIND_VALUE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callFindNode(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
keyword = Node(digest("nodetofind"))
self.protocol.callFindNode(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.FIND_NODE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callDelete(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callDelete(n, digest("Keyword"), digest("Key"), digest("Signature"))
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.DELETE)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], digest("Signature"))
def test_acceptResponse(self):
self._connecting_to_connected()
def handle_response(resp):
self.assertTrue(resp[0])
self.assertEqual(resp[1][0], "test")
self.assertTrue(message_id not in self.protocol._outstanding)
message_id = digest("msgid")
n = Node(digest("S"), self.addr1[0], self.addr1[1])
d = defer.Deferred()
self.protocol._outstanding[message_id] = (d, self.addr1, reactor.callLater(5, handle_response))
self.protocol._acceptResponse(message_id, ["test"], n)
return d.addCallback(handle_response)
def test_unknownRPC(self):
self.assertFalse(self.handler.receive_message(str(random.getrandbits(1400))))
def test_timeout(self):
def handle_response(resp, n):
self.assertFalse(resp[0])
self.assertIsNone(resp[1])
self.assertTrue(self.protocol.router.isNewNode(n))
n = Node(digest("S"), self.addr1[0], self.addr1[1])
d = defer.Deferred().addCallback(handle_response, n)
self.protocol._outstanding["msgID"] = [d, self.addr1]
self.protocol.router.addContact(n)
self.protocol.timeout(self.addr1, n)
def test_transferKeyValues(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.protocol.router.addContact(mknode())
self.protocol.storage[digest("keyword")] = (
digest("key"), self.protocol.sourceNode.getProto().SerializeToString(), 10)
self.protocol.storage[digest("keyword")] = (
digest("key2"), self.protocol.sourceNode.getProto().SerializeToString(), 10)
self.protocol.transferKeyValues(Node(digest("id"), self.addr1[0], self.addr1[1]))
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
x = message.Message()
x.ParseFromString(sent_message)
i = objects.Inv()
i.keyword = digest("keyword")
i.valueKey = digest("key")
i2 = objects.Inv()
i2.keyword = digest("keyword")
i2.valueKey = digest("key2")
m = message.Message()
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("INV")
m.protoVer = self.version
m.arguments.append(i.SerializeToString())
m.arguments.append(i2.SerializeToString())
self.assertEqual(x.sender, m.sender)
self.assertEqual(x.command, m.command)
self.assertTrue(x.arguments[0] in m.arguments)
self.assertTrue(x.arguments[1] in m.arguments)
def test_refreshIDs(self):
node1 = Node(digest("id1"), "127.0.0.1", 12345, signed_pubkey=digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, signed_pubkey=digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, signed_pubkey=digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
for b in self.protocol.router.buckets:
b.lastUpdated = (time.time() - 5000)
ids = self.protocol.getRefreshIDs()
self.assertTrue(len(ids) == 1)
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(
42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True
)
self.con.receive_packet(remote_synack_packet)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1
def test_badRPCDelete(self):
n = mknode()
val = self.protocol.rpc_delete(n, 'testkeyword', 'key', 'testsig')
self.assertEqual(val, ["False"])
val = self.protocol.rpc_delete(n, '', '', '')
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, node, db, signing_key, ksize=20, alpha=3, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
node: The node instance for this peer. It must contain (at minimum) an ID,
public key, ip address, and port.
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
storage: An instance that implements :interface:`~dht.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = node
self.protocol = KademliaProtocol(self.node, self.storage, ksize, db,
signing_key)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling::
reactor.listenUDP(port, server.protocol)
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
refresh_ids = self.protocol.getRefreshIDs()
refresh_ids.append(digest(
random.getrandbits(255))) # random node so we get more diversity
for rid in refresh_ids:
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
ds.append(spider.find())
def republishKeys(_):
for bucket in self.protocol.router.buckets:
for node in bucket.nodes.values():
self.protocol.transferKeyValues(node)
return defer.gatherResults(ds).addCallback(republishKeys)
def querySeed(self, list_seed_pubkey):
"""
Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.
Args:
Receives a list of one or more tuples Example [(seed, pubkey)]
seed: A `string` consisting of "ip:port" or "hostname:port"
pubkey: The hex encoded public key to verify the signature on the response
"""
nodes = []
if not list_seed_pubkey:
self.log.error('failed to query seed {0} from ob.cfg'.format(
list_seed_pubkey))
return nodes
else:
for sp in list_seed_pubkey:
seed, pubkey = sp
try:
self.log.info("querying %s for peers" % seed)
c = httplib.HTTPConnection(seed)
c.request("GET", "/")
response = c.getresponse()
self.log.debug("Http response from %s: %s, %s" %
(seed, response.status, response.reason))
data = response.read()
reread_data = data.decode("zlib")
proto = peers.PeerSeeds()
proto.ParseFromString(reread_data)
for peer in proto.serializedNode:
n = objects.Node()
n.ParseFromString(peer)
tup = (str(n.nodeAddress.ip), n.nodeAddress.port)
nodes.append(tup)
verify_key = nacl.signing.VerifyKey(
pubkey, encoder=nacl.encoding.HexEncoder)
verify_key.verify("".join(proto.serializedNode),
proto.signature)
self.log.info("%s returned %s addresses" %
(seed, len(nodes)))
except Exception, e:
self.log.error("failed to query seed: %s" % str(e))
return nodes
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, node, ksize=20, alpha=3, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
node: The node instance for this peer. It must contain (at minimum) an ID,
public key, ip address, and port.
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
storage: An instance that implements :interface:`~dht.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = node
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling::
reactor.listenUDP(port, server.protocol)
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for rid in self.protocol.getRefreshIDs():
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for keyword in self.storage.iterkeys():
for k, v in self.storage.iteritems(keyword):
if self.storage.get_ttl(keyword, k) < 601200:
ds.append(self.set(keyword, k, v))
return defer.gatherResults(ds).addCallback(republishKeys)
def querySeed(self, seed, pubkey):
"""
Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.
Args:
seed: A `string` consisting of "ip:port" or "hostname:port"
pubkey: The hex encoded public key to verify the signature on the response
"""
nodes = []
c = httplib.HTTPConnection(seed)
c.request("GET", "/")
response = c.getresponse()
self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason))
data = response.read()
reread_data = data.decode("zlib")
proto = peers.PeerSeeds()
try:
proto.ParseFromString(reread_data)
for peer in proto.peer_data:
p = peers.PeerData()
p.ParseFromString(peer)
tup = (str(p.ip_address), p.port)
nodes.append(tup)
verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder)
verify_key.verify(proto.signature + "".join(proto.peer_data))
return nodes
except Exception:
self.log.error("Error parsing seed response.")
return nodes
def bootstrappableNeighbors(self):
"""
Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument
to the bootstrap method.
The server should have been bootstrapped
already - this is just a utility for getting some neighbors and then
storing them if this server is going down for a while. When it comes
back up, the list of nodes can be used to bootstrap.
"""
neighbors = self.protocol.router.findNeighbors(self.node)
return [tuple(n)[-2:] for n in neighbors]
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.multiplexer.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(n.signedPublicKey)
h = nacl.hash.sha512(n.signedPublicKey)
hash_pow = h[64:128]
if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
raise Exception('Invalid GUID')
nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
except Exception:
self.log.msg("Bootstrap node returned invalid GUID")
spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
return spider.find()
ds = {}
for addr in addrs:
ds[addr] = self.protocol.ping((addr[0], addr[1]))
return deferredDict(ds).addCallback(initTable)
def inetVisibleIP(self):
"""
Get the internet visible IP's of this node as other nodes see it.
Returns:
A `list` of IP's. If no one can be contacted, then the `list` will be empty.
"""
def handle(results):
ips = []
for result in results:
if result[0]:
ips.append((result[1][0], int(result[1][1])))
self.log.debug("other nodes think our ip is %s" % str(ips))
return ips
ds = []
for neighbor in self.bootstrappableNeighbors():
ds.append(self.protocol.stun(neighbor))
return defer.gatherResults(ds).addCallback(handle)
def get(self, keyword):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(keyword)
if self.storage.get(dkey) is not None:
return defer.succeed(self.storage.get(dkey))
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % keyword)
return defer.succeed(None)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find()
def set(self, keyword, key, value):
"""
Set the given key/value tuple at the hash of the given keyword.
All values stored in the DHT are stored as dictionaries of key/value
pairs. If a value already exists for a given keyword, the new key/value
pair will be appended to the dictionary.
Args:
keyword: a `string` keyword. The SHA1 hash of which will be used as
the key when inserting in the DHT.
key: the 20 byte hash of the data.
value: a serialized `protos.objects.Node` object which serves as a
pointer to the node storing the data.
Return: True if at least one peer responded. False if the store rpc
completely failed.
"""
self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value)))
dkey = digest(keyword)
def store(nodes):
self.log.info("setting '%s' on %s" % (keyword, [str(i) for i in nodes]))
ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes]
keynode = Node(dkey)
if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
self.storage[dkey] = (key, value)
self.log.debug("got a store request from %s, storing value" % str(self.node))
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def delete(self, keyword, key, signature):
"""
Delete the given key/value pair from the keyword dictionary on the network.
To delete you must provide a signature covering the key that you wish to
delete. It will be verified against the public key stored in the value. We
use our ksize as alpha to make sure we reach as many nodes storing our value
as possible.
Args:
keyword: the `string` keyword where the data being deleted is stored.
key: the 20 byte hash of the data.
signature: a signature covering the key.
"""
self.log.info("deleting '%s':'%s' from the network" % (keyword, hexlify(key)))
dkey = digest(keyword)
def delete(nodes):
self.log.debug("deleting '%s' on %s" % (key, [str(i) for i in nodes]))
ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes]
if self.storage.getSpecific(dkey, key) is not None:
self.storage.delete(dkey, key)
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to delete key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize)
return spider.find().addCallback(delete)
def resolve(self, guid):
"""
Given a guid return a `Node` object containing its ip and port or none if it's
not found.
Args:
guid: the 20 raw bytes representing the guid.
"""
node_to_find = Node(guid)
def check_for_node(nodes):
for node in nodes:
if node.id == node_to_find.id:
return node
return None
index = self.protocol.router.getBucketFor(node_to_find)
nodes = self.protocol.router.buckets[index].getNodes()
for node in nodes:
if node.id == node_to_find.id:
return defer.succeed(node)
nearest = self.protocol.router.findNeighbors(node_to_find)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
return defer.succeed(None)
spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
return spider.find().addCallback(check_for_node)
def saveState(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
data = {'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.id,
'vendor': self.node.vendor,
'signed_pubkey': self.node.signed_pubkey,
'neighbors': self.bootstrappableNeighbors()}
if len(data['neighbors']) == 0:
self.log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'w') as f:
pickle.dump(data, f)
@classmethod
def loadState(cls, fname, ip_address, port, multiplexer, callback=None, storage=None):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
with open(fname, 'r') as f:
data = pickle.load(f)
n = Node(data['id'], ip_address, port, data['signed_pubkey'], data['vendor'])
s = Server(n, data['ksize'], data['alpha'], storage=storage)
s.protocol.connect_multiplexer(multiplexer)
if len(data['neighbors']) > 0:
if callback is not None:
s.bootstrap(data['neighbors']).addCallback(callback)
else:
s.bootstrap(data['neighbors'])
return s
def saveStateRegularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
filename.
Args:
fname: File name to save retularly to
frequencey: Frequency in seconds that the state should be saved.
By default, 10 minutes.
"""
loop = LoopingCall(self.saveState, fname)
loop.start(frequency)
return loop
class KademliaProtocolTest(unittest.TestCase):
def setUp(self):
self.version = PROTOCOL_VERSION
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "63d901c4d57cde34fc1f1e28b9af5d56ed342cae5c2fb470046d0130a4226b0c"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
h = nacl.hash.sha512(verify_key.encode())
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port,
verify_key.encode(), None, objects.FULL_CONE, True)
self.db = datastore.Database(filepath="test.db")
self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db, self.signing_key)
self.wire_protocol = OpenBazaarProtocol(self.db, self.own_addr, objects.FULL_CONE)
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol, None)
self.handler.connection = self.con
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
def tearDown(self):
if self.con.state != connection.State.SHUTDOWN:
self.con.shutdown()
self.wire_protocol.shutdown()
os.remove("test.db")
def test_invalid_datagram(self):
self.assertFalse(self.handler.receive_message("hi"))
self.assertFalse(self.handler.receive_message("hihihihihihihihihihihihihihihihihihihihih"))
def test_rpc_ping(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("PING")
m.protoVer = self.version
m.testnet = False
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
m.ClearField("signature")
expected_message = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
m2 = message.Message()
m2.ParseFromString(received_message)
m2.ClearField("signature")
received_message = m2.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_store(self):
self._connecting_to_connected()
self.protocol.router.addContact(self.protocol.sourceNode)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
del m.arguments[-4:]
m.arguments.append("True")
m.ClearField("signature")
expected_message = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
m2 = message.Message()
m2.ParseFromString(received_message)
m2.ClearField("signature")
received_message = m2.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
def test_bad_rpc_store(self):
r = self.protocol.rpc_store(self.node, 'testkeyword', 'kw', 'val', 10)
self.assertEqual(r, ['False'])
def test_rpc_delete(self):
self._connecting_to_connected()
self.protocol.router.addContact(self.protocol.sourceNode)
# Set a keyword to store
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
del m.arguments[-4:]
m.arguments.append("True")
m.ClearField("signature")
expected_message1 = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
# Test bad signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key", "Bad Signature"])
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("False")
m.ClearField("signature")
expected_message2 = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list
)
m2 = message.Message()
m2.ParseFromString(sent_packets[0].payload)
m2.ClearField("signature")
received_message1 = m2.SerializeToString()
m3 = message.Message()
m3.ParseFromString(sent_packets[1].payload)
m3.ClearField("signature")
received_message2 = m3.SerializeToString()
self.assertEqual(received_message1, expected_message1)
self.assertEqual(received_message2, expected_message2)
self.proto_mock.send_datagram.call_args_list = []
# Test good signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.protoVer = self.version
m.testnet = False
m.arguments.extend([digest("Keyword"), "Key", self.signing_key.sign("Key")[:64]])
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("True")
m.ClearField("signature")
expected_message3 = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
m4 = message.Message()
m4.ParseFromString(sent_packet.payload)
m4.ClearField("signature")
received_message = m4.SerializeToString()
self.assertEqual(received_message, expected_message3)
self.assertTrue(self.storage.getSpecific(digest("Keyword"), "Key") is None)
def test_rpc_stun(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STUN")
m.protoVer = self.version
m.testnet = False
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
m.arguments.extend([self.public_ip, str(self.port)])
m.ClearField("signature")
expected_message = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
a.ClearField("signature")
received_message = a.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_node(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"), nat_type=objects.FULL_CONE)
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"), nat_type=objects.FULL_CONE)
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"), nat_type=objects.FULL_CONE)
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_NODE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("nodetofind"))
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
del m.arguments[-1]
m.arguments.extend([node2.getProto().SerializeToString(), node1.getProto().SerializeToString(),
node3.getProto().SerializeToString()])
m.ClearField("signature")
expected_message = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
a.ClearField("signature")
received_message = a.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_value(self):
self._connecting_to_connected()
self.protocol.router.addContact(self.protocol.sourceNode)
# Set a value to find
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.protoVer = self.version
m.arguments.extend([digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
# Send the find_value rpc
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("Keyword"))
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
value = objects.Value()
value.valueKey = "Key"
value.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
value.ttl = 10
m.arguments.append("value")
m.arguments.append(value.SerializeToString())
m.ClearField("signature")
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list
)
received_message = sent_packets[1].payload
a = message.Message()
a.ParseFromString(received_message)
a.ClearField("signature")
received_message = a.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 3)
def test_rpc_find_without_value(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"), nat_type=objects.FULL_CONE)
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"), nat_type=objects.FULL_CONE)
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"), nat_type=objects.FULL_CONE)
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.protoVer = self.version
m.testnet = False
m.arguments.append(digest("Keyword"))
m.signature = self.signing_key.sign(m.SerializeToString())[:64]
data = m.SerializeToString()
self.handler.on_connection_made()
self.handler.receive_message(data)
del m.arguments[-1]
m.arguments.extend([node3.getProto().SerializeToString(), node1.getProto().SerializeToString(),
node2.getProto().SerializeToString()])
m.ClearField("signature")
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
a.ClearField("signature")
received_message = a.SerializeToString()
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_callPing(self):
self._connecting_to_connected()
n = Node(digest("guid"), self.addr1[0], self.addr1[1], digest("pubkey"), None, objects.FULL_CONE, False)
self.wire_protocol[self.addr1] = self.con
self.protocol.callPing(n)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
self.assertTrue(m.command == message.PING)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
def test_callStore(self):
self._connecting_to_connected()
n = Node(digest("guid"), self.addr1[0], self.addr1[1], digest("pubkey"), None, objects.FULL_CONE, False)
self.wire_protocol[self.addr1] = self.con
self.protocol.callStore(n, digest("Keyword"), digest("Key"),
self.protocol.sourceNode.getProto().SerializeToString(), 10)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
self.assertTrue(m.command == message.STORE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], self.protocol.sourceNode.getProto().SerializeToString())
def test_callFindValue(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
keyword = Node(digest("Keyword"))
self.protocol.callFindValue(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
self.assertTrue(m.command == message.FIND_VALUE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callFindNode(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
keyword = Node(digest("nodetofind"))
self.protocol.callFindNode(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
self.assertTrue(m.command == message.FIND_NODE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callDelete(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callDelete(n, digest("Keyword"), digest("Key"), digest("Signature"))
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
self.assertTrue(m.command == message.DELETE)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], digest("Signature"))
def test_acceptResponse(self):
self._connecting_to_connected()
def handle_response(resp):
self.assertTrue(resp[0])
self.assertEqual(resp[1][0], "test")
self.assertTrue(message_id not in self.protocol._outstanding)
message_id = digest("msgid")
n = Node(digest("S"), self.addr1[0], self.addr1[1])
d = defer.Deferred()
self.protocol._outstanding[message_id] = (d, self.addr1, reactor.callLater(5, handle_response))
self.protocol._acceptResponse(message_id, ["test"], n)
return d.addCallback(handle_response)
def test_unknownRPC(self):
self.assertFalse(self.handler.receive_message(str(random.getrandbits(1400))))
def test_timeout(self):
def handle_response(resp, n):
self.assertFalse(resp[0])
self.assertIsNone(resp[1])
n = Node(digest("S"), self.addr1[0], self.addr1[1])
d = defer.Deferred().addCallback(handle_response, n)
self.protocol._outstanding["msgID"] = [d, self.addr1, reactor.callLater(5, handle_response)]
self.protocol.router.addContact(n)
self.protocol.timeout(n)
def test_transferKeyValues(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.protocol.storage[digest("keyword")] = (
digest("key"), self.protocol.sourceNode.getProto().SerializeToString(), 10)
self.protocol.storage[digest("keyword")] = (
digest("key2"), self.protocol.sourceNode.getProto().SerializeToString(), 10)
self.protocol.transferKeyValues(Node(digest("id"), self.addr1[0], self.addr1[1]))
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
x = message.Message()
x.ParseFromString(sent_message)
i = objects.Inv()
i.keyword = digest("keyword")
i.valueKey = digest("key")
i2 = objects.Inv()
i2.keyword = digest("keyword")
i2.valueKey = digest("key2")
m = message.Message()
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("INV")
m.protoVer = self.version
m.arguments.append(i.SerializeToString())
m.arguments.append(i2.SerializeToString())
self.assertEqual(x.sender.guid, m.sender.guid)
self.assertEqual(x.command, m.command)
self.assertTrue(x.arguments[0] in m.arguments)
self.assertTrue(x.arguments[1] in m.arguments)
def test_refreshIDs(self):
node1 = Node(digest("id1"), "127.0.0.1", 12345, pubkey=digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, pubkey=digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, pubkey=digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
for b in self.protocol.router.buckets:
b.lastUpdated = (time.time() - 5000)
ids = self.protocol.getRefreshIDs()
self.assertTrue(len(ids) == 1)
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(
42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True
)
self.con.receive_packet(remote_synack_packet, self.addr1)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1
def test_badRPCDelete(self):
n = Node(digest("S"), self.addr1[0], self.addr1[1])
val = self.protocol.rpc_delete(n, 'testkeyword', 'key', 'testsig')
self.assertEqual(val, ["False"])
val = self.protocol.rpc_delete(n, '', '', '')
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, node, db, signing_key, ksize=20, alpha=3, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
node: The node instance for this peer. It must contain (at minimum) an ID,
public key, ip address, and port.
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
storage: An instance that implements :interface:`~dht.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = node
self.protocol = KademliaProtocol(self.node, self.storage, ksize, db, signing_key)
self.refreshLoop = LoopingCall(self.refreshTable)
reactor.callLater(1800, self.refreshLoop.start, 3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling::
reactor.listenUDP(port, server.protocol)
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
refresh_ids = self.protocol.getRefreshIDs()
refresh_ids.append(digest(random.getrandbits(255))) # random node so we get more diversity
for rid in refresh_ids:
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
ds.append(spider.find())
def republishKeys(_):
self.log.debug("Republishing key/values...")
neighbors = self.protocol.router.findNeighbors(self.node, exclude=self.node)
for node in neighbors:
self.protocol.transferKeyValues(node)
return defer.gatherResults(ds).addCallback(republishKeys)
def querySeed(self, list_seed_pubkey):
"""
Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.
Args:
Receives a list of one or more tuples Example [(seed, pubkey)]
seed: A `string` consisting of "ip:port" or "hostname:port"
pubkey: The hex encoded public key to verify the signature on the response
"""
nodes = []
if not list_seed_pubkey:
self.log.error('failed to query seed {0} from ob.cfg'.format(list_seed_pubkey))
return nodes
else:
for sp in list_seed_pubkey:
seed, pubkey = sp
try:
self.log.info("querying %s for peers" % seed)
c = httplib.HTTPConnection(seed)
c.request("GET", "/")
response = c.getresponse()
self.log.debug("Http response from %s: %s, %s" % (seed, response.status, response.reason))
data = response.read()
reread_data = data.decode("zlib")
proto = peers.PeerSeeds()
proto.ParseFromString(reread_data)
for peer in proto.serializedNode:
n = objects.Node()
n.ParseFromString(peer)
tup = (str(n.nodeAddress.ip), n.nodeAddress.port)
nodes.append(tup)
verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder)
verify_key.verify("".join(proto.serializedNode), proto.signature)
self.log.info("%s returned %s addresses" % (seed, len(nodes)))
except Exception, e:
self.log.error("failed to query seed: %s" % str(e))
return nodes
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, node, db, ksize=20, alpha=3, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
node: The node instance for this peer. It must contain (at minimum) an ID,
public key, ip address, and port.
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
storage: An instance that implements :interface:`~dht.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = node
self.protocol = KademliaProtocol(self.node, self.storage, ksize, db)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling::
reactor.listenUDP(port, server.protocol)
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for rid in self.protocol.getRefreshIDs():
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for keyword in self.storage.iterkeys():
for k, v in self.storage.iteritems(keyword):
if self.storage.get_ttl(keyword, k) < 601200:
ds.append(self.set(keyword, k, v))
return defer.gatherResults(ds).addCallback(republishKeys)
def querySeed(self, seed, pubkey):
"""
Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.
Args:
seed: A `string` consisting of "ip:port" or "hostname:port"
pubkey: The hex encoded public key to verify the signature on the response
"""
try:
nodes = []
c = httplib.HTTPConnection(seed)
c.request("GET", "/")
response = c.getresponse()
self.log.info("Https response from %s: %s, %s" %
(seed, response.status, response.reason))
data = response.read()
reread_data = data.decode("zlib")
proto = peers.PeerSeeds()
proto.ParseFromString(reread_data)
for peer in proto.peer_data:
p = peers.PeerData()
p.ParseFromString(peer)
tup = (str(p.ip_address), p.port)
nodes.append(tup)
verify_key = nacl.signing.VerifyKey(
pubkey, encoder=nacl.encoding.HexEncoder)
verify_key.verify("".join(proto.peer_data), proto.signature)
return nodes
except Exception, e:
self.log.error("Failed to query seed: %s" % str(e))
class KademliaProtocolTest(unittest.TestCase):
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(self.proto_mock, self.handler_mock,
self.own_addr, self.addr1)
valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
self.signing_key = nacl.signing.SigningKey(
valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
signed_pubkey = self.signing_key.sign(str(verify_key))
h = nacl.hash.sha512(signed_pubkey)
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port,
signed_pubkey, True)
self.protocol = KademliaProtocol(self.node, self.storage, 20)
self.wire_protocol = OpenBazaarProtocol(self.own_addr)
self.wire_protocol.register_processor(self.protocol)
self.protocol.connect_multiplexer(self.wire_protocol)
self.handler = self.wire_protocol.ConnHandler([self.protocol])
self.handler.connection = self.con
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.wire_protocol.makeConnection(transport)
def tearDown(self):
self.con.shutdown()
self.wire_protocol.shutdown()
def test_invalid_datagram(self):
self.assertFalse(self.handler.receive_message("hi"))
self.assertFalse(
self.handler.receive_message(
"hihihihihihihihihihihihihihihihihihihihih"))
def test_rpc_ping(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("PING")
data = m.SerializeToString()
m.arguments.append(
self.protocol.sourceNode.getProto().SerializeToString())
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_store(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.extend([
digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString()
])
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("True")
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
def test_rpc_delete(self):
self._connecting_to_connected()
# Set a keyword to store
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.extend([
digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString()
])
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("True")
expected_message1 = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
# Test bad signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.arguments.extend([digest("Keyword"), "Key", "Bad Signature"])
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("False")
expected_message2 = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list)
self.assertEqual(sent_packets[0].payload, expected_message1)
self.assertEqual(sent_packets[1].payload, expected_message2)
self.proto_mock.send_datagram.call_args_list = []
# Test good signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.arguments.extend(
[digest("Keyword"), "Key",
self.signing_key.sign("Key")[:64]])
data = m.SerializeToString()
del m.arguments[-3:]
m.arguments.append("True")
expected_message3 = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
self.assertEqual(sent_packet.payload, expected_message3)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") is None)
def test_rpc_stun(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STUN")
data = m.SerializeToString()
m.arguments.extend([self.public_ip, str(self.port)])
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_node(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_NODE")
m.arguments.append(digest("nodetofind"))
data = m.SerializeToString()
del m.arguments[-1]
m.arguments.extend([
node2.getProto().SerializeToString(),
node1.getProto().SerializeToString(),
node3.getProto().SerializeToString()
])
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
a = message.Message()
a.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_value(self):
self._connecting_to_connected()
# Set a value to find
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.extend([
digest("Keyword"), "Key",
self.protocol.sourceNode.getProto().SerializeToString()
])
data = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(
self.storage.getSpecific(digest("Keyword"), "Key") ==
self.protocol.sourceNode.getProto().SerializeToString())
# Send the find_value rpc
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.arguments.append(digest("Keyword"))
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
value = objects.Value()
value.valueKey = "Key"
value.serializedData = self.protocol.sourceNode.getProto(
).SerializeToString()
m.arguments.append("value")
m.arguments.append(value.SerializeToString())
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list)
received_message = sent_packets[1].payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 3)
def test_rpc_find_without_value(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.arguments.append(digest("Keyword"))
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
m.arguments.extend([
node3.getProto().SerializeToString(),
node1.getProto().SerializeToString(),
node2.getProto().SerializeToString()
])
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
m = message.Message()
m.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_callPing(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callPing(n)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
m.sender.signedPublicKey)
self.assertTrue(m.command == message.PING)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
def test_callStore(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callStore(
n, digest("Keyword"), digest("Key"),
self.protocol.sourceNode.getProto().SerializeToString())
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
m.sender.signedPublicKey)
self.assertTrue(m.command == message.STORE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(
m.arguments[2],
self.protocol.sourceNode.getProto().SerializeToString())
def test_callFindValue(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
keyword = Node(digest("Keyword"))
self.protocol.callFindValue(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
m.sender.signedPublicKey)
self.assertTrue(m.command == message.FIND_VALUE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callFindNode(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
keyword = Node(digest("nodetofind"))
self.protocol.callFindNode(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
m.sender.signedPublicKey)
self.assertTrue(m.command == message.FIND_NODE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callDelete(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
self.protocol.callDelete(n, digest("Keyword"), digest("Key"),
digest("Signature"))
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
self.addr1)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid,
m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
m.sender.signedPublicKey)
self.assertTrue(m.command == message.DELETE)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], digest("Signature"))
def test_acceptResponse(self):
self._connecting_to_connected()
def handle_response(resp):
self.assertTrue(resp[0])
self.assertEqual(resp[1][0], self.protocol.sourceNode.id)
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.wire_protocol[self.addr1] = self.con
d = self.protocol.callPing(n)
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
timeout = reactor.callLater(5, self.protocol._timeout, m.messageID)
self.protocol._outstanding[m.messageID] = (d, timeout)
m.arguments.append(self.protocol.sourceNode.id)
self.handler.receive_message(m.SerializeToString())
return d.addCallback(handle_response)
def test_unknownRPC(self):
self.assertFalse(
self.handler.receive_message(str(random.getrandbits(1400))))
def test_timeout(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
def test_remove_outstanding():
self.assertTrue(len(self.protocol._outstanding) == 0)
def test_deffered(d):
self.assertFalse(d[0])
test_remove_outstanding()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
d = self.protocol.callPing(n)
self.clock.advance(6)
connection.REACTOR.runUntilCurrent()
self.clock.advance(6)
return d.addCallback(test_deffered)
def test_transferKeyValues(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.protocol.addToRouter(mknode())
self.protocol.storage[digest("keyword")] = (
digest("key"),
self.protocol.sourceNode.getProto().SerializeToString())
self.protocol.transferKeyValues(
Node(digest("id"), self.addr1[0], self.addr1[1]))
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(
self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
x = message.Message()
x.ParseFromString(sent_message)
m = message.Message()
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.append(digest("keyword"))
m.arguments.append(digest("key"))
m.arguments.append(
self.protocol.sourceNode.getProto().SerializeToString())
self.assertEqual(x.sender, m.sender)
self.assertEqual(x.command, m.command)
self.assertEqual(x.arguments[0], m.arguments[0])
self.assertEqual(x.arguments[1], m.arguments[1])
self.assertEqual(x.arguments[2], m.arguments[2])
def test_refreshIDs(self):
node1 = Node(digest("id1"),
"127.0.0.1",
12345,
signed_pubkey=digest("key1"))
node2 = Node(digest("id2"),
"127.0.0.1",
22222,
signed_pubkey=digest("key2"))
node3 = Node(digest("id3"),
"127.0.0.1",
77777,
signed_pubkey=digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
for b in self.protocol.router.buckets:
b.lastUpdated = (time.time() - 5000)
ids = self.protocol.getRefreshIDs()
self.assertTrue(len(ids) == 1)
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True)
self.con.receive_packet(remote_synack_packet)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1
def test_badRPCDelete(self):
n = mknode()
val = self.protocol.rpc_delete(n, 'testkeyword', 'key', 'testsig')
self.assertEqual(val, ["False"])
val = self.protocol.rpc_delete(n, '', '', '')
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, node, ksize=20, alpha=3, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
node: The node instance for this peer. It must contain (at minimum) an ID,
public key, ip address, and port.
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
storage: An instance that implements :interface:`~dht.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = node
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling::
reactor.listenUDP(port, server.protocol)
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for id in self.protocol.getRefreshIDs():
node = Node(id)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for keyword in self.storage.iterkeys():
for k, v in self.storage.iteritems(keyword):
if self.storage.get_ttl(keyword, k) < 601200:
ds.append(self.set(keyword, k, v))
return defer.gatherResults(ds).addCallback(republishKeys)
def querySeed(self, seed, pubkey):
"""
Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.
Args:
seed: A `string` consisting of "ip:port" or "hostname:port"
pubkey: The hex encoded public key to verify the signature on the response
"""
nodes = []
c = httplib.HTTPConnection(seed)
c.request("GET", "/")
response = c.getresponse()
self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason))
data = response.read()
reread_data = data.decode("zlib")
seeds = peers.PeerSeeds()
try:
seeds.ParseFromString(reread_data)
for peer in seeds.peer_data:
p = peers.PeerData()
p.ParseFromString(peer)
tup = (str(p.ip_address), p.port)
nodes.append(tup)
verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder)
verify_key.verify(seed.signature + "".join(seeds.peer_data))
except:
self.log.error("Error parsing seed response.")
return nodes
def bootstrappableNeighbors(self):
"""
Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument
to the bootstrap method.
The server should have been bootstrapped
already - this is just a utility for getting some neighbors and then
storing them if this server is going down for a while. When it comes
back up, the list of nodes can be used to bootstrap.
"""
neighbors = self.protocol.router.findNeighbors(self.node)
return [tuple(n)[-2:] for n in neighbors]
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.multiplexer.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(n.signedPublicKey)
h = nacl.hash.sha512(n.signedPublicKey)
pow = h[64:128]
if int(pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
raise Exception('Invalid GUID')
nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
except:
self.log.msg("Bootstrap node returned invalid GUID")
spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
return spider.find()
ds = {}
for addr in addrs:
ds[addr] = self.protocol.ping((addr[0], addr[1]))
return deferredDict(ds).addCallback(initTable)
def inetVisibleIP(self):
"""
Get the internet visible IP's of this node as other nodes see it.
Returns:
A `list` of IP's. If no one can be contacted, then the `list` will be empty.
"""
def handle(results):
ips = []
for result in results:
if result[0]:
ips.append((result[1][0], int(result[1][1])))
self.log.debug("other nodes think our ip is %s" % str(ips))
return ips
ds = []
for neighbor in self.bootstrappableNeighbors():
ds.append(self.protocol.stun(neighbor))
return defer.gatherResults(ds).addCallback(handle)
def get(self, keyword):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(keyword)
if self.storage.get(dkey) is not None:
return defer.succeed(self.storage.get(dkey))
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % keyword)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find()
def set(self, keyword, key, value):
"""
Set the given key/value tuple at the hash of the given keyword.
All values stored in the DHT are stored as dictionaries of key/value
pairs. If a value already exists for a given keyword, the new key/value
pair will be appended to the dictionary.
Args:
keyword: a `string` keyword. The SHA1 hash of which will be used as
the key when inserting in the DHT.
key: the 20 byte hash of the data.
value: a serialized `protos.objects.Node` object which serves as a
pointer to the node storing the data.
Return: True if at least one peer responded. False if the store rpc
completely failed.
"""
self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value)))
dkey = digest(keyword)
def store(nodes):
self.log.info("setting '%s' on %s" % (keyword, map(str, nodes)))
ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes]
keynode = Node(dkey)
if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
self.storage[dkey] = (key, value)
self.log.debug("got a store request from %s, storing value" % str(self.node))
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def delete(self, keyword, key, signature):
"""
Delete the given key/value pair from the keyword dictionary on the network.
To delete you must provide a signature covering the key that you wish to
delete. It will be verified against the public key stored in the value. We
use our ksize as alpha to make sure we reach as many nodes storing our value
as possible.
Args:
keyword: the `string` keyword where the data being deleted is stored.
key: the 20 byte hash of the data.
signature: a signature covering the key.
"""
self.log.debug("deleting '%s':'%s' from the network" % (keyword, hexlify(key)))
dkey = digest(keyword)
def delete(nodes):
self.log.info("deleting '%s' on %s" % (key, map(str, nodes)))
ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes]
if self.storage.getSpecific(keyword, key) is not None:
self.storage.delete(keyword, key)
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to delete key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize)
return spider.find().addCallback(delete)
def get_node(self, guid):
"""
Given a guid return a `Node` object containing its ip and port or none if it's
not found.
Args:
guid: the 20 raw bytes representing the guid.
"""
node_to_find = Node(guid)
def check_for_node(nodes):
for node in nodes:
if node.id == node_to_find.id:
return node
return None
index = self.protocol.router.getBucketFor(node_to_find)
nodes = self.protocol.router.buckets[index].getNodes()
for node in nodes:
if node.id == node_to_find.id:
return defer.succeed(node)
nearest = self.protocol.router.findNeighbors(node_to_find)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
return defer.succeed(None)
spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
return spider.find().addCallback(check_for_node)
def _anyRespondSuccess(self, responses):
"""
Given the result of a DeferredList of calls to peers, ensure that at least
one of them was contacted and responded with a Truthy result.
"""
for deferSuccess, result in responses:
peerReached, peerResponse = result
if deferSuccess and peerReached and peerResponse:
return True
return False
def saveState(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
data = {'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.id,
'signed_pubkey': self.node.signed_pubkey,
'neighbors': self.bootstrappableNeighbors()}
if len(data['neighbors']) == 0:
self.log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'w') as f:
pickle.dump(data, f)
@classmethod
def loadState(self, fname, ip_address, port, multiplexer, storage=None):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
with open(fname, 'r') as f:
data = pickle.load(f)
n = Node(data['id'], ip_address, port, data['signed_pubkey'])
s = Server(n, data['ksize'], data['alpha'], storage=storage)
s.protocol.connect_multiplexer(multiplexer)
if len(data['neighbors']) > 0:
s.bootstrap(data['neighbors'])
return s
def saveStateRegularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
filename.
Args:
fname: File name to save retularly to
frequencey: Frequency in seconds that the state should be saved.
By default, 10 minutes.
"""
loop = LoopingCall(self.saveState, fname)
loop.start(frequency)
return loop
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, node, db, ksize=20, alpha=3, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
node: The node instance for this peer. It must contain (at minimum) an ID,
public key, ip address, and port.
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
storage: An instance that implements :interface:`~dht.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = node
self.protocol = KademliaProtocol(self.node, self.storage, ksize, db)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling::
reactor.listenUDP(port, server.protocol)
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for rid in self.protocol.getRefreshIDs():
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for keyword in self.storage.iterkeys():
for k, v in self.storage.iteritems(keyword):
if self.storage.get_ttl(keyword, k) < 601200:
ds.append(self.set(keyword, k, v))
return defer.gatherResults(ds).addCallback(republishKeys)
def querySeed(self, seed, pubkey):
"""
Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.
Args:
seed: A `string` consisting of "ip:port" or "hostname:port"
pubkey: The hex encoded public key to verify the signature on the response
"""
try:
nodes = []
c = httplib.HTTPConnection(seed)
c.request("GET", "/")
response = c.getresponse()
self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason))
data = response.read()
reread_data = data.decode("zlib")
proto = peers.PeerSeeds()
proto.ParseFromString(reread_data)
for peer in proto.peer_data:
p = peers.PeerData()
p.ParseFromString(peer)
tup = (str(p.ip_address), p.port)
nodes.append(tup)
verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder)
verify_key.verify("".join(proto.peer_data), proto.signature)
return nodes
except Exception, e:
self.log.error("Failed to query seed: %s" % str(e))
class KademliaProtocolTest(unittest.TestCase):
def setUp(self):
self.public_ip = '123.45.67.89'
self.port = 12345
self.own_addr = (self.public_ip, self.port)
self.addr1 = ('132.54.76.98', 54321)
self.addr2 = ('231.76.45.89', 15243)
self.clock = task.Clock()
connection.REACTOR.callLater = self.clock.callLater
self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
self.handler_mock = mock.Mock(spec_set=connection.Handler)
self.con = connection.Connection(
self.proto_mock,
self.handler_mock,
self.own_addr,
self.addr1
)
valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
verify_key = self.signing_key.verify_key
signed_pubkey = self.signing_key.sign(str(verify_key))
h = nacl.hash.sha512(signed_pubkey)
self.storage = ForgetfulStorage()
self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, signed_pubkey, True)
self.protocol = KademliaProtocol(self.node, self.storage, 20)
self.handler = self.protocol.RPCHandler(False, 5, self.protocol._outstanding, self.protocol)
self.handler.connection = self.con
transport = mock.Mock(spec_set=udp.Port)
ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
self.protocol.makeConnection(transport)
def tearDown(self):
self.con.shutdown()
self.protocol.shutdown()
def test_invalid_datagram(self):
self.assertFalse(self.handler.receive_message("hi"))
self.assertFalse(self.handler.receive_message("hihihihihihihihihihihihihihihihihihihihih"))
def test_rpc_ping(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("PING")
data = m.SerializeToString()
m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_store(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.extend(["Keyword", "Key", self.protocol.sourceNode.getProto().SerializeToString()])
data = m.SerializeToString()
for i in range(0, 3):
del m.arguments[-1]
m.arguments.append("True")
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())
def test_rpc_delete(self):
self._connecting_to_connected()
# Set a keyword to store
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.extend(["Keyword", "Key", self.protocol.sourceNode.getProto().SerializeToString()])
data = m.SerializeToString()
for i in range(0, 3):
del m.arguments[-1]
m.arguments.append("True")
expected_message1 = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())
# Test bad signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.arguments.extend(["Keyword", "Key", "Bad Signature"])
data = m.SerializeToString()
for i in range(0, 3):
del m.arguments[-1]
m.arguments.append("False")
expected_message2 = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list
)
self.assertEqual(sent_packets[0].payload, expected_message1)
self.assertEqual(sent_packets[1].payload, expected_message2)
self.proto_mock.send_datagram.call_args_list = []
# Test good signature
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("DELETE")
m.arguments.extend(["Keyword", "Key", self.signing_key.sign("Key")[:64]])
data = m.SerializeToString()
for i in range(0, 3):
del m.arguments[-1]
m.arguments.append("True")
expected_message3 = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
self.assertEqual(sent_packet.payload, expected_message3)
self.assertTrue(self.storage.getSpecific("Keyword", "Key") is None)
def test_rpc_stun(self):
self._connecting_to_connected()
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STUN")
data = m.SerializeToString()
m.arguments.extend([self.addr1[0], str(self.addr1[1])])
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_node(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_NODE")
m.arguments.append(digest("nodetofind"))
data = m.SerializeToString()
del m.arguments[-1]
m.arguments.extend([node3.getProto().SerializeToString(), node2.getProto().SerializeToString(), node1.getProto().SerializeToString()])
expected_message = m.SerializeToString()
self.handler.receive_message(data)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_rpc_find_value(self):
self._connecting_to_connected()
# Set a value to find
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.extend(["Keyword", "Key", self.protocol.sourceNode.getProto().SerializeToString()])
data = m.SerializeToString()
self.handler.receive_message(data)
self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())
# Send the find_value rpc
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.arguments.append("Keyword")
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
value = message.Value()
value.valueKey = "Key"
value.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
m.arguments.append("value")
m.arguments.append(value.SerializeToString())
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packets = tuple(
packet.Packet.from_bytes(call[0][0])
for call in self.proto_mock.send_datagram.call_args_list
)
received_message = sent_packets[1].payload
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 3)
def test_rpc_find_without_value(self):
self._connecting_to_connected()
node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
m = message.Message()
m.messageID = digest("msgid")
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("FIND_VALUE")
m.arguments.append(digest("Keyword"))
data = m.SerializeToString()
self.handler.receive_message(data)
del m.arguments[-1]
m.arguments.extend([node2.getProto().SerializeToString(), node3.getProto().SerializeToString(), node1.getProto().SerializeToString()])
expected_message = m.SerializeToString()
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
m_calls = self.proto_mock.send_datagram.call_args_list
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
received_message = sent_packet.payload
m = message.Message()
m.ParseFromString(received_message)
self.assertEqual(received_message, expected_message)
self.assertEqual(len(m_calls), 2)
def test_callPing(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.protocol[self.addr1] = self.con
self.protocol.callPing(n)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.PING)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
def test_callStore(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.protocol[self.addr1] = self.con
self.protocol.callStore(n, digest("Keyword"), digest("Key"), self.protocol.sourceNode.getProto().SerializeToString())
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.STORE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], self.protocol.sourceNode.getProto().SerializeToString())
def test_callFindValue(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.protocol[self.addr1] = self.con
keyword = Node(digest("Keyword"))
self.protocol.callFindValue(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.FIND_VALUE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callFindNode(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.protocol[self.addr1] = self.con
keyword = Node(digest("nodetofind"))
self.protocol.callFindNode(n, keyword)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.FIND_NODE)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertEqual(m.arguments[0], keyword.id)
def test_callDelete(self):
self._connecting_to_connected()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.protocol[self.addr1] = self.con
self.protocol.callDelete(n, digest("Keyword"), digest("Key"), digest("Signature"))
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
self.assertTrue(len(m.messageID) == 20)
self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
self.assertTrue(m.command == message.DELETE)
self.assertEqual(m.arguments[0], digest("Keyword"))
self.assertEqual(m.arguments[1], digest("Key"))
self.assertEqual(m.arguments[2], digest("Signature"))
def test_acceptResponse(self):
self._connecting_to_connected()
def handle_response(resp):
self.assertTrue(resp[0])
self.assertEqual(resp[1][0], self.protocol.sourceNode.id)
n = Node(digest("S"), self.addr1[0], self.addr1[1])
self.protocol[self.addr1] = self.con
d = self.protocol.callPing(n)
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
m = message.Message()
m.ParseFromString(sent_message)
timeout = reactor.callLater(5, self.protocol._timeout, m.messageID)
self.handler._outstanding[m.messageID] = (d, timeout)
m.arguments.append(self.protocol.sourceNode.id)
self.handler.receive_message(m.SerializeToString())
return d.addCallback(handle_response)
def test_unknownRPC(self):
self.assertFalse(self.handler._acceptRequest(digest("msgid"), "unknown", [digest("argument")], Node(digest("nodeid"))))
def test_timeout(self):
self._connecting_to_connected()
self.protocol[self.addr1] = self.con
def test_remove_outstanding():
self.assertTrue(len(self.protocol._outstanding) == 0)
def test_deffered(d):
self.assertFalse(d[0])
test_remove_outstanding()
n = Node(digest("S"), self.addr1[0], self.addr1[1])
d = self.protocol.callPing(n)
self.clock.advance(6)
return d.addCallback(test_deffered)
def test_transferKeyValues(self):
self._connecting_to_connected()
self.protocol[self.addr1] = self.con
self.protocol.storage[digest("keyword")] = (digest("key"), self.protocol.sourceNode.getProto().SerializeToString())
self.protocol.transferKeyValues(Node(digest("id"), self.addr1[0], self.addr1[1]))
self.clock.advance(1)
connection.REACTOR.runUntilCurrent()
sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
sent_message = sent_packet.payload
x = message.Message()
x.ParseFromString(sent_message)
m = message.Message()
m.sender.MergeFrom(self.protocol.sourceNode.getProto())
m.command = message.Command.Value("STORE")
m.arguments.append(digest("keyword"))
m.arguments.append(digest("key"))
m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
self.assertEqual(x.sender, m.sender)
self.assertEqual(x.command, m.command)
self.assertEqual(x.arguments[0], m.arguments[0])
self.assertEqual(x.arguments[1], m.arguments[1])
self.assertEqual(x.arguments[2], m.arguments[2])
def test_refreshIDs(self):
node1 = Node(digest("id1"), "127.0.0.1", 12345, signed_pubkey=digest("key1"))
node2 = Node(digest("id2"), "127.0.0.1", 22222, signed_pubkey=digest("key2"))
node3 = Node(digest("id3"), "127.0.0.1", 77777, signed_pubkey=digest("key3"))
self.protocol.router.addContact(node1)
self.protocol.router.addContact(node2)
self.protocol.router.addContact(node3)
for b in self.protocol.router.buckets:
b.lastUpdated = (time.time() - 5000)
ids = self.protocol.getRefreshIDs()
self.assertTrue(len(ids) == 1)
def _connecting_to_connected(self):
remote_synack_packet = packet.Packet.from_data(
42,
self.con.own_addr,
self.con.dest_addr,
ack=0,
syn=True
)
self.con.receive_packet(remote_synack_packet)
self.clock.advance(0)
connection.REACTOR.runUntilCurrent()
self.next_remote_seqnum = 43
m_calls = self.proto_mock.send_datagram.call_args_list
sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
seqnum = sent_syn_packet.sequence_number
self.handler_mock.reset_mock()
self.proto_mock.reset_mock()
self.next_seqnum = seqnum + 1