Esempio n. 1
0
class KademliaProtocolTest(unittest.TestCase):
    def setUp(self):
        self.version = PROTOCOL_VERSION
        self.public_ip = '123.45.67.89'
        self.port = 12345
        self.own_addr = (self.public_ip, self.port)
        self.addr1 = ('132.54.76.98', 54321)
        self.addr2 = ('231.76.45.89', 15243)

        self.clock = task.Clock()
        connection.REACTOR.callLater = self.clock.callLater

        self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
        self.handler_mock = mock.Mock(spec_set=connection.Handler)
        self.con = connection.Connection(
            self.proto_mock,
            self.handler_mock,
            self.own_addr,
            self.addr1
        )

        valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
        self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
        verify_key = self.signing_key.verify_key
        signed_pubkey = self.signing_key.sign(str(verify_key))
        h = nacl.hash.sha512(signed_pubkey)
        self.storage = ForgetfulStorage()
        self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, signed_pubkey, True)
        self.db = datastore.Database(filepath="test.db")
        self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db)

        self.wire_protocol = OpenBazaarProtocol(self.own_addr, "Full Cone")
        self.wire_protocol.register_processor(self.protocol)

        self.protocol.connect_multiplexer(self.wire_protocol)
        self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol)
        self.handler.connection = self.con

        transport = mock.Mock(spec_set=udp.Port)
        ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
        transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
        self.wire_protocol.makeConnection(transport)

    def tearDown(self):
        if self.con.state != connection.State.SHUTDOWN:
            self.con.shutdown()
        self.wire_protocol.shutdown()
        os.remove("test.db")

    def test_invalid_datagram(self):
        self.assertFalse(self.handler.receive_message("hi"))
        self.assertFalse(self.handler.receive_message("hihihihihihihihihihihihihihihihihihihihih"))

    def test_rpc_ping(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("PING")
        m.protoVer = self.version
        m.testnet = False
        data = m.SerializeToString()
        m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_store(self):
        self._connecting_to_connected()
        self.protocol.router.addContact(self.protocol.sourceNode)

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key",
                            self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
        data = m.SerializeToString()
        del m.arguments[-4:]
        m.arguments.append("True")
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

    def test_bad_rpc_store(self):
        r = self.protocol.rpc_store(self.node, 'testkeyword', 'kw', 'val', 10)
        self.assertEqual(r, ['False'])

    def test_rpc_delete(self):
        self._connecting_to_connected()
        self.protocol.router.addContact(self.protocol.sourceNode)

        # Set a keyword to store
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key",
                            self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
        data = m.SerializeToString()
        del m.arguments[-4:]
        m.arguments.append("True")
        expected_message1 = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        # Test bad signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key", "Bad Signature"])
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("False")
        expected_message2 = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list
        )
        self.assertEqual(sent_packets[0].payload, expected_message1)
        self.assertEqual(sent_packets[1].payload, expected_message2)
        self.proto_mock.send_datagram.call_args_list = []

        # Test good signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key", self.signing_key.sign("Key")[:64]])
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("True")
        expected_message3 = m.SerializeToString()
        self.handler.receive_message(data)
        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        self.assertEqual(sent_packet.payload, expected_message3)
        self.assertTrue(self.storage.getSpecific(digest("Keyword"), "Key") is None)

    def test_rpc_stun(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STUN")
        m.protoVer = self.version
        m.testnet = False
        data = m.SerializeToString()
        m.arguments.extend([self.public_ip, str(self.port)])
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        a = message.Message()
        a.ParseFromString(received_message)
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_node(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_NODE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.append(digest("nodetofind"))
        data = m.SerializeToString()
        del m.arguments[-1]
        m.arguments.extend([node2.getProto().SerializeToString(), node1.getProto().SerializeToString(),
                            node3.getProto().SerializeToString()])
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        a = message.Message()
        a.ParseFromString(received_message)
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_value(self):
        self._connecting_to_connected()
        self.protocol.router.addContact(self.protocol.sourceNode)

        # Set a value to find
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.protoVer = self.version
        m.arguments.extend([digest("Keyword"), "Key",
                            self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
        data = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        # Send the find_value rpc
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.append(digest("Keyword"))
        data = m.SerializeToString()
        self.handler.receive_message(data)

        del m.arguments[-1]
        value = objects.Value()
        value.valueKey = "Key"
        value.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
        value.ttl = 10
        m.arguments.append("value")
        m.arguments.append(value.SerializeToString())
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list
        )
        received_message = sent_packets[1].payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 3)

    def test_rpc_find_without_value(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.append(digest("Keyword"))
        data = m.SerializeToString()
        self.handler.receive_message(data)

        del m.arguments[-1]
        m.arguments.extend([node3.getProto().SerializeToString(), node1.getProto().SerializeToString(),
                            node2.getProto().SerializeToString()])
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        m = message.Message()
        m.ParseFromString(received_message)

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_callPing(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callPing(n)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.PING)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)

    def test_callStore(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callStore(n, digest("Keyword"), digest("Key"),
                                self.protocol.sourceNode.getProto().SerializeToString(), 10)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.STORE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(m.arguments[2], self.protocol.sourceNode.getProto().SerializeToString())

    def test_callFindValue(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        keyword = Node(digest("Keyword"))
        self.protocol.callFindValue(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.FIND_VALUE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callFindNode(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        keyword = Node(digest("nodetofind"))
        self.protocol.callFindNode(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.FIND_NODE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callDelete(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callDelete(n, digest("Keyword"), digest("Key"), digest("Signature"))

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.DELETE)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(m.arguments[2], digest("Signature"))

    def test_acceptResponse(self):
        self._connecting_to_connected()

        def handle_response(resp):
            self.assertTrue(resp[0])
            self.assertEqual(resp[1][0], "test")
            self.assertTrue(message_id not in self.protocol._outstanding)

        message_id = digest("msgid")
        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        d = defer.Deferred()
        self.protocol._outstanding[message_id] = (d, self.addr1, reactor.callLater(5, handle_response))
        self.protocol._acceptResponse(message_id, ["test"], n)

        return d.addCallback(handle_response)

    def test_unknownRPC(self):
        self.assertFalse(self.handler.receive_message(str(random.getrandbits(1400))))

    def test_timeout(self):

        def handle_response(resp, n):
            self.assertFalse(resp[0])
            self.assertIsNone(resp[1])
            self.assertTrue(self.protocol.router.isNewNode(n))

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        d = defer.Deferred().addCallback(handle_response, n)
        self.protocol._outstanding["msgID"] = [d, self.addr1]
        self.protocol.router.addContact(n)
        self.protocol.timeout(self.addr1, n)

    def test_transferKeyValues(self):
        self._connecting_to_connected()
        self.wire_protocol[self.addr1] = self.con

        self.protocol.router.addContact(mknode())

        self.protocol.storage[digest("keyword")] = (
            digest("key"), self.protocol.sourceNode.getProto().SerializeToString(), 10)
        self.protocol.storage[digest("keyword")] = (
            digest("key2"), self.protocol.sourceNode.getProto().SerializeToString(), 10)

        self.protocol.transferKeyValues(Node(digest("id"), self.addr1[0], self.addr1[1]))

        self.clock.advance(1)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload
        x = message.Message()
        x.ParseFromString(sent_message)

        i = objects.Inv()
        i.keyword = digest("keyword")
        i.valueKey = digest("key")

        i2 = objects.Inv()
        i2.keyword = digest("keyword")
        i2.valueKey = digest("key2")

        m = message.Message()
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("INV")
        m.protoVer = self.version
        m.arguments.append(i.SerializeToString())
        m.arguments.append(i2.SerializeToString())
        self.assertEqual(x.sender, m.sender)
        self.assertEqual(x.command, m.command)
        self.assertTrue(x.arguments[0] in m.arguments)
        self.assertTrue(x.arguments[1] in m.arguments)

    def test_refreshIDs(self):
        node1 = Node(digest("id1"), "127.0.0.1", 12345, signed_pubkey=digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, signed_pubkey=digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, signed_pubkey=digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        for b in self.protocol.router.buckets:
            b.lastUpdated = (time.time() - 5000)
        ids = self.protocol.getRefreshIDs()
        self.assertTrue(len(ids) == 1)

    def _connecting_to_connected(self):
        remote_synack_packet = packet.Packet.from_data(
            42,
            self.con.own_addr,
            self.con.dest_addr,
            ack=0,
            syn=True
        )
        self.con.receive_packet(remote_synack_packet)

        self.clock.advance(0)
        connection.REACTOR.runUntilCurrent()

        self.next_remote_seqnum = 43

        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
        seqnum = sent_syn_packet.sequence_number

        self.handler_mock.reset_mock()
        self.proto_mock.reset_mock()

        self.next_seqnum = seqnum + 1

    def test_badRPCDelete(self):
        n = mknode()
        val = self.protocol.rpc_delete(n, 'testkeyword', 'key', 'testsig')
        self.assertEqual(val, ["False"])
        val = self.protocol.rpc_delete(n, '', '', '')
Esempio n. 2
0
class Server(object):
    """
    High level view of a node instance.  This is the object that should be created
    to start listening as an active node on the network.
    """

    def __init__(self, node, ksize=20, alpha=3, storage=None):
        """
        Create a server instance.  This will start listening on the given port.

        Args:
            node: The node instance for this peer. It must contain (at minimum) an ID,
                public key, ip address, and port.
            ksize (int): The k parameter from the paper
            alpha (int): The alpha parameter from the paper
            storage: An instance that implements :interface:`~dht.storage.IStorage`
        """
        self.ksize = ksize
        self.alpha = alpha
        self.log = Logger(system=self)
        self.storage = storage or ForgetfulStorage()
        self.node = node
        self.protocol = KademliaProtocol(self.node, self.storage, ksize)
        self.refreshLoop = LoopingCall(self.refreshTable).start(3600)

    def listen(self, port):
        """
        Start listening on the given port.

        This is the same as calling::

            reactor.listenUDP(port, server.protocol)
        """
        return reactor.listenUDP(port, self.protocol)

    def refreshTable(self):
        """
        Refresh buckets that haven't had any lookups in the last hour
        (per section 2.3 of the paper).
        """
        ds = []
        for rid in self.protocol.getRefreshIDs():
            node = Node(rid)
            nearest = self.protocol.router.findNeighbors(node, self.alpha)
            spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
            ds.append(spider.find())

        def republishKeys(_):
            ds = []
            # Republish keys older than one hour
            for keyword in self.storage.iterkeys():
                for k, v in self.storage.iteritems(keyword):
                    if self.storage.get_ttl(keyword, k) < 601200:
                        ds.append(self.set(keyword, k, v))

        return defer.gatherResults(ds).addCallback(republishKeys)

    def querySeed(self, seed, pubkey):
        """
        Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.

        Args:
           seed: A `string` consisting of "ip:port" or "hostname:port"
           pubkey: The hex encoded public key to verify the signature on the response
        """
        nodes = []
        c = httplib.HTTPConnection(seed)
        c.request("GET", "/")
        response = c.getresponse()
        self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason))
        data = response.read()
        reread_data = data.decode("zlib")
        proto = peers.PeerSeeds()
        try:
            proto.ParseFromString(reread_data)
            for peer in proto.peer_data:
                p = peers.PeerData()
                p.ParseFromString(peer)
                tup = (str(p.ip_address), p.port)
                nodes.append(tup)
            verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder)
            verify_key.verify(proto.signature + "".join(proto.peer_data))
            return nodes
        except Exception:
            self.log.error("Error parsing seed response.")
        return nodes

    def bootstrappableNeighbors(self):
        """
        Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument
        to the bootstrap method.

        The server should have been bootstrapped
        already - this is just a utility for getting some neighbors and then
        storing them if this server is going down for a while.  When it comes
        back up, the list of nodes can be used to bootstrap.
        """
        neighbors = self.protocol.router.findNeighbors(self.node)
        return [tuple(n)[-2:] for n in neighbors]

    def bootstrap(self, addrs):
        """
        Bootstrap the server by connecting to other known nodes in the network.

        Args:
            addrs: A `list` of (ip, port) `tuple` pairs.  Note that only IP addresses
                   are acceptable - hostnames will cause an error.
        """

        # if the transport hasn't been initialized yet, wait a second
        if self.protocol.multiplexer.transport is None:
            return task.deferLater(reactor, 1, self.bootstrap, addrs)

        def initTable(results):
            nodes = []
            for addr, result in results.items():
                if result[0]:
                    n = objects.Node()
                    try:
                        n.ParseFromString(result[1][0])
                        pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
                        verify_key = nacl.signing.VerifyKey(pubkey)
                        verify_key.verify(n.signedPublicKey)
                        h = nacl.hash.sha512(n.signedPublicKey)
                        hash_pow = h[64:128]
                        if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
                            raise Exception('Invalid GUID')
                        nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
                    except Exception:
                        self.log.msg("Bootstrap node returned invalid GUID")
            spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
            return spider.find()

        ds = {}
        for addr in addrs:
            ds[addr] = self.protocol.ping((addr[0], addr[1]))
        return deferredDict(ds).addCallback(initTable)

    def inetVisibleIP(self):
        """
        Get the internet visible IP's of this node as other nodes see it.

        Returns:
            A `list` of IP's.  If no one can be contacted, then the `list` will be empty.
        """

        def handle(results):
            ips = []
            for result in results:
                if result[0]:
                    ips.append((result[1][0], int(result[1][1])))
            self.log.debug("other nodes think our ip is %s" % str(ips))
            return ips

        ds = []
        for neighbor in self.bootstrappableNeighbors():
            ds.append(self.protocol.stun(neighbor))
        return defer.gatherResults(ds).addCallback(handle)

    def get(self, keyword):
        """
        Get a key if the network has it.

        Returns:
            :class:`None` if not found, the value otherwise.
        """
        dkey = digest(keyword)
        if self.storage.get(dkey) is not None:
            return defer.succeed(self.storage.get(dkey))
        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to get key %s" % keyword)
            return defer.succeed(None)
        spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find()

    def set(self, keyword, key, value):
        """
        Set the given key/value tuple at the hash of the given keyword.
        All values stored in the DHT are stored as dictionaries of key/value
        pairs. If a value already exists for a given keyword, the new key/value
        pair will be appended to the dictionary.

        Args:
            keyword: a `string` keyword. The SHA1 hash of which will be used as
                the key when inserting in the DHT.
            key: the 20 byte hash of the data.
            value: a serialized `protos.objects.Node` object which serves as a
                pointer to the node storing the data.

        Return: True if at least one peer responded. False if the store rpc
            completely failed.
        """
        self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value)))
        dkey = digest(keyword)

        def store(nodes):
            self.log.info("setting '%s' on %s" % (keyword, [str(i) for i in nodes]))
            ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes]

            keynode = Node(dkey)
            if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
                self.storage[dkey] = (key, value)
                self.log.debug("got a store request from %s, storing value" % str(self.node))

            return defer.DeferredList(ds).addCallback(_anyRespondSuccess)

        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to set key %s" % key)
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(store)

    def delete(self, keyword, key, signature):
        """
        Delete the given key/value pair from the keyword dictionary on the network.
        To delete you must provide a signature covering the key that you wish to
        delete. It will be verified against the public key stored in the value. We
        use our ksize as alpha to make sure we reach as many nodes storing our value
        as possible.

        Args:
            keyword: the `string` keyword where the data being deleted is stored.
            key: the 20 byte hash of the data.
            signature: a signature covering the key.

        """
        self.log.info("deleting '%s':'%s' from the network" % (keyword, hexlify(key)))
        dkey = digest(keyword)

        def delete(nodes):
            self.log.debug("deleting '%s' on %s" % (key, [str(i) for i in nodes]))
            ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes]

            if self.storage.getSpecific(dkey, key) is not None:
                self.storage.delete(dkey, key)

            return defer.DeferredList(ds).addCallback(_anyRespondSuccess)

        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to delete key %s" % key)
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize)
        return spider.find().addCallback(delete)

    def resolve(self, guid):
        """
        Given a guid return a `Node` object containing its ip and port or none if it's
        not found.

        Args:
            guid: the 20 raw bytes representing the guid.
        """
        node_to_find = Node(guid)

        def check_for_node(nodes):
            for node in nodes:
                if node.id == node_to_find.id:
                    return node
            return None
        index = self.protocol.router.getBucketFor(node_to_find)
        nodes = self.protocol.router.buckets[index].getNodes()
        for node in nodes:
            if node.id == node_to_find.id:
                return defer.succeed(node)
        nearest = self.protocol.router.findNeighbors(node_to_find)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
            return defer.succeed(None)
        spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(check_for_node)

    def saveState(self, fname):
        """
        Save the state of this node (the alpha/ksize/id/immediate neighbors)
        to a cache file with the given fname.
        """
        data = {'ksize': self.ksize,
                'alpha': self.alpha,
                'id': self.node.id,
                'vendor': self.node.vendor,
                'signed_pubkey': self.node.signed_pubkey,
                'neighbors': self.bootstrappableNeighbors()}
        if len(data['neighbors']) == 0:
            self.log.warning("No known neighbors, so not writing to cache.")
            return
        with open(fname, 'w') as f:
            pickle.dump(data, f)

    @classmethod
    def loadState(cls, fname, ip_address, port, multiplexer, callback=None, storage=None):
        """
        Load the state of this node (the alpha/ksize/id/immediate neighbors)
        from a cache file with the given fname.
        """
        with open(fname, 'r') as f:
            data = pickle.load(f)
        n = Node(data['id'], ip_address, port, data['signed_pubkey'], data['vendor'])
        s = Server(n, data['ksize'], data['alpha'], storage=storage)
        s.protocol.connect_multiplexer(multiplexer)
        if len(data['neighbors']) > 0:
            if callback is not None:
                s.bootstrap(data['neighbors']).addCallback(callback)
            else:
                s.bootstrap(data['neighbors'])
        return s

    def saveStateRegularly(self, fname, frequency=600):
        """
        Save the state of node with a given regularity to the given
        filename.

        Args:
            fname: File name to save retularly to
            frequencey: Frequency in seconds that the state should be saved.
                        By default, 10 minutes.
        """
        loop = LoopingCall(self.saveState, fname)
        loop.start(frequency)
        return loop
Esempio n. 3
0
class KademliaProtocolTest(unittest.TestCase):
    def setUp(self):
        self.version = PROTOCOL_VERSION
        self.public_ip = '123.45.67.89'
        self.port = 12345
        self.own_addr = (self.public_ip, self.port)
        self.addr1 = ('132.54.76.98', 54321)
        self.addr2 = ('231.76.45.89', 15243)

        self.clock = task.Clock()
        connection.REACTOR.callLater = self.clock.callLater

        self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
        self.handler_mock = mock.Mock(spec_set=connection.Handler)
        self.con = connection.Connection(
            self.proto_mock,
            self.handler_mock,
            self.own_addr,
            self.addr1
        )

        valid_key = "63d901c4d57cde34fc1f1e28b9af5d56ed342cae5c2fb470046d0130a4226b0c"
        self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
        verify_key = self.signing_key.verify_key
        h = nacl.hash.sha512(verify_key.encode())
        self.storage = ForgetfulStorage()
        self.node = Node(unhexlify(h[:40]), self.public_ip, self.port,
                         verify_key.encode(), None, objects.FULL_CONE, True)
        self.db = datastore.Database(filepath="test.db")
        self.protocol = KademliaProtocol(self.node, self.storage, 20, self.db, self.signing_key)

        self.wire_protocol = OpenBazaarProtocol(self.db, self.own_addr, objects.FULL_CONE)
        self.wire_protocol.register_processor(self.protocol)

        self.protocol.connect_multiplexer(self.wire_protocol)
        self.handler = self.wire_protocol.ConnHandler([self.protocol], self.wire_protocol, None)
        self.handler.connection = self.con

        transport = mock.Mock(spec_set=udp.Port)
        ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
        transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
        self.wire_protocol.makeConnection(transport)

    def tearDown(self):
        if self.con.state != connection.State.SHUTDOWN:
            self.con.shutdown()
        self.wire_protocol.shutdown()
        os.remove("test.db")

    def test_invalid_datagram(self):
        self.assertFalse(self.handler.receive_message("hi"))
        self.assertFalse(self.handler.receive_message("hihihihihihihihihihihihihihihihihihihihih"))

    def test_rpc_ping(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("PING")
        m.protoVer = self.version
        m.testnet = False
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
        m.ClearField("signature")
        expected_message = m.SerializeToString()
        self.handler.on_connection_made()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        m2 = message.Message()
        m2.ParseFromString(received_message)
        m2.ClearField("signature")
        received_message = m2.SerializeToString()

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_store(self):
        self._connecting_to_connected()
        self.protocol.router.addContact(self.protocol.sourceNode)

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key",
                            self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        del m.arguments[-4:]
        m.arguments.append("True")
        m.ClearField("signature")
        expected_message = m.SerializeToString()
        self.handler.on_connection_made()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        m2 = message.Message()
        m2.ParseFromString(received_message)
        m2.ClearField("signature")
        received_message = m2.SerializeToString()
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

    def test_bad_rpc_store(self):
        r = self.protocol.rpc_store(self.node, 'testkeyword', 'kw', 'val', 10)
        self.assertEqual(r, ['False'])

    def test_rpc_delete(self):
        self._connecting_to_connected()
        self.protocol.router.addContact(self.protocol.sourceNode)

        # Set a keyword to store
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key",
                            self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        del m.arguments[-4:]
        m.arguments.append("True")
        m.ClearField("signature")
        expected_message1 = m.SerializeToString()
        self.handler.on_connection_made()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        # Test bad signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key", "Bad Signature"])
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("False")
        m.ClearField("signature")
        expected_message2 = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list
        )
        m2 = message.Message()
        m2.ParseFromString(sent_packets[0].payload)
        m2.ClearField("signature")
        received_message1 = m2.SerializeToString()
        m3 = message.Message()
        m3.ParseFromString(sent_packets[1].payload)
        m3.ClearField("signature")
        received_message2 = m3.SerializeToString()
        self.assertEqual(received_message1, expected_message1)
        self.assertEqual(received_message2, expected_message2)
        self.proto_mock.send_datagram.call_args_list = []

        # Test good signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.extend([digest("Keyword"), "Key", self.signing_key.sign("Key")[:64]])
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("True")
        m.ClearField("signature")
        expected_message3 = m.SerializeToString()
        self.handler.receive_message(data)
        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        m4 = message.Message()
        m4.ParseFromString(sent_packet.payload)
        m4.ClearField("signature")
        received_message = m4.SerializeToString()
        self.assertEqual(received_message, expected_message3)
        self.assertTrue(self.storage.getSpecific(digest("Keyword"), "Key") is None)

    def test_rpc_stun(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STUN")
        m.protoVer = self.version
        m.testnet = False
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        m.arguments.extend([self.public_ip, str(self.port)])
        m.ClearField("signature")
        expected_message = m.SerializeToString()
        self.handler.on_connection_made()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        a = message.Message()
        a.ParseFromString(received_message)
        a.ClearField("signature")
        received_message = a.SerializeToString()
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_node(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"), nat_type=objects.FULL_CONE)
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"), nat_type=objects.FULL_CONE)
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"), nat_type=objects.FULL_CONE)
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_NODE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.append(digest("nodetofind"))
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        del m.arguments[-1]
        m.arguments.extend([node2.getProto().SerializeToString(), node1.getProto().SerializeToString(),
                            node3.getProto().SerializeToString()])
        m.ClearField("signature")
        expected_message = m.SerializeToString()
        self.handler.on_connection_made()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        a = message.Message()
        a.ParseFromString(received_message)
        a.ClearField("signature")
        received_message = a.SerializeToString()
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_value(self):
        self._connecting_to_connected()
        self.protocol.router.addContact(self.protocol.sourceNode)

        # Set a value to find
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.protoVer = self.version
        m.arguments.extend([digest("Keyword"), "Key",
                            self.protocol.sourceNode.getProto().SerializeToString(), str(10)])
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        self.handler.on_connection_made()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        # Send the find_value rpc
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.append(digest("Keyword"))
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        self.handler.receive_message(data)

        del m.arguments[-1]
        value = objects.Value()
        value.valueKey = "Key"
        value.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
        value.ttl = 10
        m.arguments.append("value")
        m.arguments.append(value.SerializeToString())
        m.ClearField("signature")
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list
        )
        received_message = sent_packets[1].payload
        a = message.Message()
        a.ParseFromString(received_message)
        a.ClearField("signature")
        received_message = a.SerializeToString()

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 3)

    def test_rpc_find_without_value(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"), nat_type=objects.FULL_CONE)
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"), nat_type=objects.FULL_CONE)
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"), nat_type=objects.FULL_CONE)
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.protoVer = self.version
        m.testnet = False
        m.arguments.append(digest("Keyword"))
        m.signature = self.signing_key.sign(m.SerializeToString())[:64]
        data = m.SerializeToString()
        self.handler.on_connection_made()
        self.handler.receive_message(data)

        del m.arguments[-1]
        m.arguments.extend([node3.getProto().SerializeToString(), node1.getProto().SerializeToString(),
                            node2.getProto().SerializeToString()])
        m.ClearField("signature")
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        a = message.Message()
        a.ParseFromString(received_message)
        a.ClearField("signature")
        received_message = a.SerializeToString()

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_callPing(self):
        self._connecting_to_connected()

        n = Node(digest("guid"), self.addr1[0], self.addr1[1], digest("pubkey"), None, objects.FULL_CONE, False)
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callPing(n)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
        self.assertTrue(m.command == message.PING)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)

    def test_callStore(self):
        self._connecting_to_connected()

        n = Node(digest("guid"), self.addr1[0], self.addr1[1], digest("pubkey"), None, objects.FULL_CONE, False)
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callStore(n, digest("Keyword"), digest("Key"),
                                self.protocol.sourceNode.getProto().SerializeToString(), 10)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
        self.assertTrue(m.command == message.STORE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(m.arguments[2], self.protocol.sourceNode.getProto().SerializeToString())

    def test_callFindValue(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        keyword = Node(digest("Keyword"))
        self.protocol.callFindValue(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
        self.assertTrue(m.command == message.FIND_VALUE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callFindNode(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        keyword = Node(digest("nodetofind"))
        self.protocol.callFindNode(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
        self.assertTrue(m.command == message.FIND_NODE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callDelete(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callDelete(n, digest("Keyword"), digest("Key"), digest("Signature"))

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().publicKey, m.sender.publicKey)
        self.assertTrue(m.command == message.DELETE)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(m.arguments[2], digest("Signature"))

    def test_acceptResponse(self):
        self._connecting_to_connected()

        def handle_response(resp):
            self.assertTrue(resp[0])
            self.assertEqual(resp[1][0], "test")
            self.assertTrue(message_id not in self.protocol._outstanding)

        message_id = digest("msgid")
        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        d = defer.Deferred()
        self.protocol._outstanding[message_id] = (d, self.addr1, reactor.callLater(5, handle_response))
        self.protocol._acceptResponse(message_id, ["test"], n)

        return d.addCallback(handle_response)

    def test_unknownRPC(self):
        self.assertFalse(self.handler.receive_message(str(random.getrandbits(1400))))

    def test_timeout(self):

        def handle_response(resp, n):
            self.assertFalse(resp[0])
            self.assertIsNone(resp[1])

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        d = defer.Deferred().addCallback(handle_response, n)
        self.protocol._outstanding["msgID"] = [d, self.addr1, reactor.callLater(5, handle_response)]
        self.protocol.router.addContact(n)
        self.protocol.timeout(n)

    def test_transferKeyValues(self):
        self._connecting_to_connected()
        self.wire_protocol[self.addr1] = self.con

        self.protocol.storage[digest("keyword")] = (
            digest("key"), self.protocol.sourceNode.getProto().SerializeToString(), 10)
        self.protocol.storage[digest("keyword")] = (
            digest("key2"), self.protocol.sourceNode.getProto().SerializeToString(), 10)

        self.protocol.transferKeyValues(Node(digest("id"), self.addr1[0], self.addr1[1]))

        self.clock.advance(1)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload
        x = message.Message()
        x.ParseFromString(sent_message)

        i = objects.Inv()
        i.keyword = digest("keyword")
        i.valueKey = digest("key")

        i2 = objects.Inv()
        i2.keyword = digest("keyword")
        i2.valueKey = digest("key2")

        m = message.Message()
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("INV")
        m.protoVer = self.version
        m.arguments.append(i.SerializeToString())
        m.arguments.append(i2.SerializeToString())
        self.assertEqual(x.sender.guid, m.sender.guid)
        self.assertEqual(x.command, m.command)
        self.assertTrue(x.arguments[0] in m.arguments)
        self.assertTrue(x.arguments[1] in m.arguments)

    def test_refreshIDs(self):
        node1 = Node(digest("id1"), "127.0.0.1", 12345, pubkey=digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, pubkey=digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, pubkey=digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        for b in self.protocol.router.buckets:
            b.lastUpdated = (time.time() - 5000)
        ids = self.protocol.getRefreshIDs()
        self.assertTrue(len(ids) == 1)

    def _connecting_to_connected(self):
        remote_synack_packet = packet.Packet.from_data(
            42,
            self.con.own_addr,
            self.con.dest_addr,
            ack=0,
            syn=True
        )
        self.con.receive_packet(remote_synack_packet, self.addr1)

        self.clock.advance(0)
        connection.REACTOR.runUntilCurrent()

        self.next_remote_seqnum = 43

        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
        seqnum = sent_syn_packet.sequence_number

        self.handler_mock.reset_mock()
        self.proto_mock.reset_mock()

        self.next_seqnum = seqnum + 1

    def test_badRPCDelete(self):
        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        val = self.protocol.rpc_delete(n, 'testkeyword', 'key', 'testsig')
        self.assertEqual(val, ["False"])
        val = self.protocol.rpc_delete(n, '', '', '')
class KademliaProtocolTest(unittest.TestCase):
    def setUp(self):
        self.public_ip = '123.45.67.89'
        self.port = 12345
        self.own_addr = (self.public_ip, self.port)
        self.addr1 = ('132.54.76.98', 54321)
        self.addr2 = ('231.76.45.89', 15243)

        self.clock = task.Clock()
        connection.REACTOR.callLater = self.clock.callLater

        self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
        self.handler_mock = mock.Mock(spec_set=connection.Handler)
        self.con = connection.Connection(self.proto_mock, self.handler_mock,
                                         self.own_addr, self.addr1)

        valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
        self.signing_key = nacl.signing.SigningKey(
            valid_key, encoder=nacl.encoding.HexEncoder)
        verify_key = self.signing_key.verify_key
        signed_pubkey = self.signing_key.sign(str(verify_key))
        h = nacl.hash.sha512(signed_pubkey)
        self.storage = ForgetfulStorage()
        self.node = Node(unhexlify(h[:40]), self.public_ip, self.port,
                         signed_pubkey, True)
        self.protocol = KademliaProtocol(self.node, self.storage, 20)

        self.wire_protocol = OpenBazaarProtocol(self.own_addr)
        self.wire_protocol.register_processor(self.protocol)

        self.protocol.connect_multiplexer(self.wire_protocol)
        self.handler = self.wire_protocol.ConnHandler([self.protocol])
        self.handler.connection = self.con

        transport = mock.Mock(spec_set=udp.Port)
        ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
        transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
        self.wire_protocol.makeConnection(transport)

    def tearDown(self):
        self.con.shutdown()
        self.wire_protocol.shutdown()

    def test_invalid_datagram(self):
        self.assertFalse(self.handler.receive_message("hi"))
        self.assertFalse(
            self.handler.receive_message(
                "hihihihihihihihihihihihihihihihihihihihih"))

    def test_rpc_ping(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("PING")
        data = m.SerializeToString()
        m.arguments.append(
            self.protocol.sourceNode.getProto().SerializeToString())
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_store(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.extend([
            digest("Keyword"), "Key",
            self.protocol.sourceNode.getProto().SerializeToString()
        ])
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("True")
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

    def test_rpc_delete(self):
        self._connecting_to_connected()

        # Set a keyword to store
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.extend([
            digest("Keyword"), "Key",
            self.protocol.sourceNode.getProto().SerializeToString()
        ])
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("True")
        expected_message1 = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        # Test bad signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.arguments.extend([digest("Keyword"), "Key", "Bad Signature"])
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("False")
        expected_message2 = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list)
        self.assertEqual(sent_packets[0].payload, expected_message1)
        self.assertEqual(sent_packets[1].payload, expected_message2)
        self.proto_mock.send_datagram.call_args_list = []

        # Test good signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.arguments.extend(
            [digest("Keyword"), "Key",
             self.signing_key.sign("Key")[:64]])
        data = m.SerializeToString()
        del m.arguments[-3:]
        m.arguments.append("True")
        expected_message3 = m.SerializeToString()
        self.handler.receive_message(data)
        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        self.assertEqual(sent_packet.payload, expected_message3)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") is None)

    def test_rpc_stun(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STUN")
        data = m.SerializeToString()
        m.arguments.extend([self.public_ip, str(self.port)])
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        a = message.Message()
        a.ParseFromString(received_message)
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_node(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_NODE")
        m.arguments.append(digest("nodetofind"))
        data = m.SerializeToString()
        del m.arguments[-1]
        m.arguments.extend([
            node2.getProto().SerializeToString(),
            node1.getProto().SerializeToString(),
            node3.getProto().SerializeToString()
        ])
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        a = message.Message()
        a.ParseFromString(received_message)
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_value(self):
        self._connecting_to_connected()

        # Set a value to find
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.extend([
            digest("Keyword"), "Key",
            self.protocol.sourceNode.getProto().SerializeToString()
        ])
        data = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(
            self.storage.getSpecific(digest("Keyword"), "Key") ==
            self.protocol.sourceNode.getProto().SerializeToString())

        # Send the find_value rpc
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.arguments.append(digest("Keyword"))
        data = m.SerializeToString()
        self.handler.receive_message(data)

        del m.arguments[-1]
        value = objects.Value()
        value.valueKey = "Key"
        value.serializedData = self.protocol.sourceNode.getProto(
        ).SerializeToString()
        m.arguments.append("value")
        m.arguments.append(value.SerializeToString())
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list)
        received_message = sent_packets[1].payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 3)

    def test_rpc_find_without_value(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.arguments.append(digest("Keyword"))
        data = m.SerializeToString()
        self.handler.receive_message(data)

        del m.arguments[-1]
        m.arguments.extend([
            node3.getProto().SerializeToString(),
            node1.getProto().SerializeToString(),
            node2.getProto().SerializeToString()
        ])
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        m = message.Message()
        m.ParseFromString(received_message)

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_callPing(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callPing(n)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid,
                         m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
                         m.sender.signedPublicKey)
        self.assertTrue(m.command == message.PING)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
                         self.addr1)

    def test_callStore(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callStore(
            n, digest("Keyword"), digest("Key"),
            self.protocol.sourceNode.getProto().SerializeToString())

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid,
                         m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
                         m.sender.signedPublicKey)
        self.assertTrue(m.command == message.STORE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
                         self.addr1)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(
            m.arguments[2],
            self.protocol.sourceNode.getProto().SerializeToString())

    def test_callFindValue(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        keyword = Node(digest("Keyword"))
        self.protocol.callFindValue(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid,
                         m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
                         m.sender.signedPublicKey)
        self.assertTrue(m.command == message.FIND_VALUE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
                         self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callFindNode(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        keyword = Node(digest("nodetofind"))
        self.protocol.callFindNode(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid,
                         m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
                         m.sender.signedPublicKey)
        self.assertTrue(m.command == message.FIND_NODE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
                         self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callDelete(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        self.protocol.callDelete(n, digest("Keyword"), digest("Key"),
                                 digest("Signature"))

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1],
                         self.addr1)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid,
                         m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey,
                         m.sender.signedPublicKey)
        self.assertTrue(m.command == message.DELETE)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(m.arguments[2], digest("Signature"))

    def test_acceptResponse(self):
        self._connecting_to_connected()

        def handle_response(resp):
            self.assertTrue(resp[0])
            self.assertEqual(resp[1][0], self.protocol.sourceNode.id)

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.wire_protocol[self.addr1] = self.con
        d = self.protocol.callPing(n)

        self.clock.advance(1)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        timeout = reactor.callLater(5, self.protocol._timeout, m.messageID)
        self.protocol._outstanding[m.messageID] = (d, timeout)
        m.arguments.append(self.protocol.sourceNode.id)
        self.handler.receive_message(m.SerializeToString())

        return d.addCallback(handle_response)

    def test_unknownRPC(self):
        self.assertFalse(
            self.handler.receive_message(str(random.getrandbits(1400))))

    def test_timeout(self):
        self._connecting_to_connected()
        self.wire_protocol[self.addr1] = self.con

        def test_remove_outstanding():
            self.assertTrue(len(self.protocol._outstanding) == 0)

        def test_deffered(d):
            self.assertFalse(d[0])
            test_remove_outstanding()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        d = self.protocol.callPing(n)
        self.clock.advance(6)
        connection.REACTOR.runUntilCurrent()
        self.clock.advance(6)
        return d.addCallback(test_deffered)

    def test_transferKeyValues(self):
        self._connecting_to_connected()
        self.wire_protocol[self.addr1] = self.con

        self.protocol.addToRouter(mknode())

        self.protocol.storage[digest("keyword")] = (
            digest("key"),
            self.protocol.sourceNode.getProto().SerializeToString())
        self.protocol.transferKeyValues(
            Node(digest("id"), self.addr1[0], self.addr1[1]))

        self.clock.advance(1)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(
            self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload
        x = message.Message()
        x.ParseFromString(sent_message)

        m = message.Message()
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.append(digest("keyword"))
        m.arguments.append(digest("key"))
        m.arguments.append(
            self.protocol.sourceNode.getProto().SerializeToString())
        self.assertEqual(x.sender, m.sender)
        self.assertEqual(x.command, m.command)
        self.assertEqual(x.arguments[0], m.arguments[0])
        self.assertEqual(x.arguments[1], m.arguments[1])
        self.assertEqual(x.arguments[2], m.arguments[2])

    def test_refreshIDs(self):
        node1 = Node(digest("id1"),
                     "127.0.0.1",
                     12345,
                     signed_pubkey=digest("key1"))
        node2 = Node(digest("id2"),
                     "127.0.0.1",
                     22222,
                     signed_pubkey=digest("key2"))
        node3 = Node(digest("id3"),
                     "127.0.0.1",
                     77777,
                     signed_pubkey=digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        for b in self.protocol.router.buckets:
            b.lastUpdated = (time.time() - 5000)
        ids = self.protocol.getRefreshIDs()
        self.assertTrue(len(ids) == 1)

    def _connecting_to_connected(self):
        remote_synack_packet = packet.Packet.from_data(42,
                                                       self.con.own_addr,
                                                       self.con.dest_addr,
                                                       ack=0,
                                                       syn=True)
        self.con.receive_packet(remote_synack_packet)

        self.clock.advance(0)
        connection.REACTOR.runUntilCurrent()

        self.next_remote_seqnum = 43

        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
        seqnum = sent_syn_packet.sequence_number

        self.handler_mock.reset_mock()
        self.proto_mock.reset_mock()

        self.next_seqnum = seqnum + 1

    def test_badRPCDelete(self):
        n = mknode()
        val = self.protocol.rpc_delete(n, 'testkeyword', 'key', 'testsig')
        self.assertEqual(val, ["False"])
        val = self.protocol.rpc_delete(n, '', '', '')
Esempio n. 5
0
class Server(object):
    """
    High level view of a node instance.  This is the object that should be created
    to start listening as an active node on the network.
    """

    def __init__(self, node, ksize=20, alpha=3, storage=None):
        """
        Create a server instance.  This will start listening on the given port.

        Args:
            node: The node instance for this peer. It must contain (at minimum) an ID,
                public key, ip address, and port.
            ksize (int): The k parameter from the paper
            alpha (int): The alpha parameter from the paper
            storage: An instance that implements :interface:`~dht.storage.IStorage`
        """
        self.ksize = ksize
        self.alpha = alpha
        self.log = Logger(system=self)
        self.storage = storage or ForgetfulStorage()
        self.node = node
        self.protocol = KademliaProtocol(self.node, self.storage, ksize)
        self.refreshLoop = LoopingCall(self.refreshTable).start(3600)

    def listen(self, port):
        """
        Start listening on the given port.

        This is the same as calling::

            reactor.listenUDP(port, server.protocol)
        """
        return reactor.listenUDP(port, self.protocol)

    def refreshTable(self):
        """
        Refresh buckets that haven't had any lookups in the last hour
        (per section 2.3 of the paper).
        """
        ds = []
        for id in self.protocol.getRefreshIDs():
            node = Node(id)
            nearest = self.protocol.router.findNeighbors(node, self.alpha)
            spider = NodeSpiderCrawl(self.protocol, node, nearest)
            ds.append(spider.find())

        def republishKeys(_):
            ds = []
            # Republish keys older than one hour
            for keyword in self.storage.iterkeys():
                for k, v in self.storage.iteritems(keyword):
                    if self.storage.get_ttl(keyword, k) < 601200:
                        ds.append(self.set(keyword, k, v))

        return defer.gatherResults(ds).addCallback(republishKeys)

    def querySeed(self, seed, pubkey):
        """
        Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs.

        Args:
           seed: A `string` consisting of "ip:port" or "hostname:port"
           pubkey: The hex encoded public key to verify the signature on the response
        """
        nodes = []
        c = httplib.HTTPConnection(seed)
        c.request("GET", "/")
        response = c.getresponse()
        self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason))
        data = response.read()
        reread_data = data.decode("zlib")
        seeds = peers.PeerSeeds()
        try:
            seeds.ParseFromString(reread_data)
            for peer in seeds.peer_data:
                p = peers.PeerData()
                p.ParseFromString(peer)
                tup = (str(p.ip_address), p.port)
                nodes.append(tup)
            verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder)
            verify_key.verify(seed.signature + "".join(seeds.peer_data))
        except:
            self.log.error("Error parsing seed response.")
        return nodes

    def bootstrappableNeighbors(self):
        """
        Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument
        to the bootstrap method.

        The server should have been bootstrapped
        already - this is just a utility for getting some neighbors and then
        storing them if this server is going down for a while.  When it comes
        back up, the list of nodes can be used to bootstrap.
        """
        neighbors = self.protocol.router.findNeighbors(self.node)
        return [tuple(n)[-2:] for n in neighbors]

    def bootstrap(self, addrs):
        """
        Bootstrap the server by connecting to other known nodes in the network.

        Args:
            addrs: A `list` of (ip, port) `tuple` pairs.  Note that only IP addresses
                   are acceptable - hostnames will cause an error.
        """

        # if the transport hasn't been initialized yet, wait a second
        if self.protocol.multiplexer.transport is None:
            return task.deferLater(reactor, 1, self.bootstrap, addrs)

        def initTable(results):
            nodes = []
            for addr, result in results.items():
                if result[0]:
                    n = objects.Node()
                    try:
                        n.ParseFromString(result[1][0])
                        pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
                        verify_key = nacl.signing.VerifyKey(pubkey)
                        verify_key.verify(n.signedPublicKey)
                        h = nacl.hash.sha512(n.signedPublicKey)
                        pow = h[64:128]
                        if int(pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
                            raise Exception('Invalid GUID')
                        nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
                    except:
                        self.log.msg("Bootstrap node returned invalid GUID")
            spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
            return spider.find()

        ds = {}
        for addr in addrs:
            ds[addr] = self.protocol.ping((addr[0], addr[1]))
        return deferredDict(ds).addCallback(initTable)

    def inetVisibleIP(self):
        """
        Get the internet visible IP's of this node as other nodes see it.

        Returns:
            A `list` of IP's.  If no one can be contacted, then the `list` will be empty.
        """

        def handle(results):
            ips = []
            for result in results:
                if result[0]:
                    ips.append((result[1][0], int(result[1][1])))
            self.log.debug("other nodes think our ip is %s" % str(ips))
            return ips

        ds = []
        for neighbor in self.bootstrappableNeighbors():
            ds.append(self.protocol.stun(neighbor))
        return defer.gatherResults(ds).addCallback(handle)

    def get(self, keyword):
        """
        Get a key if the network has it.

        Returns:
            :class:`None` if not found, the value otherwise.
        """
        dkey = digest(keyword)
        if self.storage.get(dkey) is not None:
            return defer.succeed(self.storage.get(dkey))
        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to get key %s" % keyword)
            return None
        spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find()

    def set(self, keyword, key, value):
        """
        Set the given key/value tuple at the hash of the given keyword.
        All values stored in the DHT are stored as dictionaries of key/value
        pairs. If a value already exists for a given keyword, the new key/value
        pair will be appended to the dictionary.

        Args:
            keyword: a `string` keyword. The SHA1 hash of which will be used as
                the key when inserting in the DHT.
            key: the 20 byte hash of the data.
            value: a serialized `protos.objects.Node` object which serves as a
                pointer to the node storing the data.

        Return: True if at least one peer responded. False if the store rpc
            completely failed.
        """
        self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value)))
        dkey = digest(keyword)

        def store(nodes):
            self.log.info("setting '%s' on %s" % (keyword, map(str, nodes)))
            ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes]

            keynode = Node(dkey)
            if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
                self.storage[dkey] = (key, value)
                self.log.debug("got a store request from %s, storing value" % str(self.node))

            return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)

        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to set key %s" % key)
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(store)

    def delete(self, keyword, key, signature):
        """
        Delete the given key/value pair from the keyword dictionary on the network.
        To delete you must provide a signature covering the key that you wish to
        delete. It will be verified against the public key stored in the value. We
        use our ksize as alpha to make sure we reach as many nodes storing our value
        as possible.

        Args:
            keyword: the `string` keyword where the data being deleted is stored.
            key: the 20 byte hash of the data.
            signature: a signature covering the key.

        """
        self.log.debug("deleting '%s':'%s' from the network" % (keyword, hexlify(key)))
        dkey = digest(keyword)

        def delete(nodes):
            self.log.info("deleting '%s' on %s" % (key, map(str, nodes)))
            ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes]

            if self.storage.getSpecific(keyword, key) is not None:
                self.storage.delete(keyword, key)

            return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)

        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to delete key %s" % key)
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize)
        return spider.find().addCallback(delete)

    def get_node(self, guid):
        """
        Given a guid return a `Node` object containing its ip and port or none if it's
        not found.

        Args:
            guid: the 20 raw bytes representing the guid.
        """
        node_to_find = Node(guid)

        def check_for_node(nodes):
            for node in nodes:
                if node.id == node_to_find.id:
                    return node
            return None
        index = self.protocol.router.getBucketFor(node_to_find)
        nodes = self.protocol.router.buckets[index].getNodes()
        for node in nodes:
            if node.id == node_to_find.id:
                return defer.succeed(node)
        nearest = self.protocol.router.findNeighbors(node_to_find)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
            return defer.succeed(None)
        spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(check_for_node)

    def _anyRespondSuccess(self, responses):
        """
        Given the result of a DeferredList of calls to peers, ensure that at least
        one of them was contacted and responded with a Truthy result.
        """
        for deferSuccess, result in responses:
            peerReached, peerResponse = result
            if deferSuccess and peerReached and peerResponse:
                return True
        return False

    def saveState(self, fname):
        """
        Save the state of this node (the alpha/ksize/id/immediate neighbors)
        to a cache file with the given fname.
        """
        data = {'ksize': self.ksize,
                'alpha': self.alpha,
                'id': self.node.id,
                'signed_pubkey': self.node.signed_pubkey,
                'neighbors': self.bootstrappableNeighbors()}
        if len(data['neighbors']) == 0:
            self.log.warning("No known neighbors, so not writing to cache.")
            return
        with open(fname, 'w') as f:
            pickle.dump(data, f)

    @classmethod
    def loadState(self, fname, ip_address, port, multiplexer, storage=None):
        """
        Load the state of this node (the alpha/ksize/id/immediate neighbors)
        from a cache file with the given fname.
        """
        with open(fname, 'r') as f:
            data = pickle.load(f)
        n = Node(data['id'], ip_address, port, data['signed_pubkey'])
        s = Server(n, data['ksize'], data['alpha'], storage=storage)
        s.protocol.connect_multiplexer(multiplexer)
        if len(data['neighbors']) > 0:
            s.bootstrap(data['neighbors'])
        return s

    def saveStateRegularly(self, fname, frequency=600):
        """
        Save the state of node with a given regularity to the given
        filename.

        Args:
            fname: File name to save retularly to
            frequencey: Frequency in seconds that the state should be saved.
                        By default, 10 minutes.
        """
        loop = LoopingCall(self.saveState, fname)
        loop.start(frequency)
        return loop
Esempio n. 6
0
class KademliaProtocolTest(unittest.TestCase):

    def setUp(self):
        self.public_ip = '123.45.67.89'
        self.port = 12345
        self.own_addr = (self.public_ip, self.port)
        self.addr1 = ('132.54.76.98', 54321)
        self.addr2 = ('231.76.45.89', 15243)

        self.clock = task.Clock()
        connection.REACTOR.callLater = self.clock.callLater

        self.proto_mock = mock.Mock(spec_set=rudp.ConnectionMultiplexer)
        self.handler_mock = mock.Mock(spec_set=connection.Handler)
        self.con = connection.Connection(
            self.proto_mock,
            self.handler_mock,
            self.own_addr,
            self.addr1
        )

        valid_key = "1a5c8e67edb8d279d1ae32fa2da97e236b95e95c837dc8c3c7c2ff7a7cc29855"
        self.signing_key = nacl.signing.SigningKey(valid_key, encoder=nacl.encoding.HexEncoder)
        verify_key = self.signing_key.verify_key
        signed_pubkey = self.signing_key.sign(str(verify_key))
        h = nacl.hash.sha512(signed_pubkey)
        self.storage = ForgetfulStorage()
        self.node = Node(unhexlify(h[:40]), self.public_ip, self.port, signed_pubkey, True)
        self.protocol = KademliaProtocol(self.node, self.storage, 20)

        self.handler = self.protocol.RPCHandler(False, 5, self.protocol._outstanding, self.protocol)
        self.handler.connection = self.con

        transport = mock.Mock(spec_set=udp.Port)
        ret_val = address.IPv4Address('UDP', self.public_ip, self.port)
        transport.attach_mock(mock.Mock(return_value=ret_val), 'getHost')
        self.protocol.makeConnection(transport)

    def tearDown(self):
        self.con.shutdown()
        self.protocol.shutdown()

    def test_invalid_datagram(self):
        self.assertFalse(self.handler.receive_message("hi"))
        self.assertFalse(self.handler.receive_message("hihihihihihihihihihihihihihihihihihihihih"))

    def test_rpc_ping(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("PING")
        data = m.SerializeToString()
        m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_store(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.extend(["Keyword", "Key", self.protocol.sourceNode.getProto().SerializeToString()])
        data = m.SerializeToString()
        for i in range(0, 3):
            del m.arguments[-1]
        m.arguments.append("True")
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload
        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)
        self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())

    def test_rpc_delete(self):
        self._connecting_to_connected()

        # Set a keyword to store
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.extend(["Keyword", "Key", self.protocol.sourceNode.getProto().SerializeToString()])
        data = m.SerializeToString()
        for i in range(0, 3):
            del m.arguments[-1]
        m.arguments.append("True")
        expected_message1 = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())

        # Test bad signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.arguments.extend(["Keyword", "Key", "Bad Signature"])
        data = m.SerializeToString()
        for i in range(0, 3):
            del m.arguments[-1]
        m.arguments.append("False")
        expected_message2 = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list
        )
        self.assertEqual(sent_packets[0].payload, expected_message1)
        self.assertEqual(sent_packets[1].payload, expected_message2)
        self.proto_mock.send_datagram.call_args_list = []

        # Test good signature
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("DELETE")
        m.arguments.extend(["Keyword", "Key", self.signing_key.sign("Key")[:64]])
        data = m.SerializeToString()
        for i in range(0, 3):
            del m.arguments[-1]
        m.arguments.append("True")
        expected_message3 = m.SerializeToString()
        self.handler.receive_message(data)
        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        self.assertEqual(sent_packet.payload, expected_message3)
        self.assertTrue(self.storage.getSpecific("Keyword", "Key") is None)

    def test_rpc_stun(self):
        self._connecting_to_connected()

        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STUN")
        data = m.SerializeToString()
        m.arguments.extend([self.addr1[0], str(self.addr1[1])])
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_node(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_NODE")
        m.arguments.append(digest("nodetofind"))
        data = m.SerializeToString()
        del m.arguments[-1]
        m.arguments.extend([node3.getProto().SerializeToString(), node2.getProto().SerializeToString(), node1.getProto().SerializeToString()])
        expected_message = m.SerializeToString()
        self.handler.receive_message(data)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_rpc_find_value(self):
        self._connecting_to_connected()

        # Set a value to find
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.extend(["Keyword", "Key", self.protocol.sourceNode.getProto().SerializeToString()])
        data = m.SerializeToString()
        self.handler.receive_message(data)
        self.assertTrue(self.storage.getSpecific("Keyword", "Key") == self.protocol.sourceNode.getProto().SerializeToString())

        # Send the find_value rpc
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.arguments.append("Keyword")
        data = m.SerializeToString()
        self.handler.receive_message(data)

        del m.arguments[-1]
        value = message.Value()
        value.valueKey = "Key"
        value.serializedData = self.protocol.sourceNode.getProto().SerializeToString()
        m.arguments.append("value")
        m.arguments.append(value.SerializeToString())
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packets = tuple(
            packet.Packet.from_bytes(call[0][0])
            for call in self.proto_mock.send_datagram.call_args_list
        )
        received_message = sent_packets[1].payload

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 3)

    def test_rpc_find_without_value(self):
        self._connecting_to_connected()

        node1 = Node(digest("id1"), "127.0.0.1", 12345, digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        m = message.Message()
        m.messageID = digest("msgid")
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("FIND_VALUE")
        m.arguments.append(digest("Keyword"))
        data = m.SerializeToString()
        self.handler.receive_message(data)

        del m.arguments[-1]
        m.arguments.extend([node2.getProto().SerializeToString(), node3.getProto().SerializeToString(), node1.getProto().SerializeToString()])
        expected_message = m.SerializeToString()

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        received_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(received_message)

        self.assertEqual(received_message, expected_message)
        self.assertEqual(len(m_calls), 2)

    def test_callPing(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.protocol[self.addr1] = self.con
        self.protocol.callPing(n)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.PING)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)

    def test_callStore(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.protocol[self.addr1] = self.con
        self.protocol.callStore(n, digest("Keyword"), digest("Key"), self.protocol.sourceNode.getProto().SerializeToString())

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.STORE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(m.arguments[2], self.protocol.sourceNode.getProto().SerializeToString())

    def test_callFindValue(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.protocol[self.addr1] = self.con
        keyword = Node(digest("Keyword"))
        self.protocol.callFindValue(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.FIND_VALUE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callFindNode(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.protocol[self.addr1] = self.con
        keyword = Node(digest("nodetofind"))
        self.protocol.callFindNode(n, keyword)

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.FIND_NODE)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertEqual(m.arguments[0], keyword.id)

    def test_callDelete(self):
        self._connecting_to_connected()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.protocol[self.addr1] = self.con
        self.protocol.callDelete(n, digest("Keyword"), digest("Key"), digest("Signature"))

        self.clock.advance(100 * constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        self.assertEqual(self.proto_mock.send_datagram.call_args_list[0][0][1], self.addr1)
        self.assertTrue(len(m.messageID) == 20)
        self.assertEqual(self.protocol.sourceNode.getProto().guid, m.sender.guid)
        self.assertEqual(self.protocol.sourceNode.getProto().signedPublicKey, m.sender.signedPublicKey)
        self.assertTrue(m.command == message.DELETE)
        self.assertEqual(m.arguments[0], digest("Keyword"))
        self.assertEqual(m.arguments[1], digest("Key"))
        self.assertEqual(m.arguments[2], digest("Signature"))

    def test_acceptResponse(self):
        self._connecting_to_connected()

        def handle_response(resp):
            self.assertTrue(resp[0])
            self.assertEqual(resp[1][0], self.protocol.sourceNode.id)

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        self.protocol[self.addr1] = self.con
        d = self.protocol.callPing(n)

        self.clock.advance(1)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload

        m = message.Message()
        m.ParseFromString(sent_message)
        timeout = reactor.callLater(5, self.protocol._timeout, m.messageID)
        self.handler._outstanding[m.messageID] = (d, timeout)
        m.arguments.append(self.protocol.sourceNode.id)
        self.handler.receive_message(m.SerializeToString())

        return d.addCallback(handle_response)

    def test_unknownRPC(self):
        self.assertFalse(self.handler._acceptRequest(digest("msgid"), "unknown", [digest("argument")], Node(digest("nodeid"))))

    def test_timeout(self):
        self._connecting_to_connected()
        self.protocol[self.addr1] = self.con

        def test_remove_outstanding():
            self.assertTrue(len(self.protocol._outstanding) == 0)

        def test_deffered(d):
            self.assertFalse(d[0])
            test_remove_outstanding()

        n = Node(digest("S"), self.addr1[0], self.addr1[1])
        d = self.protocol.callPing(n)
        self.clock.advance(6)
        return d.addCallback(test_deffered)

    def test_transferKeyValues(self):
        self._connecting_to_connected()
        self.protocol[self.addr1] = self.con

        self.protocol.storage[digest("keyword")] = (digest("key"), self.protocol.sourceNode.getProto().SerializeToString())
        self.protocol.transferKeyValues(Node(digest("id"), self.addr1[0], self.addr1[1]))

        self.clock.advance(1)
        connection.REACTOR.runUntilCurrent()
        sent_packet = packet.Packet.from_bytes(self.proto_mock.send_datagram.call_args_list[0][0][0])
        sent_message = sent_packet.payload
        x = message.Message()
        x.ParseFromString(sent_message)

        m = message.Message()
        m.sender.MergeFrom(self.protocol.sourceNode.getProto())
        m.command = message.Command.Value("STORE")
        m.arguments.append(digest("keyword"))
        m.arguments.append(digest("key"))
        m.arguments.append(self.protocol.sourceNode.getProto().SerializeToString())
        self.assertEqual(x.sender, m.sender)
        self.assertEqual(x.command, m.command)
        self.assertEqual(x.arguments[0], m.arguments[0])
        self.assertEqual(x.arguments[1], m.arguments[1])
        self.assertEqual(x.arguments[2], m.arguments[2])

    def test_refreshIDs(self):
        node1 = Node(digest("id1"), "127.0.0.1", 12345, signed_pubkey=digest("key1"))
        node2 = Node(digest("id2"), "127.0.0.1", 22222, signed_pubkey=digest("key2"))
        node3 = Node(digest("id3"), "127.0.0.1", 77777, signed_pubkey=digest("key3"))
        self.protocol.router.addContact(node1)
        self.protocol.router.addContact(node2)
        self.protocol.router.addContact(node3)
        for b in self.protocol.router.buckets:
            b.lastUpdated = (time.time() - 5000)
        ids = self.protocol.getRefreshIDs()
        self.assertTrue(len(ids) == 1)

    def _connecting_to_connected(self):
        remote_synack_packet = packet.Packet.from_data(
            42,
            self.con.own_addr,
            self.con.dest_addr,
            ack=0,
            syn=True
        )
        self.con.receive_packet(remote_synack_packet)

        self.clock.advance(0)
        connection.REACTOR.runUntilCurrent()

        self.next_remote_seqnum = 43

        m_calls = self.proto_mock.send_datagram.call_args_list
        sent_syn_packet = packet.Packet.from_bytes(m_calls[0][0][0])
        seqnum = sent_syn_packet.sequence_number

        self.handler_mock.reset_mock()
        self.proto_mock.reset_mock()

        self.next_seqnum = seqnum + 1