Exemple #1
0
 def setup(self, config, cache_dir):
     """Setup all the Khashmir sub-modules.
     
     @type config: C{dictionary}
     @param config: the configuration parameters for the DHT
     @type cache_dir: C{string}
     @param cache_dir: the directory to store all files in
     """
     self.config = config
     self.port = config['PORT']
     self.store = DB(os.path.join(cache_dir, 'khashmir.' + str(self.port) + '.db'))
     self.node = self._loadSelfNode('', self.port)
     self.table = KTable(self.node, config)
     self.token_secrets = [newID()]
     self.stats = StatsLogger(self.table, self.store)
     
     # Start listening
     self.udp = krpc.hostbroker(self, self.stats, config)
     self.udp.protocol = krpc.KRPC
     self.listenport = reactor.listenUDP(self.port, self.udp)
     
     # Load the routing table and begin checkpointing
     self._loadRoutingTable()
     self.refreshTable(force = True)
     self.next_checkpoint = reactor.callLater(60, self.checkpoint)
Exemple #2
0
 def setup(self, host, port, db='khashmir.db'):
     self._findDB(db)
     self.port = port
     self.node = self._loadSelfNode(host, port)
     self.table = KTable(self.node)
     #self.app = service.Application("krpc")
     self.udp = krpc.hostbroker(self)
     self.udp.protocol = krpc.KRPC
     self.listenport = reactor.listenUDP(port, self.udp)
     self.last = time.time()
     self._loadRoutingTable()
     KeyExpirer(store=self.store)
     self.refreshTable(force=1)
     reactor.callLater(60, self.checkpoint, (1, ))
Exemple #3
0
    def _loadRoutingTable(self):
        """Load routing table from the database"""
        self.table = KTable(self.node)

        c = self.store.cursor()
        c.execute("select id, host, port from nodes")
        for row in c:
            n = self.Node().init(str(row[0]), row[1], row[2])
            self.table.insertNode(n, contacted = False)
        c.close()

        if DEBUG:
            print("Debug: DHT - nodes loaded:", self.stats())
Exemple #4
0
 def setup(self, host, port, db='khashmir.db'):
     self._findDB(db)
     self.port = port
     self.node = self._loadSelfNode(host, port)
     self.table = KTable(self.node)
     #self.app = service.Application("krpc")
     self.udp = krpc.hostbroker(self)
     self.udp.protocol = krpc.KRPC
     self.listenport = reactor.listenUDP(port, self.udp)
     self.last = time.time()
     self._loadRoutingTable()
     KeyExpirer(store=self.store)
     self.refreshTable(force=1)
     reactor.callLater(60, self.checkpoint, (1,))
Exemple #5
0
 def _load(self):
     do_load = False
     try:
         s = open(os.path.join(self.ddir, "routing_table"), 'r').read()
         dict = bdecode(s)
     except:
         id = newID()
     else:
         id = dict['id']
         do_load = True
         
     self.node = self._Node(self.udp.connectionForAddr).init(id, self.host, self.port)
     self.table = KTable(self.node)
     if do_load:
         self._loadRoutingTable(dict['rt'])
 def _load(self):
     do_load = False
     try:
         s = open(os.path.join(self.ddir, "routing_table"), 'r').read()
         dict = bdecode(s)
     except:
         id = newID()
     else:
         id = dict['id']
         do_load = True
         
     self.node = self._Node(self.udp.connectionForAddr).init(id, self.host, self.port)
     self.table = KTable(self.node)
     if do_load:
         self._loadRoutingTable(dict['rt'])
Exemple #7
0
class KhashmirBase:
    _Node = KNodeBase
    def __init__(self, host, port, data_dir, rawserver=None, max_ul_rate=1024, checkpoint=True, errfunc=None, rlcount=foo, config={'pause':False, 'max_rate_period':20}):
        if rawserver:
            self.rawserver = rawserver
        else:
            self.flag = Event()
            d = dict([(x[0],x[1]) for x in common_options + rare_options])            
            self.rawserver = RawServer(self.flag, d)
        self.max_ul_rate = max_ul_rate
        self.socket = None
        self.config = config
        self.setup(host, port, data_dir, rlcount, checkpoint)

    def setup(self, host, port, data_dir, rlcount, checkpoint=True):
        self.host = host
        self.port = port
        self.ddir = data_dir
        self.store = KStore()
        self.pingcache = {}
        self.socket = self.rawserver.create_udpsocket(self.port, self.host, False)
        self.udp = krpc.hostbroker(self, (self.host, self.port), self.socket, self.rawserver.add_task, self.max_ul_rate, self.config, rlcount)
        self._load()
        self.rawserver.start_listening_udp(self.socket, self.udp)
        self.last = time()
        KeyExpirer(self.store, self.rawserver.add_task)
        self.refreshTable(force=1)
        if checkpoint:
            self.rawserver.add_task(self.findCloseNodes, 30, (lambda a: a, True))
            self.rawserver.add_task(self.checkpoint, 60, (1,))

    def Node(self):
        n = self._Node(self.udp.connectionForAddr)
        n.table = self
        return n
    
    def __del__(self):
        if self.socket is not None:
            self.rawserver.stop_listening_udp(self.socket)
            self.socket.close()
        
    def _load(self):
        do_load = False
        try:
            s = open(os.path.join(self.ddir, "routing_table"), 'r').read()
            dict = bdecode(s)
        except:
            id = newID()
        else:
            id = dict['id']
            do_load = True
            
        self.node = self._Node(self.udp.connectionForAddr).init(id, self.host, self.port)
        self.table = KTable(self.node)
        if do_load:
            self._loadRoutingTable(dict['rt'])

        
    def checkpoint(self, auto=0):
        d = {}
        d['id'] = self.node.id
        d['rt'] = self._dumpRoutingTable()
        try:
            f = open(os.path.join(self.ddir, "routing_table"), 'wb')
            f.write(bencode(d))
            f.close()
        except:
            #XXX real error here
            print ">>> unable to dump routing table!", str(e)
            pass
        
        
        if auto:
            self.rawserver.add_task(self.checkpoint,
                                    randrange(int(const.CHECKPOINT_INTERVAL * .9),
                                              int(const.CHECKPOINT_INTERVAL * 1.1)),
                                    (1,))
        
    def _loadRoutingTable(self, nodes):
        """
            load routing table nodes from database
            it's usually a good idea to call refreshTable(force=1) after loading the table
        """
        for rec in nodes:
            n = self.Node().initWithDict(rec)
            self.table.insertNode(n, contacted=0, nocheck=True)

    def _dumpRoutingTable(self):
        """
            save routing table nodes to the database
        """
        l = []
        for bucket in self.table.buckets:
            for node in bucket.l:
                l.append({'id':node.id, 'host':node.host, 'port':node.port, 'age':int(node.age)})
        return l
        
            
    def _addContact(self, host, port, callback=None):
        """
            ping this node and add the contact info to the table on pong!
        """
        n =self.Node().init(const.NULL_ID, host, port)
        try:
            self.sendPing(n, callback=callback)
        except krpc.KRPCSelfNodeError:
            # our own node
            pass


    #######
    #######  LOCAL INTERFACE    - use these methods!
    def addContact(self, ip, port, callback=None):
        """
            ping this node and add the contact info to the table on pong!
        """
        if ip_pat.match(ip):
            self._addContact(ip, port)
        else:
            def go(ip=ip, port=port):
                ip = gethostbyname(ip)
                self.rawserver.external_add_task(self._addContact, 0, (ip, port))
            t = Thread(target=go)
            t.start()


    ## this call is async!
    def findNode(self, id, callback, errback=None):
        """ returns the contact info for node, or the k closest nodes, from the global table """
        # get K nodes out of local table/cache, or the node we want
        nodes = self.table.findNodes(id, invalid=True)
        l = [x for x in nodes if x.invalid]
        if len(l) > 4:
            nodes = sample(l , 4) + self.table.findNodes(id, invalid=False)[:4]

        d = Deferred()
        if errback:
            d.addCallbacks(callback, errback)
        else:
            d.addCallback(callback)
        if len(nodes) == 1 and nodes[0].id == id :
            d.callback(nodes)
        else:
            # create our search state
            state = FindNode(self, id, d.callback, self.rawserver.add_task)
            self.rawserver.external_add_task(state.goWithNodes, 0, (nodes,))
    
    def insertNode(self, n, contacted=1):
        """
        insert a node in our local table, pinging oldest contact in bucket, if necessary
        
        If all you have is a host/port, then use addContact, which calls this method after
        receiving the PONG from the remote node.  The reason for the seperation is we can't insert
        a node into the table without it's peer-ID.  That means of course the node passed into this
        method needs to be a properly formed Node object with a valid ID.
        """
        old = self.table.insertNode(n, contacted=contacted)
        if old and old != n:
            if not old.inPing():
                self.checkOldNode(old, n, contacted)
            else:
                l = self.pingcache.get(old.id, [])
                if len(l) < 10 or contacted:
                    l.append((n, contacted))
                    self.pingcache[old.id] = l

                

    def checkOldNode(self, old, new, contacted=False):
        ## these are the callbacks used when we ping the oldest node in a bucket

        def cmp(a, b):
            if a[1] == 1 and b[1] == 0:
                return -1
            elif b[1] == 1 and a[1] == 0:
                return 1
            else:
                return 0
            
        def _staleNodeHandler(dict, old=old, new=new, contacted=contacted):
            """ called if the pinged node never responds """
            if old.fails >= 2:
                l = self.pingcache.get(old.id, [])
                l.sort(cmp)
                if l:
                    n, nc = l[0]
                    if (not contacted) and nc:
                        l = l[1:] + [(new, contacted)]
                        new = n
                        contacted = nc
                o = self.table.replaceStaleNode(old, new)
                if o and o != new:
                    self.checkOldNode(o, new)
                    try:
                        self.pingcache[o.id] = self.pingcache[old.id]
                        del(self.pingcache[old.id])
                    except KeyError:
                        pass
                else:
                    if l:
                        del(self.pingcache[old.id])
                        l.sort(cmp)
                        for node in l:
                            self.insertNode(node[0], node[1])
            else:
                l = self.pingcache.get(old.id, [])
                if l:
                    del(self.pingcache[old.id])
                self.insertNode(new, contacted)
                for node in l:
                    self.insertNode(node[0], node[1])
                    
        def _notStaleNodeHandler(dict, old=old, new=new, contacted=contacted):
            """ called when we get a pong from the old node """
            self.table.insertNode(old, True)
            self.insertNode(new, contacted)
            l = self.pingcache.get(old.id, [])
            l.sort(cmp)
            for node in l:
                self.insertNode(node[0], node[1])
            try:
                del(self.pingcache[old.id])
            except KeyError:
                pass
        try:
            df = old.ping(self.node.id)
        except krpc.KRPCSelfNodeError:
            pass
        df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler)

    def sendPing(self, node, callback=None):
        """
            ping a node
        """
        try:
            df = node.ping(self.node.id)
        except krpc.KRPCSelfNodeError:
            pass
        else:
            ## these are the callbacks we use when we issue a PING
            def _pongHandler(dict, node=node, table=self.table, callback=callback):
                _krpc_sender = dict['_krpc_sender']
                dict = dict['rsp']
                sender = {'id' : dict['id']}
                sender['host'] = _krpc_sender[0]
                sender['port'] = _krpc_sender[1]
                n = self.Node().initWithDict(sender)
                table.insertNode(n)
                if callback:
                    callback()
            def _defaultPong(err, node=node, table=self.table, callback=callback):
                if callback:
                    callback()

            df.addCallbacks(_pongHandler,_defaultPong)

    def findCloseNodes(self, callback=lambda a: a, auto=False):
        """
            This does a findNode on the ID one away from our own.  
            This will allow us to populate our table with nodes on our network closest to our own.
            This is called as soon as we start up with an empty table
        """
        if not self.config['pause']:
            id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256)
            self.findNode(id, callback)
        if auto:
            if not self.config['pause']:
                self.refreshTable()
            self.rawserver.external_add_task(self.findCloseNodes, randrange(int(const.FIND_CLOSE_INTERVAL *0.9),
                                                                   int(const.FIND_CLOSE_INTERVAL *1.1)), (lambda a: True, True))

    def refreshTable(self, force=0):
        """
            force=1 will refresh table regardless of last bucket access time
        """
        def callback(nodes):
            pass

        refresh = [bucket for bucket in self.table.buckets if force or (len(bucket.l) < K) or len(filter(lambda a: a.invalid, bucket.l)) or (time() - bucket.lastAccessed > const.BUCKET_STALENESS)]
        for bucket in refresh:
            id = newIDInRange(bucket.min, bucket.max)
            self.findNode(id, callback)

    def stats(self):
        """
        Returns (num_contacts, num_nodes)
        num_contacts: number contacts in our routing table
        num_nodes: number of nodes estimated in the entire dht
        """
        num_contacts = reduce(lambda a, b: a + len(b.l), self.table.buckets, 0)
        num_nodes = const.K * (2**(len(self.table.buckets) - 1))
        return {'num_contacts':num_contacts, 'num_nodes':num_nodes}

    def krpc_ping(self, id, _krpc_sender):
        sender = {'id' : id}
        sender['host'] = _krpc_sender[0]
        sender['port'] = _krpc_sender[1]        
        n = self.Node().initWithDict(sender)
        self.insertNode(n, contacted=0)
        return {"id" : self.node.id}
        
    def krpc_find_node(self, target, id, _krpc_sender):
        nodes = self.table.findNodes(target, invalid=False)
        nodes = map(lambda node: node.senderDict(), nodes)
        sender = {'id' : id}
        sender['host'] = _krpc_sender[0]
        sender['port'] = _krpc_sender[1]        
        n = self.Node().initWithDict(sender)
        self.insertNode(n, contacted=0)
        return {"nodes" : packNodes(nodes), "id" : self.node.id}
Exemple #8
0
class KhashmirBase(protocol.Factory):
    __slots__ = ('listener', 'node', 'table', 'store', 'app', 'last', 'protocol')
    _Node = KNodeBase
    def __init__(self, host, port, db='khashmir.db'):
        self.setup(host, port, db)
        
    def setup(self, host, port, db='khashmir.db'):
        self._findDB(db)
        self.port = port
        self.node = self._loadSelfNode(host, port)
        self.table = KTable(self.node)
        #self.app = service.Application("krpc")
        self.udp = krpc.hostbroker(self)
        self.udp.protocol = krpc.KRPC
        self.listenport = reactor.listenUDP(port, self.udp)
        self.last = time.time()
        self._loadRoutingTable()
        KeyExpirer(store=self.store)
        self.refreshTable(force=1)
        reactor.callLater(60, self.checkpoint, (1,))

    def Node(self):
        n = self._Node()
        n.table = self.table
        return n
    
    def __del__(self):
        self.listenport.stopListening()
        
    def _loadSelfNode(self, host, port):
        c = self.store.cursor()
        c.execute('select id from self where num = 0;')
        if c.rowcount > 0:
            id = c.fetchone()[0]
        else:
            id = newID()
        return self._Node().init(id, host, port)
        
    def _saveSelfNode(self):
        c = self.store.cursor()
        c.execute('delete from self where num = 0;')
        c.execute("insert into self values (0, ?);", (sqlite.Binary(self.node.id), ))
        self.store.commit()
        
    def checkpoint(self, auto=0):
        self._saveSelfNode()
        self._dumpRoutingTable()
        self.refreshTable()
        if auto:
            reactor.callLater(randrange(int(const.CHECKPOINT_INTERVAL * .9), int(const.CHECKPOINT_INTERVAL * 1.1)), self.checkpoint, (1,))
        
    def _findDB(self, db):
        import os
        try:
            os.stat(db)
        except OSError:
            self._createNewDB(db)
        else:
            self._loadDB(db)
        self.store.text_factory = lambda x: str(x)
        self.store.row_factory = sqlite.Row
        
    def _loadDB(self, db):
        try:
            self.store = sqlite.connect(db)
            #self.store.autocommit = 0
        except:
            import traceback
            raise KhashmirDBExcept, "Couldn't open DB", traceback.exc_traceback
        
    def _createNewDB(self, db):
        self.store = sqlite.connect(db)
        s = """
            create table kv (key binary, value binary, time timestamp, primary key (key, value));
            create index kv_key on kv(key);
            create index kv_timestamp on kv(time);
            
            create table nodes (id binary primary key, host text, port number);
            
            create table self (num number primary key, id binary);
            """
        c = self.store.cursor()
        c.executescript(s)
        self.store.commit()

    def _dumpRoutingTable(self):
        """
            save routing table nodes to the database
        """
        c = self.store.cursor()
        c.execute("delete from nodes where id not NULL;")
        for bucket in self.table.buckets:
            for node in bucket.l:
                c.execute("insert into nodes values (?, ?, ?);", (sqlite.Binary(node.id), node.host, node.port))
        self.store.commit()
        
    def _loadRoutingTable(self):
        """
            load routing table nodes from database
            it's usually a good idea to call refreshTable(force=1) after loading the table
        """
        c = self.store.cursor()
        c.execute("select * from nodes;")
        for rec in c.fetchall():
            n = self.Node().initWithDict({'id':rec[0], 'host':rec[1], 'port':int(rec[2])})
            n.conn = self.udp.connectionForAddr((n.host, n.port))
            self.table.insertNode(n, contacted=0)
            

    #######
    #######  LOCAL INTERFACE    - use these methods!
    def addContact(self, host, port, callback=None):
        """
            ping this node and add the contact info to the table on pong!
        """
        n =self.Node().init(const.NULL_ID, host, port) 
        n.conn = self.udp.connectionForAddr((n.host, n.port))
        self.sendPing(n, callback=callback)

    ## this call is async!
    def findNode(self, id, callback, errback=None):
        """ returns the contact info for node, or the k closest nodes, from the global table """
        # get K nodes out of local table/cache, or the node we want
        nodes = self.table.findNodes(id)
        d = Deferred()
        if errback:
            d.addCallbacks(callback, errback)
        else:
            d.addCallback(callback)
        if len(nodes) == 1 and nodes[0].id == id :
            d.callback(nodes)
        else:
            # create our search state
            state = FindNode(self, id, d.callback)
            reactor.callFromThread(state.goWithNodes, nodes)
    
    def insertNode(self, n, contacted=1):
        """
        insert a node in our local table, pinging oldest contact in bucket, if necessary
        
        If all you have is a host/port, then use addContact, which calls this method after
        receiving the PONG from the remote node.  The reason for the seperation is we can't insert
        a node into the table without it's peer-ID.  That means of course the node passed into this
        method needs to be a properly formed Node object with a valid ID.
        """
        old = self.table.insertNode(n, contacted=contacted)
        if old and (time.time() - old.lastSeen) > const.MIN_PING_INTERVAL and old.id != self.node.id:
            # the bucket is full, check to see if old node is still around and if so, replace it
            
            ## these are the callbacks used when we ping the oldest node in a bucket
            def _staleNodeHandler(oldnode=old, newnode = n):
                """ called if the pinged node never responds """
                self.table.replaceStaleNode(old, newnode)
            
            def _notStaleNodeHandler(dict, old=old):
                """ called when we get a pong from the old node """
                dict = dict['rsp']
                if dict['id'] == old.id:
                    self.table.justSeenNode(old.id)
            
            df = old.ping(self.node.id)
            df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler)

    def sendPing(self, node, callback=None):
        """
            ping a node
        """
        df = node.ping(self.node.id)
        ## these are the callbacks we use when we issue a PING
        def _pongHandler(dict, node=node, table=self.table, callback=callback):
            _krpc_sender = dict['_krpc_sender']
            dict = dict['rsp']
            sender = {'id' : dict['id']}
            sender['host'] = _krpc_sender[0]
            sender['port'] = _krpc_sender[1]
            n = self.Node().initWithDict(sender)
            n.conn = self.udp.connectionForAddr((n.host, n.port))
            table.insertNode(n)
            if callback:
                callback()
        def _defaultPong(err, node=node, table=self.table, callback=callback):
            table.nodeFailed(node)
            if callback:
                callback()
        
        df.addCallbacks(_pongHandler,_defaultPong)

    def findCloseNodes(self, callback=lambda a: None):
        """
            This does a findNode on the ID one away from our own.  
            This will allow us to populate our table with nodes on our network closest to our own.
            This is called as soon as we start up with an empty table
        """
        id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256)
        self.findNode(id, callback)

    def refreshTable(self, force=0):
        """
            force=1 will refresh table regardless of last bucket access time
        """
        def callback(nodes):
            pass
    
        for bucket in self.table.buckets:
            if force or (time.time() - bucket.lastAccessed >= const.BUCKET_STALENESS):
                id = newIDInRange(bucket.min, bucket.max)
                self.findNode(id, callback)

    def stats(self):
        """
        Returns (num_contacts, num_nodes)
        num_contacts: number contacts in our routing table
        num_nodes: number of nodes estimated in the entire dht
        """
        num_contacts = reduce(lambda a, b: a + len(b.l), self.table.buckets, 0)
        num_nodes = const.K * (2**(len(self.table.buckets) - 1))
        return (num_contacts, num_nodes)

    def krpc_ping(self, id, _krpc_sender):
        sender = {'id' : id}
        sender['host'] = _krpc_sender[0]
        sender['port'] = _krpc_sender[1]        
        n = self.Node().initWithDict(sender)
        n.conn = self.udp.connectionForAddr((n.host, n.port))
        self.insertNode(n, contacted=0)
        return {"id" : self.node.id}
        
    def krpc_find_node(self, target, id, _krpc_sender):
        nodes = self.table.findNodes(target)
        nodes = map(lambda node: node.senderDict(), nodes)
        sender = {'id' : id}
        sender['host'] = _krpc_sender[0]
        sender['port'] = _krpc_sender[1]        
        n = self.Node().initWithDict(sender)
        n.conn = self.udp.connectionForAddr((n.host, n.port))
        self.insertNode(n, contacted=0)
        return {"nodes" : nodes, "id" : self.node.id}
Exemple #9
0
class KhashmirBase(protocol.Factory):
    __slots__ = ('listener', 'node', 'table', 'store', 'app', 'last',
                 'protocol')
    _Node = KNodeBase

    def __init__(self, host, port, db='khashmir.db'):
        self.setup(host, port, db)

    def setup(self, host, port, db='khashmir.db'):
        self._findDB(db)
        self.port = port
        self.node = self._loadSelfNode(host, port)
        self.table = KTable(self.node)
        #self.app = service.Application("krpc")
        self.udp = krpc.hostbroker(self)
        self.udp.protocol = krpc.KRPC
        self.listenport = reactor.listenUDP(port, self.udp)
        self.last = time.time()
        self._loadRoutingTable()
        KeyExpirer(store=self.store)
        self.refreshTable(force=1)
        reactor.callLater(60, self.checkpoint, (1, ))

    def Node(self):
        n = self._Node()
        n.table = self.table
        return n

    def __del__(self):
        self.listenport.stopListening()

    def _loadSelfNode(self, host, port):
        c = self.store.cursor()
        c.execute('select id from self where num = 0;')
        if c.rowcount > 0:
            id = c.fetchone()[0]
        else:
            id = newID()
        return self._Node().init(id, host, port)

    def _saveSelfNode(self):
        c = self.store.cursor()
        c.execute('delete from self where num = 0;')
        c.execute("insert into self values (0, %s);",
                  sqlite.encode(self.node.id))
        self.store.commit()

    def checkpoint(self, auto=0):
        self._saveSelfNode()
        self._dumpRoutingTable()
        self.refreshTable()
        if auto:
            reactor.callLater(
                randrange(int(const.CHECKPOINT_INTERVAL * .9),
                          int(const.CHECKPOINT_INTERVAL * 1.1)),
                self.checkpoint, (1, ))

    def _findDB(self, db):
        import os
        try:
            os.stat(db)
        except OSError:
            self._createNewDB(db)
        else:
            self._loadDB(db)

    def _loadDB(self, db):
        try:
            self.store = sqlite.connect(db=db)
            #self.store.autocommit = 0
        except:
            import traceback
            raise KhashmirDBExcept, "Couldn't open DB", traceback.exc_traceback

    def _createNewDB(self, db):
        self.store = sqlite.connect(db=db)
        s = """
            create table kv (key binary, value binary, time timestamp, primary key (key, value));
            create index kv_key on kv(key);
            create index kv_timestamp on kv(time);
            
            create table nodes (id binary primary key, host text, port number);
            
            create table self (num number primary key, id binary);
            """
        c = self.store.cursor()
        c.execute(s)
        self.store.commit()

    def _dumpRoutingTable(self):
        """
            save routing table nodes to the database
        """
        c = self.store.cursor()
        c.execute("delete from nodes where id not NULL;")
        for bucket in self.table.buckets:
            for node in bucket.l:
                c.execute("insert into nodes values (%s, %s, %s);",
                          (sqlite.encode(node.id), node.host, node.port))
        self.store.commit()

    def _loadRoutingTable(self):
        """
            load routing table nodes from database
            it's usually a good idea to call refreshTable(force=1) after loading the table
        """
        c = self.store.cursor()
        c.execute("select * from nodes;")
        for rec in c.fetchall():
            n = self.Node().initWithDict({
                'id': rec[0],
                'host': rec[1],
                'port': int(rec[2])
            })
            n.conn = self.udp.connectionForAddr((n.host, n.port))
            self.table.insertNode(n, contacted=0)

    #######
    #######  LOCAL INTERFACE    - use these methods!
    def addContact(self, host, port, callback=None):
        """
            ping this node and add the contact info to the table on pong!
        """
        n = self.Node().init(const.NULL_ID, host, port)
        n.conn = self.udp.connectionForAddr((n.host, n.port))
        self.sendPing(n, callback=callback)

    ## this call is async!
    def findNode(self, id, callback, errback=None):
        """ returns the contact info for node, or the k closest nodes, from the global table """
        # get K nodes out of local table/cache, or the node we want
        nodes = self.table.findNodes(id)
        d = Deferred()
        if errback:
            d.addCallbacks(callback, errback)
        else:
            d.addCallback(callback)
        if len(nodes) == 1 and nodes[0].id == id:
            d.callback(nodes)
        else:
            # create our search state
            state = FindNode(self, id, d.callback)
            reactor.callFromThread(state.goWithNodes, nodes)

    def insertNode(self, n, contacted=1):
        """
        insert a node in our local table, pinging oldest contact in bucket, if necessary
        
        If all you have is a host/port, then use addContact, which calls this method after
        receiving the PONG from the remote node.  The reason for the seperation is we can't insert
        a node into the table without it's peer-ID.  That means of course the node passed into this
        method needs to be a properly formed Node object with a valid ID.
        """
        old = self.table.insertNode(n, contacted=contacted)
        if old and (time.time() - old.lastSeen
                    ) > const.MIN_PING_INTERVAL and old.id != self.node.id:
            # the bucket is full, check to see if old node is still around and if so, replace it

            ## these are the callbacks used when we ping the oldest node in a bucket
            def _staleNodeHandler(oldnode=old, newnode=n):
                """ called if the pinged node never responds """
                self.table.replaceStaleNode(old, newnode)

            def _notStaleNodeHandler(dict, old=old):
                """ called when we get a pong from the old node """
                dict = dict['rsp']
                if dict['id'] == old.id:
                    self.table.justSeenNode(old.id)

            df = old.ping(self.node.id)
            df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler)

    def sendPing(self, node, callback=None):
        """
            ping a node
        """
        df = node.ping(self.node.id)

        ## these are the callbacks we use when we issue a PING
        def _pongHandler(dict, node=node, table=self.table, callback=callback):
            _krpc_sender = dict['_krpc_sender']
            dict = dict['rsp']
            sender = {'id': dict['id']}
            sender['host'] = _krpc_sender[0]
            sender['port'] = _krpc_sender[1]
            n = self.Node().initWithDict(sender)
            n.conn = self.udp.connectionForAddr((n.host, n.port))
            table.insertNode(n)
            if callback:
                callback()

        def _defaultPong(err, node=node, table=self.table, callback=callback):
            table.nodeFailed(node)
            if callback:
                callback()

        df.addCallbacks(_pongHandler, _defaultPong)

    def findCloseNodes(self, callback=lambda a: None):
        """
            This does a findNode on the ID one away from our own.  
            This will allow us to populate our table with nodes on our network closest to our own.
            This is called as soon as we start up with an empty table
        """
        id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256)
        self.findNode(id, callback)

    def refreshTable(self, force=0):
        """
            force=1 will refresh table regardless of last bucket access time
        """
        def callback(nodes):
            pass

        for bucket in self.table.buckets:
            if force or (time.time() - bucket.lastAccessed >=
                         const.BUCKET_STALENESS):
                id = newIDInRange(bucket.min, bucket.max)
                self.findNode(id, callback)

    def stats(self):
        """
        Returns (num_contacts, num_nodes)
        num_contacts: number contacts in our routing table
        num_nodes: number of nodes estimated in the entire dht
        """
        num_contacts = reduce(lambda a, b: a + len(b.l), self.table.buckets, 0)
        num_nodes = const.K * (2**(len(self.table.buckets) - 1))
        return (num_contacts, num_nodes)

    def krpc_ping(self, id, _krpc_sender):
        sender = {'id': id}
        sender['host'] = _krpc_sender[0]
        sender['port'] = _krpc_sender[1]
        n = self.Node().initWithDict(sender)
        n.conn = self.udp.connectionForAddr((n.host, n.port))
        self.insertNode(n, contacted=0)
        return {"id": self.node.id}

    def krpc_find_node(self, target, id, _krpc_sender):
        nodes = self.table.findNodes(target)
        nodes = map(lambda node: node.senderDict(), nodes)
        sender = {'id': id}
        sender['host'] = _krpc_sender[0]
        sender['port'] = _krpc_sender[1]
        n = self.Node().initWithDict(sender)
        n.conn = self.udp.connectionForAddr((n.host, n.port))
        self.insertNode(n, contacted=0)
        return {"nodes": nodes, "id": self.node.id}
Exemple #10
0
class Factory:
    """
    Background functionality
    """
    def __init__(self, host, port, dbDir, ipv6_enable = False, upnp = 0, natpmp = False):
        self.host = host
        self.port = port
        self.dbDir = dbDir
        self.store = None
        
        self.rawserver = RawServer(self, host, port, ipv6_enable, upnp, natpmp)
        self.tokensHandler = TokensHandler(self)
        self.krpc = KRPC(self)

        self.contacts = []  # [ip,] - contacts added by bittorrent
        self.announce = {}  # {ip:[hash,],} - nodes that announced us 
        
    def Node(self):
        """Create a new node"""
        return KNode(self)

    def start(self):
        """Start Factory"""
        self.rawserver.add_task(self._init)
        self.rawserver.add_task(self._checkpoint, 60)
        self.rawserver.start()
        self.rawserver.add_task(self._cleanDataBase, KEINITIAL_DELAY)
        self.rawserver.add_task(self.refreshTable, 5, [True])

    def _init(self):
        """Initialize Factory"""
        self._loadDB()
        self._loadSelfNode()
        self._loadRoutingTable()

    def _close(self):
        """Close Factory"""
        self._updateDB()
        self.rawserver.shutdown()
        
    ####################
    # Database Handler
    ####################
    def _loadDB(self):
        """Load the database"""
        if DEBUG:
            print("Debug: DHT - _loadDB")
        # connect
        self.store = sqlite3.connect(os.path.join(self.dbDir.encode("UTF-8"), "dht.db"))
        self.store.text_factory = str
        
        # create if neccacery
        c = self.store.cursor()
        statements = ["create table kv (key binary, value binary, age timestamp, primary key (key, value))",
                      "create table nodes (id binary primary key, host text, port number)",
                      "create table self (num number primary key, id binary, age timestamp)"]
        try:
            [c.execute(s) for s in statements]
        except sqlite3.OperationalError:
            pass
        else:
            self.store.commit()
        c.close()
        
    def _closeDB(self):
        """Close the database"""
        self.store.close()

    def _loadSelfNode(self):
        """Load the root node"""
        if DEBUG:
            print("Debug: DHT - loadSelfNode")

        # Get ID
        c = self.store.cursor()
        c.execute('select id, age from self where num = 0')
        data = c.fetchone()

        # Clean if too old
        if not data or time() - data[1] > 86400*5: # more than 5 days old
            id = newID()
            c.execute('delete from self')
            c.execute("insert into self values (0, ?, ?)", (sqlite3.Binary(id), time()))
            c.execute('delete from kv')
            c.execute('delete from nodes')
            self.store.commit()
        else:
            id = str(data[0])
        c.close()

        # Load self node
        self.node = self.Node().init(id, self.host, self.port)
   
    def _saveSelfNode(self):
        """Save the root node"""
        if DEBUG:
            print("Debug: DHT - saveSelfNode")
        c = self.store.cursor()
        c.execute('delete from self')
        c.execute("insert into self values (0, ?, ?)", (sqlite3.Binary(self.node.id), time()))
        self.store.commit()
        c.close()
        
    def _loadRoutingTable(self):
        """Load routing table from the database"""
        self.table = KTable(self.node)

        c = self.store.cursor()
        c.execute("select id, host, port from nodes")
        for row in c:
            n = self.Node().init(str(row[0]), row[1], row[2])
            self.table.insertNode(n, contacted = False)
        c.close()

        if DEBUG:
            print("Debug: DHT - nodes loaded:", self.stats())

    def _saveRoutingTable(self):
        """Save routing table nodes to the database"""
        if DEBUG:
            print("Debug: DHT - saveRoutingTable")
        c = self.store.cursor()
        c.execute("delete from nodes")
        for bucket in self.table.buckets:
            for node in bucket.l:
                c.execute("insert into nodes values (?, ?, ?)", (sqlite3.Binary(node.id), node.host, node.port))
        self.store.commit()
        c.close()
        
    def _updateDB(self):
        """Save info to database"""
        if DEBUG:
            print("Debug: DHT - updateDB")
        self._saveSelfNode()
        self._saveRoutingTable()
        if DEBUG:
            print("Debug: DHT - updateDB completed")

    def _flushExpired(self):
        """Clean old values from database"""
        c = self.store.cursor()
        c.execute("delete from kv where age > ?", (KE_AGE,))
        self.store.commit()
        c.close()
                                            
    def _retrieveValue(self, key):
        """Returns the value found for key in local table"""
        values = []
        c = self.store.cursor()
        c.execute("select value from kv where key = ?", (sqlite3.Binary(key),))
        for row in c:
            values.append(str(row[0]))
        c.close()
        return values[:20]
    
    def _storeValue(self, key, value):
        """Stores <key:value> pair in the database"""
        c = self.store.cursor()
        try:
            c.execute("insert into kv values (?, ?, ?);", (sqlite3.Binary(key), sqlite3.Binary(value), time()))
        except sqlite3.IntegrityError:
            c.execute("update kv set age = ? where key = ? and value = ?", (time(), sqlite3.Binary(key), sqlite3.Binary(value)))
        self.store.commit()
        c.close()

    ####################
    # Automatic updates
    ####################
    def _checkpoint(self):
        """Make some saving and refreshing once in a while"""
        if DEBUG:
            print("Debug: DHT - checkpoint")
        # Save DB to disk
        self._updateDB()
        # Find close nodes
        self.findCloseNodes()
        # Refresh Table
        self.refreshTable()
            
        self.rawserver.add_task(self._checkpoint, randrange(int(CHECKPOINT_INTERVAL * .9), int(CHECKPOINT_INTERVAL * 1.1)))
    
    def _cleanDataBase(self):
        self._flushExpired()
        self.rawserver.add_task(self._cleanDataBase, KE_DELAY)

    ####################
    # Interface
    ####################
    def addContact(self, host, port):
        """
        Ping this node and add the contact info to the table on pong!
        """
        # Validation
        if not isinstance(port, int):
            port = int(port)
        if not isinstance(host, str):
            host = str(host)
        if host in self.contacts:
            return False
        self.contacts.append(host)

        # Add Contact
        if is_valid_ip(host):
            n = self.Node().init(NULL_ID, host, port)
            self.sendPing(n)
        else:
            Thread(target = self.addRouterContact, args = [host, port]).start()
            
    def addRouterContact(self, host, port):
        try:
            host = socket.gethostbyname(host)
        except socket.error:
            return False
        n = self.Node().init(NULL_ID, host, port)
        self.sendPing(n)
        return True
    
    def insertNode(self, n, contacted = True):
        """
        Insert a node in our local table, pinging oldest contact in bucket, if necessary
        
        If all you have is a host/port, then use addContact, which calls this method after
        receiving the PONG from the remote node.  The reason for the seperation is we can't insert
        a node into the table without it's peer-ID.  That means of course the node passed into this
        method needs to be a properly formed Node object with a valid ID.
        """
        old = self.table.insertNode(n, contacted = contacted)
        if old and (clock() - old.lastSeen) > MIN_PING_INTERVAL and old.id != self.node.id:
            # the bucket is full, check to see if old node is still around and if so, replace it
            
            ## these are the callbacks used when we ping the oldest node in a bucket
            def _staleNodeHandler(oldnode=old, newnode = n):
                """ called if the pinged node never responds """
                self.table.replaceStaleNode(old, newnode)
            
            def _notStaleNodeHandler(dict, old=old):
                """ called when we get a pong from the old node """
                dict = dict['rsp']
                if dict['id'] == old.id:
                    self.table.justSeenNode(old.id)

            try:
                df = old.ping(self.node.id)
            except KrpcGenericError:
                _staleNodeHandler()
            else:
                df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler)

    def findCloseNodes(self, callback=lambda a: None):
        """
        This does a findNode on the ID one away from our own.  
        This will allow us to populate our table with nodes on our network closest to our own.
        This is called as soon as we start up with an empty table
        """
        if DEBUG:
            print("Debug: DHT - findCloseNodes")
        id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256)
        self.findNode(id, callback)

    def refreshTable(self, force = False, callback=lambda a: None):
        """
        Refresh the table
        force=True will refresh table regardless of last bucket access time
        """
        if DEBUG:
            print("Debug: DHT - refreshTable")
        for bucket in self.table.buckets:
            if force or (clock() - bucket.lastAccessed >= BUCKET_STALENESS):
                id = newIDInRange(bucket.min, bucket.max)
                self.findNode(id, callback)

    def stats(self):
        """
        Returns the number of contacts in our routing table
        """
        return reduce(lambda a, b: a + len(b.l), self.table.buckets, 0)        

    ####################
    # RPC Handler
    ####################
    def krpc_ping(self, id, _krpc_sender, **kwargs):
        """Incoming RPC: got ping"""
        if len(id) != 20:
            raise KrpcProtocolError("invalid id length: %d" % len(id))
        n = self.Node().init(id, *_krpc_sender)
        self.insertNode(n, contacted = False)
        return {"id" : self.node.id}
        
    def krpc_find_node(self, target, id, _krpc_sender, **kwargs):
        """Incoming RPC: got find_node"""
        if len(id) != 20:
            raise KrpcProtocolError("invalid id length: %d" % len(id))
        if len(target) != 20:
            raise KrpcProtocolError("invalid target id length: %d" % len(target))

        nodes = self.table.findNodes(target)
        nodes = map(lambda node: node.senderDict(), nodes)
        n = self.Node().init(id, *_krpc_sender)
        self.insertNode(n, contacted = False)
        return {"nodes" : encodeNodes(nodes), "id" : self.node.id}

    def krpc_get_peers(self, id, info_hash, _krpc_sender, **kwargs):
        """Incoming RPC: got get_peers"""
        if len(id) != 20:
            raise KrpcProtocolError("invalid id length: %d" % len(id))
        if len(info_hash) != 20:
            raise KrpcProtocolError("invalid info_hash length: %d" % len(info_hash))
        if id == NULL_ID:
            raise KrpcProtocolError("invalid id (NULL ID)")

        n = self.Node().init(id, *_krpc_sender)
        self.insertNode(n, contacted = False)
    
        l = self._retrieveValue(info_hash)
        if len(l) > 0:
            return {'values' : l, "id": self.node.id,
                    "token" : self.tokensHandler.tokenToSend(info_hash, *_krpc_sender)}
        else:
            nodes = self.table.findNodes(info_hash)
            nodes = map(lambda node: node.senderDict(), nodes)
            return {'nodes' : encodeNodes(nodes), "id": self.node.id,
                    "token" : self.tokensHandler.tokenToSend(info_hash, *_krpc_sender)}

    def krpc_announce_peer(self, id, info_hash, port, token, _krpc_sender, **kwargs):
        """Incoming RPC: got announce_peer"""
        if len(id) != 20:
            raise KrpcProtocolError("invalid id length: %d" % len(id))
        if len(info_hash) != 20:
            raise KrpcProtocolError("invalid info_hash length: %d" % len(info_hash))
        if not isinstance(port, int):
            try:
                port = int(port)
            except:
                raise KrpcProtocolError("invalid port")
        if not self.tokensHandler.checkToken(token, info_hash, *_krpc_sender):
            raise KrpcProtocolError("Got invalid token")

        # TODO: add a limit:  maximum 3 info_hash announces per peer
        ip = _krpc_sender[0]
        if ip not in self.announce:
            self.announce[ip] = []
        if info_hash not in self.announce[ip]:
            self.announce[ip].append(info_hash)
        if len(self.announce[ip]) > 3:
            raise KrpcGenericError("I only allow 3 infohash announces per peer!")

        self._storeValue(info_hash, encodePeer((_krpc_sender[0], port)))
        n = self.Node().init(id, *_krpc_sender)
        self.insertNode(n, contacted = False)
        return {"id" : self.node.id}

    def sendPing(self, node, callback=None):
        """
        Ping a node
        """        
        def _pongHandler(dict, node=node, table=self.table, callback=callback):
            _krpc_sender = dict['_krpc_sender']
            id = dict['rsp']['id']
            if len(id) == 20:
                n = self.Node().init(dict['rsp']['id'], *_krpc_sender)
                table.insertNode(n)
                if callback:
                    callback()
        def _defaultPong(err, node=node, table=self.table, callback=callback):
            table.nodeFailed(node)
            if callback:
                callback()
        try:
            df = node.ping(self.node.id)
        except KrpcGenericError:
            _defaultPong()
        else:
            df.addCallbacks(_pongHandler,_defaultPong)

    def findNode(self, id, callback, errback = None):
        """
        Returns the the k closest nodes to that ID from the global table
        """
        # get K nodes out of local table/cache
        nodes = self.table.findNodes(id)
        d = Deferred()
        if errback:
            d.addCallbacks(callback, errback)
        else:
            d.addCallback(callback)

        # create our search state
        state = FindNode(self, id, d.callback)
        self.rawserver.add_task(state.goWithNodes, 0, [nodes])

    def getPeers(self, key, callback, searchlocal = True, donecallback = None):
        """
        Get Value from global table
        """
        if not hasattr(self, "store"):
            self.rawserver.add_task(self.getPeers, 3, [key, callback, searchlocal, donecallback])
            return
        
        # get locals
        if searchlocal:
            l = self._retrieveValue(key)
            if len(l) > 0:
                l = decodePeers(l)
                self.rawserver.add_task(callback, 0, [l])
        else:
            l = []

        # create our search state
        nodes = self.table.findNodes(key)
        state = GetValue(self, key, callback, "getPeers", donecallback)
        self.rawserver.add_task(state.goWithNodes, 0, [nodes, l])
    
    def announcePeer(self, key, value, callback=None):
        """
        Store Value in global table
        """
        def _storeValueForKey(nodes, key=key, value=value, response=callback , table=self.table):
            if not response:
                # default callback
                def _storedValueHandler(sender):
                    pass
                response=_storedValueHandler
            action = StoreValue(self, key, value, response, "announcePeer")
            self.rawserver.add_task(action.goWithNodes, 0, [nodes])
            
        # this call is asynch
        self.findNode(key, _storeValueForKey)
              

    def getPeersAndAnnounce(self, key, value, callback, searchlocal = True):
        """
        Get value and store it
        """
        def doneCallback(nodes, key = key, value = value):
            action = StoreValue(self, key, value, None, "announcePeer")
            self.rawserver.add_task(action.goWithNodes, 0, [nodes])
        self.getPeers(key, callback, searchlocal, doneCallback)
Exemple #11
0
class KhashmirBase(protocol.Factory):
    """The base Khashmir class, with base functionality and find node, no key-value mappings.
    
    @type _Node: L{node.Node}
    @ivar _Node: the knode implementation to use for this class of DHT
    @type config: C{dictionary}
    @ivar config: the configuration parameters for the DHT
    @type pinging: C{dictionary}
    @ivar pinging: the node's that are currently being pinged, keys are the
        node id's, values are the Deferred or DelayedCall objects
    @type port: C{int}
    @ivar port: the port to listen on
    @type store: L{db.DB}
    @ivar store: the database to store nodes and key/value pairs in
    @type node: L{node.Node}
    @ivar node: this node
    @type table: L{ktable.KTable}
    @ivar table: the routing table
    @type token_secrets: C{list} of C{string}
    @ivar token_secrets: the current secrets to use to create tokens
    @type stats: L{stats.StatsLogger}
    @ivar stats: the statistics gatherer
    @type udp: L{krpc.hostbroker}
    @ivar udp: the factory for the KRPC protocol
    @type listenport: L{twisted.internet.interfaces.IListeningPort}
    @ivar listenport: the UDP listening port
    @type next_checkpoint: L{twisted.internet.interfaces.IDelayedCall}
    @ivar next_checkpoint: the delayed call for the next checkpoint
    """
    
    _Node = KNodeBase
    
    def __init__(self, config, cache_dir='/tmp'):
        """Initialize the Khashmir class and call the L{setup} method.
        
        @type config: C{dictionary}
        @param config: the configuration parameters for the DHT
        @type cache_dir: C{string}
        @param cache_dir: the directory to store all files in
            (optional, defaults to the /tmp directory)
        """
        self.config = None
        self.pinging = {}
        self.setup(config, cache_dir)
        
    def setup(self, config, cache_dir):
        """Setup all the Khashmir sub-modules.
        
        @type config: C{dictionary}
        @param config: the configuration parameters for the DHT
        @type cache_dir: C{string}
        @param cache_dir: the directory to store all files in
        """
        self.config = config
        self.port = config['PORT']
        self.store = DB(os.path.join(cache_dir, 'khashmir.' + str(self.port) + '.db'))
        self.node = self._loadSelfNode('', self.port)
        self.table = KTable(self.node, config)
        self.token_secrets = [newID()]
        self.stats = StatsLogger(self.table, self.store)
        
        # Start listening
        self.udp = krpc.hostbroker(self, self.stats, config)
        self.udp.protocol = krpc.KRPC
        self.listenport = reactor.listenUDP(self.port, self.udp)
        
        # Load the routing table and begin checkpointing
        self._loadRoutingTable()
        self.refreshTable(force = True)
        self.next_checkpoint = reactor.callLater(60, self.checkpoint)

    def Node(self, id, host = None, port = None):
        """Create a new node.
        
        @see: L{node.Node.__init__}
        """
        n = self._Node(id, host, port)
        n.table = self.table
        n.conn = self.udp.connectionForAddr((n.host, n.port))
        return n
    
    def __del__(self):
        """Stop listening for packets."""
        self.listenport.stopListening()
        
    def _loadSelfNode(self, host, port):
        """Create this node, loading any previously saved one."""
        id = self.store.getSelfNode()
        if not id or not id.endswith(self.config['VERSION']):
            id = newID(self.config['VERSION'])
        return self._Node(id, host, port)
        
    def checkpoint(self):
        """Perform some periodic maintenance operations."""
        # Create a new token secret
        self.token_secrets.insert(0, newID())
        if len(self.token_secrets) > 3:
            self.token_secrets.pop()
            
        # Save some parameters for reloading
        self.store.saveSelfNode(self.node.id)
        self.store.dumpRoutingTable(self.table.buckets)
        
        # DHT maintenance
        self.store.expireValues(self.config['KEY_EXPIRE'])
        self.refreshTable()
        
        self.next_checkpoint = reactor.callLater(randrange(int(self.config['CHECKPOINT_INTERVAL'] * .9), 
                                                           int(self.config['CHECKPOINT_INTERVAL'] * 1.1)), 
                                                 self.checkpoint)
        
    def _loadRoutingTable(self):
        """Load the previous routing table nodes from the database.
        
        It's usually a good idea to call refreshTable(force = True) after
        loading the table.
        """
        nodes = self.store.getRoutingTable()
        for rec in nodes:
            n = self.Node(rec[0], rec[1], int(rec[2]))
            self.table.insertNode(n, contacted = False)
            
    #{ Local interface
    def addContact(self, host, port, callback=None, errback=None):
        """Ping this node and add the contact info to the table on pong.
        
        @type host: C{string}
        @param host: the IP address of the node to contact
        @type port: C{int}
        @param port:the port of the node to contact
        @type callback: C{method}
        @param callback: the method to call with the results, it must take 1
            parameter, the contact info returned by the node
            (optional, defaults to doing nothing with the results)
        @type errback: C{method}
        @param errback: the method to call if an error occurs
            (optional, defaults to calling the callback with the error)
        """
        n = self.Node(NULL_ID, host, port)
        self.sendJoin(n, callback=callback, errback=errback)

    def findNode(self, id, callback):
        """Find the contact info for the K closest nodes in the global table.
        
        @type id: C{string}
        @param id: the target ID to find the K closest nodes of
        @type callback: C{method}
        @param callback: the method to call with the results, it must take 1
            parameter, the list of K closest nodes
        """
        # Mark the bucket as having been accessed
        self.table.touch(id)
        
        # Start with our node
        nodes = [copy(self.node)]

        # Start the finding nodes action
        state = FindNode(self, id, callback, self.config, self.stats)
        reactor.callLater(0, state.goWithNodes, nodes)
    
    def insertNode(self, node, contacted = True):
        """Try to insert a node in our local table, pinging oldest contact if necessary.
        
        If all you have is a host/port, then use L{addContact}, which calls this
        method after receiving the PONG from the remote node. The reason for
        the separation is we can't insert a node into the table without its
        node ID. That means of course the node passed into this method needs
        to be a properly formed Node object with a valid ID.

        @type node: L{node.Node}
        @param node: the new node to try and insert
        @type contacted: C{boolean}
        @param contacted: whether the new node is known to be good, i.e.
            responded to a request (optional, defaults to True)
        """
        # Don't add any local nodes to the routing table
        if not self.config['LOCAL_OK'] and isLocal.match(node.host):
            log.msg('Not adding local node to table: %s/%s' % (node.host, node.port))
            return
        
        old = self.table.insertNode(node, contacted=contacted)

        if (isinstance(old, self._Node) and old.id != self.node.id and
            (datetime.now() - old.lastSeen) > 
             timedelta(seconds=self.config['MIN_PING_INTERVAL'])):
            
            # Bucket is full, check to see if old node is still available
            df = self.sendPing(old)
            df.addErrback(self._staleNodeHandler, old, node, contacted)
        elif not old and not contacted:
            # There's room, we just need to contact the node first
            df = self.sendPing(node)
            # Also schedule a future ping to make sure the node works
            def rePing(newnode, self = self):
                if newnode.id not in self.pinging:
                    self.pinging[newnode.id] = reactor.callLater(self.config['MIN_PING_INTERVAL'],
                                                                 self.sendPing, newnode)
                return newnode
            df.addCallback(rePing)

    def _staleNodeHandler(self, err, old, node, contacted):
        """The pinged node never responded, so replace it."""
        self.table.invalidateNode(old)
        self.insertNode(node, contacted)
        return err
    
    def nodeFailed(self, node):
        """Mark a node as having failed a request and schedule a future check.
        
        @type node: L{node.Node}
        @param node: the new node to try and insert
        """
        exists = self.table.nodeFailed(node)
        
        # If in the table, schedule a ping, if one isn't already sent/scheduled
        if exists and node.id not in self.pinging:
            self.pinging[node.id] = reactor.callLater(self.config['MIN_PING_INTERVAL'],
                                                      self.sendPing, node)
    
    def sendPing(self, node):
        """Ping the node to see if it's still alive.
        
        @type node: L{node.Node}
        @param node: the node to send the join to
        """
        # Check for a ping already underway
        if (isinstance(self.pinging.get(node.id, None), DelayedCall) and
            self.pinging[node.id].active()):
            self.pinging[node.id].cancel()
        elif isinstance(self.pinging.get(node.id, None), Deferred):
            return self.pinging[node.id]

        self.stats.startedAction('ping')
        df = node.ping(self.node.id)
        self.pinging[node.id] = df
        df.addCallbacks(self._pingHandler, self._pingError,
                        callbackArgs = (node, datetime.now()),
                        errbackArgs = (node, datetime.now()))
        return df

    def _pingHandler(self, dict, node, start):
        """Node responded properly, update it and return the node object."""
        self.stats.completedAction('ping', start)
        del self.pinging[node.id]
        # Create the node using the returned contact info
        n = self.Node(dict['id'], dict['_krpc_sender'][0], dict['_krpc_sender'][1])
        reactor.callLater(0, self.insertNode, n)
        return n

    def _pingError(self, err, node, start):
        """Error occurred, fail node."""
        log.msg("action ping failed on %s/%s: %s" % (node.host, node.port, err.getErrorMessage()))
        self.stats.completedAction('ping', start)
        
        # Consume unhandled errors
        self.pinging[node.id].addErrback(lambda ping_err: None)
        del self.pinging[node.id]
        
        self.nodeFailed(node)
        return err
        
    def sendJoin(self, node, callback=None, errback=None):
        """Join the DHT by pinging a bootstrap node.
        
        @type node: L{node.Node}
        @param node: the node to send the join to
        @type callback: C{method}
        @param callback: the method to call with the results, it must take 1
            parameter, the contact info returned by the node
            (optional, defaults to doing nothing with the results)
        @type errback: C{method}
        @param errback: the method to call if an error occurs
            (optional, defaults to calling the callback with the error)
        """
        if errback is None:
            errback = callback
        self.stats.startedAction('join')
        df = node.join(self.node.id)
        df.addCallbacks(self._joinHandler, self._joinError,
                        callbackArgs = (node, datetime.now()),
                        errbackArgs = (node, datetime.now()))
        if callback:
            df.addCallbacks(callback, errback)

    def _joinHandler(self, dict, node, start):
        """Node responded properly, extract the response."""
        self.stats.completedAction('join', start)
        # Create the node using the returned contact info
        n = self.Node(dict['id'], dict['_krpc_sender'][0], dict['_krpc_sender'][1])
        reactor.callLater(0, self.insertNode, n)
        return (dict['ip_addr'], dict['port'])

    def _joinError(self, err, node, start):
        """Error occurred, fail node."""
        log.msg("action join failed on %s/%s: %s" % (node.host, node.port, err.getErrorMessage()))
        self.stats.completedAction('join', start)
        self.nodeFailed(node)
        return err
        
    def findCloseNodes(self, callback=lambda a: None):
        """Perform a findNode on the ID one away from our own.

        This will allow us to populate our table with nodes on our network
        closest to our own. This is called as soon as we start up with an
        empty table.

        @type callback: C{method}
        @param callback: the method to call with the results, it must take 1
            parameter, the list of K closest nodes
            (optional, defaults to doing nothing with the results)
        """
        id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256)
        self.findNode(id, callback)

    def refreshTable(self, force = False):
        """Check all the buckets for those that need refreshing.
        
        @param force: refresh all buckets regardless of last bucket access time
            (optional, defaults to False)
        """
        def callback(nodes):
            pass
    
        for bucket in self.table.buckets:
            if force or (datetime.now() - bucket.lastAccessed > 
                         timedelta(seconds=self.config['BUCKET_STALENESS'])):
                # Choose a random ID in the bucket and try and find it
                id = newIDInRange(bucket.min, bucket.max)
                self.findNode(id, callback)

    def shutdown(self):
        """Closes the port and cancels pending later calls."""
        self.listenport.stopListening()
        try:
            self.next_checkpoint.cancel()
        except:
            pass
        for nodeid in self.pinging.keys():
            if isinstance(self.pinging[nodeid], DelayedCall) and self.pinging[nodeid].active():
                self.pinging[nodeid].cancel()
                del self.pinging[nodeid]
        self.store.close()
    
    def getStats(self):
        """Gather the statistics for the DHT."""
        return self.stats.formatHTML()

    #{ Remote interface
    def krpc_ping(self, id, _krpc_sender = None):
        """Pong with our ID.
        
        @type id: C{string}
        @param id: the node ID of the sender node
        @type _krpc_sender: (C{string}, C{int})
        @param _krpc_sender: the sender node's IP address and port
        """
        if _krpc_sender is not None:
            n = self.Node(id, _krpc_sender[0], _krpc_sender[1])
            reactor.callLater(0, self.insertNode, n, False)

        return {"id" : self.node.id}
        
    def krpc_join(self, id, _krpc_sender = None):
        """Add the node by responding with its address and port.
        
        @type id: C{string}
        @param id: the node ID of the sender node
        @type _krpc_sender: (C{string}, C{int})
        @param _krpc_sender: the sender node's IP address and port
        """
        if _krpc_sender is not None:
            n = self.Node(id, _krpc_sender[0], _krpc_sender[1])
            reactor.callLater(0, self.insertNode, n, False)
        else:
            _krpc_sender = ('127.0.0.1', self.port)

        return {"ip_addr" : _krpc_sender[0], "port" : _krpc_sender[1], "id" : self.node.id}
        
    def krpc_find_node(self, id, target, _krpc_sender = None):
        """Find the K closest nodes to the target in the local routing table.
        
        @type target: C{string}
        @param target: the target ID to find nodes for
        @type id: C{string}
        @param id: the node ID of the sender node
        @type _krpc_sender: (C{string}, C{int})
        @param _krpc_sender: the sender node's IP address and port
        """
        if _krpc_sender is not None:
            n = self.Node(id, _krpc_sender[0], _krpc_sender[1])
            reactor.callLater(0, self.insertNode, n, False)
        else:
            _krpc_sender = ('127.0.0.1', self.port)

        nodes = self.table.findNodes(target)
        nodes = map(lambda node: node.contactInfo(), nodes)
        token = sha(self.token_secrets[0] + _krpc_sender[0]).digest()
        return {"nodes" : nodes, "token" : token, "id" : self.node.id}