def testStoreRetrieve(self):
for i in range(10):
K = newID()
V = newID()
for a in range(3):
self.done = 0
def _scb(key, value, result):
self.done = 1
self.l[randrange(0, self.num)].storeValueForKey(K, V, _scb)
while not self.done:
reactor.iterate()
def _rcb(key, val):
if not val:
self.done = 1
self.failUnlessEqual(self.got, 1)
elif V in val:
self.got = 1
for x in range(3):
self.got = 0
self.done = 0
self.l[randrange(0, self.num)].valueForKey(K, _rcb)
while not self.done:
reactor.iterate()
def testStoreRetrieve(self):
for i in range(10):
K = khash.newID()
V = khash.newID()
for a in range(3):
self.done = 0
def _scb(val):
self.done = 1
self.l[randrange(0, self.num)].storeValueForKey(K, V, _scb)
while not self.done:
self.r.listen_once(1)
def _rcb(val):
if not val:
self.done = 1
self.assertEqual(self.got, 1)
elif V in val:
self.got = 1
for x in range(3):
self.got = 0
self.done = 0
self.l[randrange(0, self.num)].valueForKey(K, _rcb)
while not self.done:
self.r.listen_once(1)
def sendRequest(self, method, args):
"""Send a request to the remote node.
@type method: C{string}
@param method: the method name to call on the remote node
@param args: the arguments to send to the remote node's method
"""
if self.stopped:
return defer.fail(KrpcError(KRPC_ERROR_PROTOCOL_STOPPED,
"cannot send, connection has been stopped"))
# Create the request message
newTID = newID()
msg = {TID : newTID, TYP : REQ, REQ : method, ARG : args}
if self.config.get('SPEW', False):
log.msg("%d sending to %r: %s" % (self.factory.port, self.addr, msg))
data = bencode(msg)
# Create the request object and save it with the TID
req = KrpcRequest(self, newTID, method, data, self.config)
self.tids[newTID] = req
# Save the conclusion of the action
req.addCallbacks(self.stats.responseAction, self.stats.failedAction,
callbackArgs = (method, datetime.now()),
errbackArgs = (method, datetime.now()))
return req
def setup(self, config, cache_dir):
"""Setup all the Khashmir sub-modules.
@type config: C{dictionary}
@param config: the configuration parameters for the DHT
@type cache_dir: C{string}
@param cache_dir: the directory to store all files in
"""
self.config = config
self.port = config['PORT']
self.store = DB(os.path.join(cache_dir, 'khashmir.' + str(self.port) + '.db'))
self.node = self._loadSelfNode('', self.port)
self.table = KTable(self.node, config)
self.token_secrets = [newID()]
self.stats = StatsLogger(self.table, self.store)
# Start listening
self.udp = krpc.hostbroker(self, self.stats, config)
self.udp.protocol = krpc.KRPC
self.listenport = reactor.listenUDP(self.port, self.udp)
# Load the routing table and begin checkpointing
self._loadRoutingTable()
self.refreshTable(force = True)
self.next_checkpoint = reactor.callLater(60, self.checkpoint)
def newSecret(self):
"""
Generate new secret every 5 minutes
"""
self.oldSecret = self.secret
self.secret = newID(digest=False)
self.factory.rawserver.add_task(self.newSecret, TOKEN_UPDATE)
def _loadSelfNode(self, host, port):
c = self.store.cursor()
c.execute('select id from self where num = 0;')
if c.rowcount > 0:
id = c.fetchone()[0]
else:
id = newID()
return self._Node().init(id, host, port)
def testMergeBuckets(self):
for i in xrange(1000):
b = Node(khash.newID(), "127.0.0.1", 2003 + i)
self.t.insertNode(b)
num = len(self.t.buckets)
i = self.t._bucketIndexForInt(self.a.num)
for b in self.t.buckets[i].nodes[:]:
self.t.invalidateNode(b)
self.failUnlessEqual(len(self.t.buckets), num - 1)
def _load(self):
do_load = False
try:
s = open(os.path.join(self.ddir, "routing_table"), 'r').read()
dict = bdecode(s)
except:
id = newID()
else:
id = dict['id']
do_load = True
self.node = self._Node(self.udp.connectionForAddr).init(id, self.host, self.port)
self.table = KTable(self.node)
if do_load:
self._loadRoutingTable(dict['rt'])
def _load(self):
do_load = False
try:
s = open(os.path.join(self.ddir, "routing_table"), 'r').read()
dict = bdecode(s)
except:
id = newID()
else:
id = dict['id']
do_load = True
self.node = self._Node(self.udp.connectionForAddr).init(id, self.host, self.port)
self.table = KTable(self.node)
if do_load:
self._loadRoutingTable(dict['rt'])
def checkpoint(self):
"""Perform some periodic maintenance operations."""
# Create a new token secret
self.token_secrets.insert(0, newID())
if len(self.token_secrets) > 3:
self.token_secrets.pop()
# Save some parameters for reloading
self.store.saveSelfNode(self.node.id)
self.store.dumpRoutingTable(self.table.buckets)
# DHT maintenance
self.store.expireValues(self.config['KEY_EXPIRE'])
self.refreshTable()
self.next_checkpoint = reactor.callLater(randrange(int(self.config['CHECKPOINT_INTERVAL'] * .9),
int(self.config['CHECKPOINT_INTERVAL'] * 1.1)),
self.checkpoint)
def _loadSelfNode(self):
"""Load the root node"""
if DEBUG:
print("Debug: DHT - loadSelfNode")
# Get ID
c = self.store.cursor()
c.execute('select id, age from self where num = 0')
data = c.fetchone()
# Clean if too old
if not data or time() - data[1] > 86400*5: # more than 5 days old
id = newID()
c.execute('delete from self')
c.execute("insert into self values (0, ?, ?)", (sqlite3.Binary(id), time()))
c.execute('delete from kv')
c.execute('delete from nodes')
self.store.commit()
else:
id = str(data[0])
c.close()
# Load self node
self.node = self.Node().init(id, self.host, self.port)
def gen_token(self, loop=False):
self.last_token = self.cur_token
self.cur_token = sha(newID())
if loop:
self.rawserver.external_add_task(self.gen_token, TOKEN_UPDATE_INTERVAL, (True,))
def testAddNode(self):
self.b = Node().init(hash.newID(), 'localhost', 2003)
self.t.insertNode(self.b)
self.assertEqual(len(self.t.buckets[0].l), 1)
self.assertEqual(self.t.buckets[0].l[0], self.b)
def setUp(self):
self.a = Node().init(hash.newID(), 'localhost', 2002)
self.t = KTable(self.a)
def setUp(self):
self.node = Node(khash.newID(), "127.0.0.1", 2002)
def _loadSelfNode(self, host, port):
"""Create this node, loading any previously saved one."""
id = self.store.getSelfNode()
if not id or not id.endswith(self.config['VERSION']):
id = newID(self.config['VERSION'])
return self._Node(id, host, port)
def setUp(self):
self.node = Node().init(khash.newID(), 'localhost', 2002)
def setUp(self):
self.a = Node(khash.newID(), "127.0.0.1", 2002)
self.t = KTable(self.a, {"MAX_FAILURES": 3})
def gen_token(self, loop=False):
self.last_token = self.cur_token
self.cur_token = sha(newID())
if loop:
self.add_task(TOKEN_UPDATE_INTERVAL,
self.gen_token, True)
def gen_token(self, loop=False):
self.last_token = self.cur_token
self.cur_token = sha(newID())
if loop:
self.add_task(TOKEN_UPDATE_INTERVAL, self.gen_token, True)
def testAddNode(self):
self.b = Node(khash.newID(), "127.0.0.1", 2003)
self.t.insertNode(self.b)
self.failUnlessEqual(len(self.t.buckets[0].nodes), 1)
self.failUnlessEqual(self.t.buckets[0].nodes[0], self.b)
