def __init__(self, contents, min, max):
self.l = contents
self.index = {}
self.invalid = {}
self.min = min
self.max = max
self.lastAccessed = time()
def run(self, check=False):
t = time()
self.expire(t)
self.curr -= (t - self.last) * self.rate
self.last = t
if check:
self.curr = max(self.curr, 0 - self.rate)
shuffle(self.q)
while self.q and self.curr <= 0:
x, tup = self.q.pop()
size = len(tup[0])
self.curr += size
try:
self.transport.sendto(*tup)
self.sent += 1
self.rlcount(size)
self.measure.update_rate(size)
except:
if tup[2][1] != 0:
print ">>> sendto exception", tup
print_exc()
self.q.sort()
if self.q or self.curr > 0:
self.running = True
# sleep for at least a half second
self.call_later(self.run, max(self.curr / self.rate, 0.5))
else:
self.running = False
def run(self, check=False):
t = time()
self.expire(t)
self.curr -= (t - self.last) * self.rate
self.last = t
if check:
self.curr = max(self.curr, 0 - self.rate)
shuffle(self.q)
while self.q and self.curr <= 0:
x, tup = self.q.pop()
size = len(tup[0])
self.curr += size
try:
self.transport.sendto(*tup)
self.sent+=1
self.rlcount(size)
self.measure.update_rate(size)
except:
if tup[2][1] != 0:
print ">>> sendto exception", tup
print_exc()
self.q.sort()
if self.q or self.curr > 0:
self.running = True
# sleep for at least a half second
self.call_later(self.run, max(self.curr / self.rate, 0.5))
else:
self.running = False
def __init__(self, contents, min, max):
self.l = contents
self.index = {}
self.invalid = {}
self.min = min
self.max = max
self.lastAccessed = time()
def krpc_store_value(self, key, value, id, _krpc_sender):
t = "%0.6f" % time()
self.store[key] = value
sender = {'id': id}
sender['host'] = _krpc_sender[0]
sender['port'] = _krpc_sender[1]
n = self.Node().initWithDict(sender)
self.insertNode(n, contacted=0)
return {"id": self.node.id}
def krpc_store_value(self, key, value, id, _krpc_sender):
t = "%0.6f" % time()
self.store[key] = value
sender = {'id' : id}
sender['host'] = _krpc_sender[0]
sender['port'] = _krpc_sender[1]
n = self.Node().initWithDict(sender)
self.insertNode(n, contacted=0)
return {"id" : self.node.id}
def setup(self, host, port, data_dir, rlcount, checkpoint=True):
self.host = host
self.port = port
self.ddir = data_dir
self.store = KStore()
self.pingcache = {}
self.socket = self.rawserver.create_udpsocket(self.port, self.host, False)
self.udp = krpc.hostbroker(self, (self.host, self.port), self.socket, self.rawserver.add_task, self.max_ul_rate, self.config, rlcount)
self._load()
self.rawserver.start_listening_udp(self.socket, self.udp)
self.last = time()
KeyExpirer(self.store, self.rawserver.add_task)
self.refreshTable(force=1)
if checkpoint:
self.rawserver.add_task(self.findCloseNodes, 30, (lambda a: a, True))
self.rawserver.add_task(self.checkpoint, 60, (1,))
def refreshTable(self, force=0):
"""
force=1 will refresh table regardless of last bucket access time
"""
def callback(nodes):
pass
refresh = [
bucket for bucket in self.table.buckets
if force or (len(bucket.l) < K)
or len(filter(lambda a: a.invalid, bucket.l)) or (
time() - bucket.lastAccessed > const.BUCKET_STALENESS)
]
for bucket in refresh:
id = newIDInRange(bucket.min, bucket.max)
self.findNode(id, callback)
def setup(self, host, port, data_dir, rlcount, checkpoint=True):
self.host = host
self.port = port
self.ddir = data_dir
self.store = KStore()
self.pingcache = {}
self.socket = self.rawserver.create_udpsocket(self.port, self.host,
False)
self.udp = krpc.hostbroker(self, (self.host, self.port), self.socket,
self.rawserver.add_task, self.max_ul_rate,
self.config, rlcount)
self._load()
self.rawserver.start_listening_udp(self.socket, self.udp)
self.last = time()
KeyExpirer(self.store, self.rawserver.add_task)
self.refreshTable(force=1)
if checkpoint:
self.rawserver.add_task(self.findCloseNodes, 30,
(lambda a: a, True))
self.rawserver.add_task(self.checkpoint, 60, (1, ))
def touch(self):
self.lastAccessed = time()
def __init__(self):
self.fails = 0
self.lastSeen = 0
self.invalid = True
self.id = self.host = self.port = ''
self.age = time()
def insertNode(self, node, contacted=1, nocheck=False):
"""
this insert the node, returning None if successful, returns the oldest node in the bucket if it's full
the caller responsible for pinging the returned node and calling replaceStaleNode if it is found to be stale!!
contacted means that yes, we contacted THEM and we know the node is reachable
"""
if node.id == NULL_ID or node.id == self.node.id:
return
if contacted:
node.updateLastSeen()
# get the bucket for this node
i = self._bucketIndexForInt(node.num)
# check to see if node is in the bucket already
if self.buckets[i].hasNode(node):
it = self.buckets[i].l.index(node.num)
xnode = self.buckets[i].l[it]
if contacted:
node.age = xnode.age
self.buckets[i].seenNode(node)
elif xnode.lastSeen != 0 and xnode.port == node.port and xnode.host == node.host:
xnode.updateLastSeen()
return
# we don't have this node, check to see if the bucket is full
if not self.buckets[i].bucketFull():
# no, append this node and return
self.buckets[i].addNode(node)
return
# full bucket, check to see if any nodes are invalid
t = time()
invalid = [x for x in self.buckets[i].invalid.values() if x.invalid]
if len(invalid) and not nocheck:
invalid.sort(ls)
while invalid and not self.buckets[i].hasNode(invalid[0]):
del (self.buckets[i].invalid[invalid[0].num])
invalid = invalid[1:]
if invalid and (invalid[0].lastSeen == 0
and invalid[0].fails < MAX_FAILURES):
return invalid[0]
elif invalid:
self.replaceStaleNode(invalid[0], node)
return
stale = [
n for n in self.buckets[i].l
if (t - n.lastSeen) > MIN_PING_INTERVAL
]
if len(stale) and not nocheck:
stale.sort(ls)
return stale[0]
# bucket is full and all nodes are valid, check to see if self.node is in the bucket
if not (self.buckets[i].min <= self.node < self.buckets[i].max):
return
# this bucket is full and contains our node, split the bucket
if len(self.buckets) >= HASH_LENGTH:
# our table is FULL, this is really unlikely
print "Hash Table is FULL! Increase K!"
return
self._splitBucket(self.buckets[i])
# now that the bucket is split and balanced, try to insert the node again
return self.insertNode(node, contacted)
def updateLastSeen(self):
self.lastSeen = time()
self.fails = 0
self.invalid = False
def doExpire(self):
self.cut = time() - const.KE_AGE
self.store.expire(self.cut)
self.callLater(self.doExpire, const.KE_DELAY)
def expire(self, t=time()):
if self.q:
expire_time = t - self.age
while self.q and self.q[0][0] < expire_time:
self.q.pop(0)
self.dropped += 1
def __repr__(self):
return `(self.k, self.v, time() - self.t)`
def insertNode(self, node, contacted=1, nocheck=False):
"""
this insert the node, returning None if successful, returns the oldest node in the bucket if it's full
the caller responsible for pinging the returned node and calling replaceStaleNode if it is found to be stale!!
contacted means that yes, we contacted THEM and we know the node is reachable
"""
if node.id == NULL_ID or node.id == self.node.id:
return
if contacted:
node.updateLastSeen()
# get the bucket for this node
i = self._bucketIndexForInt(node.num)
# check to see if node is in the bucket already
if self.buckets[i].hasNode(node):
it = self.buckets[i].l.index(node.num)
xnode = self.buckets[i].l[it]
if contacted:
node.age = xnode.age
self.buckets[i].seenNode(node)
elif xnode.lastSeen != 0 and xnode.port == node.port and xnode.host == node.host:
xnode.updateLastSeen()
return
# we don't have this node, check to see if the bucket is full
if not self.buckets[i].bucketFull():
# no, append this node and return
self.buckets[i].addNode(node)
return
# full bucket, check to see if any nodes are invalid
t = time()
invalid = [x for x in self.buckets[i].invalid.values() if x.invalid]
if len(invalid) and not nocheck:
invalid.sort(ls)
while invalid and not self.buckets[i].hasNode(invalid[0]):
del(self.buckets[i].invalid[invalid[0].num])
invalid = invalid[1:]
if invalid and (invalid[0].lastSeen == 0 and invalid[0].fails < MAX_FAILURES):
return invalid[0]
elif invalid:
self.replaceStaleNode(invalid[0], node)
return
stale = [n for n in self.buckets[i].l if (t - n.lastSeen) > MIN_PING_INTERVAL]
if len(stale) and not nocheck:
stale.sort(ls)
return stale[0]
# bucket is full and all nodes are valid, check to see if self.node is in the bucket
if not (self.buckets[i].min <= self.node < self.buckets[i].max):
return
# this bucket is full and contains our node, split the bucket
if len(self.buckets) >= HASH_LENGTH:
# our table is FULL, this is really unlikely
print "Hash Table is FULL! Increase K!"
return
self._splitBucket(self.buckets[i])
# now that the bucket is split and balanced, try to insert the node again
return self.insertNode(node, contacted)
def expire(self, t=time()):
if self.q:
expire_time = t - self.age
while self.q and self.q[0][0] < expire_time:
self.q.pop(0)
self.dropped+=1
def __setitem__(self, key, value):
t = time()
self.data[key] = (t, value)
self.q.insert(0, (t, key, value))
def sendto(self, s, i, addr):
self.q.append((time(), (s, i, addr)))
if not self.running:
self.run(check=True)
def doExpire(self):
self.cut = time() - const.KE_AGE
self.store.expire(self.cut)
self.callLater(self.doExpire, const.KE_DELAY)
def sendto(self, s, i, addr):
self.q.append((time(), (s, i, addr)))
if not self.running:
self.run(check=True)
def __setitem__(self, key, value):
t = time()
self.data[key] = (t, value)
self.q.insert(0, (t, key, value))
def touch(self):
self.lastAccessed = time()
def refreshTable(self, force=0):
"""
force=1 will refresh table regardless of last bucket access time
"""
def callback(nodes):
pass
refresh = [bucket for bucket in self.table.buckets if force or (len(bucket.l) < K) or len(filter(lambda a: a.invalid, bucket.l)) or (time() - bucket.lastAccessed > const.BUCKET_STALENESS)]
for bucket in refresh:
id = newIDInRange(bucket.min, bucket.max)
self.findNode(id, callback)
def updateLastSeen(self):
self.lastSeen = time()
self.fails = 0
self.invalid = False
def __init__(self, key, value):
self.t = time()
self.k = key
self.v = value
def __init__(self):
self.fails = 0
self.lastSeen = 0
self.invalid = True
self.id = self.host = self.port = ''
self.age = time()