def end_of_interval(self): # Note: at low bitrates it is possible and correct for the # token bucket to have significantly negative values. At low # bit rates, the interval length can be on the same order # or even larger than the packet burst interarrival times, the token # size becomes smaller than zero due to a burst of packet # arrivals. This is okay. The observed rate will sawtooth around # the correct rate. # compute token size based on the time that really elapsed. now = time() if self._prev_time is None: token = int(self._max_download_rate * self._interval) else: token = int(self._max_download_rate * (now - self._prev_time)) self._prev_time = time() # update token bucket. self._token_bucket += token if self._token_bucket > self._max_token_bytes: self._token_bucket = self._max_token_bytes # if token bucket not in deficit then safe to begin sending requests. if self._token_bucket > 0: self.unthrottle() else: self.throttle()
def end_of_interval(self): # Note: at low bitrates it is possible and correct for the # token bucket to have significantly negative values. At low # bit rates, the interval length can be on the same order # or even larger than the packet burst interarrival times, the token # size becomes smaller than zero due to a burst of packet # arrivals. This is okay. The observed rate will sawtooth around # the correct rate. # compute token size based on the time that really elapsed. now = time() if self._prev_time is None: token = int(self._max_download_rate*self._interval) else: token = int(self._max_download_rate*(now-self._prev_time)) self._prev_time = time() # update token bucket. self._token_bucket += token if self._token_bucket > self._max_token_bytes: self._token_bucket = self._max_token_bytes # if token bucket not in deficit then safe to begin sending requests. if self._token_bucket > 0: self.unthrottle() else: self.throttle()
def __init__(self, contents, min, max): self.l = contents self.index = {} self.invalid = {} self.min = min self.max = max self.lastAccessed = time()
def __setitem__(self, key, value): # O(log n). actually O(log n + RECENT_SIZE) assert self._ttl > 0, "no default TTL defined. Perhaps the caller should call set " \ "rather than __setitem__." t = time() if self._data.has_key(key): ttl,_ = self._data[key] else: ttl = self._ttl self.set(key,value,ttl) if self._recent: self._recent.add(key) # perform cache replacement if necessary. if self._max_items is not None and len(self._data) > self._max_items: to_remove = [] for t,k in self._exp.iteritems(): # worst case is O(RECENT_SIZE), but it is highly unlikely # that all members of the recent access set are the oldest # in the cache. if k not in self._recent: to_remove.append(k) if len(to_remove) >= len(self._data) - self._max_items: break for k in to_remove: del self[k]
def __setitem__(self, key, value): # O(log n). actually O(log n + RECENT_SIZE) assert self._ttl > 0, "no default TTL defined. Perhaps the caller should call set " \ "rather than __setitem__." t = time() if self._data.has_key(key): ttl, _ = self._data[key] else: ttl = self._ttl self.set(key, value, ttl) if self._recent: self._recent.add(key) # perform cache replacement if necessary. if self._max_items is not None and len(self._data) > self._max_items: to_remove = [] for t, k in self._exp.iteritems(): # worst case is O(RECENT_SIZE), but it is highly unlikely # that all members of the recent access set are the oldest # in the cache. if k not in self._recent: to_remove.append(k) if len(to_remove) >= len(self._data) - self._max_items: break for k in to_remove: del self[k]
def run(self, check=False): t = time() self.expire(t) self.curr -= (t - self.last) * self.rate self.last = t if check: self.curr = max(self.curr, 0 - self.rate) shuffle(self.q) while self.q and self.curr <= 0: x, tup = self.q.pop() size = len(tup[0]) self.curr += size try: self.transport.sendto(*tup) self.sent += 1 self.rlcount(size) self.measure.update_rate(size) except: if tup[2][1] != 0: print ">>> sendto exception", tup print_exc() self.q.sort() if self.q or self.curr > 0: self.running = True # sleep for at least a half second self.call_later(max(self.curr / self.rate, 0.5), self.run) else: self.running = False
def run(self, check=False): t = time() self.expire(t) self.curr -= (t - self.last) * self.rate self.last = t if check: self.curr = max(self.curr, 0 - self.rate) shuffle(self.q) while self.q and self.curr <= 0: x, tup = self.q.pop() size = len(tup[0]) self.curr += size try: self.transport.sendto(*tup) self.sent+=1 self.rlcount(size) self.measure.update_rate(size) except: if tup[2][1] != 0: print ">>> sendto exception", tup print_exc() self.q.sort() if self.q or self.curr > 0: self.running = True # sleep for at least a half second self.call_later(max(self.curr / self.rate, 0.5), self.run) else: self.running = False
def krpc_store_value(self, key, value, id, _krpc_sender): t = "%0.6f" % time() self.store[key] = value sender = {'id': id} sender['host'] = _krpc_sender[0] sender['port'] = _krpc_sender[1] n = self.Node().initWithDict(sender) self.insertNode(n, contacted=0) return {"id": self.node.id}
def krpc_store_value(self, key, value, id, _krpc_sender): t = "%0.6f" % time() self.store[key] = value sender = {'id' : id} sender['host'] = _krpc_sender[0] sender['port'] = _krpc_sender[1] n = self.Node().initWithDict(sender) self.insertNode(n, contacted=0) return {"id" : self.node.id}
def set(self, key, value, ttl): """Set using non-default TTL. ttl is a duration, not an absolute time.""" t = time() self._data[key] = (ttl, value) i = self._exp.find_key_by_value(key) if i.at_end(): self._exp[t + ttl] = key else: assert i.value() == key self._exp.update_key(i, t + ttl)
def set(self, key, value, ttl): """Set using non-default TTL. ttl is a duration, not an absolute time.""" t = time() self._data[key] = (ttl, value) i = self._exp.find_key_by_value(key) if i.at_end(): self._exp[t+ttl] = key else: assert i.value() == key self._exp.update_key(i,t+ttl)
def add(self, key): # O(log n) i = self._data.find_key_by_value(key) t = time() if i.at_end(): self._data[t] = key else: self._data.update_key(i,t) while len(self._data) > self._max_items: j = self._data.begin() assert not j.at_end() self._data.erase(j)
def add(self, key): # O(log n) i = self._data.find_key_by_value(key) t = time() if i.at_end(): self._data[t] = key else: self._data.update_key(i, t) while len(self._data) > self._max_items: j = self._data.begin() assert not j.at_end() self._data.erase(j)
def _expire2(self): t = time() #try: while True: i = self._exp.begin() if i.at_end(): break if i.key() < t: key = i.value() self._exp.erase(i) del self._data[key] else: break assert len(self._data) == len(self._exp)
def setup(self, host, port, data_dir, rlcount, checkpoint=True): self.host = host self.port = port self.ddir = data_dir self.store = KStore() self.pingcache = {} self.socket = self.rawserver.create_udpsocket(self.port, self.host) self.udp = krpc.hostbroker(self, (self.host, self.port), self.socket, self.rawserver.add_task, self.max_ul_rate, self.config, rlcount) self._load() self.rawserver.start_listening_udp(self.socket, self.udp) self.last = time() KeyExpirer(self.store, self.rawserver.add_task) self.refreshTable(force=1) if checkpoint: self.rawserver.add_task(30, self.findCloseNodes, lambda a: a, True) self.rawserver.add_task(60, self.checkpoint, 1)
def refreshTable(self, force=0): """ force=1 will refresh table regardless of last bucket access time """ def callback(nodes): pass refresh = [ bucket for bucket in self.table.buckets if force or (len(bucket.l) < K) or len(filter(lambda a: a.invalid, bucket.l)) or ( time() - bucket.lastAccessed > const.BUCKET_STALENESS) ] for bucket in refresh: id = newIDInRange(bucket.min, bucket.max) self.findNode(id, callback)
def __getitem__(self, key): # O(1) if not touch and not newly expired, else O(log n) """Raises KeyError if the key is not in the cache. This can happen if the entry was deleted or expired.""" ttl,v = self._data[key] # ttl is duration, not absolute time. i = self._exp.find_key_by_value(key) # O(1). Key in exp is time. 'key' variable # is exp's value. :-) if i.at_end(): raise KeyError() t = time() if i.key() < t: # expired. del self[key] # O(log n) raise KeyError() if self._recent: self._recent.add(key) if self._touch: self._exp.update_key(i,t+ttl) # O(1) if no reordering else O(log n) return v
def __getitem__( self, key): # O(1) if not touch and not newly expired, else O(log n) """Raises KeyError if the key is not in the cache. This can happen if the entry was deleted or expired.""" ttl, v = self._data[key] # ttl is duration, not absolute time. i = self._exp.find_key_by_value( key) # O(1). Key in exp is time. 'key' variable # is exp's value. :-) if i.at_end(): raise KeyError() t = time() if i.key() < t: # expired. del self[key] # O(log n) raise KeyError() if self._recent: self._recent.add(key) if self._touch: self._exp.update_key(i, t + ttl) # O(1) if no reordering else O(log n) return v
def insertNode(self, node, contacted=1, nocheck=False): """ this insert the node, returning None if successful, returns the oldest node in the bucket if it's full the caller responsible for pinging the returned node and calling replaceStaleNode if it is found to be stale!! contacted means that yes, we contacted THEM and we know the node is reachable """ if node.id == NULL_ID or node.id == self.node.id: return if contacted: node.updateLastSeen() # get the bucket for this node i = self._bucketIndexForInt(node.num) # check to see if node is in the bucket already if self.buckets[i].hasNode(node): it = self.buckets[i].l.index(node.num) xnode = self.buckets[i].l[it] if contacted: node.age = xnode.age self.buckets[i].seenNode(node) elif xnode.lastSeen != 0 and xnode.port == node.port and xnode.host == node.host: xnode.updateLastSeen() return # we don't have this node, check to see if the bucket is full if not self.buckets[i].bucketFull(): # no, append this node and return self.buckets[i].addNode(node) return # full bucket, check to see if any nodes are invalid t = time() invalid = [x for x in self.buckets[i].invalid.values() if x.invalid] if len(invalid) and not nocheck: invalid.sort(ls) while invalid and not self.buckets[i].hasNode(invalid[0]): del(self.buckets[i].invalid[invalid[0].num]) invalid = invalid[1:] if invalid and (invalid[0].lastSeen == 0 and invalid[0].fails < MAX_FAILURES): return invalid[0] elif invalid: self.replaceStaleNode(invalid[0], node) return stale = [n for n in self.buckets[i].l if (t - n.lastSeen) > MIN_PING_INTERVAL] if len(stale) and not nocheck: stale.sort(ls) return stale[0] # bucket is full and all nodes are valid, check to see if self.node is in the bucket if not (self.buckets[i].min <= self.node < self.buckets[i].max): return # this bucket is full and contains our node, split the bucket if len(self.buckets) >= HASH_LENGTH: # our table is FULL, this is really unlikely print "Hash Table is FULL! Increase K!" return self._splitBucket(self.buckets[i]) # now that the bucket is split and balanced, try to insert the node again return self.insertNode(node, contacted)
def touch(self): self.lastAccessed = time()
def updateLastSeen(self): self.lastSeen = time() self.fails = 0 self.invalid = False
def __init__(self): self.fails = 0 self.lastSeen = 0 self.invalid = True self.id = self.host = self.port = '' self.age = time()
def __init__(self, key, value): self.t = time() self.k = key self.v = value
def sendto(self, s, i, addr): self.q.append((time(), (s, i, addr))) if not self.running: self.run(check=True)
def expire(self, t=time()): if self.q: expire_time = t - self.age while self.q and self.q[0][0] < expire_time: self.q.pop(0) self.dropped+=1
def __repr__(self): return ` (self.k, self.v, time() - self.t) `
def doExpire(self): self.cut = time() - const.KE_AGE self.store.expire(self.cut) self.callLater(const.KE_DELAY, self.doExpire)
def expire(self, t=time()): if self.q: expire_time = t - self.age while self.q and self.q[0][0] < expire_time: self.q.pop(0) self.dropped += 1
def __repr__(self): return `(self.k, self.v, time() - self.t)`
def __setitem__(self, key, value): t = time() self.data[key] = (t, value) self.q.appendleft((t, key, value))
def refreshTable(self, force=0): """ force=1 will refresh table regardless of last bucket access time """ def callback(nodes): pass refresh = [bucket for bucket in self.table.buckets if force or (len(bucket.l) < K) or len(filter(lambda a: a.invalid, bucket.l)) or (time() - bucket.lastAccessed > const.BUCKET_STALENESS)] for bucket in refresh: id = newIDInRange(bucket.min, bucket.max) self.findNode(id, callback)
def insertNode(self, node, contacted=1, nocheck=False): """ this insert the node, returning None if successful, returns the oldest node in the bucket if it's full the caller responsible for pinging the returned node and calling replaceStaleNode if it is found to be stale!! contacted means that yes, we contacted THEM and we know the node is reachable """ if node.id == NULL_ID or node.id == self.node.id: return if contacted: node.updateLastSeen() # get the bucket for this node i = self._bucketIndexForInt(node.num) # check to see if node is in the bucket already if self.buckets[i].hasNode(node): it = self.buckets[i].l.index(node.num) xnode = self.buckets[i].l[it] if contacted: node.age = xnode.age self.buckets[i].seenNode(node) elif xnode.lastSeen != 0 and xnode.port == node.port and xnode.host == node.host: xnode.updateLastSeen() return # we don't have this node, check to see if the bucket is full if not self.buckets[i].bucketFull(): # no, append this node and return self.buckets[i].addNode(node) return # full bucket, check to see if any nodes are invalid t = time() invalid = [x for x in self.buckets[i].invalid.values() if x.invalid] if len(invalid) and not nocheck: invalid.sort(ls) while invalid and not self.buckets[i].hasNode(invalid[0]): del (self.buckets[i].invalid[invalid[0].num]) invalid = invalid[1:] if invalid and (invalid[0].lastSeen == 0 and invalid[0].fails < MAX_FAILURES): return invalid[0] elif invalid: self.replaceStaleNode(invalid[0], node) return stale = [ n for n in self.buckets[i].l if (t - n.lastSeen) > MIN_PING_INTERVAL ] if len(stale) and not nocheck: stale.sort(ls) return stale[0] # bucket is full and all nodes are valid, check to see if self.node is in the bucket if not (self.buckets[i].min <= self.node < self.buckets[i].max): return # this bucket is full and contains our node, split the bucket if len(self.buckets) >= HASH_LENGTH: # our table is FULL, this is really unlikely print "Hash Table is FULL! Increase K!" return self._splitBucket(self.buckets[i]) # now that the bucket is split and balanced, try to insert the node again return self.insertNode(node, contacted)
def _run(): TTL = 1 SET_TTL = 2 # TTL used when explicitly setting TTL using "def set." EXPIRE_INTERVAL = .3 EPSILON = .5 ### # BoundedCacheSet correctness tests. c = _BoundedCacheSet(2) c.add(10) assert 10 in c c.add(15) assert 15 in c c.add(16) assert 16 in c assert 10 not in c assert 15 in c c.remove(15) assert 15 not in c try: c.remove(23) assert False except KeyError: pass ### # basic CacheMap correctness tests. c = CacheMap(default_ttl=TTL,expire_interval=EPSILON) class K(object): def __init__(self): self.x = range(10000) class V(object): def __init__(self): self.x = range(10000) k = K() v = V() t = time() c.set(k, v, SET_TTL) assert len(c) == 1 assert c.num_unexpired() == 1 assert c._exp.begin().key() < t + SET_TTL + EPSILON and \ c._exp.begin().key() > t + SET_TTL - EPSILON, \ "First item in c._exp should have expiration time that is close to the " \ "current time + SET_TTL which is %s, but the expiration time is %s." \ % (t+SET_TTL, c._exp.begin().key()) assert c.has_key(k) assert not c.has_key( "blah" ) assert c[k] == v c._expire2() # should not expire anything because little time has passed. assert len(c) == 1 assert c.num_unexpired() == 1 try: y = c[10] assert False, "should've raised KeyError." except KeyError: pass v2 = V() c[k] = v2 assert c._exp.begin().key() < t + SET_TTL + EPSILON and \ c._exp.begin().key() > t + SET_TTL - EPSILON, \ "First item in c._exp should have expiration time that is close to the " \ "current time + SET_TTL, but the expiration time is %s." % c._exp.begin().key() assert not c[k] == v assert c[k] == v2 assert len(c) == 1 assert c.num_unexpired() == 1 k2 = K() t = time() c[k2] = v2 assert c._exp.begin().key() < t + TTL + EPSILON and \ c._exp.begin().key() > t + TTL - EPSILON, \ "First item in c._exp should have expiration time that is close to the " \ "current time + TTL, but the expiration time is %s." % c._exp.begin().key() assert c[k2] == v2 assert not c[k] == v # shouldn't be a problem with two items having the same value. assert len(c) == 2 assert c.num_unexpired() == 2 # wait long enough for the cache entries to expire. df = Deferred() reactor.callLater(SET_TTL+EPSILON, df.callback, None) yield df df.getResult() assert c.num_unexpired() == 0, "Should have expired all entries, but there are %d " \ "unexpired items and %d items in c._data. " % (c.num_unexpired(), len(c._data)) assert len(c) == 0 assert len(c._exp) == 0 assert len(c._data) == 0 assert k not in c assert k2 not in c # basic correctness of bounded-size cache map. c = CacheMap(default_ttl=TTL,expire_interval=1000,max_items = 2) c[k] = v assert len(c) == 1 assert c[k] == v c[k2] = v2 assert len(c) == 2 assert c[k2] == v2 c[10] = 15 assert len(c) == 2 assert c[10] == 15 assert c[k2] == v2 # order from most recent access is now [(k2,v2), (10,15), (k,v)]. try: a = c[k] assert False, "when cache with size bound of 2 exceeded 2 elements, " \ "the oldest should've been removed." except KeyError: pass c[56] = 1 # order from most recent access ... assert len(c) == 2 assert 56 in c assert 10 not in c ### # test expirations and for memory leaks. # Watch memory consumption (e.g., using top) and see if it grows. if LEAK_TEST: c = CacheMap(default_ttl=TTL,expire_interval=EPSILON) i = 0 while True: for x in xrange(100): i += 1 if i % 20 == 0: print len(c) c[i] = K() if i % 5 == 0: try: l = len(c) del c[i] assert len(c) == l-1 except KeyError: pass # allow time for expirations. df = Deferred() reactor.callLater(TTL+EPSILON,df.callback,None) yield df df.getResult()
def _run(): TTL = 1 SET_TTL = 2 # TTL used when explicitly setting TTL using "def set." EXPIRE_INTERVAL = .3 EPSILON = .5 ### # BoundedCacheSet correctness tests. c = _BoundedCacheSet(2) c.add(10) assert 10 in c c.add(15) assert 15 in c c.add(16) assert 16 in c assert 10 not in c assert 15 in c c.remove(15) assert 15 not in c try: c.remove(23) assert False except KeyError: pass ### # basic CacheMap correctness tests. c = CacheMap(default_ttl=TTL, expire_interval=EPSILON) class K(object): def __init__(self): self.x = range(10000) class V(object): def __init__(self): self.x = range(10000) k = K() v = V() t = time() c.set(k, v, SET_TTL) assert len(c) == 1 assert c.num_unexpired() == 1 assert c._exp.begin().key() < t + SET_TTL + EPSILON and \ c._exp.begin().key() > t + SET_TTL - EPSILON, \ "First item in c._exp should have expiration time that is close to the " \ "current time + SET_TTL which is %s, but the expiration time is %s." \ % (t+SET_TTL, c._exp.begin().key()) assert c.has_key(k) assert not c.has_key("blah") assert c[k] == v c._expire2( ) # should not expire anything because little time has passed. assert len(c) == 1 assert c.num_unexpired() == 1 try: y = c[10] assert False, "should've raised KeyError." except KeyError: pass v2 = V() c[k] = v2 assert c._exp.begin().key() < t + SET_TTL + EPSILON and \ c._exp.begin().key() > t + SET_TTL - EPSILON, \ "First item in c._exp should have expiration time that is close to the " \ "current time + SET_TTL, but the expiration time is %s." % c._exp.begin().key() assert not c[k] == v assert c[k] == v2 assert len(c) == 1 assert c.num_unexpired() == 1 k2 = K() t = time() c[k2] = v2 assert c._exp.begin().key() < t + TTL + EPSILON and \ c._exp.begin().key() > t + TTL - EPSILON, \ "First item in c._exp should have expiration time that is close to the " \ "current time + TTL, but the expiration time is %s." % c._exp.begin().key() assert c[k2] == v2 assert not c[ k] == v # shouldn't be a problem with two items having the same value. assert len(c) == 2 assert c.num_unexpired() == 2 # wait long enough for the cache entries to expire. df = Deferred() reactor.callLater(SET_TTL + EPSILON, df.callback, None) yield df df.getResult() assert c.num_unexpired() == 0, "Should have expired all entries, but there are %d " \ "unexpired items and %d items in c._data. " % (c.num_unexpired(), len(c._data)) assert len(c) == 0 assert len(c._exp) == 0 assert len(c._data) == 0 assert k not in c assert k2 not in c # basic correctness of bounded-size cache map. c = CacheMap(default_ttl=TTL, expire_interval=1000, max_items=2) c[k] = v assert len(c) == 1 assert c[k] == v c[k2] = v2 assert len(c) == 2 assert c[k2] == v2 c[10] = 15 assert len(c) == 2 assert c[10] == 15 assert c[ k2] == v2 # order from most recent access is now [(k2,v2), (10,15), (k,v)]. try: a = c[k] assert False, "when cache with size bound of 2 exceeded 2 elements, " \ "the oldest should've been removed." except KeyError: pass c[56] = 1 # order from most recent access ... assert len(c) == 2 assert 56 in c assert 10 not in c ### # test expirations and for memory leaks. # Watch memory consumption (e.g., using top) and see if it grows. if LEAK_TEST: c = CacheMap(default_ttl=TTL, expire_interval=EPSILON) i = 0 while True: for x in xrange(100): i += 1 if i % 20 == 0: print len(c) c[i] = K() if i % 5 == 0: try: l = len(c) del c[i] assert len(c) == l - 1 except KeyError: pass # allow time for expirations. df = Deferred() reactor.callLater(TTL + EPSILON, df.callback, None) yield df df.getResult()