class Deterministic_Zipfian_Lambda(object): def __init__(self, count, theta): zetan = zipfian_zeta(theta, count) zeta2theta = zipfian_zeta(theta, 2) self.count = count self.theta = theta freqs = [float(i)**-theta for i in range(1, count + 1)] s = sum(freqs) self.lambdas = [s / i for i in freqs] self.t = 0 self.next_arrivals = SortedCollection(iterable=enumerate(self.lambdas), key=itemgetter(1)) def get_popularity(self, item): return (1.0 / self.lambdas[item]) def __get_next(self): self.t += 1 item = self.next_arrivals[0] del self.next_arrivals[0] item_lambda = self.lambdas[item[0]] item_next = (item[0], item_lambda + self.t) self.next_arrivals.insert_right(item_next) return (item_next[0], 1.0 / item_lambda) def get_next(self, rand_float): return self.__get_next()
def __init__(self, driver = None, name = "PK_Freq", **kwargs): self.name = name self.nodes = SortedCollection(key=attrgetter('priority')) self.error_numer = 0 self.error_denom = 1 self.driver_access = driver self.watch_for_shift = (driver_access and "just_shifted" in driver_access.__dict__)
class GD_PQ_PCoarse(object): def __init__(self, n_buckets, name = None): if name: self.name = name else: self.name = self.__class__.__name__ self.nodes_cur = SortedCollection(key = attrgetter("priority")) self.nodes_next = SortedCollection(key = attrgetter("priority")) self.H = 0 self.n_buckets = n_buckets self.error_numer = 0 self.error_denom = 1 def touch(self, node): if node in self.nodes_cur: self.nodes_cur.remove(node) else: self.nodes_next.remove(node) self._insert_with_p_(node) def add_node(self, value, cost): if cost >= self.n_buckets: print("cost(%f) > n_buckets(%d)") % (cost, self.n_buckets) assert False new_node = PQNode(value, cost) self._insert_with_p_(new_node) return new_node def _insert_with_p_(self, node): p = int(self.H + node.cost + 0.5) if p >= self.n_buckets: p -= self.n_buckets target_q = self.nodes_next else: target_q = self.nodes_cur node.priority = p target_q.insert_right(node) def evict_node(self): if len(self.nodes_cur) == 0: t = self.nodes_cur self.nodes_cur = self.nodes_next self.nodes_next = t to_evict = self.nodes_cur[0] del self.nodes_cur[0] self.H = to_evict.priority return to_evict
def __init__(self, n_buckets, name = None): if name: self.name = name else: self.name = self.__class__.__name__ self.nodes_cur = SortedCollection(key = attrgetter("priority")) self.nodes_next = SortedCollection(key = attrgetter("priority")) self.H = 0 self.n_buckets = n_buckets self.error_numer = 0 self.error_denom = 1
def __init__(self, bucket_bounds, name = None, **kwargs): if name: self.name = name else: self.name = self.__class__.__name__ self.bucket_bounds = bucket_bounds self.bucket_costs = [float(a + b)/2 for a,b in zip([0] + bucket_bounds[:-1], bucket_bounds)] self.bucket_frac = [(0,0) for _ in bucket_bounds] self.nodes = SortedCollection(key=self.objective_f) self.H = 0 self.time = 0
def __init__(self, count, theta): zetan = zipfian_zeta(theta, count) zeta2theta = zipfian_zeta(theta, 2) self.count = count self.theta = theta freqs = [float(i)**-theta for i in range(1, count + 1)] s = sum(freqs) self.lambdas = [s / i for i in freqs] self.t = 0 self.next_arrivals = SortedCollection(iterable=enumerate(self.lambdas), key=itemgetter(1))
def __init__(self, bucket_bounds, name = None): if name: self.name = name else: self.name = self.__class__.__name__ self.bucket_bounds = bucket_bounds self.bucket_costs = [float(a + b)/2 for a,b in zip([0] + bucket_bounds[:-1], bucket_bounds)] self.bucket_frac = [(0,0) for _ in bucket_bounds] self.bucket_evictions = [0 for _ in self.bucket_bounds] self.bucket_misses = [0 for _ in self.bucket_bounds] self.nodes = SortedCollection(key=attrgetter("priority")) self.H = 0 self.time = 0 self.error_numer = 0 self.error_denom = 1
def __init__(self, bucket_bounds): super(CostBucket_Frequency, self).__init__(0) self.bucket_bounds = bucket_bounds self.bucket_queues = [ SortedCollection(key=self._get_count) for _ in bucket_bounds ] self.bucket_values = [(0, 0) for _ in bucket_bounds] self.counts = defaultdict(lambda: 0) self.generator = None self.Nodes = DummyList()
class PerfectKnowledge_PQ(object): needs_driver = True def __init__(self, driver = None, name = "PK_Freq", **kwargs): self.name = name self.nodes = SortedCollection(key=attrgetter('priority')) self.error_numer = 0 self.error_denom = 1 self.driver_access = driver self.watch_for_shift = (driver_access and "just_shifted" in driver_access.__dict__) def handle_shift(self): for node in self.nodes: old_p = node.priority node.priority = self.driver_access.get_item_pop(node.value) * node.cost self.nodes.reorder() def add_node(self, item, cost, popularity): new_node = PQNode(item, cost) new_node.priority = popularity * new_node.cost self.nodes.insert_right(new_node) if self.watch_for_shift and self.driver_access.just_shifted: self.handle_shift() self.driver_access.just_shifted = False return new_node def touch(self, node): if self.watch_for_shift and self.driver_access.just_shifted: self.handle_shift() self.driver_access.just_shifted = False self.nodes.remove(node) self.nodes.insert_right(node) def evict_node(self): to_evict = self.nodes[0] del self.nodes[0] return to_evict
class LRU_Ranked(LastAccess): def __init__(self, S, retain=0): super(LRU_Ranked, self).__init__(S, retain=retain) self.Nodes = SortedCollection(key=attrgetter("LA")) # using error to track the average eviction rank self.error_denom = 0 self.error_numer = 0 def objective_f(self, node): return node.LA def evict_node(self): # grab the minimum x = self.get_minimum(self.sample()) to_evict = self.Nodes[x] self.error_denom += 1 self.error_numer += x del self.Nodes[x] return to_evict def touch(self, node): self.Nodes.remove(node) node.LA = self.time self.time += 1 self.Nodes.insert_right(node) def add_node(self, value, cost): new_node = LastAccessNode(value, self.time) self.time += 1 self.Nodes.insert_right(new_node) return new_node
def __init__(self, S, retain=0): super(LRU_Ranked, self).__init__(S, retain=retain) self.Nodes = SortedCollection(key=attrgetter("LA")) # using error to track the average eviction rank self.error_denom = 0 self.error_numer = 0
class GD_PQ_Bucketing(object): def __init__(self, bucket_bounds, name = None): if name: self.name = name else: self.name = self.__class__.__name__ self.bucket_bounds = bucket_bounds self.bucket_costs = [float(a + b)/2 for a,b in zip([0] + bucket_bounds[:-1], bucket_bounds)] self.bucket_frac = [(0,0) for _ in bucket_bounds] self.bucket_evictions = [0 for _ in self.bucket_bounds] self.bucket_misses = [0 for _ in self.bucket_bounds] self.nodes = SortedCollection(key=attrgetter("priority")) self.H = 0 self.time = 0 self.error_numer = 0 self.error_denom = 1 def update_bucket(self, bucket_ix, new_cost): # average of current contents bucket_frac = self.bucket_frac[bucket_ix] new_frac = (bucket_frac[0] + new_cost, bucket_frac[1] + 1) self.bucket_frac[bucket_ix] = new_frac self.bucket_costs[bucket_ix] = float(new_frac[0])/new_frac[1] return self.bucket_costs[bucket_ix] def touch(self, node): self.nodes.remove(node) b_cost = self.bucket_costs[node.bucket] node.cost = b_cost node.priority = (self.H + b_cost, self.time) self.time += 1 self.error_numer += abs(b_cost - node.real_c) self.error_denom += 1 self.nodes.insert(node) def add_node(self, value, cost): bucket_ix = bisect_left(self.bucket_bounds, cost) b_cost = self.update_bucket(bucket_ix, cost) new_node = PQNode(value, b_cost) new_node.priority = (self.H + b_cost, self.time) new_node.real_c = cost new_node.bucket = bucket_ix self.bucket_misses[bucket_ix] += cost self.time += 1 self.nodes.insert(new_node) self.error_numer += abs(b_cost - cost) self.error_denom += 1 return new_node def evict_node(self): to_evict = self.nodes[0] del self.nodes[0] self.H = to_evict.cost # update the bucket average bucket_ix = to_evict.bucket bucket_frac = self.bucket_frac[bucket_ix] new_frac = (bucket_frac[0] - to_evict.real_c, bucket_frac[1] - 1) self.bucket_frac[bucket_ix] = new_frac if(new_frac[1] == 0): self.bucket_costs[bucket_ix] = 0 else: self.bucket_costs[bucket_ix] = float(new_frac[0])/new_frac[1] self.bucket_evictions[to_evict.bucket] += to_evict.real_c return to_evict
class GD_PQ_Bucketing_Old(object): def __init__(self, bucket_bounds, name = None, **kwargs): if name: self.name = name else: self.name = self.__class__.__name__ self.bucket_bounds = bucket_bounds self.bucket_costs = [float(a + b)/2 for a,b in zip([0] + bucket_bounds[:-1], bucket_bounds)] self.bucket_frac = [(0,0) for _ in bucket_bounds] self.nodes = SortedCollection(key=self.objective_f) self.H = 0 self.time = 0 def objective_f(self, node): node_L, bucket_ix = node.cost return (self.bucket_costs[bucket_ix] + node_L, node.priority) # such an absurd objf. def update_ordering(self): # self.nodes.key = self.objective_f # will resort the collection. pass def update_bucket(self, bucket_ix, new_cost): # default:: just set bucket to newest. # self.bucket_costs[bucket_ix] = float(new_cost) # do a moving window? bucket_frac = self.bucket_frac[bucket_ix] new_frac = (bucket_frac[0] + new_cost, bucket_frac[1] + 1) self.bucket_frac[bucket_ix] = new_frac self.bucket_costs[bucket_ix] = float(new_frac[0])/new_frac[1] return self.bucket_costs[bucket_ix] def touch(self, node): self.nodes.remove(node) node.cost = (self.H, node.cost[1]) node.priority = self.time self.time += 1 self.nodes.insert(node) def add_node(self, value, cost): bucket_ix = bisect_left(self.bucket_bounds, cost) self.update_bucket(bucket_ix, cost) self.update_ordering() new_node = PQNode(value, (self.H, bucket_ix)) new_node.priority = self.time new_node.real_c = cost self.time += 1 self.nodes.insert(new_node) return new_node def evict_node(self): to_evict = self.nodes[0] del self.nodes[0] self.H = self.objective_f(to_evict)[0] # update the bucket average bucket_ix = to_evict.cost[1] bucket_frac = self.bucket_frac[bucket_ix] new_frac = (bucket_frac[0] - to_evict.real_c, bucket_frac[1] - 1) self.bucket_frac[bucket_ix] = new_frac if(new_frac[1] == 0): self.bucket_costs[bucket_ix] = 0 else: self.bucket_costs[bucket_ix] = float(new_frac[0])/new_frac[1] self.update_ordering() return to_evict