class Group: def __init__(self, name, preferences, quota): ''' @preferences: IE (2, 0, 1). Tuple of group's preferences over applicants with preferences[0] being the most preferred applicant. @rankings: IE {0:1, 2:0, 1:2}. Dictionary of rank of each applicant. ''' # self.preferences = preferences self.applicantToRank = makeRankings(preferences) # ["andrew", "will"] self.name = name self.quota = quota self.waitList = SortedDict() # rank --> applicant def addToWaitList(self, applicant): rank = self.applicantToRank[applicant] self.waitList[rank] = applicant def acceptQuota(self): accepted = self.waitList.islice(0, self.quota) acceptedApplicants = [self.waitList[rank] for rank in accepted] rejected = self.waitList.islice(self.quota) rejectedApplicants = [self.waitList.pop(rank) for rank in rejected] return acceptedApplicants, rejectedApplicants
def test_islice(): mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)] temp = SortedDict(7, mapping) for start in range(30): for stop in range(30): assert list(temp.islice(start, stop)) == list(string.ascii_lowercase[start:stop])
class sorted_cached_db: def __init__(self): self.sorted_map = SortedDict() def if_exists(self, key): return key in self.sorted_map def add(self, key, value): self.sorted_map[key] = value def delete(self, key): if not self.if_exists(key): return "key not present" del self.sorted_map[key] redis.query.filter_by(key=key, type="sorted").delete() db.session.commit() return "deleted" def find_rank(self, key): if not self.if_exists(key): return "does'not exists" return jsonify(self.sorted_map.index(key)) def get_range(self, start, end): n = len(self.sorted_map) start = (n + start) % n end = (n + end) % n + 1 return jsonify([ self.sorted_map[key]['value'] for key in self.sorted_map.islice(start, end) ])
class StreamChangeCache: """Keeps track of the stream positions of the latest change in a set of entities. Typically the entity will be a room or user id. Given a list of entities and a stream position, it will give a subset of entities that may have changed since that position. If position key is too old then the cache will simply return all given entities. """ def __init__( self, name: str, current_stream_pos: int, max_size=10000, prefilled_cache: Optional[Mapping[EntityType, int]] = None, ): self._original_max_size = max_size self._max_size = math.floor(max_size) self._entity_to_key = {} # type: Dict[EntityType, int] # map from stream id to the a set of entities which changed at that stream id. self._cache = SortedDict() # type: SortedDict[int, Set[EntityType]] # the earliest stream_pos for which we can reliably answer # get_all_entities_changed. In other words, one less than the earliest # stream_pos for which we know _cache is valid. # self._earliest_known_stream_pos = current_stream_pos self.name = name self.metrics = caches.register_cache( "cache", self.name, self._cache, resize_callback=self.set_cache_factor) if prefilled_cache: for entity, stream_pos in prefilled_cache.items(): self.entity_has_changed(entity, stream_pos) def set_cache_factor(self, factor: float) -> bool: """ Set the cache factor for this individual cache. This will trigger a resize if it changes, which may require evicting items from the cache. Returns: bool: Whether the cache changed size or not. """ new_size = math.floor(self._original_max_size * factor) if new_size != self._max_size: self.max_size = new_size self._evict() return True return False def has_entity_changed(self, entity: EntityType, stream_pos: int) -> bool: """Returns True if the entity may have been updated since stream_pos """ assert type(stream_pos) in integer_types if stream_pos < self._earliest_known_stream_pos: self.metrics.inc_misses() return True latest_entity_change_pos = self._entity_to_key.get(entity, None) if latest_entity_change_pos is None: self.metrics.inc_hits() return False if stream_pos < latest_entity_change_pos: self.metrics.inc_misses() return True self.metrics.inc_hits() return False def get_entities_changed( self, entities: Collection[EntityType], stream_pos: int) -> Union[Set[EntityType], FrozenSet[EntityType]]: """ Returns subset of entities that have had new things since the given position. Entities unknown to the cache will be returned. If the position is too old it will just return the given list. """ changed_entities = self.get_all_entities_changed(stream_pos) if changed_entities is not None: # We now do an intersection, trying to do so in the most efficient # way possible (some of these sets are *large*). First check in the # given iterable is already set that we can reuse, otherwise we # create a set of the *smallest* of the two iterables and call # `intersection(..)` on it (this can be twice as fast as the reverse). if isinstance(entities, (set, frozenset)): result = entities.intersection(changed_entities) elif len(changed_entities) < len(entities): result = set(changed_entities).intersection(entities) else: result = set(entities).intersection(changed_entities) self.metrics.inc_hits() else: result = set(entities) self.metrics.inc_misses() return result def has_any_entity_changed(self, stream_pos: int) -> bool: """Returns if any entity has changed """ assert type(stream_pos) is int if not self._cache: # If the cache is empty, nothing can have changed. return False if stream_pos >= self._earliest_known_stream_pos: self.metrics.inc_hits() return self._cache.bisect_right(stream_pos) < len(self._cache) else: self.metrics.inc_misses() return True def get_all_entities_changed( self, stream_pos: int) -> Optional[List[EntityType]]: """Returns all entities that have had new things since the given position. If the position is too old it will return None. Returns the entities in the order that they were changed. """ assert type(stream_pos) is int if stream_pos < self._earliest_known_stream_pos: return None changed_entities = [] # type: List[EntityType] for k in self._cache.islice( start=self._cache.bisect_right(stream_pos)): changed_entities.extend(self._cache[k]) return changed_entities def entity_has_changed(self, entity: EntityType, stream_pos: int) -> None: """Informs the cache that the entity has been changed at the given position. """ assert type(stream_pos) is int if stream_pos <= self._earliest_known_stream_pos: return old_pos = self._entity_to_key.get(entity, None) if old_pos is not None: if old_pos >= stream_pos: # nothing to do return e = self._cache[old_pos] e.remove(entity) if not e: # cache at this point is now empty del self._cache[old_pos] e1 = self._cache.get(stream_pos) if e1 is None: e1 = self._cache[stream_pos] = set() e1.add(entity) self._entity_to_key[entity] = stream_pos self._evict() # if the cache is too big, remove entries while len(self._cache) > self._max_size: k, r = self._cache.popitem(0) self._earliest_known_stream_pos = max( k, self._earliest_known_stream_pos) for entity in r: del self._entity_to_key[entity] def _evict(self): while len(self._cache) > self._max_size: k, r = self._cache.popitem(0) self._earliest_known_stream_pos = max( k, self._earliest_known_stream_pos) for entity in r: self._entity_to_key.pop(entity, None) def get_max_pos_of_last_change(self, entity: EntityType) -> int: """Returns an upper bound of the stream id of the last change to an entity. """ return self._entity_to_key.get(entity, self._earliest_known_stream_pos)
class StepVector(): @classmethod def sliced(cls, other, start, end): newobj = cls(other.datatype, _tree=other._t, _bounds=(start, end)) return newobj def __init__(self, datatype, _tree=None, _bounds=None): self.datatype = datatype if _tree is not None: self._t = _tree else: self._t = SortedDict() if _bounds is not None: self._bounds = _bounds else: self._bounds = (None, None) # set upon slicing/subsetting def __getitem__(self, key): if type(key) == slice: if (key.step is not None) and (key.step != 1): raise ValueError("Invalid step value") start = key.start end = key.stop if self._bounds[0] is not None: if start is None: start = self._bounds[0] else: if start < self._bounds[0]: raise ValueError("Start out of bounds") if self._bounds[1] is not None: if end is None: end = self._bounds[1] else: if end > self._bounds[1]: raise ValueError("End out of bounds") return self.sliced(self, start, end) else: assert type(key) == int if self._bounds[0] is not None: if key < self._bounds[0]: raise ValueError("Key out of bounds") if self._bounds[1] is not None: if key >= self._bounds[0]: raise ValueError("Key out of bounds") if self._t: try: prevkey = self._floor_key(key) return self._t[prevkey] except KeyError: # no item smaller than or equal to key return self.datatype() else: # empty tree return self.datatype() def __setitem__(self, key, value): if type(key) == slice: start = key.start end = key.stop else: assert type(key) == int start = key end = key + 1 assert start is not None assert end is not None assert type(value) == self.datatype assert end >= start if start == end: return # check next val if self._t: try: nkey = self._floor_key(end, bisect="right") nvalue = self._t[nkey] except KeyError: nkey = None nvalue = None else: # empty tree nkey = None nvalue = None # check prev val if self._t: try: pkey = self._floor_key(start) pvalue = self._t[pkey] except KeyError: pkey = None pvalue = None else: pkey = None pvalue = None # remove intermediate steps if any if self._t: a = self._t.bisect_left(start) b = self._t.bisect(end) assert a <= b del self._t.iloc[a:b] # set an end marker if necessary if nkey is None: self._t[end] = self.datatype() elif nvalue != value: self._t[end] = nvalue # set a start marker if necessary if pkey is None or pvalue != value: self._t[start] = value def __iter__(self): start, end = self._bounds if not self._t: # empty tree if start is None or end is None: raise StopIteration # FIXME: can't figure out a better thing to do if only one is None else: if start < end: yield (start, end, self.datatype()) raise StopIteration if start is None: a = 0 else: a = max(0, self._bisect_right(start) - 1) if end is None: b = len(self._t) else: b = self._bisect_right(end) assert b >= a if a == b: if a is None: start = self._t[a] if b is None: end = self._t[b] if start < end: yield (start, end, self.datatype()) raise StopIteration it = self._t.islice(a, b) currkey = next(it) currvalue = self._t[currkey] if start is not None: currkey = max(start, currkey) if start < currkey: yield (start, currkey, self.datatype()) prevkey, prevvalue = currkey, currvalue for currkey in it: currvalue = self._t[currkey] yield (prevkey, currkey, prevvalue) prevkey = currkey prevvalue = currvalue if end is not None: if currkey < end: yield (currkey, end, prevvalue) def add_value(self, start, end, value): assert type(value) == self.datatype # can't modify self while iterating over values; will change the tree, and thus f**k up iteration items = list(self[start:end]) for a, b, x in items: if self.datatype == set: y = x.copy() y.update(value) else: y = x + value self[a:b] = y def _bisect_left(self, key): return self._t.bisect_left(key) def _bisect_right(self, key): return self._t.bisect_right(key) def _floor_key(self, key, bisect="left"): """ Returns the greatest key less than or equal to key """ if bisect == "right": p = self._bisect_right(key) else: p = self._bisect_left(key) if p == 0: raise KeyError else: return self._t.iloc[p - 1]
class StreamChangeCache(object): """Keeps track of the stream positions of the latest change in a set of entities. Typically the entity will be a room or user id. Given a list of entities and a stream position, it will give a subset of entities that may have changed since that position. If position key is too old then the cache will simply return all given entities. """ def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache=None): self._max_size = int(max_size * caches.CACHE_SIZE_FACTOR) self._entity_to_key = {} self._cache = SortedDict() self._earliest_known_stream_pos = current_stream_pos self.name = name self.metrics = caches.register_cache("cache", self.name, self._cache) if prefilled_cache: for entity, stream_pos in prefilled_cache.items(): self.entity_has_changed(entity, stream_pos) def has_entity_changed(self, entity, stream_pos): """Returns True if the entity may have been updated since stream_pos """ assert type(stream_pos) in integer_types if stream_pos < self._earliest_known_stream_pos: self.metrics.inc_misses() return True latest_entity_change_pos = self._entity_to_key.get(entity, None) if latest_entity_change_pos is None: self.metrics.inc_hits() return False if stream_pos < latest_entity_change_pos: self.metrics.inc_misses() return True self.metrics.inc_hits() return False def get_entities_changed(self, entities, stream_pos): """ Returns subset of entities that have had new things since the given position. Entities unknown to the cache will be returned. If the position is too old it will just return the given list. """ assert type(stream_pos) is int if stream_pos >= self._earliest_known_stream_pos: changed_entities = { self._cache[k] for k in self._cache.islice( start=self._cache.bisect_right(stream_pos), ) } result = changed_entities.intersection(entities) self.metrics.inc_hits() else: result = set(entities) self.metrics.inc_misses() return result def has_any_entity_changed(self, stream_pos): """Returns if any entity has changed """ assert type(stream_pos) is int if not self._cache: # If we have no cache, nothing can have changed. return False if stream_pos >= self._earliest_known_stream_pos: self.metrics.inc_hits() return self._cache.bisect_right(stream_pos) < len(self._cache) else: self.metrics.inc_misses() return True def get_all_entities_changed(self, stream_pos): """Returns all entites that have had new things since the given position. If the position is too old it will return None. """ assert type(stream_pos) is int if stream_pos >= self._earliest_known_stream_pos: return [self._cache[k] for k in self._cache.islice( start=self._cache.bisect_right(stream_pos))] else: return None def entity_has_changed(self, entity, stream_pos): """Informs the cache that the entity has been changed at the given position. """ assert type(stream_pos) is int if stream_pos > self._earliest_known_stream_pos: old_pos = self._entity_to_key.get(entity, None) if old_pos is not None: stream_pos = max(stream_pos, old_pos) self._cache.pop(old_pos, None) self._cache[stream_pos] = entity self._entity_to_key[entity] = stream_pos while len(self._cache) > self._max_size: k, r = self._cache.popitem(0) self._earliest_known_stream_pos = max( k, self._earliest_known_stream_pos, ) self._entity_to_key.pop(r, None) def get_max_pos_of_last_change(self, entity): """Returns an upper bound of the stream id of the last change to an entity. """ return self._entity_to_key.get(entity, self._earliest_known_stream_pos)
class BaseColorCodePatchBuilder(ASAxesPatchBuilder, PickablePatchBuilder): """ The patch generator build the matplotlib patches for each capability node. The nodes are rendered as lines with a different color depending on the permission bits of the capability. The builder produces a LineCollection for each combination of permission bits and creates the lines for the nodes. """ def __init__(self, figure, pgm): """ Constructor :param figure: the figure to attache the click callback :param pgm: the provenance graph model """ super().__init__(figure=figure) self._pgm = pgm """The provenance graph model""" self._collection_map = defaultdict(lambda: []) """ Map capability permission to the set where the line should go. Any combination of capability permissions is used as key for a list of (start, end) values that are used to build LineCollections. The key "call" is used for system call nodes, the int(0) key is used for no permission. """ self._colors = {} """ Map capability permission to line colors. XXX: keep this for now, move to a colormap """ self._bbox = [np.inf, np.inf, 0, 0] """Bounding box of the patches as (xmin, ymin, xmax, ymax).""" self._node_map = SortedDict() """Maps the Y axis coordinate to the graph node at that position""" def _clickable_element(self, vertex, y): """remember the node at the given Y for faster indexing.""" data = self._pgm.data[vertex] self._node_map[y] = data def _add_bbox(self, xmin, xmax, y): """Update the view bbox.""" if self._bbox[0] > xmin: self._bbox[0] = xmin if self._bbox[1] > y: self._bbox[1] = y if self._bbox[2] < xmax: self._bbox[2] = xmax if self._bbox[3] < y: self._bbox[3] = y def _get_patch_collections(self, axes): """Return a generator of collections of patches to add to the axes.""" pass def get_patches(self, axes): """ Return a collection of lines from the collection_map. """ super().get_patches(axes) for coll in self._get_patch_collections(axes): axes.add_collection(coll) def get_bbox(self): return Bbox.from_extents(*self._bbox) def on_click(self, event): """ Attempt to retreive the data in less than O(n) for better interactivity at the expense of having to hold a dictionary of references to nodes for each t_alloc. Note that t_alloc is unique for each capability node as it is the cycle count, so it can be used as the key. """ ax = event.inaxes if ax is None: return # back to data coords without scaling y_coord = int(event.ydata) y_max = self._bbox[3] # tolerance for y distance, 0.1 * 10^6 cycles epsilon = 0.1 * 10**6 # try to get the node closer to the y_coord # in the fast way # For now fall-back to a reduced linear search but would be # useful to be able to index lines with an R-tree? idx_min = self._node_map.bisect_left(max(0, y_coord - epsilon)) idx_max = self._node_map.bisect_right(min(y_max, y_coord + epsilon)) iter_keys = self._node_map.islice(idx_min, idx_max) # find the closest node to the click position pick_target = None for key in iter_keys: node = self._node_map[key] if (node.cap.base <= event.xdata and node.cap.bound >= event.xdata): # the click event is within the node bounds and # the node Y is closer to the click event than # the previous pick_target if (pick_target is None or abs(y_coord - key) < abs(y_coord - pick_target.cap.t_alloc)): pick_target = node if pick_target is not None: ax.set_status_message(pick_target) else: ax.set_status_message("")
class StreamChangeCache(object): """Keeps track of the stream positions of the latest change in a set of entities. Typically the entity will be a room or user id. Given a list of entities and a stream position, it will give a subset of entities that may have changed since that position. If position key is too old then the cache will simply return all given entities. """ def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache=None): self._max_size = int(max_size * caches.CACHE_SIZE_FACTOR) self._entity_to_key = {} self._cache = SortedDict() self._earliest_known_stream_pos = current_stream_pos self.name = name self.metrics = caches.register_cache("cache", self.name, self._cache) if prefilled_cache: for entity, stream_pos in prefilled_cache.items(): self.entity_has_changed(entity, stream_pos) def has_entity_changed(self, entity, stream_pos): """Returns True if the entity may have been updated since stream_pos """ assert type(stream_pos) is int or type(stream_pos) is long if stream_pos < self._earliest_known_stream_pos: self.metrics.inc_misses() return True latest_entity_change_pos = self._entity_to_key.get(entity, None) if latest_entity_change_pos is None: self.metrics.inc_hits() return False if stream_pos < latest_entity_change_pos: self.metrics.inc_misses() return True self.metrics.inc_hits() return False def get_entities_changed(self, entities, stream_pos): """ Returns subset of entities that have had new things since the given position. Entities unknown to the cache will be returned. If the position is too old it will just return the given list. """ assert type(stream_pos) is int if stream_pos >= self._earliest_known_stream_pos: not_known_entities = set(entities) - set(self._entity_to_key) result = ({ self._cache[k] for k in self._cache.islice( start=self._cache.bisect_right(stream_pos)) }.intersection(entities).union(not_known_entities)) self.metrics.inc_hits() else: result = set(entities) self.metrics.inc_misses() return result def has_any_entity_changed(self, stream_pos): """Returns if any entity has changed """ assert type(stream_pos) is int if not self._cache: # If we have no cache, nothing can have changed. return False if stream_pos >= self._earliest_known_stream_pos: self.metrics.inc_hits() return self._cache.bisect_right(stream_pos) < len(self._cache) else: self.metrics.inc_misses() return True def get_all_entities_changed(self, stream_pos): """Returns all entites that have had new things since the given position. If the position is too old it will return None. """ assert type(stream_pos) is int if stream_pos >= self._earliest_known_stream_pos: return [ self._cache[k] for k in self._cache.islice( start=self._cache.bisect_right(stream_pos)) ] else: return None def entity_has_changed(self, entity, stream_pos): """Informs the cache that the entity has been changed at the given position. """ assert type(stream_pos) is int if stream_pos > self._earliest_known_stream_pos: old_pos = self._entity_to_key.get(entity, None) if old_pos is not None: stream_pos = max(stream_pos, old_pos) self._cache.pop(old_pos, None) self._cache[stream_pos] = entity self._entity_to_key[entity] = stream_pos while len(self._cache) > self._max_size: k, r = self._cache.popitem(0) self._earliest_known_stream_pos = max( k, self._earliest_known_stream_pos, ) self._entity_to_key.pop(r, None) def get_max_pos_of_last_change(self, entity): """Returns an upper bound of the stream id of the last change to an entity. """ return self._entity_to_key.get(entity, self._earliest_known_stream_pos)
def _make_slice(orders: SortedDict, start=None, stop=None): return [(key, orders[key]) for key in orders.islice(start=start, stop=stop)]
class ColorCodePatchBuilder(PickablePatchBuilder): """ The patch generator build the matplotlib patches for each capability node. The nodes are rendered as lines with a different color depending on the permission bits of the capability. The builder produces a LineCollection for each combination of permission bits and creates the lines for the nodes. """ def __init__(self, figure): super(ColorCodePatchBuilder, self).__init__(figure, None) self.y_unit = 10**-6 """Unit on the y-axis""" # permission composition shorthands load_store = CheriCapPerm.LOAD | CheriCapPerm.STORE load_exec = CheriCapPerm.LOAD | CheriCapPerm.EXEC store_exec = CheriCapPerm.STORE | CheriCapPerm.EXEC load_store_exec = (CheriCapPerm.STORE | CheriCapPerm.LOAD | CheriCapPerm.EXEC) self._collection_map = { 0: [], CheriCapPerm.LOAD: [], CheriCapPerm.STORE: [], CheriCapPerm.EXEC: [], load_store: [], load_exec: [], store_exec: [], load_store_exec: [], "call": [], } """Map capability permission to the set where the line should go""" self._colors = { 0: colorConverter.to_rgb("#bcbcbc"), CheriCapPerm.LOAD: colorConverter.to_rgb("k"), CheriCapPerm.STORE: colorConverter.to_rgb("y"), CheriCapPerm.EXEC: colorConverter.to_rgb("m"), load_store: colorConverter.to_rgb("c"), load_exec: colorConverter.to_rgb("b"), store_exec: colorConverter.to_rgb("g"), load_store_exec: colorConverter.to_rgb("r"), "call": colorConverter.to_rgb("#31c648"), } """Map capability permission to line colors""" self._patches = None """List of generated patches""" self._node_map = SortedDict() """Maps the Y axis coordinate to the graph node at that position""" def _build_patch(self, node_range, y, perms): """ Build patch for the given range and type and add it to the patch collection for drawing """ line = [(node_range.start, y), (node_range.end, y)] if perms is None: perms = 0 rwx_perm = perms & (CheriCapPerm.LOAD | CheriCapPerm.STORE | CheriCapPerm.EXEC) self._collection_map[rwx_perm].append(line) def _build_call_patch(self, node_range, y, origin): """ Build patch for a node representing a system call This is added to a different collection so it can be colored differently. """ line = [(node_range.start, y), (node_range.end, y)] self._collection_map["call"].append(line) def inspect(self, node): """ Inspect a graph vertex and create the patches for it. """ if node.cap.bound < node.cap.base: logger.warning("Skip overflowed node %s", node) return node_y = node.cap.t_alloc * self.y_unit node_box = transforms.Bbox.from_extents(node.cap.base, node_y, node.cap.bound, node_y) self._bbox = transforms.Bbox.union([self._bbox, node_box]) keep_range = Range(node.cap.base, node.cap.bound, Range.T_KEEP) if node.origin == CheriNodeOrigin.SYS_MMAP: self._build_call_patch(keep_range, node_y, node.origin) else: self._build_patch(keep_range, node_y, node.cap.permissions) self._node_map[node.cap.t_alloc] = node #invalidate collections self._patches = None def get_patches(self): if self._patches: return self._patches self._patches = [] for key, collection in self._collection_map.items(): coll = collections.LineCollection(collection, colors=[self._colors[key]], linestyle="solid") self._patches.append(coll) return self._patches def get_legend(self): if not self._patches: self.get_patches() legend = ([], []) for patch, key in zip(self._patches, self._collection_map.keys()): legend[0].append(patch) if key == "call": legend[1].append("mmap") else: perm_string = "" if key & CheriCapPerm.LOAD: perm_string += "R" if key & CheriCapPerm.STORE: perm_string += "W" if key & CheriCapPerm.EXEC: perm_string += "X" if perm_string == "": perm_string = "None" legend[1].append(perm_string) return legend def on_click(self, event): """ Attempt to retreive the data in less than O(n) for better interactivity at the expense of having to hold a dictionary of references to nodes for each t_alloc. Note that t_alloc is unique for each capability node as it is the cycle count, so it can be used as the key. """ ax = event.inaxes if ax is None: return # back to data coords without scaling y_coord = int(event.ydata / self.y_unit) y_max = self._bbox.ymax / self.y_unit # tolerance for y distance, 0.25 units epsilon = 0.25 / self.y_unit # try to get the node closer to the y_coord # in the fast way # For now fall-back to a reduced linear search but would be # useful to be able to index lines with an R-tree? idx_min = self._node_map.bisect_left(max(0, y_coord - epsilon)) idx_max = self._node_map.bisect_right(min(y_max, y_coord + epsilon)) iter_keys = self._node_map.islice(idx_min, idx_max) # the closest node to the click position # initialize it with the first node in the search range try: pick_target = self._node_map[next(iter_keys)] except StopIteration: # no match found ax.set_status_message("") return for key in iter_keys: node = self._node_map[key] if (node.cap.base <= event.xdata and node.cap.bound >= event.xdata and abs(y_coord - key) < abs(y_coord - pick_target.cap.t_alloc)): # the click event is within the node bounds and # the node Y is closer to the click event than # the previous pick_target pick_target = node ax.set_status_message(pick_target)
class StreamChangeCache: """Keeps track of the stream positions of the latest change in a set of entities. Typically the entity will be a room or user id. Given a list of entities and a stream position, it will give a subset of entities that may have changed since that position. If position key is too old then the cache will simply return all given entities. """ def __init__( self, name: str, current_stream_pos: int, max_size=10000, prefilled_cache: Optional[Mapping[EntityType, int]] = None, ): self._max_size = int(max_size * caches.CACHE_SIZE_FACTOR) self._entity_to_key = {} # type: Dict[EntityType, int] # map from stream id to the a set of entities which changed at that stream id. self._cache = SortedDict() # type: SortedDict[int, Set[EntityType]] # the earliest stream_pos for which we can reliably answer # get_all_entities_changed. In other words, one less than the earliest # stream_pos for which we know _cache is valid. # self._earliest_known_stream_pos = current_stream_pos self.name = name self.metrics = caches.register_cache("cache", self.name, self._cache) if prefilled_cache: for entity, stream_pos in prefilled_cache.items(): self.entity_has_changed(entity, stream_pos) def has_entity_changed(self, entity: EntityType, stream_pos: int) -> bool: """Returns True if the entity may have been updated since stream_pos """ assert type(stream_pos) in integer_types if stream_pos < self._earliest_known_stream_pos: self.metrics.inc_misses() return True latest_entity_change_pos = self._entity_to_key.get(entity, None) if latest_entity_change_pos is None: self.metrics.inc_hits() return False if stream_pos < latest_entity_change_pos: self.metrics.inc_misses() return True self.metrics.inc_hits() return False def get_entities_changed(self, entities: Iterable[EntityType], stream_pos: int) -> Set[EntityType]: """ Returns subset of entities that have had new things since the given position. Entities unknown to the cache will be returned. If the position is too old it will just return the given list. """ changed_entities = self.get_all_entities_changed(stream_pos) if changed_entities is not None: result = set(changed_entities).intersection(entities) self.metrics.inc_hits() else: result = set(entities) self.metrics.inc_misses() return result def has_any_entity_changed(self, stream_pos: int) -> bool: """Returns if any entity has changed """ assert type(stream_pos) is int if not self._cache: # If the cache is empty, nothing can have changed. return False if stream_pos >= self._earliest_known_stream_pos: self.metrics.inc_hits() return self._cache.bisect_right(stream_pos) < len(self._cache) else: self.metrics.inc_misses() return True def get_all_entities_changed( self, stream_pos: int) -> Optional[List[EntityType]]: """Returns all entities that have had new things since the given position. If the position is too old it will return None. Returns the entities in the order that they were changed. """ assert type(stream_pos) is int if stream_pos < self._earliest_known_stream_pos: return None changed_entities = [] # type: List[EntityType] for k in self._cache.islice( start=self._cache.bisect_right(stream_pos)): changed_entities.extend(self._cache[k]) return changed_entities def entity_has_changed(self, entity: EntityType, stream_pos: int) -> None: """Informs the cache that the entity has been changed at the given position. """ assert type(stream_pos) is int if stream_pos <= self._earliest_known_stream_pos: return old_pos = self._entity_to_key.get(entity, None) if old_pos is not None: if old_pos >= stream_pos: # nothing to do return e = self._cache[old_pos] e.remove(entity) if not e: # cache at this point is now empty del self._cache[old_pos] e1 = self._cache.get(stream_pos) if e1 is None: e1 = self._cache[stream_pos] = set() e1.add(entity) self._entity_to_key[entity] = stream_pos # if the cache is too big, remove entries while len(self._cache) > self._max_size: k, r = self._cache.popitem(0) self._earliest_known_stream_pos = max( k, self._earliest_known_stream_pos) for entity in r: del self._entity_to_key[entity] def get_max_pos_of_last_change(self, entity: EntityType) -> int: """Returns an upper bound of the stream id of the last change to an entity. """ return self._entity_to_key.get(entity, self._earliest_known_stream_pos)