class AdapterRegistry(object): """ Registry of adapters""" _sentinel = object() def __init__(self): self.underlying = adapter.AdapterRegistry() self.cache = LRUCache(500) def lookup_adapter(self, typ): """ Lookup adapter for ``typ``""" adapter = self.cache.get(typ, self._sentinel) if adapter is self._sentinel: adapter = self.underlying.lookup([typ], IJSONSerializeable, "") self.cache.put(typ, adapter) return adapter def register_adapter(self, typ, adapter=None): """ Register ``adapter`` for type ``typ`` If no ``adapter`` supplied then this method returns decorator. """ if adapter is None: def decorator(adapter): self.register_adapter_impl(typ, adapter) return adapter return decorator return self.register_adapter_impl(typ, adapter) def register_adapter_impl(self, typ, adapter): self.underlying.register([implementedBy(typ)], IJSONSerializeable, "", adapter) self.cache.clear()
class AdapterRegistry(object): """ Registry of adapters""" _sentinel = object() def __init__(self): self.underlying = adapter.AdapterRegistry() self.cache = LRUCache(500) def lookup_adapter(self, typ): """ Lookup adapter for ``typ``""" adapter = self.cache.get(typ, self._sentinel) if adapter is self._sentinel: adapter = self.underlying.lookup([typ], IJSONSerializeable, "") self.cache.put(typ, adapter) return adapter def register_adapter(self, typ, adapter=None): """ Register ``adapter`` for type ``typ`` If no ``adapter`` supplied then this method returns decorator. """ if adapter is None: def decorator(adapter): self.register_adapter_impl(typ, adapter) return adapter return decorator return self.register_adapter_impl(typ, adapter) def register_adapter_impl(self, typ, adapter): self.underlying.register( [implementedBy(typ)], IJSONSerializeable, "", adapter) self.cache.clear()
class ClusterCache(object): def __init__(self, cache_size): self.lru = LRUCache(cache_size) self.hits = 0 self.misses = 0 def get(self, file_buffer, ptr): v = self.lru.get((file_buffer, ptr)) if v is not None: self.hits += 1 return v v = ClusterData(file_buffer, ptr) self.lru.put((file_buffer, ptr), v) self.misses += 1 return v def clear(self): logger.debug("CACHE HITS " + str(self.hits) + " VS MISSES " + str(self.misses)) self.lru.clear()
class LRUShelf(Shelf): """An in-memory Least-Recently Used shelf up to `maxsize`..""" def __init__(self, maxsize=1000): self.store = LRUCache(int(maxsize)) def getitem(self, key): value = self.store.get(key, UNSET) if value is UNSET: raise KeyError(key) return value def setitem(self, key, value): self.store.put(key, value) def delitem(self, key): self.store.invalidate(key) def clear(self): self.store.clear()
class EDBag(Counter): def __init__(self): super(EDBag, self).__init__() self.cache1 = LRUCache(256) # values where distance=1 self.cache2 = LRUCache(256) # values where distance>1 def add(self, x): if not x in self: self.cache2.clear() self[x] += 1 def closest_by_edit_distance(self, x): if x in self: # Optimization: if x is in multiset, then closest # edit dist = 0. Nothing can be any closer. return (x, 0) # Optimization: If we've looked up this value before, # return previously computed answer. cached_answer = self.cache1.get(x) if cached_answer: return cached_answer cached_answer = self.cache2.get(x) if cached_answer: return cached_answer closest = None closest_dist = None for y,_ in self.most_common(): d = editdistance.eval(x, y) if not closest_dist or d < closest_dist: closest = y closest_dist = d if d == 1: # Optimization: nothing can be any closer, as # we know there's nothing at edit distance 0 (x is not # in the multiset). self.cache1.put(x, (closest, closest_dist)) return (closest, closest_dist) self.cache2.put(x, (closest, closest_dist)) return (closest, closest_dist)
class FeatureLoader(object): def __init__(self): self.points_cache = LRUCache(1000) self.colors_cache = LRUCache(1000) self.features_cache = LRUCache(200) self.words_cache = LRUCache(200) self.masks_cache = LRUCache(1000) self.index_cache = LRUCache(200) self.masked_index_cache = LRUCache(200) def clear_cache(self): self.points_cache.clear() self.colors_cache.clear() self.features_cache.clear() self.words_cache.clear() self.masks_cache.clear() def load_mask(self, data, image, points=None): masks = self.masks_cache.get(image) if masks is None: if points is None: points, _ = self.load_points_colors(data, image, masked=False) masks = data.load_features_mask(image, points[:, :2]) self.masks_cache.put(image, masks) return masks def load_points_colors(self, data, image, masked=False): points = self.points_cache.get(image) colors = self.colors_cache.get(image) if points is None or colors is None: points, _, colors = self._load_features_nocache(data, image) self.points_cache.put(image, points) self.colors_cache.put(image, colors) if masked: mask = self.load_mask(data, image, points) if mask is not None: points = points[mask] colors = colors[mask] return points, colors def load_points_features_colors(self, data, image, masked=False): points = self.points_cache.get(image) features = self.features_cache.get(image) colors = self.colors_cache.get(image) if points is None or features is None or colors is None: points, features, colors = self._load_features_nocache(data, image) self.points_cache.put(image, points) self.features_cache.put(image, features) self.colors_cache.put(image, colors) if masked: mask = self.load_mask(data, image, points) if mask is not None: points = points[mask] features = features[mask] colors = colors[mask] return points, features, colors def load_features_index(self, data, image, masked=False): cache = self.masked_index_cache if masked else self.index_cache cached = cache.get(image) if cached is None: _, features, _ = self.load_points_features_colors(data, image, masked) index = ft.build_flann_index(features, data.config) cache.put(image, (features, index)) else: features, index = cached return index def load_words(self, data, image, masked): words = self.words_cache.get(image) if words is None: words = data.load_words(image) self.words_cache.put(image, words) if masked and words is not None: mask = self.load_mask(data, image) if mask is not None: words = words[mask] return words def _load_features_nocache(self, data, image): points, features, colors = data.load_features(image) if points is None: logger.error('Could not load features for image {}'.format(image)) else: points = np.array(points[:, :3], dtype=float) return points, features, colors
class Registry(object): """ A component registry. The component registry supports the Python mapping interface and can be used as you might a regular dictionary. It also support more advanced registrations and lookups that include a ``requires`` argument and a ``name`` via its ``register`` and ``lookup`` methods. It may be treated as an component registry by using its ``resolve`` method.""" def __init__(self, dict=None, **kwargs): self.data = {} self._lkpcache = LRUCache(1000) if dict is not None: self.update(dict) if len(kwargs): self.update(kwargs) self.listener_registered = False # at least one listener registered @property def _dictmembers(self): D = {} norequires = self.data.get((), {}) for k, v in norequires.items(): provides, name = k if name == '': D[provides] = v return D def __cmp__(self, dict): if isinstance(dict, Registry): return cmp(self.data, dict.data) else: return cmp(self._dictmembers, dict) def __len__(self): return len(self._dictmembers) def __getitem__(self, key): notrequires = self.data.get((), {}) return notrequires[(key, '')] def __setitem__(self, key, val): self.register(key, val) def __delitem__(self, key): self._lkpcache.clear() notrequires = self.data.get((), {}) try: del notrequires[(key, '')] except KeyError: raise KeyError(key) def clear(self, full=False): if full: self.data = {} else: notrequires = self.data.get((), {}) for k, v in notrequires.items(): provides, name = k if name == '': del notrequires[k] self._lkpcache.clear() def copy(self): import copy return copy.copy(self) def items(self): return self._dictmembers.items() def keys(self): return self._dictmembers.keys() def values(self): return self._dictmembers.values() def iteritems(self): return iter(self.items()) def iterkeys(self): return iter(self.keys()) def itervalues(self): return iter(self.values()) def __contains__(self, key): return key in self._dictmembers has_key = __contains__ def get(self, key, default=None): try: return self[key] except KeyError: return default @classmethod def fromkeys(cls, iterable, value=None): d = cls() for key in iterable: d[key] = value return d def update(self, dict=None, **kw): if dict is not None: for k, v in dict.items(): self.register(k, v) for k, v in kw.items(): self.register(k, v) def setdefault(self, key, failobj=None): self._lkpcache.clear() val = self.get(key, default=failobj) if val is failobj: self[key] = failobj return self[key] def __iter__(self): return iter(self._dictmembers) def pop(self, key, *args): if len(args) > 1: raise TypeError, "pop expected at most 2 arguments, got "\ + repr(1 + len(args)) try: value = self[key] except KeyError: if args: return args[0] raise del self[key] return value def popitem(self): try: k, v = self.iteritems().next() except StopIteration: raise KeyError, 'container is empty' del self[k] return (k, v) def register(self, provides, component, *requires, **kw): """ Register a component """ name = kw.get('name', '') if name is ALL: raise ValueError('ALL cannot be used in a registration as a name') self._lkpcache.clear() if provides is _subscribers: self.listener_registered = True info = self.data.setdefault(requires, {}) info[(provides, name)] = component all = info.setdefault((provides, ALL), []) all.append(component) def unregister(self, provides, component, *requires, **kw): self._lkpcache.clear() name = kw.get('name', '') if name is ALL: del self.data[requires] return info = self.data.get(requires, {}) del info[(provides, name)] all = info.get((provides, ALL), []) all.remove(component) if not all: del self.data[requires] def subscribe(self, fn, *requires, **kw): name = kw.get('name', '') if name is ALL: raise ValueError('ALL may not be used as a name to subscribe') newkw = {'name': name, 'default': _marker} subscribers = self.lookup(_subscribers, *requires, **newkw) if subscribers is _marker: subscribers = [] subscribers.append(fn) self.register(_subscribers, subscribers, *requires, **kw) def unsubscribe(self, fn, *requires, **kw): name = kw.get('name', '') if name is ALL: raise ValueError('ALL may not be used as a name to unsubscribe') newkw = {'name': name, 'default': _marker} subscribers = self.lookup(_subscribers, *requires, **newkw) if subscribers is _marker: subscribers = [] if fn in subscribers: subscribers.remove(fn) def notify(self, *objects, **kw): if not self.listener_registered: return # optimization subscribers = self.resolve(_subscribers, *objects, **kw) name = kw.get('name', '') if subscribers is not None: if name is ALL: for subscriberlist in subscribers: for subscriber in subscriberlist: subscriber(*objects) else: for subscriber in subscribers: subscriber(*objects) def _lookup(self, provides, name, default, requires, default_requires): # the requires and default_requires arguments *must* be # hashable sequences of tuples composed of hashable objects reg = self.data cachekey = (provides, requires, name, default_requires) cached = self._lkpcache.get(cachekey, _marker) if cached is _marker: combinations = cached_augmented_product(requires, default_requires) regkey = (provides, name) for combo in combinations: try: result = reg[combo][regkey] self._lkpcache.put(cachekey, result) return result except KeyError: pass self._lkpcache.put(cachekey, _notfound) cached = _notfound if cached is _notfound: if default is _missing: raise LookupError( "Couldn't find a component providing %s for requires " "args %r with name `%s`" % (provides, list(requires), name)) return default return cached def lookup(self, provides, *requires, **kw): req = [] for val in requires: if not hasattr(val, '__iter__'): req.append((val, )) else: req.append(tuple(val)) name = kw.get('name', '') extras = ((None, ), ) * len(req) default = kw.get('default', _missing) return self._lookup(provides, name, default, tuple(req), extras) def resolve(self, provides, *objects, **kw): requires = tuple( [directlyprovidedby(obj) + alsoprovidedby(obj) for obj in objects]) extras = tuple([defaultprovidedby(obj) for obj in objects]) name = kw.get('name', '') default = kw.get('default', _missing) return self._lookup(provides, name, default, requires, extras)
class FeatureLoader(object): def __init__(self): self.points_cache = LRUCache(1000) self.colors_cache = LRUCache(1000) self.features_cache = LRUCache(200) self.words_cache = LRUCache(200) self.masks_cache = LRUCache(1000) self.index_cache = LRUCache(200) def clear_cache(self): self.points_cache.clear() self.colors_cache.clear() self.features_cache.clear() self.words_cache.clear() self.masks_cache.clear() def load_points_colors(self, data, image): points = self.points_cache.get(image) colors = self.colors_cache.get(image) if points is None or colors is None: points, _, colors = self._load_features_nocache(data, image) self.points_cache.put(image, points) self.colors_cache.put(image, colors) return points, colors def load_masks(self, data, image): points, _ = self.load_points_colors(data, image) masks = self.masks_cache.get(image) if masks is None: masks = data.load_features_mask(image, points[:, :2]) self.masks_cache.put(image, masks) return masks def load_features_index(self, data, image, features): index = self.index_cache.get(image) current_features = self.load_points_features_colors(data, image) use_load = len(current_features) == len(features) and index is None use_rebuild = len(current_features) != len(features) if use_load: index = data.load_feature_index(image, features) if use_rebuild: index = ft.build_flann_index(features, data.config) if use_load or use_rebuild: self.index_cache.put(image, index) return index def load_points_features_colors(self, data, image): points = self.points_cache.get(image) features = self.features_cache.get(image) colors = self.colors_cache.get(image) if points is None or features is None or colors is None: points, features, colors = self._load_features_nocache(data, image) self.points_cache.put(image, points) self.features_cache.put(image, features) self.colors_cache.put(image, colors) return points, features, colors def load_words(self, data, image): words = self.words_cache.get(image) if words is None: words = data.load_words(image) self.words_cache.put(image, words) return words def _load_features_nocache(self, data, image): points, features, colors = data.load_features(image) if points is None: logger.error('Could not load features for image {}'.format(image)) else: points = np.array(points[:, :3], dtype=float) return points, features, colors
class Registry(object): """ A component registry. The component registry supports the Python mapping interface and can be used as you might a regular dictionary. It also support more advanced registrations and lookups that include a ``requires`` argument and a ``name`` via its ``register`` and ``lookup`` methods. It may be treated as an component registry by using its ``resolve`` method.""" def __init__(self, dict=None, **kwargs): self.data = {} self._lkpcache = LRUCache(1000) if dict is not None: self.update(dict) if len(kwargs): self.update(kwargs) self.listener_registered = False # at least one listener registered @property def _dictmembers(self): D = {} norequires = self.data.get((), {}) for k, v in norequires.items(): provides, name = k if name == '': D[provides] = v return D def __cmp__(self, dict): if isinstance(dict, Registry): return cmp(self.data, dict.data) else: return cmp(self._dictmembers, dict) def __len__(self): return len(self._dictmembers) def __getitem__(self, key): notrequires = self.data.get((), {}) return notrequires[(key, '')] def __setitem__(self, key, val): self.register(key, val) def __delitem__(self, key): self._lkpcache.clear() notrequires = self.data.get((), {}) try: del notrequires[(key, '')] except KeyError: raise KeyError(key) def clear(self, full=False): if full: self.data = {} else: notrequires = self.data.get((), {}) for k, v in notrequires.items(): provides, name = k if name == '': del notrequires[k] self._lkpcache.clear() def copy(self): import copy return copy.copy(self) def items(self): return self._dictmembers.items() def keys(self): return self._dictmembers.keys() def values(self): return self._dictmembers.values() def iteritems(self): return iter(self.items()) def iterkeys(self): return iter(self.keys()) def itervalues(self): return iter(self.values()) def __contains__(self, key): return key in self._dictmembers has_key = __contains__ def get(self, key, default=None): try: return self[key] except KeyError: return default @classmethod def fromkeys(cls, iterable, value=None): d = cls() for key in iterable: d[key] = value return d def update(self, dict=None, **kw): if dict is not None: for k, v in dict.items(): self.register(k, v) for k, v in kw.items(): self.register(k, v) def setdefault(self, key, failobj=None): self._lkpcache.clear() val = self.get(key, default=failobj) if val is failobj: self[key] = failobj return self[key] def __iter__(self): return iter(self._dictmembers) def pop(self, key, *args): if len(args) > 1: raise TypeError, "pop expected at most 2 arguments, got "\ + repr(1 + len(args)) try: value = self[key] except KeyError: if args: return args[0] raise del self[key] return value def popitem(self): try: k, v = self.iteritems().next() except StopIteration: raise KeyError, 'container is empty' del self[k] return (k, v) def register(self, provides, component, *requires, **kw): """ Register a component """ name = kw.get('name', '') if name is ALL: raise ValueError('ALL cannot be used in a registration as a name') self._lkpcache.clear() if provides is _subscribers: self.listener_registered = True info = self.data.setdefault(requires, {}) info[(provides, name)] = component all = info.setdefault((provides, ALL), []) all.append(component) def unregister(self, provides, component, *requires, **kw): self._lkpcache.clear() name = kw.get('name', '') if name is ALL: del self.data[requires] return info = self.data.get(requires, {}) del info[(provides, name)] all = info.get((provides, ALL), []) all.remove(component) if not all: del self.data[requires] def subscribe(self, fn, *requires, **kw): name = kw.get('name', '') if name is ALL: raise ValueError('ALL may not be used as a name to subscribe') newkw = {'name':name, 'default':_marker} subscribers = self.lookup(_subscribers, *requires, **newkw) if subscribers is _marker: subscribers = [] subscribers.append(fn) self.register(_subscribers, subscribers, *requires, **kw) def unsubscribe(self, fn, *requires, **kw): name = kw.get('name', '') if name is ALL: raise ValueError('ALL may not be used as a name to unsubscribe') newkw = {'name':name, 'default':_marker} subscribers = self.lookup(_subscribers, *requires, **newkw) if subscribers is _marker: subscribers = [] if fn in subscribers: subscribers.remove(fn) def notify(self, *objects, **kw): if not self.listener_registered: return # optimization subscribers = self.resolve(_subscribers, *objects, **kw) name = kw.get('name', '') if subscribers is not None: if name is ALL: for subscriberlist in subscribers: for subscriber in subscriberlist: subscriber(*objects) else: for subscriber in subscribers: subscriber(*objects) def _lookup(self, provides, name, default, requires, default_requires): # the requires and default_requires arguments *must* be # hashable sequences of tuples composed of hashable objects reg = self.data cachekey = (provides, requires, name, default_requires) cached = self._lkpcache.get(cachekey, _marker) if cached is _marker: combinations = cached_augmented_product(requires, default_requires) regkey = (provides, name) for combo in combinations: try: result = reg[combo][regkey] self._lkpcache.put(cachekey, result) return result except KeyError: pass self._lkpcache.put(cachekey, _notfound) cached = _notfound if cached is _notfound: if default is _missing: raise LookupError( "Couldn't find a component providing %s for requires " "args %r with name `%s`" % (provides, list(requires), name)) return default return cached def lookup(self, provides, *requires, **kw): req = [] for val in requires: if not hasattr(val, '__iter__'): req.append((val,)) else: req.append(tuple(val)) name = kw.get('name', '') extras = ((None,),) * len(req) default = kw.get('default', _missing) return self._lookup(provides, name, default, tuple(req), extras) def resolve(self, provides, *objects, **kw): requires = tuple( [directlyprovidedby(obj)+alsoprovidedby(obj) for obj in objects ]) extras = tuple([defaultprovidedby(obj) for obj in objects]) name = kw.get('name', '') default = kw.get('default', _missing) return self._lookup(provides, name, default, requires, extras)