class AdapterRegistry(object):
""" Registry of adapters"""
_sentinel = object()
def __init__(self):
self.underlying = adapter.AdapterRegistry()
self.cache = LRUCache(500)
def lookup_adapter(self, typ):
""" Lookup adapter for ``typ``"""
adapter = self.cache.get(typ, self._sentinel)
if adapter is self._sentinel:
adapter = self.underlying.lookup([typ], IJSONSerializeable, "")
self.cache.put(typ, adapter)
return adapter
def register_adapter(self, typ, adapter=None):
""" Register ``adapter`` for type ``typ``
If no ``adapter`` supplied then this method returns decorator.
"""
if adapter is None:
def decorator(adapter):
self.register_adapter_impl(typ, adapter)
return adapter
return decorator
return self.register_adapter_impl(typ, adapter)
def register_adapter_impl(self, typ, adapter):
self.underlying.register([implementedBy(typ)], IJSONSerializeable, "",
adapter)
self.cache.clear()
class AdapterRegistry(object):
""" Registry of adapters"""
_sentinel = object()
def __init__(self):
self.underlying = adapter.AdapterRegistry()
self.cache = LRUCache(500)
def lookup_adapter(self, typ):
""" Lookup adapter for ``typ``"""
adapter = self.cache.get(typ, self._sentinel)
if adapter is self._sentinel:
adapter = self.underlying.lookup([typ], IJSONSerializeable, "")
self.cache.put(typ, adapter)
return adapter
def register_adapter(self, typ, adapter=None):
""" Register ``adapter`` for type ``typ``
If no ``adapter`` supplied then this method returns decorator.
"""
if adapter is None:
def decorator(adapter):
self.register_adapter_impl(typ, adapter)
return adapter
return decorator
return self.register_adapter_impl(typ, adapter)
def register_adapter_impl(self, typ, adapter):
self.underlying.register(
[implementedBy(typ)], IJSONSerializeable, "", adapter)
self.cache.clear()
class ClusterCache(object):
def __init__(self, cache_size):
self.lru = LRUCache(cache_size)
self.hits = 0
self.misses = 0
def get(self, file_buffer, ptr):
v = self.lru.get((file_buffer, ptr))
if v is not None:
self.hits += 1
return v
v = ClusterData(file_buffer, ptr)
self.lru.put((file_buffer, ptr), v)
self.misses += 1
return v
def clear(self):
logger.debug("CACHE HITS " + str(self.hits) + " VS MISSES " + str(self.misses))
self.lru.clear()
class LRUShelf(Shelf):
"""An in-memory Least-Recently Used shelf up to `maxsize`.."""
def __init__(self, maxsize=1000):
self.store = LRUCache(int(maxsize))
def getitem(self, key):
value = self.store.get(key, UNSET)
if value is UNSET:
raise KeyError(key)
return value
def setitem(self, key, value):
self.store.put(key, value)
def delitem(self, key):
self.store.invalidate(key)
def clear(self):
self.store.clear()
class EDBag(Counter):
def __init__(self):
super(EDBag, self).__init__()
self.cache1 = LRUCache(256) # values where distance=1
self.cache2 = LRUCache(256) # values where distance>1
def add(self, x):
if not x in self:
self.cache2.clear()
self[x] += 1
def closest_by_edit_distance(self, x):
if x in self:
# Optimization: if x is in multiset, then closest
# edit dist = 0. Nothing can be any closer.
return (x, 0)
# Optimization: If we've looked up this value before,
# return previously computed answer.
cached_answer = self.cache1.get(x)
if cached_answer:
return cached_answer
cached_answer = self.cache2.get(x)
if cached_answer:
return cached_answer
closest = None
closest_dist = None
for y,_ in self.most_common():
d = editdistance.eval(x, y)
if not closest_dist or d < closest_dist:
closest = y
closest_dist = d
if d == 1:
# Optimization: nothing can be any closer, as
# we know there's nothing at edit distance 0 (x is not
# in the multiset).
self.cache1.put(x, (closest, closest_dist))
return (closest, closest_dist)
self.cache2.put(x, (closest, closest_dist))
return (closest, closest_dist)
class LRUShelf(Shelf):
"""An in-memory Least-Recently Used shelf up to `maxsize`.."""
def __init__(self, maxsize=1000):
self.store = LRUCache(int(maxsize))
def getitem(self, key):
value = self.store.get(key, UNSET)
if value is UNSET:
raise KeyError(key)
return value
def setitem(self, key, value):
self.store.put(key, value)
def delitem(self, key):
self.store.invalidate(key)
def clear(self):
self.store.clear()
class ClusterCache(object):
def __init__(self, cache_size):
self.lru = LRUCache(cache_size)
self.hits = 0
self.misses = 0
def get(self, file_buffer, ptr):
v = self.lru.get((file_buffer, ptr))
if v is not None:
self.hits += 1
return v
v = ClusterData(file_buffer, ptr)
self.lru.put((file_buffer, ptr), v)
self.misses += 1
return v
def clear(self):
logger.debug("CACHE HITS " + str(self.hits) + " VS MISSES " +
str(self.misses))
self.lru.clear()
class FeatureLoader(object):
def __init__(self):
self.points_cache = LRUCache(1000)
self.colors_cache = LRUCache(1000)
self.features_cache = LRUCache(200)
self.words_cache = LRUCache(200)
self.masks_cache = LRUCache(1000)
self.index_cache = LRUCache(200)
self.masked_index_cache = LRUCache(200)
def clear_cache(self):
self.points_cache.clear()
self.colors_cache.clear()
self.features_cache.clear()
self.words_cache.clear()
self.masks_cache.clear()
def load_mask(self, data, image, points=None):
masks = self.masks_cache.get(image)
if masks is None:
if points is None:
points, _ = self.load_points_colors(data, image, masked=False)
masks = data.load_features_mask(image, points[:, :2])
self.masks_cache.put(image, masks)
return masks
def load_points_colors(self, data, image, masked=False):
points = self.points_cache.get(image)
colors = self.colors_cache.get(image)
if points is None or colors is None:
points, _, colors = self._load_features_nocache(data, image)
self.points_cache.put(image, points)
self.colors_cache.put(image, colors)
if masked:
mask = self.load_mask(data, image, points)
if mask is not None:
points = points[mask]
colors = colors[mask]
return points, colors
def load_points_features_colors(self, data, image, masked=False):
points = self.points_cache.get(image)
features = self.features_cache.get(image)
colors = self.colors_cache.get(image)
if points is None or features is None or colors is None:
points, features, colors = self._load_features_nocache(data, image)
self.points_cache.put(image, points)
self.features_cache.put(image, features)
self.colors_cache.put(image, colors)
if masked:
mask = self.load_mask(data, image, points)
if mask is not None:
points = points[mask]
features = features[mask]
colors = colors[mask]
return points, features, colors
def load_features_index(self, data, image, masked=False):
cache = self.masked_index_cache if masked else self.index_cache
cached = cache.get(image)
if cached is None:
_, features, _ = self.load_points_features_colors(data, image,
masked)
index = ft.build_flann_index(features, data.config)
cache.put(image, (features, index))
else:
features, index = cached
return index
def load_words(self, data, image, masked):
words = self.words_cache.get(image)
if words is None:
words = data.load_words(image)
self.words_cache.put(image, words)
if masked and words is not None:
mask = self.load_mask(data, image)
if mask is not None:
words = words[mask]
return words
def _load_features_nocache(self, data, image):
points, features, colors = data.load_features(image)
if points is None:
logger.error('Could not load features for image {}'.format(image))
else:
points = np.array(points[:, :3], dtype=float)
return points, features, colors
class Registry(object):
""" A component registry. The component registry supports the
Python mapping interface and can be used as you might a regular
dictionary. It also support more advanced registrations and
lookups that include a ``requires`` argument and a ``name`` via
its ``register`` and ``lookup`` methods. It may be treated as an
component registry by using its ``resolve`` method."""
def __init__(self, dict=None, **kwargs):
self.data = {}
self._lkpcache = LRUCache(1000)
if dict is not None:
self.update(dict)
if len(kwargs):
self.update(kwargs)
self.listener_registered = False # at least one listener registered
@property
def _dictmembers(self):
D = {}
norequires = self.data.get((), {})
for k, v in norequires.items():
provides, name = k
if name == '':
D[provides] = v
return D
def __cmp__(self, dict):
if isinstance(dict, Registry):
return cmp(self.data, dict.data)
else:
return cmp(self._dictmembers, dict)
def __len__(self):
return len(self._dictmembers)
def __getitem__(self, key):
notrequires = self.data.get((), {})
return notrequires[(key, '')]
def __setitem__(self, key, val):
self.register(key, val)
def __delitem__(self, key):
self._lkpcache.clear()
notrequires = self.data.get((), {})
try:
del notrequires[(key, '')]
except KeyError:
raise KeyError(key)
def clear(self, full=False):
if full:
self.data = {}
else:
notrequires = self.data.get((), {})
for k, v in notrequires.items():
provides, name = k
if name == '':
del notrequires[k]
self._lkpcache.clear()
def copy(self):
import copy
return copy.copy(self)
def items(self):
return self._dictmembers.items()
def keys(self):
return self._dictmembers.keys()
def values(self):
return self._dictmembers.values()
def iteritems(self):
return iter(self.items())
def iterkeys(self):
return iter(self.keys())
def itervalues(self):
return iter(self.values())
def __contains__(self, key):
return key in self._dictmembers
has_key = __contains__
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
@classmethod
def fromkeys(cls, iterable, value=None):
d = cls()
for key in iterable:
d[key] = value
return d
def update(self, dict=None, **kw):
if dict is not None:
for k, v in dict.items():
self.register(k, v)
for k, v in kw.items():
self.register(k, v)
def setdefault(self, key, failobj=None):
self._lkpcache.clear()
val = self.get(key, default=failobj)
if val is failobj:
self[key] = failobj
return self[key]
def __iter__(self):
return iter(self._dictmembers)
def pop(self, key, *args):
if len(args) > 1:
raise TypeError, "pop expected at most 2 arguments, got "\
+ repr(1 + len(args))
try:
value = self[key]
except KeyError:
if args:
return args[0]
raise
del self[key]
return value
def popitem(self):
try:
k, v = self.iteritems().next()
except StopIteration:
raise KeyError, 'container is empty'
del self[k]
return (k, v)
def register(self, provides, component, *requires, **kw):
""" Register a component """
name = kw.get('name', '')
if name is ALL:
raise ValueError('ALL cannot be used in a registration as a name')
self._lkpcache.clear()
if provides is _subscribers:
self.listener_registered = True
info = self.data.setdefault(requires, {})
info[(provides, name)] = component
all = info.setdefault((provides, ALL), [])
all.append(component)
def unregister(self, provides, component, *requires, **kw):
self._lkpcache.clear()
name = kw.get('name', '')
if name is ALL:
del self.data[requires]
return
info = self.data.get(requires, {})
del info[(provides, name)]
all = info.get((provides, ALL), [])
all.remove(component)
if not all:
del self.data[requires]
def subscribe(self, fn, *requires, **kw):
name = kw.get('name', '')
if name is ALL:
raise ValueError('ALL may not be used as a name to subscribe')
newkw = {'name': name, 'default': _marker}
subscribers = self.lookup(_subscribers, *requires, **newkw)
if subscribers is _marker:
subscribers = []
subscribers.append(fn)
self.register(_subscribers, subscribers, *requires, **kw)
def unsubscribe(self, fn, *requires, **kw):
name = kw.get('name', '')
if name is ALL:
raise ValueError('ALL may not be used as a name to unsubscribe')
newkw = {'name': name, 'default': _marker}
subscribers = self.lookup(_subscribers, *requires, **newkw)
if subscribers is _marker:
subscribers = []
if fn in subscribers:
subscribers.remove(fn)
def notify(self, *objects, **kw):
if not self.listener_registered:
return # optimization
subscribers = self.resolve(_subscribers, *objects, **kw)
name = kw.get('name', '')
if subscribers is not None:
if name is ALL:
for subscriberlist in subscribers:
for subscriber in subscriberlist:
subscriber(*objects)
else:
for subscriber in subscribers:
subscriber(*objects)
def _lookup(self, provides, name, default, requires, default_requires):
# the requires and default_requires arguments *must* be
# hashable sequences of tuples composed of hashable objects
reg = self.data
cachekey = (provides, requires, name, default_requires)
cached = self._lkpcache.get(cachekey, _marker)
if cached is _marker:
combinations = cached_augmented_product(requires, default_requires)
regkey = (provides, name)
for combo in combinations:
try:
result = reg[combo][regkey]
self._lkpcache.put(cachekey, result)
return result
except KeyError:
pass
self._lkpcache.put(cachekey, _notfound)
cached = _notfound
if cached is _notfound:
if default is _missing:
raise LookupError(
"Couldn't find a component providing %s for requires "
"args %r with name `%s`" %
(provides, list(requires), name))
return default
return cached
def lookup(self, provides, *requires, **kw):
req = []
for val in requires:
if not hasattr(val, '__iter__'):
req.append((val, ))
else:
req.append(tuple(val))
name = kw.get('name', '')
extras = ((None, ), ) * len(req)
default = kw.get('default', _missing)
return self._lookup(provides, name, default, tuple(req), extras)
def resolve(self, provides, *objects, **kw):
requires = tuple(
[directlyprovidedby(obj) + alsoprovidedby(obj) for obj in objects])
extras = tuple([defaultprovidedby(obj) for obj in objects])
name = kw.get('name', '')
default = kw.get('default', _missing)
return self._lookup(provides, name, default, requires, extras)
class FeatureLoader(object):
def __init__(self):
self.points_cache = LRUCache(1000)
self.colors_cache = LRUCache(1000)
self.features_cache = LRUCache(200)
self.words_cache = LRUCache(200)
self.masks_cache = LRUCache(1000)
self.index_cache = LRUCache(200)
def clear_cache(self):
self.points_cache.clear()
self.colors_cache.clear()
self.features_cache.clear()
self.words_cache.clear()
self.masks_cache.clear()
def load_points_colors(self, data, image):
points = self.points_cache.get(image)
colors = self.colors_cache.get(image)
if points is None or colors is None:
points, _, colors = self._load_features_nocache(data, image)
self.points_cache.put(image, points)
self.colors_cache.put(image, colors)
return points, colors
def load_masks(self, data, image):
points, _ = self.load_points_colors(data, image)
masks = self.masks_cache.get(image)
if masks is None:
masks = data.load_features_mask(image, points[:, :2])
self.masks_cache.put(image, masks)
return masks
def load_features_index(self, data, image, features):
index = self.index_cache.get(image)
current_features = self.load_points_features_colors(data, image)
use_load = len(current_features) == len(features) and index is None
use_rebuild = len(current_features) != len(features)
if use_load:
index = data.load_feature_index(image, features)
if use_rebuild:
index = ft.build_flann_index(features, data.config)
if use_load or use_rebuild:
self.index_cache.put(image, index)
return index
def load_points_features_colors(self, data, image):
points = self.points_cache.get(image)
features = self.features_cache.get(image)
colors = self.colors_cache.get(image)
if points is None or features is None or colors is None:
points, features, colors = self._load_features_nocache(data, image)
self.points_cache.put(image, points)
self.features_cache.put(image, features)
self.colors_cache.put(image, colors)
return points, features, colors
def load_words(self, data, image):
words = self.words_cache.get(image)
if words is None:
words = data.load_words(image)
self.words_cache.put(image, words)
return words
def _load_features_nocache(self, data, image):
points, features, colors = data.load_features(image)
if points is None:
logger.error('Could not load features for image {}'.format(image))
else:
points = np.array(points[:, :3], dtype=float)
return points, features, colors
class Registry(object):
""" A component registry. The component registry supports the
Python mapping interface and can be used as you might a regular
dictionary. It also support more advanced registrations and
lookups that include a ``requires`` argument and a ``name`` via
its ``register`` and ``lookup`` methods. It may be treated as an
component registry by using its ``resolve`` method."""
def __init__(self, dict=None, **kwargs):
self.data = {}
self._lkpcache = LRUCache(1000)
if dict is not None:
self.update(dict)
if len(kwargs):
self.update(kwargs)
self.listener_registered = False # at least one listener registered
@property
def _dictmembers(self):
D = {}
norequires = self.data.get((), {})
for k, v in norequires.items():
provides, name = k
if name == '':
D[provides] = v
return D
def __cmp__(self, dict):
if isinstance(dict, Registry):
return cmp(self.data, dict.data)
else:
return cmp(self._dictmembers, dict)
def __len__(self):
return len(self._dictmembers)
def __getitem__(self, key):
notrequires = self.data.get((), {})
return notrequires[(key, '')]
def __setitem__(self, key, val):
self.register(key, val)
def __delitem__(self, key):
self._lkpcache.clear()
notrequires = self.data.get((), {})
try:
del notrequires[(key, '')]
except KeyError:
raise KeyError(key)
def clear(self, full=False):
if full:
self.data = {}
else:
notrequires = self.data.get((), {})
for k, v in notrequires.items():
provides, name = k
if name == '':
del notrequires[k]
self._lkpcache.clear()
def copy(self):
import copy
return copy.copy(self)
def items(self):
return self._dictmembers.items()
def keys(self):
return self._dictmembers.keys()
def values(self):
return self._dictmembers.values()
def iteritems(self):
return iter(self.items())
def iterkeys(self):
return iter(self.keys())
def itervalues(self):
return iter(self.values())
def __contains__(self, key):
return key in self._dictmembers
has_key = __contains__
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
@classmethod
def fromkeys(cls, iterable, value=None):
d = cls()
for key in iterable:
d[key] = value
return d
def update(self, dict=None, **kw):
if dict is not None:
for k, v in dict.items():
self.register(k, v)
for k, v in kw.items():
self.register(k, v)
def setdefault(self, key, failobj=None):
self._lkpcache.clear()
val = self.get(key, default=failobj)
if val is failobj:
self[key] = failobj
return self[key]
def __iter__(self):
return iter(self._dictmembers)
def pop(self, key, *args):
if len(args) > 1:
raise TypeError, "pop expected at most 2 arguments, got "\
+ repr(1 + len(args))
try:
value = self[key]
except KeyError:
if args:
return args[0]
raise
del self[key]
return value
def popitem(self):
try:
k, v = self.iteritems().next()
except StopIteration:
raise KeyError, 'container is empty'
del self[k]
return (k, v)
def register(self, provides, component, *requires, **kw):
""" Register a component """
name = kw.get('name', '')
if name is ALL:
raise ValueError('ALL cannot be used in a registration as a name')
self._lkpcache.clear()
if provides is _subscribers:
self.listener_registered = True
info = self.data.setdefault(requires, {})
info[(provides, name)] = component
all = info.setdefault((provides, ALL), [])
all.append(component)
def unregister(self, provides, component, *requires, **kw):
self._lkpcache.clear()
name = kw.get('name', '')
if name is ALL:
del self.data[requires]
return
info = self.data.get(requires, {})
del info[(provides, name)]
all = info.get((provides, ALL), [])
all.remove(component)
if not all:
del self.data[requires]
def subscribe(self, fn, *requires, **kw):
name = kw.get('name', '')
if name is ALL:
raise ValueError('ALL may not be used as a name to subscribe')
newkw = {'name':name, 'default':_marker}
subscribers = self.lookup(_subscribers, *requires, **newkw)
if subscribers is _marker:
subscribers = []
subscribers.append(fn)
self.register(_subscribers, subscribers, *requires, **kw)
def unsubscribe(self, fn, *requires, **kw):
name = kw.get('name', '')
if name is ALL:
raise ValueError('ALL may not be used as a name to unsubscribe')
newkw = {'name':name, 'default':_marker}
subscribers = self.lookup(_subscribers, *requires, **newkw)
if subscribers is _marker:
subscribers = []
if fn in subscribers:
subscribers.remove(fn)
def notify(self, *objects, **kw):
if not self.listener_registered:
return # optimization
subscribers = self.resolve(_subscribers, *objects, **kw)
name = kw.get('name', '')
if subscribers is not None:
if name is ALL:
for subscriberlist in subscribers:
for subscriber in subscriberlist:
subscriber(*objects)
else:
for subscriber in subscribers:
subscriber(*objects)
def _lookup(self, provides, name, default, requires, default_requires):
# the requires and default_requires arguments *must* be
# hashable sequences of tuples composed of hashable objects
reg = self.data
cachekey = (provides, requires, name, default_requires)
cached = self._lkpcache.get(cachekey, _marker)
if cached is _marker:
combinations = cached_augmented_product(requires, default_requires)
regkey = (provides, name)
for combo in combinations:
try:
result = reg[combo][regkey]
self._lkpcache.put(cachekey, result)
return result
except KeyError:
pass
self._lkpcache.put(cachekey, _notfound)
cached = _notfound
if cached is _notfound:
if default is _missing:
raise LookupError(
"Couldn't find a component providing %s for requires "
"args %r with name `%s`" % (provides, list(requires), name))
return default
return cached
def lookup(self, provides, *requires, **kw):
req = []
for val in requires:
if not hasattr(val, '__iter__'):
req.append((val,))
else:
req.append(tuple(val))
name = kw.get('name', '')
extras = ((None,),) * len(req)
default = kw.get('default', _missing)
return self._lookup(provides, name, default, tuple(req), extras)
def resolve(self, provides, *objects, **kw):
requires = tuple(
[directlyprovidedby(obj)+alsoprovidedby(obj) for obj in objects ])
extras = tuple([defaultprovidedby(obj) for obj in objects])
name = kw.get('name', '')
default = kw.get('default', _missing)
return self._lookup(provides, name, default, requires, extras)
