class InMemoryDataModelCache(DataModelCache):
""" Implementation of the data model cache backed by an in-memory dictionary. """
def __init__(self):
self.cache = ExpiresDict()
def retrieve(self, cache_key, loader, should_cache=is_not_none):
not_found = [None]
logger.debug('Checking cache for key %s', cache_key.key)
result = self.cache.get(cache_key.key, default_value=not_found)
if result != not_found:
logger.debug('Found result in cache for key %s: %s', cache_key.key,
result)
return json.loads(result)
logger.debug('Found no result in cache for key %s; calling loader',
cache_key.key)
result = loader()
logger.debug('Got loaded result for key %s: %s', cache_key.key, result)
if should_cache(result):
logger.debug(
'Caching loaded result for key %s with expiration %s: %s',
cache_key.key, result, cache_key.expiration)
expires = convert_to_timedelta(
cache_key.expiration) + datetime.now()
self.cache.set(cache_key.key, json.dumps(result), expires=expires)
logger.debug(
'Cached loaded result for key %s with expiration %s: %s',
cache_key.key, result, cache_key.expiration)
else:
logger.debug('Not caching loaded result for key %s: %s',
cache_key.key, result)
return result
class MemoryOrchestrator(Orchestrator):
def __init__(self, **kwargs):
self.state = ExpiresDict()
self.callbacks = {}
def _callbacks_prefixed(self, key):
return (callback for (prefix, callback) in self.callbacks.items() if key.startswith(prefix))
def on_key_change(self, key, callback, restarter=None):
self.callbacks[key] = callback
def get_prefixed_keys(self, prefix):
return {
k: value for (k, value) in list(self.state.items())
if k.startswith(prefix) and not k.endswith(REDIS_EXPIRED_SUFFIX) and not k.endswith(REDIS_EXPIRING_SUFFIX)
}
def get_key(self, key):
return self.state[key]
def set_key(self, key, value, overwrite=False, expiration=None):
preexisting_key = key in self.state
if preexisting_key and not overwrite:
raise KeyError(key)
# Simulate redis' behavior when using xx and the key does not exist.
if not preexisting_key and overwrite:
return
absolute_expiration = None
if expiration is not None:
absolute_expiration = datetime.datetime.now() + datetime.timedelta(seconds=expiration)
self.state.set(key, value, expires=absolute_expiration)
self.state.set(slash_join(key, REDIS_EXPIRING_SUFFIX), value, expires=absolute_expiration)
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
for callback in self._callbacks_prefixed(key):
callback(KeyChange(event, key, value))
def delete_key(self, key):
value = self.state[key]
del self.state[key]
for callback in self._callbacks_prefixed(key):
callback(KeyChange(KeyEvent.DELETE, key, value))
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
try:
self.set_key(key, "", overwrite=False, expiration=expiration)
except KeyError:
return False
return True
def shutdown(self):
self.state = None
self.callbacks = None
class InMemoryDataModelCache(DataModelCache):
"""
Implementation of the data model cache backed by an in-memory dictionary.
"""
def __init__(self, cache_config):
super(InMemoryDataModelCache, self).__init__(cache_config)
self.cache = ExpiresDict()
def empty_for_testing(self):
self.cache = ExpiresDict()
def retrieve(self, cache_key, loader, should_cache=is_not_none):
not_found = [None]
logger.debug("Checking cache for key %s", cache_key.key)
result = self.cache.get(cache_key.key, default_value=not_found)
if result != not_found:
logger.debug("Found result in cache for key %s: %s", cache_key.key,
result)
cache_count.labels("hit").inc()
return json.loads(result)
else:
cache_count.labels("miss").inc()
logger.debug("Found no result in cache for key %s; calling loader",
cache_key.key)
result = loader()
logger.debug("Got loaded result for key %s: %s", cache_key.key, result)
if should_cache(result):
logger.debug(
"Caching loaded result for key %s with expiration %s: %s",
cache_key.key,
result,
cache_key.expiration,
)
expires = convert_to_timedelta(
cache_key.expiration) + datetime.now()
self.cache.set(cache_key.key, json.dumps(result), expires=expires)
logger.debug(
"Cached loaded result for key %s with expiration %s: %s",
cache_key.key,
result,
cache_key.expiration,
)
else:
logger.debug("Not caching loaded result for key %s: %s",
cache_key.key, result)
return result
class MemoryOrchestrator(Orchestrator):
def __init__(self, **kwargs):
self.state = ExpiresDict()
self.callbacks = {}
def _callbacks_prefixed(self, prefix):
return (callback for (key, callback) in iteritems(self.callbacks)
if key.startswith(prefix))
def on_key_change(self, key, callback, restarter=None):
self.callbacks[key] = callback
@coroutine
def get_prefixed_keys(self, prefix):
raise Return({
k: value
for (k, value) in self.state.items() if k.startswith(prefix)
})
@coroutine
def get_key(self, key):
raise Return(self.state[key])
@coroutine
def set_key(self, key, value, overwrite=False, expiration=None):
preexisting_key = "key" in self.state
if preexisting_key and not overwrite:
raise KeyError
absolute_expiration = None
if expiration is not None:
absolute_expiration = datetime.datetime.now() + datetime.timedelta(
seconds=expiration)
self.state.set(key, value, expires=absolute_expiration)
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
for callback in self._callbacks_prefixed(key):
yield From(callback(KeyChange(event, key, value)))
def set_key_sync(self, key, value, overwrite=False, expiration=None):
"""
set_key, but without trollius coroutines.
"""
preexisting_key = "key" in self.state
if preexisting_key and not overwrite:
raise KeyError
absolute_expiration = None
if expiration is not None:
absolute_expiration = datetime.datetime.now() + datetime.timedelta(
seconds=expiration)
self.state.set(key, value, expires=absolute_expiration)
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
for callback in self._callbacks_prefixed(key):
callback(KeyChange(event, key, value))
@coroutine
def delete_key(self, key):
value = self.state[key]
del self.state[key]
for callback in self._callbacks_prefixed(key):
yield From(callback(KeyChange(KeyEvent.DELETE, key, value)))
@coroutine
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
if key in self.state:
raise Return(False)
self.state.set(key, None, expires=expiration)
raise Return(True)
def shutdown(self):
self.state = None
self.callbacks = None