class InMemoryDataModelCache(DataModelCache):
""" Implementation of the data model cache backed by an in-memory dictionary. """
def __init__(self):
self.cache = ExpiresDict()
def retrieve(self, cache_key, loader, should_cache=is_not_none):
not_found = [None]
logger.debug('Checking cache for key %s', cache_key.key)
result = self.cache.get(cache_key.key, default_value=not_found)
if result != not_found:
logger.debug('Found result in cache for key %s: %s', cache_key.key,
result)
return json.loads(result)
logger.debug('Found no result in cache for key %s; calling loader',
cache_key.key)
result = loader()
logger.debug('Got loaded result for key %s: %s', cache_key.key, result)
if should_cache(result):
logger.debug(
'Caching loaded result for key %s with expiration %s: %s',
cache_key.key, result, cache_key.expiration)
expires = convert_to_timedelta(
cache_key.expiration) + datetime.now()
self.cache.set(cache_key.key, json.dumps(result), expires=expires)
logger.debug(
'Cached loaded result for key %s with expiration %s: %s',
cache_key.key, result, cache_key.expiration)
else:
logger.debug('Not caching loaded result for key %s: %s',
cache_key.key, result)
return result
class MemoryOrchestrator(Orchestrator):
def __init__(self, **kwargs):
self.state = ExpiresDict()
self.callbacks = {}
def _callbacks_prefixed(self, key):
return (callback for (prefix, callback) in self.callbacks.items() if key.startswith(prefix))
def on_key_change(self, key, callback, restarter=None):
self.callbacks[key] = callback
def get_prefixed_keys(self, prefix):
return {
k: value for (k, value) in list(self.state.items())
if k.startswith(prefix) and not k.endswith(REDIS_EXPIRED_SUFFIX) and not k.endswith(REDIS_EXPIRING_SUFFIX)
}
def get_key(self, key):
return self.state[key]
def set_key(self, key, value, overwrite=False, expiration=None):
preexisting_key = key in self.state
if preexisting_key and not overwrite:
raise KeyError(key)
# Simulate redis' behavior when using xx and the key does not exist.
if not preexisting_key and overwrite:
return
absolute_expiration = None
if expiration is not None:
absolute_expiration = datetime.datetime.now() + datetime.timedelta(seconds=expiration)
self.state.set(key, value, expires=absolute_expiration)
self.state.set(slash_join(key, REDIS_EXPIRING_SUFFIX), value, expires=absolute_expiration)
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
for callback in self._callbacks_prefixed(key):
callback(KeyChange(event, key, value))
def delete_key(self, key):
value = self.state[key]
del self.state[key]
for callback in self._callbacks_prefixed(key):
callback(KeyChange(KeyEvent.DELETE, key, value))
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
try:
self.set_key(key, "", overwrite=False, expiration=expiration)
except KeyError:
return False
return True
def shutdown(self):
self.state = None
self.callbacks = None
class InMemoryDataModelCache(DataModelCache):
"""
Implementation of the data model cache backed by an in-memory dictionary.
"""
def __init__(self, cache_config):
super(InMemoryDataModelCache, self).__init__(cache_config)
self.cache = ExpiresDict()
def empty_for_testing(self):
self.cache = ExpiresDict()
def retrieve(self, cache_key, loader, should_cache=is_not_none):
not_found = [None]
logger.debug("Checking cache for key %s", cache_key.key)
result = self.cache.get(cache_key.key, default_value=not_found)
if result != not_found:
logger.debug("Found result in cache for key %s: %s", cache_key.key,
result)
cache_count.labels("hit").inc()
return json.loads(result)
else:
cache_count.labels("miss").inc()
logger.debug("Found no result in cache for key %s; calling loader",
cache_key.key)
result = loader()
logger.debug("Got loaded result for key %s: %s", cache_key.key, result)
if should_cache(result):
logger.debug(
"Caching loaded result for key %s with expiration %s: %s",
cache_key.key,
result,
cache_key.expiration,
)
expires = convert_to_timedelta(
cache_key.expiration) + datetime.now()
self.cache.set(cache_key.key, json.dumps(result), expires=expires)
logger.debug(
"Cached loaded result for key %s with expiration %s: %s",
cache_key.key,
result,
cache_key.expiration,
)
else:
logger.debug("Not caching loaded result for key %s: %s",
cache_key.key, result)
return result
class InstanceKeys(object):
""" InstanceKeys defines a helper class for interacting with the Quay instance service keys
used for JWT signing of registry tokens as well as requests from Quay to other services
such as Clair. Each container will have a single registered instance key.
"""
def __init__(self, app):
self.app = app
self.instance_keys = ExpiresDict(self._load_instance_keys)
def clear_cache(self):
""" Clears the cache of instance keys. """
self.instance_keys = ExpiresDict(self._load_instance_keys)
def _load_instance_keys(self):
# Load all the instance keys.
keys = {}
for key in model.service_keys.list_service_keys(self.service_name):
keys[key.kid] = ExpiresEntry(CachingKey(key), key.expiration_date)
return keys
@property
def service_name(self):
""" Returns the name of the instance key's service (i.e. 'quay'). """
return self.app.config["INSTANCE_SERVICE_KEY_SERVICE"]
@property
def service_key_expiration(self):
""" Returns the defined expiration for instance service keys, in minutes. """
return self.app.config.get("INSTANCE_SERVICE_KEY_EXPIRATION", 120)
@property
@lru_cache(maxsize=1)
def local_key_id(self):
""" Returns the ID of the local instance service key. """
return _load_file_contents(
self.app.config["INSTANCE_SERVICE_KEY_KID_LOCATION"])
@property
@lru_cache(maxsize=1)
def local_private_key(self):
""" Returns the private key of the local instance service key. """
return _load_file_contents(
self.app.config["INSTANCE_SERVICE_KEY_LOCATION"])
def get_service_key_public_key(self, kid):
""" Returns the public key associated with the given instance service key or None if none. """
caching_key = self.instance_keys.get(kid)
if caching_key is None:
return None
return caching_key.public_key
def __init__(self, **kwargs):
self.state = ExpiresDict()
self.callbacks = {}
def empty_for_testing(self):
self.cache = ExpiresDict()
def __init__(self):
self.cache = ExpiresDict()
def clear_cache(self):
"""
Clears the cache of instance keys.
"""
self.instance_keys = ExpiresDict(self._load_instance_keys)
def __init__(self, app):
self.app = app
self.instance_keys = ExpiresDict(self._load_instance_keys)
class MemoryOrchestrator(Orchestrator):
def __init__(self, **kwargs):
self.state = ExpiresDict()
self.callbacks = {}
def _callbacks_prefixed(self, prefix):
return (callback for (key, callback) in iteritems(self.callbacks)
if key.startswith(prefix))
def on_key_change(self, key, callback, restarter=None):
self.callbacks[key] = callback
@coroutine
def get_prefixed_keys(self, prefix):
raise Return({
k: value
for (k, value) in self.state.items() if k.startswith(prefix)
})
@coroutine
def get_key(self, key):
raise Return(self.state[key])
@coroutine
def set_key(self, key, value, overwrite=False, expiration=None):
preexisting_key = "key" in self.state
if preexisting_key and not overwrite:
raise KeyError
absolute_expiration = None
if expiration is not None:
absolute_expiration = datetime.datetime.now() + datetime.timedelta(
seconds=expiration)
self.state.set(key, value, expires=absolute_expiration)
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
for callback in self._callbacks_prefixed(key):
yield From(callback(KeyChange(event, key, value)))
def set_key_sync(self, key, value, overwrite=False, expiration=None):
"""
set_key, but without trollius coroutines.
"""
preexisting_key = "key" in self.state
if preexisting_key and not overwrite:
raise KeyError
absolute_expiration = None
if expiration is not None:
absolute_expiration = datetime.datetime.now() + datetime.timedelta(
seconds=expiration)
self.state.set(key, value, expires=absolute_expiration)
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
for callback in self._callbacks_prefixed(key):
callback(KeyChange(event, key, value))
@coroutine
def delete_key(self, key):
value = self.state[key]
del self.state[key]
for callback in self._callbacks_prefixed(key):
yield From(callback(KeyChange(KeyEvent.DELETE, key, value)))
@coroutine
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
if key in self.state:
raise Return(False)
self.state.set(key, None, expires=expiration)
raise Return(True)
def shutdown(self):
self.state = None
self.callbacks = None
def __init__(self, cache_config):
super(InMemoryDataModelCache, self).__init__(cache_config)
self.cache = ExpiresDict()