class ObjectCache(object): """Artifact cache handling that encapsulates artifacts handling on worker side >>> epv_cache = ObjectCache.get(ecosystem='npm', name='serve-static', version='1.7.1') >>> extracted_tarball_path = epv_cache.get_extracted_source_tarball() """ _cache = {} _base_cache_dir = get_configuration().worker_data_dir def __init__(self): raise NotImplementedError() @classmethod def wipe(cls): """Wipe all files that are stored in the current cache""" for item in cls._cache.values(): item.remove_files() cls._cache = {} @classmethod def _cache_dir(cls, ecosystem, name, version): """Get cache dir for the given EPV""" return os.path.join(cls._base_cache_dir, ecosystem, name, version) @classmethod def get(cls, ecosystem, name, version): """Get EPVCache for the given EPV""" # This code just stores info about downloaded objects, once we will want to optimize number of retrievals and # do some caching, remove wipe() call in the base task and implement caching logic here key = (ecosystem, name, version) if key not in cls._cache: cache_dir = cls._cache_dir(ecosystem, name, version) # Artifacts bucket used for caching can be expanded based on env variables item = EPVCache(ecosystem, name, version, cache_dir) cls._cache[key] = item return item else: return cls._cache[key] @classmethod def get_from_dict(cls, dictionary): """Sugar for self.get() that respects arguments from a dict""" return cls.get(dictionary['ecosystem'], dictionary['name'], dictionary['version'])
import os import logging from urllib.parse import quote from cucoslib.conf import get_configuration, get_postgres_connection_string _logger = logging.getLogger(__name__) configuration = get_configuration() def _use_sqs(): """ :return: True if worker should use Amazon SQS """ key_id = len(os.environ.get('AWS_SQS_ACCESS_KEY_ID', '')) > 0 access_key = len(os.environ.get('AWS_SQS_SECRET_ACCESS_KEY', '')) > 0 res = int(key_id) + int(access_key) if res == 1: raise RuntimeError( "In order to use AWS SQS you have to provide both 'AWS_SQS_ACCESS_KEY_ID' and " "'AWS_SQS_SECRET_ACCESS_KEY' environment variables") # Make sure we do not pass these env variables - according to Celery docs they can be used only with 'sqs://' if "AWS_ACCESS_KEY_ID" in os.environ: raise RuntimeError( "Do not use AWS_ACCESS_KEY_ID in order to access SQS, use 'AWS_SQS_ACCESS_KEY_ID'" ) if "AWS_SECRET_ACCESS_KEY" in os.environ: raise RuntimeError(
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.log = get_task_logger(self.__class__.__name__) self.configuration = get_configuration()
def test_configuration_is_a_singleton(): assert id(get_configuration()) == id(get_configuration())