def find_undocumented_settings(directive='.. setting:: '): settings = dict(flatten(NAMESPACES)) all = set(settings) documented = set(line.strip()[len(directive):].strip() for line in _input() if line.strip().startswith(directive)) return [setting for setting in all ^ documented if not is_ignored(setting, settings[setting])]
def find_undocumented_settings(directive=".. setting:: "): settings = dict(flatten(NAMESPACES)) all = set(settings) documented = set(line.strip()[len(directive):].strip() for line in input() if line.strip().startswith(directive)) return [setting for setting in all ^ documented if not is_ignored(setting, settings[setting])]
def convert_to_celery_options(config): ''' Converts string representation of configuration to its expected data type :param config: dict of configurations ''' type_map = { 'any': ast.literal_eval, 'int': int, 'string': str, 'bool': strtobool, 'float': float, 'dict': ast.literal_eval, 'tuple': ast.literal_eval, 'list': ast.literal_eval } mapping = {} # Read from celery.app.defaults to get the expected data type for each configurable property for (key, value) in defaults.flatten(defaults.NAMESPACES): __type = type_map[value.type] if __type: mapping[key] = __type # For each config that need to configure, cast/convert to the expected data type for (key, value) in config.items(): # EJ, BROKER_USE_SSL is not really bool. it is allowed to be a dict to hold cert information. # because we want our celery worker to share same configuration file as smarter. we need to make it parsable # so celery worker can use it # see # http://stackoverflow.com/questions/16406498/is-there-a-way-to-validate-the-brokers-ssl-certificate-in-django-celery # for BROKER_USE_SSL if key in mapping and key != 'BROKER_USE_SSL': config[key] = mapping[key](value) else: config[key] = ast.literal_eval(value) return config
def convert_to_celery_options(config): ''' Converts string representation of configuration to its expected data type :param config: dict of configurations ''' type_map = {'any': ast.literal_eval, 'int': int, 'string': str, 'bool': strtobool, 'float': float, 'dict': ast.literal_eval, 'tuple': ast.literal_eval, 'list': ast.literal_eval } mapping = {} # Read from celery.app.defaults to get the expected data type for each configurable property for (key, value) in defaults.flatten(defaults.NAMESPACES): __type = type_map[value.type] if __type: mapping[key] = __type # For each config that need to configure, cast/convert to the expected data type for (key, value) in config.items(): # EJ, BROKER_USE_SSL is not really bool. it is allowed to be a dict to hold cert information. # because we want our celery worker to share same configuration file as smarter. we need to make it parsable # so celery worker can use it # see # http://stackoverflow.com/questions/16406498/is-there-a-way-to-validate-the-brokers-ssl-certificate-in-django-celery # for BROKER_USE_SSL if key in mapping and key != 'BROKER_USE_SSL': config[key] = mapping[key](value) else: config[key] = ast.literal_eval(value) return config
def configcheck_project_settings(): from celery.app.defaults import NAMESPACES, flatten settings.update(dict(flatten(NAMESPACES))) return set(settings)
config[key] = config[key].replace("/n", "") TYPES_TO_OBJ = { "any": (object, None), "bool": (bool, str_to_bool), "dict": (dict, eval), "float": (float, float), "int": (int, int), "list": (list, eval), "tuple": (tuple, eval), "string": (str, str), } OPTIONS = dict((key, TYPES_TO_OBJ[opt.type]) for key, opt in defaults.flatten(defaults.NAMESPACES)) def convert_celery_options(config): """ Converts celery options to apropriate types """ for key, value in config.iteritems(): opt_type = OPTIONS.get(key) if opt_type: if opt_type[0] == str: clean_quoted_config(config, key) elif opt_type[0] is object: try: config[key] = eval(value)
TYPES_TO_OBJ = { 'any': (object, None), 'bool': (bool, str_to_bool), 'dict': (dict, eval), 'float': (float, float), 'int': (int, int), 'list': (list, eval), 'tuple': (tuple, eval), 'string': (str, str), } OPTIONS = dict( (key, TYPES_TO_OBJ[opt.type]) for key, opt in defaults.flatten(defaults.NAMESPACES) ) def convert_celery_options(config): """ Converts celery options to apropriate types """ for key, value in config.iteritems(): opt_type = OPTIONS.get(key) if opt_type: if opt_type[0] == str: clean_quoted_config(config, key) elif opt_type[0] is object: try:
def __init__( self, name, dist, config_sections, main, on_configure, watch, tasks, services_service, reloader_service=None, **config ): """Initialization In: - ``host`` -- address of the memcache server - ``port`` -- port of the memcache server """ super(_CeleryService, self).__init__(name, dist, **config) self.watch = watch self.reloader = reloader_service self.services = services_service self.files = set() nb_cpus = multiprocessing.cpu_count() config['worker']['concurrency'] = eval(config['worker']['concurrency'], {}, {'NB_CPUS': nb_cpus}) celery_config = {} app_tasks = {name: {} for name in tasks} for section, parameters in list(config.items()): if isinstance(parameters, dict): if (section not in config_sections): app_tasks[section] = config[section] = {k: v for k, v in parameters.items() if v is not None} else: celery_config[section] = parameters.copy() else: celery_config[section] = parameters celery_config['task']['routes'] = { section: celery_config['task'].pop(section)['queue'] for section, parameters in list(celery_config['task'].items()) if isinstance(parameters, dict) and (section != 'publish_retry_policy') } schedules = {} for section, parameters in list(celery_config['beat'].items()): if isinstance(parameters, dict): del celery_config['beat'][section] schedule = {'task': parameters['task'], 'args': eval(parameters['args'])} if parameters['schedule']: schedule['schedule'] = parameters['schedule'] crontab_params = {param: parameters[param] for param in CRONTAB_PARAMS if parameters[param] is not None} if crontab_params: schedule['schedule'] = crontab(**crontab_params) schedules[section] = schedule celery_config = collections.AttributeDict(defaults.flatten(celery_config)) celery_config['beat_schedule'] = schedules if celery_config.pop('result_asynchronous_callbacks'): asynchronous.register_drainer('default')(Drainer) if on_configure: reference.load_object(on_configure)[0](celery_config) self.celery = self.CELERY_FACTORY(main, log='nagare.services.celery:Logging', config_source=celery_config) for task, parameters in app_tasks.items(): self.register_task(reference.load_object(task)[0], **parameters)