def find_undocumented_settings(directive='.. setting:: '):
settings = dict(flatten(NAMESPACES))
all = set(settings)
documented = set(line.strip()[len(directive):].strip()
for line in _input()
if line.strip().startswith(directive))
return [setting for setting in all ^ documented
if not is_ignored(setting, settings[setting])]
def find_undocumented_settings(directive=".. setting:: "):
settings = dict(flatten(NAMESPACES))
all = set(settings)
documented = set(line.strip()[len(directive):].strip()
for line in input()
if line.strip().startswith(directive))
return [setting for setting in all ^ documented
if not is_ignored(setting, settings[setting])]
def convert_to_celery_options(config):
'''
Converts string representation of configuration to its expected data type
:param config: dict of configurations
'''
type_map = {
'any': ast.literal_eval,
'int': int,
'string': str,
'bool': strtobool,
'float': float,
'dict': ast.literal_eval,
'tuple': ast.literal_eval,
'list': ast.literal_eval
}
mapping = {}
# Read from celery.app.defaults to get the expected data type for each configurable property
for (key, value) in defaults.flatten(defaults.NAMESPACES):
__type = type_map[value.type]
if __type:
mapping[key] = __type
# For each config that need to configure, cast/convert to the expected data type
for (key, value) in config.items():
# EJ, BROKER_USE_SSL is not really bool. it is allowed to be a dict to hold cert information.
# because we want our celery worker to share same configuration file as smarter. we need to make it parsable
# so celery worker can use it
# see
# http://stackoverflow.com/questions/16406498/is-there-a-way-to-validate-the-brokers-ssl-certificate-in-django-celery
# for BROKER_USE_SSL
if key in mapping and key != 'BROKER_USE_SSL':
config[key] = mapping[key](value)
else:
config[key] = ast.literal_eval(value)
return config
def convert_to_celery_options(config):
'''
Converts string representation of configuration to its expected data type
:param config: dict of configurations
'''
type_map = {'any': ast.literal_eval,
'int': int,
'string': str,
'bool': strtobool,
'float': float,
'dict': ast.literal_eval,
'tuple': ast.literal_eval,
'list': ast.literal_eval
}
mapping = {}
# Read from celery.app.defaults to get the expected data type for each configurable property
for (key, value) in defaults.flatten(defaults.NAMESPACES):
__type = type_map[value.type]
if __type:
mapping[key] = __type
# For each config that need to configure, cast/convert to the expected data type
for (key, value) in config.items():
# EJ, BROKER_USE_SSL is not really bool. it is allowed to be a dict to hold cert information.
# because we want our celery worker to share same configuration file as smarter. we need to make it parsable
# so celery worker can use it
# see
# http://stackoverflow.com/questions/16406498/is-there-a-way-to-validate-the-brokers-ssl-certificate-in-django-celery
# for BROKER_USE_SSL
if key in mapping and key != 'BROKER_USE_SSL':
config[key] = mapping[key](value)
else:
config[key] = ast.literal_eval(value)
return config
def configcheck_project_settings():
from celery.app.defaults import NAMESPACES, flatten
settings.update(dict(flatten(NAMESPACES)))
return set(settings)
config[key] = config[key].replace("/n", "")
TYPES_TO_OBJ = {
"any": (object, None),
"bool": (bool, str_to_bool),
"dict": (dict, eval),
"float": (float, float),
"int": (int, int),
"list": (list, eval),
"tuple": (tuple, eval),
"string": (str, str),
}
OPTIONS = dict((key, TYPES_TO_OBJ[opt.type]) for key, opt in defaults.flatten(defaults.NAMESPACES))
def convert_celery_options(config):
"""
Converts celery options to apropriate types
"""
for key, value in config.iteritems():
opt_type = OPTIONS.get(key)
if opt_type:
if opt_type[0] == str:
clean_quoted_config(config, key)
elif opt_type[0] is object:
try:
config[key] = eval(value)
TYPES_TO_OBJ = {
'any': (object, None),
'bool': (bool, str_to_bool),
'dict': (dict, eval),
'float': (float, float),
'int': (int, int),
'list': (list, eval),
'tuple': (tuple, eval),
'string': (str, str),
}
OPTIONS = dict(
(key, TYPES_TO_OBJ[opt.type])
for key, opt in defaults.flatten(defaults.NAMESPACES)
)
def convert_celery_options(config):
"""
Converts celery options to apropriate types
"""
for key, value in config.iteritems():
opt_type = OPTIONS.get(key)
if opt_type:
if opt_type[0] == str:
clean_quoted_config(config, key)
elif opt_type[0] is object:
try:
def configcheck_project_settings():
from celery.app.defaults import NAMESPACES, flatten
settings.update(dict(flatten(NAMESPACES)))
return set(settings)
def __init__(
self,
name, dist, config_sections,
main, on_configure, watch, tasks,
services_service, reloader_service=None,
**config
):
"""Initialization
In:
- ``host`` -- address of the memcache server
- ``port`` -- port of the memcache server
"""
super(_CeleryService, self).__init__(name, dist, **config)
self.watch = watch
self.reloader = reloader_service
self.services = services_service
self.files = set()
nb_cpus = multiprocessing.cpu_count()
config['worker']['concurrency'] = eval(config['worker']['concurrency'], {}, {'NB_CPUS': nb_cpus})
celery_config = {}
app_tasks = {name: {} for name in tasks}
for section, parameters in list(config.items()):
if isinstance(parameters, dict):
if (section not in config_sections):
app_tasks[section] = config[section] = {k: v for k, v in parameters.items() if v is not None}
else:
celery_config[section] = parameters.copy()
else:
celery_config[section] = parameters
celery_config['task']['routes'] = {
section: celery_config['task'].pop(section)['queue']
for section, parameters in list(celery_config['task'].items())
if isinstance(parameters, dict) and (section != 'publish_retry_policy')
}
schedules = {}
for section, parameters in list(celery_config['beat'].items()):
if isinstance(parameters, dict):
del celery_config['beat'][section]
schedule = {'task': parameters['task'], 'args': eval(parameters['args'])}
if parameters['schedule']:
schedule['schedule'] = parameters['schedule']
crontab_params = {param: parameters[param] for param in CRONTAB_PARAMS if parameters[param] is not None}
if crontab_params:
schedule['schedule'] = crontab(**crontab_params)
schedules[section] = schedule
celery_config = collections.AttributeDict(defaults.flatten(celery_config))
celery_config['beat_schedule'] = schedules
if celery_config.pop('result_asynchronous_callbacks'):
asynchronous.register_drainer('default')(Drainer)
if on_configure:
reference.load_object(on_configure)[0](celery_config)
self.celery = self.CELERY_FACTORY(main, log='nagare.services.celery:Logging', config_source=celery_config)
for task, parameters in app_tasks.items():
self.register_task(reference.load_object(task)[0], **parameters)
