def make_app(config=None):
config = check.opt_inst_param(config, 'config', (CeleryConfig, CeleryK8sJobConfig))
app_args = config.app_args() if config is not None else {}
app_ = Celery('dagster', **app_args)
if config is None:
app_.config_from_object('dagster_celery.defaults', force=True)
if is_module_available('dagster_celery_config'):
# pylint: disable=protected-access
obj = force_mapping(app_.loader._smart_import('dagster_celery_config'))
app_.conf.update(obj)
app_.loader.import_module('celery.contrib.testing.tasks')
app_.conf.task_queues = [
Queue('dagster', routing_key='dagster.#', queue_arguments={'x-max-priority': 10})
]
app_.conf.task_routes = {
'execute_plan': {'queue': 'dagster', 'routing_key': 'dagster.execute_plan'},
'execute_step_k8s_job': {'queue': 'dagster', 'routing_key': 'dagster.execute_step_k8s_job'},
}
app_.conf.task_queue_max_priority = 10
app_.conf.task_default_priority = 5
return app_
def config_from_object(self, obj, silent=False):
if isinstance(obj, string_t):
try:
obj = self._smart_import(obj, imp=self.import_from_cwd)
except (ImportError, AttributeError):
if silent:
return False
raise
self._conf = force_mapping(obj)
return True
def config_from_object(self, obj, silent=False):
if isinstance(obj, string_t):
try:
obj = self._smart_import(obj, imp=self.import_from_cwd)
except (ImportError, AttributeError):
if silent:
return False
raise
self._conf = force_mapping(obj)
return True
def config_from_object(self, obj, silent=False):
if isinstance(obj, str):
try:
obj = self._smart_import(obj, imp=self.import_from_cwd)
except (ImportError, AttributeError):
if silent:
return False
raise
self._conf = force_mapping(obj)
if self._conf.get('override_backends') is not None:
self.override_backends = self._conf['override_backends']
return True
def make_app_with_task_routes(task_routes, app_args=None):
app_ = Celery('dagster', **(app_args if app_args else {}))
if app_args is None:
app_.config_from_object('dagster_celery.defaults', force=True)
if is_module_available('dagster_celery_config'):
# pylint: disable=protected-access
obj = force_mapping(app_.loader._smart_import('dagster_celery_config'))
app_.conf.update(obj)
app_.loader.import_module('celery.contrib.testing.tasks')
app_.conf.task_queues = [
Queue('dagster', routing_key='dagster.#', queue_arguments={'x-max-priority': 10})
]
app_.conf.task_routes = task_routes
app_.conf.task_queue_max_priority = 10
app_.conf.task_default_priority = 5
return app_
