def init(with_result_backend=False): """ Init Celery and Selinon :param with_result_backend: true if the application should connect to the result backend :return: Celery application instance """ conf = { 'broker_url': os.environ.get('BROKER_URL', 'amqp://broker:5672'), } if with_result_backend: conf['result_backend'] = os.environ.get('RESULT_BACKEND_URL', 'redis://redis:6379/0') app = Celery('myapp') app.config_from_object(conf) flow_definition_files = [] # Add all config files for flows for conf_file in os.listdir(os.path.join(_BASE_NAME, 'flows')): if conf_file.endswith('.yaml') and not conf_file.startswith('.'): flow_definition_files.append(os.path.join(_BASE_NAME, 'flows', conf_file)) # Set Selinon configuration Config.set_config_yaml(os.path.join(_BASE_NAME, 'nodes.yaml'), flow_definition_files) # Prepare Celery Config.set_celery_app(app) return app
def test_set_config_yaml_simple(self): test_file = os.path.join(self.DATA_DIR, 'test_set_config.yaml') Config.set_config_yaml(test_file, flow_definition_files=[test_file]) # TODO edge table is not tested # TODO: make sure we inspect propagated values tasks_available = {'task1', 'task2', 'task3'} flows_available = {'flow1'} storages_available = {'MyStorage'} assert tasks_available == set(Config.task_classes) assert tasks_available == Config.task_queues.keys() assert flows_available == Config.dispatcher_queues.keys() assert tasks_available == set(Config.task2storage_mapping.keys()) assert storages_available == set(Config.storage_mapping.keys()) assert storages_available == set(Config.storage2storage_cache.keys()) assert flows_available == set(Config.node_args_from_first.keys()) assert flows_available == set(Config.propagate_node_args.keys()) assert flows_available == set(Config.propagate_finished.keys()) assert flows_available == set(Config.propagate_parent.keys()) assert flows_available == set(Config.propagate_compound_finished.keys()) assert tasks_available == set(Config.throttle_tasks.keys()) assert flows_available == set(Config.throttle_flows.keys()) assert tasks_available | flows_available == set(Config.max_retry.keys()) assert tasks_available | flows_available == set(Config.retry_countdown.keys()) assert tasks_available == set(Config.storage_readonly.keys()) assert flows_available == set(Config.nowait_nodes.keys()) assert flows_available == set(Config.strategies.keys()) assert 'flow1' in Config.failures assert {'task1'} == set(Config.nowait_nodes.get('flow1')) assert 'schema.json' == Config.output_schemas.get('task1')
def __init__(self, nodes_definition, flow_definitions, concurrency=DEFAULT_CONCURRENCY, sleep_time=DEFAULT_SLEEP_TIME, config_py=None, keep_config_py=False, show_progressbar=True): """Instantiate execute. :param nodes_definition: path to nodes.yaml file :type nodes_definition: str :param flow_definitions: a list of YAML files describing flows :type flow_definitions: list :param concurrency: executor concurrency :type concurrency: int :param sleep_time: number of seconds to wait before querying queue :type sleep_time: float :param config_py: a path to file where Python code configuration should be generated :type config_py: str :param keep_config_py: if true, do not delete generated config.py :type keep_config_py: bool :param show_progressbar: show progressbar on executor run :type show_progressbar: bool """ Config.set_config_yaml(nodes_definition, flow_definitions, config_py=config_py, keep_config_py=keep_config_py) self.concurrency = concurrency self.sleep_time = sleep_time self.show_progressbar = show_progressbar if concurrency != 1: raise NotImplementedError("Concurrency is now unsupported")
def test_trace_setup(self): assert len(Trace._trace_functions) == 0 test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml') Config.set_config_yaml(test_file, flow_definition_files=[test_file]) # logging, sentry, storage and a custom function - 4 callbacks registered assert len(Trace._trace_functions) == 4 assert Trace._logger is not None
def init_selinon(app=None): """Init Selinon configuration. :param app: celery application, if omitted Selinon flow handling tasks will not be registered """ if app is not None: Config.set_celery_app(app) nodes_config, flows_config = get_dispatcher_config_files() Config.set_config_yaml(nodes_config, flows_config)
def test_trace_setup(self): assert len(Trace._trace_functions) == 0 test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml') Config.set_config_yaml(test_file, flow_definition_files=[test_file]) # logging, sentry, storage and a custom function - 3 callbacks registered # By default there is configured logging module to be used with Sentry. assert len(Trace._trace_functions) == 3 assert Trace._logger is not None
def init(with_result_backend=False): """Init Celery and Selinon. :param with_result_backend: true if the application should connect to the result backend :return: Celery application instance """ # Avoid exception on CLI run. from celery import Celery conf = {'broker_url': os.environ['BROKER_URL']} if with_result_backend: conf['result_backend'] = os.environ['RESULT_BACKEND_URL'] app = Celery('app') app.config_from_object(conf) # Set Selinon configuration. Config.set_config_yaml(*get_config_files()) # Prepare Celery Config.set_celery_app(app) return app
def init_celery(app=None, result_backend=True): """ Init Celery configuration :param app: celery configuration, if omitted, application will be instantiated :param result_backend: True if Celery should connect to result backend """ # Keep this for debugging purposes for now _logger.debug(">>> Selinon version is %s" % selinon_version) _logger.debug(">>> Selinonlib version is %s" % selinonlib_version) _logger.debug(">>> Celery version is %s" % celery_version) if not result_backend: CelerySettings.disable_result_backend() if app is None: app = Celery('tasks') app.config_from_object(CelerySettings) else: app.config_from_object(CelerySettings) Config.set_celery_app(app) nodes_config, flows_config = get_dispatcher_config_files() Config.set_config_yaml(nodes_config, flows_config)
def dispatcher_setup(): """ Setup environment for Dispatcher if needed """ nodes_yaml, flows_yaml = get_dispatcher_config_files() Config.set_config_yaml(nodes_yaml, flows_yaml)