def init(with_result_backend=False): """ Init Celery and Selinon :param with_result_backend: true if the application should connect to the result backend :return: Celery application instance """ conf = { 'broker_url': os.environ.get('BROKER_URL', 'amqp://broker:5672'), } if with_result_backend: conf['result_backend'] = os.environ.get('RESULT_BACKEND_URL', 'redis://redis:6379/0') app = Celery('myapp') app.config_from_object(conf) flow_definition_files = [] # Add all config files for flows for conf_file in os.listdir(os.path.join(_BASE_NAME, 'flows')): if conf_file.endswith('.yaml') and not conf_file.startswith('.'): flow_definition_files.append(os.path.join(_BASE_NAME, 'flows', conf_file)) # Set Selinon configuration Config.set_config_yaml(os.path.join(_BASE_NAME, 'nodes.yaml'), flow_definition_files) # Prepare Celery Config.set_celery_app(app) return app
def test_set_config_yaml_simple(self): test_file = os.path.join(self.DATA_DIR, 'test_set_config.yaml') Config.set_config_yaml(test_file, flow_definition_files=[test_file]) # TODO edge table is not tested # TODO: make sure we inspect propagated values tasks_available = {'task1', 'task2', 'task3'} flows_available = {'flow1'} storages_available = {'MyStorage'} assert tasks_available == set(Config.task_classes) assert tasks_available == Config.task_queues.keys() assert flows_available == Config.dispatcher_queues.keys() assert tasks_available == set(Config.task2storage_mapping.keys()) assert storages_available == set(Config.storage_mapping.keys()) assert storages_available == set(Config.storage2storage_cache.keys()) assert flows_available == set(Config.node_args_from_first.keys()) assert flows_available == set(Config.propagate_node_args.keys()) assert flows_available == set(Config.propagate_finished.keys()) assert flows_available == set(Config.propagate_parent.keys()) assert flows_available == set(Config.propagate_compound_finished.keys()) assert tasks_available == set(Config.throttle_tasks.keys()) assert flows_available == set(Config.throttle_flows.keys()) assert tasks_available | flows_available == set(Config.max_retry.keys()) assert tasks_available | flows_available == set(Config.retry_countdown.keys()) assert tasks_available == set(Config.storage_readonly.keys()) assert flows_available == set(Config.nowait_nodes.keys()) assert flows_available == set(Config.strategies.keys()) assert 'flow1' in Config.failures assert {'task1'} == set(Config.nowait_nodes.get('flow1')) assert 'schema.json' == Config.output_schemas.get('task1')
def test_set_config_yaml_simple(self): test_file = os.path.join(self.DATA_DIR, 'test_set_config.yaml') Config.set_config_yaml(test_file, flow_definition_files=[test_file]) # TODO edge table is not tested # TODO: make sure we inspect propagated values tasks_available = {'task1', 'task2', 'task3'} flows_available = {'flow1'} storages_available = {'MyStorage'} assert tasks_available == set(Config.task_classes) assert tasks_available == Config.task_queues.keys() assert flows_available == Config.dispatcher_queues.keys() assert tasks_available == set(Config.task2storage_mapping.keys()) assert storages_available == set(Config.storage_mapping.keys()) assert storages_available == set(Config.storage2storage_cache.keys()) assert flows_available == set(Config.node_args_from_first.keys()) assert flows_available == set(Config.propagate_node_args.keys()) assert flows_available == set(Config.propagate_finished.keys()) assert flows_available == set(Config.propagate_parent.keys()) assert flows_available == set(Config.propagate_compound_finished.keys()) assert tasks_available == set(Config.throttle_tasks.keys()) assert flows_available == set(Config.throttle_flows.keys()) assert tasks_available | flows_available == set(Config.max_retry.keys()) assert tasks_available | flows_available == set(Config.retry_countdown.keys()) assert tasks_available == set(Config.storage_readonly.keys()) assert flows_available == set(Config.nowait_nodes.keys()) assert flows_available == set(Config.strategies.keys()) assert 'flow1' in Config.failures assert {'task1'} == set(Config.nowait_nodes.get('flow1')) assert 'schema.json' == Config.output_schemas.get('task1')
def test_set_config_dict_simple(self): nodes = { 'tasks': [{ 'name': 'Task1', 'import': 'testapp.tasks' }, { 'name': 'task2', 'import': 'testapp.tasks' }], 'flows': ['flow1'] } flows = [{ 'flow-definitions': [{ 'name': 'flow1', 'edges': [{ 'from': None, 'to': 'Task1' }, { 'from': 'Task1', 'to': 'task2' }] }] }] Config.set_config_dict(nodes, [flows])
def __init__(self, nodes_definition, flow_definitions, concurrency=DEFAULT_CONCURRENCY, sleep_time=DEFAULT_SLEEP_TIME, config_py=None, keep_config_py=False, show_progressbar=True): """Instantiate execute. :param nodes_definition: path to nodes.yaml file :type nodes_definition: str :param flow_definitions: a list of YAML files describing flows :type flow_definitions: list :param concurrency: executor concurrency :type concurrency: int :param sleep_time: number of seconds to wait before querying queue :type sleep_time: float :param config_py: a path to file where Python code configuration should be generated :type config_py: str :param keep_config_py: if true, do not delete generated config.py :type keep_config_py: bool :param show_progressbar: show progressbar on executor run :type show_progressbar: bool """ Config.set_config_yaml(nodes_definition, flow_definitions, config_py=config_py, keep_config_py=keep_config_py) self.concurrency = concurrency self.sleep_time = sleep_time self.show_progressbar = show_progressbar if concurrency != 1: raise NotImplementedError("Concurrency is now unsupported")
def test_trace_setup(self): assert len(Trace._trace_functions) == 0 test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml') Config.set_config_yaml(test_file, flow_definition_files=[test_file]) # logging, sentry, storage and a custom function - 4 callbacks registered assert len(Trace._trace_functions) == 4 assert Trace._logger is not None
def init_selinon(app=None): """Init Selinon configuration. :param app: celery application, if omitted Selinon flow handling tasks will not be registered """ if app is not None: Config.set_celery_app(app) nodes_config, flows_config = get_dispatcher_config_files() Config.set_config_yaml(nodes_config, flows_config)
def test_trace_setup(self): assert len(Trace._trace_functions) == 0 test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml') Config.set_config_yaml(test_file, flow_definition_files=[test_file]) # logging, sentry, storage and a custom function - 3 callbacks registered # By default there is configured logging module to be used with Sentry. assert len(Trace._trace_functions) == 3 assert Trace._logger is not None
def test_set_config_dict_simple(self): nodes = { 'tasks': [ { 'name': 'Task1', 'import': 'testapp.tasks' }, { 'name': 'task2', 'import': 'testapp.tasks' } ], 'flows': [ 'flow1' ] } flows = [ { 'flow-definitions': [ { 'name': 'flow1', 'edges': [ { 'from': None, 'to': 'Task1' }, { 'from': 'Task1', 'to': 'task2' } ] } ] } ] Config.set_config_dict(nodes, [flows])
def init(with_result_backend=False): """Init Celery and Selinon. :param with_result_backend: true if the application should connect to the result backend :return: Celery application instance """ # Avoid exception on CLI run. from celery import Celery conf = {'broker_url': os.environ['BROKER_URL']} if with_result_backend: conf['result_backend'] = os.environ['RESULT_BACKEND_URL'] app = Celery('app') app.config_from_object(conf) # Set Selinon configuration. Config.set_config_yaml(*get_config_files()) # Prepare Celery Config.set_celery_app(app) return app
def __init__(self, nodes_definition, flow_definitions, concurrency=DEFAULT_CONCURRENCY, sleep_time=DEFAULT_SLEEP_TIME, config_py=None, keep_config_py=False, show_progressbar=True): """Instantiate execute. :param nodes_definition: path to nodes.yaml file :type nodes_definition: str :param flow_definitions: a list of YAML files describing flows :type flow_definitions: list :param concurrency: executor concurrency :type concurrency: int :param sleep_time: number of seconds to wait before querying queue :type sleep_time: float :param config_py: a path to file where Python code configuration should be generated :type config_py: str :param keep_config_py: if true, do not delete generated config.py :type keep_config_py: bool :param show_progressbar: show progressbar on executor run :type show_progressbar: bool """ Config.set_config_yaml(nodes_definition, flow_definitions, config_py=config_py, keep_config_py=keep_config_py) self.concurrency = concurrency self.sleep_time = sleep_time self.show_progressbar = show_progressbar if concurrency != 1: raise NotImplementedError("Concurrency is now unsupported")
def init_celery(app=None, result_backend=True): """ Init Celery configuration :param app: celery configuration, if omitted, application will be instantiated :param result_backend: True if Celery should connect to result backend """ # Keep this for debugging purposes for now _logger.debug(">>> Selinon version is %s" % selinon_version) _logger.debug(">>> Selinonlib version is %s" % selinonlib_version) _logger.debug(">>> Celery version is %s" % celery_version) if not result_backend: CelerySettings.disable_result_backend() if app is None: app = Celery('tasks') app.config_from_object(CelerySettings) else: app.config_from_object(CelerySettings) Config.set_celery_app(app) nodes_config, flows_config = get_dispatcher_config_files() Config.set_config_yaml(nodes_config, flows_config)
def dispatcher_setup(): """ Setup environment for Dispatcher if needed """ nodes_yaml, flows_yaml = get_dispatcher_config_files() Config.set_config_yaml(nodes_yaml, flows_yaml)
def send_selinon(request): node_dict = { 'tasks': [{ 'name': 'CheckMessage', 'queue': 'hello_task', 'import': 'tasks', 'max_retry': 0, 'storage': 'Redis' }, { 'name': 'MessageLength', 'queue': 'hello_task', 'import': 'tasks', 'max_retry': 0, 'storage': 'Redis' }, { 'name': 'SuccessAction', 'queue': 'hello_task', 'import': 'tasks', 'max_retry': 0, 'storage': 'Redis' }, { 'name': 'FailureAction', 'queue': 'hello_task', 'import': 'tasks', 'max_retry': 0, 'storage': 'Redis' }], 'flows': ['my_new_flow'], 'storages': [{ 'name': 'Redis', 'import': 'selinon.storages.redis', 'configuration': { 'host': 'localhost', 'port': 6379, 'db': 1, 'charset': 'utf-8' } }], 'global': { 'trace': { 'json': True } }, 'migration_dir': 'migration_dir' } flow_definition = [{ 'flow-definitions': [{ 'name': 'my_new_flow', 'queue': 'hello_task', 'sampling': { 'name': 'constant', 'args': { 'retry': 10 } }, 'edges': [{ 'from': '', 'to': 'CheckMessage' }, { 'from': 'CheckMessage', 'to': 'MessageLength' }, { 'from': 'MessageLength', 'to': 'SuccessAction', 'condition': { 'name': 'fieldEqual', 'args': { 'key': 'status', 'value': True } } }, { 'from': 'MessageLength', 'to': 'FailureAction', 'condition': { 'name': 'fieldEqual', 'args': { 'key': 'status', 'value': False } } }], 'failures': [{ 'nodes': 'MessageLength', 'fallback': 'FailureAction' }] }] }] flow_defi_revse = [{ 'flow-definitions': [{ 'name': 'my_new_flow', 'queue': 'hello_task', 'sampling': { 'name': 'constant', 'args': { 'retry': 10 } }, 'edges': [{ 'from': '', 'to': 'CheckMessage' }, { 'from': 'CheckMessage', 'to': 'MessageLength' }, { 'from': 'MessageLength', 'to': 'SuccessAction', 'condition': { 'and': [{ 'name': 'fieldEqual', 'args': { 'key': 'status', 'value': True } }, { 'name': 'fieldEqual', 'args': { 'key': 'checkvalue', 'value': True } }, { 'name': 'fieldEqual', 'args': { 'key': 'length', 'value': True } }] } }, { 'from': 'MessageLength', 'to': 'FailureAction', 'condition': { 'name': 'fieldEqual', 'args': { 'key': 'status', 'value': False } } }] }] }] node_dict1 = copy.deepcopy(node_dict) flow_defi_revse1 = copy.deepcopy(flow_defi_revse) Config.set_config_dict(node_dict, flow_defi_revse) # print(Config.__dict__) # from selinon_demo import celery_config # Config.set_celery_app(celery_config.app) # celery_config.app.tasks.register(Dispatcher()) # celery_config.app.autodiscover_tasks() # import ipdb;ipdb.set_trace() dispatcher_id = run_flow('my_new_flow', {}, node_dict1, flow_defi_revse1) dispatcher_id1 = run_flow('my_new_flow', {}, node_dict1, flow_definition) return JsonResponse({'status': 'success'})
from __future__ import absolute_import, unicode_literals import os from selinon import Config, run_flow from celery import Celery from django.conf import settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'selinon_demo.settings') app = Celery('selinon_demo') app.config_from_object('django.conf:settings', namespace='CELERY') # Config.set_pickel_config_object(MyDBConfig()) Config.set_celery_app(app) app.autodiscover_tasks() app.conf.task_routes = {'*': {'queue': 'hello_task'}} # node_dict = { # 'tasks': [{'name': 'CheckMessage', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'}, # {'name': 'MessageLength', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'}, # {'name': 'SuccessAction', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'}, # {'name': 'FailureAction', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'}], # 'flows': ['my_new_flow'], # 'storages': [{'name': 'Redis', 'import': 'selinon.storages.redis', 'configuration': {'host': 'localhost', 'port':6379, 'db':1, 'charset': 'utf-8'}}], # 'global': {'trace': {'json': True}}, # 'migration_dir': 'migration_dir'} # flow_definition = [{'flow-definitions': [{'name': 'my_new_flow',