def init(with_result_backend=False):
""" Init Celery and Selinon
:param with_result_backend: true if the application should connect to the result backend
:return: Celery application instance
"""
conf = {
'broker_url': os.environ.get('BROKER_URL', 'amqp://broker:5672'),
}
if with_result_backend:
conf['result_backend'] = os.environ.get('RESULT_BACKEND_URL', 'redis://redis:6379/0')
app = Celery('myapp')
app.config_from_object(conf)
flow_definition_files = []
# Add all config files for flows
for conf_file in os.listdir(os.path.join(_BASE_NAME, 'flows')):
if conf_file.endswith('.yaml') and not conf_file.startswith('.'):
flow_definition_files.append(os.path.join(_BASE_NAME, 'flows', conf_file))
# Set Selinon configuration
Config.set_config_yaml(os.path.join(_BASE_NAME, 'nodes.yaml'), flow_definition_files)
# Prepare Celery
Config.set_celery_app(app)
return app
def test_set_config_yaml_simple(self):
test_file = os.path.join(self.DATA_DIR, 'test_set_config.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# TODO edge table is not tested
# TODO: make sure we inspect propagated values
tasks_available = {'task1', 'task2', 'task3'}
flows_available = {'flow1'}
storages_available = {'MyStorage'}
assert tasks_available == set(Config.task_classes)
assert tasks_available == Config.task_queues.keys()
assert flows_available == Config.dispatcher_queues.keys()
assert tasks_available == set(Config.task2storage_mapping.keys())
assert storages_available == set(Config.storage_mapping.keys())
assert storages_available == set(Config.storage2storage_cache.keys())
assert flows_available == set(Config.node_args_from_first.keys())
assert flows_available == set(Config.propagate_node_args.keys())
assert flows_available == set(Config.propagate_finished.keys())
assert flows_available == set(Config.propagate_parent.keys())
assert flows_available == set(Config.propagate_compound_finished.keys())
assert tasks_available == set(Config.throttle_tasks.keys())
assert flows_available == set(Config.throttle_flows.keys())
assert tasks_available | flows_available == set(Config.max_retry.keys())
assert tasks_available | flows_available == set(Config.retry_countdown.keys())
assert tasks_available == set(Config.storage_readonly.keys())
assert flows_available == set(Config.nowait_nodes.keys())
assert flows_available == set(Config.strategies.keys())
assert 'flow1' in Config.failures
assert {'task1'} == set(Config.nowait_nodes.get('flow1'))
assert 'schema.json' == Config.output_schemas.get('task1')
def test_set_config_yaml_simple(self):
test_file = os.path.join(self.DATA_DIR, 'test_set_config.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# TODO edge table is not tested
# TODO: make sure we inspect propagated values
tasks_available = {'task1', 'task2', 'task3'}
flows_available = {'flow1'}
storages_available = {'MyStorage'}
assert tasks_available == set(Config.task_classes)
assert tasks_available == Config.task_queues.keys()
assert flows_available == Config.dispatcher_queues.keys()
assert tasks_available == set(Config.task2storage_mapping.keys())
assert storages_available == set(Config.storage_mapping.keys())
assert storages_available == set(Config.storage2storage_cache.keys())
assert flows_available == set(Config.node_args_from_first.keys())
assert flows_available == set(Config.propagate_node_args.keys())
assert flows_available == set(Config.propagate_finished.keys())
assert flows_available == set(Config.propagate_parent.keys())
assert flows_available == set(Config.propagate_compound_finished.keys())
assert tasks_available == set(Config.throttle_tasks.keys())
assert flows_available == set(Config.throttle_flows.keys())
assert tasks_available | flows_available == set(Config.max_retry.keys())
assert tasks_available | flows_available == set(Config.retry_countdown.keys())
assert tasks_available == set(Config.storage_readonly.keys())
assert flows_available == set(Config.nowait_nodes.keys())
assert flows_available == set(Config.strategies.keys())
assert 'flow1' in Config.failures
assert {'task1'} == set(Config.nowait_nodes.get('flow1'))
assert 'schema.json' == Config.output_schemas.get('task1')
def __init__(self, nodes_definition, flow_definitions,
concurrency=DEFAULT_CONCURRENCY, sleep_time=DEFAULT_SLEEP_TIME,
config_py=None, keep_config_py=False, show_progressbar=True):
"""Instantiate execute.
:param nodes_definition: path to nodes.yaml file
:type nodes_definition: str
:param flow_definitions: a list of YAML files describing flows
:type flow_definitions: list
:param concurrency: executor concurrency
:type concurrency: int
:param sleep_time: number of seconds to wait before querying queue
:type sleep_time: float
:param config_py: a path to file where Python code configuration should be generated
:type config_py: str
:param keep_config_py: if true, do not delete generated config.py
:type keep_config_py: bool
:param show_progressbar: show progressbar on executor run
:type show_progressbar: bool
"""
Config.set_config_yaml(nodes_definition, flow_definitions,
config_py=config_py,
keep_config_py=keep_config_py)
self.concurrency = concurrency
self.sleep_time = sleep_time
self.show_progressbar = show_progressbar
if concurrency != 1:
raise NotImplementedError("Concurrency is now unsupported")
def test_trace_setup(self):
assert len(Trace._trace_functions) == 0
test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# logging, sentry, storage and a custom function - 4 callbacks registered
assert len(Trace._trace_functions) == 4
assert Trace._logger is not None
def init_selinon(app=None):
"""Init Selinon configuration.
:param app: celery application, if omitted Selinon flow handling tasks will not be registered
"""
if app is not None:
Config.set_celery_app(app)
nodes_config, flows_config = get_dispatcher_config_files()
Config.set_config_yaml(nodes_config, flows_config)
def test_trace_setup(self):
assert len(Trace._trace_functions) == 0
test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# logging, sentry, storage and a custom function - 3 callbacks registered
# By default there is configured logging module to be used with Sentry.
assert len(Trace._trace_functions) == 3
assert Trace._logger is not None
def init(with_result_backend=False):
"""Init Celery and Selinon.
:param with_result_backend: true if the application should connect to the result backend
:return: Celery application instance
"""
# Avoid exception on CLI run.
from celery import Celery
conf = {'broker_url': os.environ['BROKER_URL']}
if with_result_backend:
conf['result_backend'] = os.environ['RESULT_BACKEND_URL']
app = Celery('app')
app.config_from_object(conf)
# Set Selinon configuration.
Config.set_config_yaml(*get_config_files())
# Prepare Celery
Config.set_celery_app(app)
return app
def __init__(self,
nodes_definition,
flow_definitions,
concurrency=DEFAULT_CONCURRENCY,
sleep_time=DEFAULT_SLEEP_TIME,
config_py=None,
keep_config_py=False,
show_progressbar=True):
"""Instantiate execute.
:param nodes_definition: path to nodes.yaml file
:type nodes_definition: str
:param flow_definitions: a list of YAML files describing flows
:type flow_definitions: list
:param concurrency: executor concurrency
:type concurrency: int
:param sleep_time: number of seconds to wait before querying queue
:type sleep_time: float
:param config_py: a path to file where Python code configuration should be generated
:type config_py: str
:param keep_config_py: if true, do not delete generated config.py
:type keep_config_py: bool
:param show_progressbar: show progressbar on executor run
:type show_progressbar: bool
"""
Config.set_config_yaml(nodes_definition,
flow_definitions,
config_py=config_py,
keep_config_py=keep_config_py)
self.concurrency = concurrency
self.sleep_time = sleep_time
self.show_progressbar = show_progressbar
if concurrency != 1:
raise NotImplementedError("Concurrency is now unsupported")
def init_celery(app=None, result_backend=True):
"""
Init Celery configuration
:param app: celery configuration, if omitted, application will be instantiated
:param result_backend: True if Celery should connect to result backend
"""
# Keep this for debugging purposes for now
_logger.debug(">>> Selinon version is %s" % selinon_version)
_logger.debug(">>> Selinonlib version is %s" % selinonlib_version)
_logger.debug(">>> Celery version is %s" % celery_version)
if not result_backend:
CelerySettings.disable_result_backend()
if app is None:
app = Celery('tasks')
app.config_from_object(CelerySettings)
else:
app.config_from_object(CelerySettings)
Config.set_celery_app(app)
nodes_config, flows_config = get_dispatcher_config_files()
Config.set_config_yaml(nodes_config, flows_config)
def dispatcher_setup():
""" Setup environment for Dispatcher if needed """
nodes_yaml, flows_yaml = get_dispatcher_config_files()
Config.set_config_yaml(nodes_yaml, flows_yaml)
