def init(with_result_backend=False):
""" Init Celery and Selinon
:param with_result_backend: true if the application should connect to the result backend
:return: Celery application instance
"""
conf = {
'broker_url': os.environ.get('BROKER_URL', 'amqp://broker:5672'),
}
if with_result_backend:
conf['result_backend'] = os.environ.get('RESULT_BACKEND_URL', 'redis://redis:6379/0')
app = Celery('myapp')
app.config_from_object(conf)
flow_definition_files = []
# Add all config files for flows
for conf_file in os.listdir(os.path.join(_BASE_NAME, 'flows')):
if conf_file.endswith('.yaml') and not conf_file.startswith('.'):
flow_definition_files.append(os.path.join(_BASE_NAME, 'flows', conf_file))
# Set Selinon configuration
Config.set_config_yaml(os.path.join(_BASE_NAME, 'nodes.yaml'), flow_definition_files)
# Prepare Celery
Config.set_celery_app(app)
return app
def test_set_config_yaml_simple(self):
test_file = os.path.join(self.DATA_DIR, 'test_set_config.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# TODO edge table is not tested
# TODO: make sure we inspect propagated values
tasks_available = {'task1', 'task2', 'task3'}
flows_available = {'flow1'}
storages_available = {'MyStorage'}
assert tasks_available == set(Config.task_classes)
assert tasks_available == Config.task_queues.keys()
assert flows_available == Config.dispatcher_queues.keys()
assert tasks_available == set(Config.task2storage_mapping.keys())
assert storages_available == set(Config.storage_mapping.keys())
assert storages_available == set(Config.storage2storage_cache.keys())
assert flows_available == set(Config.node_args_from_first.keys())
assert flows_available == set(Config.propagate_node_args.keys())
assert flows_available == set(Config.propagate_finished.keys())
assert flows_available == set(Config.propagate_parent.keys())
assert flows_available == set(Config.propagate_compound_finished.keys())
assert tasks_available == set(Config.throttle_tasks.keys())
assert flows_available == set(Config.throttle_flows.keys())
assert tasks_available | flows_available == set(Config.max_retry.keys())
assert tasks_available | flows_available == set(Config.retry_countdown.keys())
assert tasks_available == set(Config.storage_readonly.keys())
assert flows_available == set(Config.nowait_nodes.keys())
assert flows_available == set(Config.strategies.keys())
assert 'flow1' in Config.failures
assert {'task1'} == set(Config.nowait_nodes.get('flow1'))
assert 'schema.json' == Config.output_schemas.get('task1')
def test_set_config_yaml_simple(self):
test_file = os.path.join(self.DATA_DIR, 'test_set_config.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# TODO edge table is not tested
# TODO: make sure we inspect propagated values
tasks_available = {'task1', 'task2', 'task3'}
flows_available = {'flow1'}
storages_available = {'MyStorage'}
assert tasks_available == set(Config.task_classes)
assert tasks_available == Config.task_queues.keys()
assert flows_available == Config.dispatcher_queues.keys()
assert tasks_available == set(Config.task2storage_mapping.keys())
assert storages_available == set(Config.storage_mapping.keys())
assert storages_available == set(Config.storage2storage_cache.keys())
assert flows_available == set(Config.node_args_from_first.keys())
assert flows_available == set(Config.propagate_node_args.keys())
assert flows_available == set(Config.propagate_finished.keys())
assert flows_available == set(Config.propagate_parent.keys())
assert flows_available == set(Config.propagate_compound_finished.keys())
assert tasks_available == set(Config.throttle_tasks.keys())
assert flows_available == set(Config.throttle_flows.keys())
assert tasks_available | flows_available == set(Config.max_retry.keys())
assert tasks_available | flows_available == set(Config.retry_countdown.keys())
assert tasks_available == set(Config.storage_readonly.keys())
assert flows_available == set(Config.nowait_nodes.keys())
assert flows_available == set(Config.strategies.keys())
assert 'flow1' in Config.failures
assert {'task1'} == set(Config.nowait_nodes.get('flow1'))
assert 'schema.json' == Config.output_schemas.get('task1')
def test_set_config_dict_simple(self):
nodes = {
'tasks': [{
'name': 'Task1',
'import': 'testapp.tasks'
}, {
'name': 'task2',
'import': 'testapp.tasks'
}],
'flows': ['flow1']
}
flows = [{
'flow-definitions': [{
'name':
'flow1',
'edges': [{
'from': None,
'to': 'Task1'
}, {
'from': 'Task1',
'to': 'task2'
}]
}]
}]
Config.set_config_dict(nodes, [flows])
def __init__(self, nodes_definition, flow_definitions,
concurrency=DEFAULT_CONCURRENCY, sleep_time=DEFAULT_SLEEP_TIME,
config_py=None, keep_config_py=False, show_progressbar=True):
"""Instantiate execute.
:param nodes_definition: path to nodes.yaml file
:type nodes_definition: str
:param flow_definitions: a list of YAML files describing flows
:type flow_definitions: list
:param concurrency: executor concurrency
:type concurrency: int
:param sleep_time: number of seconds to wait before querying queue
:type sleep_time: float
:param config_py: a path to file where Python code configuration should be generated
:type config_py: str
:param keep_config_py: if true, do not delete generated config.py
:type keep_config_py: bool
:param show_progressbar: show progressbar on executor run
:type show_progressbar: bool
"""
Config.set_config_yaml(nodes_definition, flow_definitions,
config_py=config_py,
keep_config_py=keep_config_py)
self.concurrency = concurrency
self.sleep_time = sleep_time
self.show_progressbar = show_progressbar
if concurrency != 1:
raise NotImplementedError("Concurrency is now unsupported")
def test_trace_setup(self):
assert len(Trace._trace_functions) == 0
test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# logging, sentry, storage and a custom function - 4 callbacks registered
assert len(Trace._trace_functions) == 4
assert Trace._logger is not None
def init_selinon(app=None):
"""Init Selinon configuration.
:param app: celery application, if omitted Selinon flow handling tasks will not be registered
"""
if app is not None:
Config.set_celery_app(app)
nodes_config, flows_config = get_dispatcher_config_files()
Config.set_config_yaml(nodes_config, flows_config)
def test_trace_setup(self):
assert len(Trace._trace_functions) == 0
test_file = os.path.join(self.DATA_DIR, 'test_trace_setup.yaml')
Config.set_config_yaml(test_file, flow_definition_files=[test_file])
# logging, sentry, storage and a custom function - 3 callbacks registered
# By default there is configured logging module to be used with Sentry.
assert len(Trace._trace_functions) == 3
assert Trace._logger is not None
def test_set_config_dict_simple(self):
nodes = {
'tasks': [
{
'name': 'Task1',
'import': 'testapp.tasks'
},
{
'name': 'task2',
'import': 'testapp.tasks'
}
],
'flows': [
'flow1'
]
}
flows = [
{
'flow-definitions': [
{
'name': 'flow1',
'edges': [
{
'from': None,
'to': 'Task1'
},
{
'from': 'Task1',
'to': 'task2'
}
]
}
]
}
]
Config.set_config_dict(nodes, [flows])
def init(with_result_backend=False):
"""Init Celery and Selinon.
:param with_result_backend: true if the application should connect to the result backend
:return: Celery application instance
"""
# Avoid exception on CLI run.
from celery import Celery
conf = {'broker_url': os.environ['BROKER_URL']}
if with_result_backend:
conf['result_backend'] = os.environ['RESULT_BACKEND_URL']
app = Celery('app')
app.config_from_object(conf)
# Set Selinon configuration.
Config.set_config_yaml(*get_config_files())
# Prepare Celery
Config.set_celery_app(app)
return app
def __init__(self,
nodes_definition,
flow_definitions,
concurrency=DEFAULT_CONCURRENCY,
sleep_time=DEFAULT_SLEEP_TIME,
config_py=None,
keep_config_py=False,
show_progressbar=True):
"""Instantiate execute.
:param nodes_definition: path to nodes.yaml file
:type nodes_definition: str
:param flow_definitions: a list of YAML files describing flows
:type flow_definitions: list
:param concurrency: executor concurrency
:type concurrency: int
:param sleep_time: number of seconds to wait before querying queue
:type sleep_time: float
:param config_py: a path to file where Python code configuration should be generated
:type config_py: str
:param keep_config_py: if true, do not delete generated config.py
:type keep_config_py: bool
:param show_progressbar: show progressbar on executor run
:type show_progressbar: bool
"""
Config.set_config_yaml(nodes_definition,
flow_definitions,
config_py=config_py,
keep_config_py=keep_config_py)
self.concurrency = concurrency
self.sleep_time = sleep_time
self.show_progressbar = show_progressbar
if concurrency != 1:
raise NotImplementedError("Concurrency is now unsupported")
def init_celery(app=None, result_backend=True):
"""
Init Celery configuration
:param app: celery configuration, if omitted, application will be instantiated
:param result_backend: True if Celery should connect to result backend
"""
# Keep this for debugging purposes for now
_logger.debug(">>> Selinon version is %s" % selinon_version)
_logger.debug(">>> Selinonlib version is %s" % selinonlib_version)
_logger.debug(">>> Celery version is %s" % celery_version)
if not result_backend:
CelerySettings.disable_result_backend()
if app is None:
app = Celery('tasks')
app.config_from_object(CelerySettings)
else:
app.config_from_object(CelerySettings)
Config.set_celery_app(app)
nodes_config, flows_config = get_dispatcher_config_files()
Config.set_config_yaml(nodes_config, flows_config)
def dispatcher_setup():
""" Setup environment for Dispatcher if needed """
nodes_yaml, flows_yaml = get_dispatcher_config_files()
Config.set_config_yaml(nodes_yaml, flows_yaml)
def send_selinon(request):
node_dict = {
'tasks': [{
'name': 'CheckMessage',
'queue': 'hello_task',
'import': 'tasks',
'max_retry': 0,
'storage': 'Redis'
}, {
'name': 'MessageLength',
'queue': 'hello_task',
'import': 'tasks',
'max_retry': 0,
'storage': 'Redis'
}, {
'name': 'SuccessAction',
'queue': 'hello_task',
'import': 'tasks',
'max_retry': 0,
'storage': 'Redis'
}, {
'name': 'FailureAction',
'queue': 'hello_task',
'import': 'tasks',
'max_retry': 0,
'storage': 'Redis'
}],
'flows': ['my_new_flow'],
'storages': [{
'name': 'Redis',
'import': 'selinon.storages.redis',
'configuration': {
'host': 'localhost',
'port': 6379,
'db': 1,
'charset': 'utf-8'
}
}],
'global': {
'trace': {
'json': True
}
},
'migration_dir':
'migration_dir'
}
flow_definition = [{
'flow-definitions': [{
'name':
'my_new_flow',
'queue':
'hello_task',
'sampling': {
'name': 'constant',
'args': {
'retry': 10
}
},
'edges': [{
'from': '',
'to': 'CheckMessage'
}, {
'from': 'CheckMessage',
'to': 'MessageLength'
}, {
'from': 'MessageLength',
'to': 'SuccessAction',
'condition': {
'name': 'fieldEqual',
'args': {
'key': 'status',
'value': True
}
}
}, {
'from': 'MessageLength',
'to': 'FailureAction',
'condition': {
'name': 'fieldEqual',
'args': {
'key': 'status',
'value': False
}
}
}],
'failures': [{
'nodes': 'MessageLength',
'fallback': 'FailureAction'
}]
}]
}]
flow_defi_revse = [{
'flow-definitions': [{
'name':
'my_new_flow',
'queue':
'hello_task',
'sampling': {
'name': 'constant',
'args': {
'retry': 10
}
},
'edges': [{
'from': '',
'to': 'CheckMessage'
}, {
'from': 'CheckMessage',
'to': 'MessageLength'
}, {
'from': 'MessageLength',
'to': 'SuccessAction',
'condition': {
'and': [{
'name': 'fieldEqual',
'args': {
'key': 'status',
'value': True
}
}, {
'name': 'fieldEqual',
'args': {
'key': 'checkvalue',
'value': True
}
}, {
'name': 'fieldEqual',
'args': {
'key': 'length',
'value': True
}
}]
}
}, {
'from': 'MessageLength',
'to': 'FailureAction',
'condition': {
'name': 'fieldEqual',
'args': {
'key': 'status',
'value': False
}
}
}]
}]
}]
node_dict1 = copy.deepcopy(node_dict)
flow_defi_revse1 = copy.deepcopy(flow_defi_revse)
Config.set_config_dict(node_dict, flow_defi_revse)
# print(Config.__dict__)
# from selinon_demo import celery_config
# Config.set_celery_app(celery_config.app)
# celery_config.app.tasks.register(Dispatcher())
# celery_config.app.autodiscover_tasks()
# import ipdb;ipdb.set_trace()
dispatcher_id = run_flow('my_new_flow', {}, node_dict1, flow_defi_revse1)
dispatcher_id1 = run_flow('my_new_flow', {}, node_dict1, flow_definition)
return JsonResponse({'status': 'success'})
from __future__ import absolute_import, unicode_literals
import os
from selinon import Config, run_flow
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'selinon_demo.settings')
app = Celery('selinon_demo')
app.config_from_object('django.conf:settings', namespace='CELERY')
# Config.set_pickel_config_object(MyDBConfig())
Config.set_celery_app(app)
app.autodiscover_tasks()
app.conf.task_routes = {'*': {'queue': 'hello_task'}}
# node_dict = {
# 'tasks': [{'name': 'CheckMessage', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'},
# {'name': 'MessageLength', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'},
# {'name': 'SuccessAction', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'},
# {'name': 'FailureAction', 'queue': 'hello_task', 'import': 'tasks', 'max_retry':0, 'storage': 'Redis'}],
# 'flows': ['my_new_flow'],
# 'storages': [{'name': 'Redis', 'import': 'selinon.storages.redis', 'configuration': {'host': 'localhost', 'port':6379, 'db':1, 'charset': 'utf-8'}}],
# 'global': {'trace': {'json': True}},
# 'migration_dir': 'migration_dir'}
# flow_definition = [{'flow-definitions': [{'name': 'my_new_flow',
