def test_new_task_unknown(self):
"""
Trying to create a new task with an unknown name must raise a
`UnknownTaskName` exception.
"""
task_tmpl = TaskTemplate('dummy')
with self.assertRaises(UnknownTaskName):
task_tmpl.new_task('junk-data')
def test_new_task_unknown(self):
"""
Trying to create a new task with an unknown name must raise a
`UnknownTaskName` exception.
"""
task_tmpl = TaskTemplate('dummy')
with self.assertRaises(UnknownTaskName):
task_tmpl.new_task('junk-data')
def test_build_from_dict_without_name(self):
"""
The only required argument to create a task template is the task
name.
"""
task_dict = {
"id": "1234",
}
with self.assertRaises(KeyError):
TaskTemplate.from_dict(task_dict)
def test_build_from_dict_without_name(self):
"""
The only required argument to create a task template is the task
name.
"""
task_dict = {
"id": "1234",
}
with self.assertRaises(KeyError):
TaskTemplate.from_dict(task_dict)
def test_build_from_dict(self):
"""
Create a new task template from a dictionary
"""
task_dict = {
"id": "1234",
"name": "dummy"
}
task_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(task_tmpl, TaskTemplate)
self.assertEqual(task_tmpl.uid, '1234')
self.assertEqual(task_tmpl.name, 'dummy')
self.assertEqual(task_tmpl.topics, [])
self.assertEqual(task_tmpl.config, {})
# Can also create a task template without ID defined in the dict
task_dict = {
"name": "dummy",
"topics": ["yummy"],
"config": {"hello": "world"},
"foo": None,
"bar": [1, 2, 3, 4]
}
task_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(task_tmpl, TaskTemplate)
# Even if no ID is provided, it must be generated
self.assertIsInstance(task_tmpl.uid, str)
self.assertEqual(len(task_tmpl.uid), 36)
self.assertEqual(task_tmpl.name, 'dummy')
self.assertEqual(task_tmpl.topics, ["yummy"])
self.assertEqual(task_tmpl.config, {"hello": "world"})
# Additional keys in the dict don't harm
task_dict = {
"name": "dummy",
"foo": None,
"bar": [1, 2, 3, 4]
}
task_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(task_tmpl, TaskTemplate)
# Even if no ID is provided, it must be generated
self.assertIsInstance(task_tmpl.uid, str)
self.assertEqual(len(task_tmpl.uid), 36)
self.assertEqual(task_tmpl.name, 'dummy')
self.assertEqual(task_tmpl.topics, [])
self.assertEqual(task_tmpl.config, {})
def test_build_from_dict(self):
"""
Create a new task template from a dictionary
"""
task_dict = {
"id": "1234",
"name": "dummy"
}
task_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(task_tmpl, TaskTemplate)
self.assertEqual(task_tmpl.uid, '1234')
self.assertEqual(task_tmpl.name, 'dummy')
self.assertEqual(task_tmpl.topics, [])
self.assertEqual(task_tmpl.config, {})
# Can also create a task template without ID defined in the dict
task_dict = {
"name": "dummy",
"topics": ["yummy"],
"config": {"hello": "world"},
"foo": None,
"bar": [1, 2, 3, 4]
}
task_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(task_tmpl, TaskTemplate)
# Even if no ID is provided, it must be generated
self.assertIsInstance(task_tmpl.uid, str)
self.assertEqual(len(task_tmpl.uid), 36)
self.assertEqual(task_tmpl.name, 'dummy')
self.assertEqual(task_tmpl.topics, ["yummy"])
self.assertEqual(task_tmpl.config, {"hello": "world"})
# Additional keys in the dict don't harm
task_dict = {
"name": "dummy",
"foo": None,
"bar": [1, 2, 3, 4]
}
task_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(task_tmpl, TaskTemplate)
# Even if no ID is provided, it must be generated
self.assertIsInstance(task_tmpl.uid, str)
self.assertEqual(len(task_tmpl.uid), 36)
self.assertEqual(task_tmpl.name, 'dummy')
self.assertEqual(task_tmpl.topics, [])
self.assertEqual(task_tmpl.config, {})
def test_new_task(self):
"""
Can create new asyncio tasks from the task template
"""
self.loop.set_task_factory(tukio_factory)
# Create a task from a registered task holder
task_tmpl = TaskTemplate('my-task-holder')
task = task_tmpl.new_task('dummy-data', loop=self.loop)
self.assertIsInstance(task, TukioTask)
self.assertEqual(task.inputs, 'dummy-data')
# Also works with a registered coroutine
task_tmpl = TaskTemplate('dummy-coro')
task = task_tmpl.new_task(data='junk-data', loop=self.loop)
self.assertIsInstance(task, TukioTask)
self.assertEqual(task.inputs, 'junk-data')
def from_dict(cls, wf_dict):
"""
Build a new workflow description from the given dictionary.
The dictionary takes the form of:
{
"id": <workflow-uid>,
"topics": [<a-topic>, <another-topic>],
"policy": <policy>,
"tasks": [
{"id": <task-uid>, "name": <name>, "config": <cfg-dict>},
...
],
"graph": {
<t1-uid>: [t2-uid, <t3-uid>],
<t2-uid>: [],
...
}
}
See below the conditions applied to trigger a workflow according to the
value of 'topics':
{"topics": None}
try to trigger a workflow each time data is received by the engine
** default behavior **
{"topics": []}
never try to trigger a workflow when data is received by the engine
{"topics": ["blob", "foo"]}
try to trigger a workflow when data is received by the engine in
topics "blob" and "foo" only
"""
wf_tmpl = cls(uid=wf_dict.get('id'),
policy=wf_dict.get('policy'),
topics=wf_dict.get('topics'))
# Tasks
task_ids = dict()
for task_dict in wf_dict.get('tasks', []):
task_tmpl = TaskTemplate.from_dict(task_dict)
wf_tmpl.add(task_tmpl)
task_ids[task_tmpl.uid] = task_tmpl
# Graph
try:
for up_id, down_ids_set in wf_dict.get('graph', {}).items():
up_tmpl = task_ids[up_id]
for down_id in down_ids_set:
down_tmpl = task_ids[down_id]
wf_tmpl.link(up_tmpl, down_tmpl)
except KeyError as exc:
raise TemplateGraphError(exc.args[0]) from exc
return wf_tmpl
def test_dump_as_dict(self):
"""
A TaskTemplate instance can be dumped as a dictionary
"""
task_tmpl = TaskTemplate('my-task-holder', config={'hello': 'world'})
task_tmpl.topics = ['my-topic']
expected_dict = {
"name": "my-task-holder",
"topics": ["my-topic"],
"config": {'hello': 'world'}
}
task_dict = task_tmpl.as_dict()
del task_dict['id']
self.assertEqual(task_dict, expected_dict)
# The dumped dict must be loadable by the `from_dict()` classmethod
other_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(other_tmpl, TaskTemplate)
def test_dump_as_dict(self):
"""
A TaskTemplate instance can be dumped as a dictionary
"""
task_tmpl = TaskTemplate('my-task-holder', config={'hello': 'world'})
task_tmpl.topics = ['my-topic']
expected_dict = {
'name': 'my-task-holder',
'timeout': None,
'topics': ['my-topic'],
'config': {'hello': 'world'}
}
task_dict = task_tmpl.as_dict()
del task_dict['id']
self.assertEqual(task_dict, expected_dict)
# The dumped dict must be loadable by the `from_dict()` classmethod
other_tmpl = TaskTemplate.from_dict(task_dict)
self.assertIsInstance(other_tmpl, TaskTemplate)
def test_new_task_bad_args(self):
"""
Trying to create a new task with invalid arguments must raise a
`TypeError` exception.
"""
# Missing argument
task_tmpl = TaskTemplate('my-task-holder')
with self.assertRaisesRegex(TypeError, 'positional argument'):
task_tmpl.new_task()
# Too many arguments
task_tmpl = TaskTemplate('bad-coro-task')
with self.assertRaisesRegex(TypeError, 'positional argument'):
task_tmpl.new_task('junk')
def test_new_template(self):
"""
Valid new task template operations
"""
# The simplest case: only a name is required
# Note: task name is not checked!
name = 'dummy'
task_tmpl = TaskTemplate(name)
self.assertEqual(task_tmpl.name, name)
self.assertEqual(task_tmpl.config, {})
self.assertEqual(task_tmpl.topics, [])
# Even if no ID is provided, it must be generated
self.assertIsInstance(task_tmpl.uid, str)
self.assertEqual(len(task_tmpl.uid), 36)
def test_new_task_bad_args(self):
"""
Trying to create a new task with invalid arguments must raise a
`TypeError` exception.
"""
# Missing argument
task_tmpl = TaskTemplate('my-task-holder')
with self.assertRaisesRegex(TypeError, 'positional argument'):
task_tmpl.new_task()
# Too many arguments
task_tmpl = TaskTemplate('bad-coro-task')
with self.assertRaisesRegex(TypeError, 'positional argument'):
task_tmpl.new_task('junk')
def test_new_task(self):
"""
Can create new asyncio tasks from the task template
"""
self.loop.set_task_factory(tukio_factory)
# Create a task from a registered task holder
task_tmpl = TaskTemplate('my-task-holder')
task = task_tmpl.new_task('dummy-data', loop=self.loop)
self.assertIsInstance(task, TukioTask)
self.assertEqual(task.inputs, 'dummy-data')
# Also works with a registered coroutine
task_tmpl = TaskTemplate('dummy-coro')
task = task_tmpl.new_task(data='junk-data', loop=self.loop)
self.assertIsInstance(task, TukioTask)
self.assertEqual(task.inputs, 'junk-data')
def from_dict(cls, wf_dict):
"""
Build a new workflow description from the given dictionary.
The dictionary takes the form of:
{
"id": <workflow-uid>,
"topics": [<a-topic>, <another-topic>],
"policy": <policy>,
"timeout": <timeout>,
"schema": <int>,
"tasks": [
{"id": <task-uid>, "name": <name>, "config": <cfg-dict>},
...
],
"graph": {
<t1-uid>: [t2-uid, <t3-uid>],
<t2-uid>: [],
...
}
}
See below the conditions applied to trigger a workflow according to the
value of 'topics':
{"topics": None}
try to trigger a workflow each time data is received by the engine
** default behavior **
{"topics": []}
never try to trigger a workflow when data is received by the engine
{"topics": ["blob", "foo"]}
try to trigger a workflow when data is received by the engine in
topics "blob" and "foo" only
"""
wf_tmpl = cls(
uid=wf_dict.get('id'),
policy=wf_dict.get('policy'),
topics=wf_dict.get('topics'),
timeout=wf_dict.get('timeout'),
schema=wf_dict.get('schema'),
)
# Tasks
task_ids = dict()
for task_dict in wf_dict.get('tasks', []):
task_tmpl = TaskTemplate.from_dict(task_dict)
wf_tmpl.add(task_tmpl)
task_ids[task_tmpl.uid] = task_tmpl
# Graph
for up_id, down_ids_set in wf_dict.get('graph', {}).items():
up_tmpl = task_ids[up_id]
for down_id in down_ids_set:
down_tmpl = task_ids[down_id]
wf_tmpl.link(up_tmpl, down_tmpl)
# Graph validation
try:
wf_tmpl.dag.validate()
except KeyError as exc:
raise TemplateGraphError(exc.args[0]) from exc
return wf_tmpl