def test_convert_with_items_expr_matching_regex_str(self):
wi_str = 'b in <% [3, 4, 5] %>'
converter = workflows_base.WorkflowConverter()
actual = converter.convert_with_items_expr(wi_str, yql.YaqlExpressionConverter)
expected = 'b in <% [3, 4, 5] %>'
self.assertEqual(expected, actual)
wi_str = 'i in <% $.items %>'
converter = workflows_base.WorkflowConverter()
actual = converter.convert_with_items_expr(wi_str, yql.YaqlExpressionConverter)
expected = 'i in <% ctx().items %>'
self.assertEqual(expected, actual)
def test_convert_tasks_yaql(self):
converter = workflows_base.WorkflowConverter()
expr_converter = yql.YaqlExpressionConverter()
mistral_tasks = OrderedMap([
('jinja_task', OrderedMap([
('action', 'mypack.actionname'),
('input', OrderedMap([
('cmd', '<% $.test %>'),
])),
('on-success', ['next_task']),
]))
])
result = converter.convert_tasks(mistral_tasks, expr_converter, set())
self.assertEqual(result, OrderedMap([
('jinja_task', OrderedMap([
('action', 'mypack.actionname'),
('input', OrderedMap([
('cmd', '<% ctx().test %>'),
])),
('next', [
OrderedMap([
('when', '<% succeeded() %>'),
('do', [
'next_task',
])
]),
]),
]))
]))
def test_convert_task_transition_simple_yaql(self):
converter = workflows_base.WorkflowConverter()
transitions = ['a', 'b', 'c']
publish = OrderedMap([('key_plain', 'data'),
('key_expr', '{{ _.test }}')])
orquesta_expr = 'succeeded()'
expr_converter = yql.YaqlExpressionConverter()
result = converter.convert_task_transition_simple(transitions,
publish,
orquesta_expr,
expr_converter)
# note: only the orquesta_expr is converted using YAQL, the expressions
# in the rest are converted independently because they may each
# be a different type
expected = OrderedMap([
('when', '<% succeeded() %>'),
('publish', [
{'key_plain': 'data'},
{'key_expr': '{{ ctx().test }}'}
]),
('do', ['a', 'b', 'c']),
])
self.assertEqual(result, expected)
def test_convert_tasks_unsupported_attributes(self):
converter = workflows_base.WorkflowConverter()
expr_converter = yql.YaqlExpressionConverter()
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('keep-result', True))
converter.convert_tasks(mistral_tasks, expr_converter, set())
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('pause-before', True))
converter.convert_tasks(mistral_tasks, expr_converter, set())
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('safe-rerun', True))
converter.convert_tasks(mistral_tasks, expr_converter, set())
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('target', 'some-node'))
converter.convert_tasks(mistral_tasks, expr_converter, set())
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('timeout', 60))
converter.convert_tasks(mistral_tasks, expr_converter, set())
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('wait-after', 60))
converter.convert_tasks(mistral_tasks, expr_converter, set())
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('wait-before', 60))
converter.convert_tasks(mistral_tasks, expr_converter, set())
with self.assertRaises(NotImplementedError):
mistral_tasks = self._create_task(('workflow', 'someworkflowname'))
converter.convert_tasks(mistral_tasks, expr_converter, set())
def test_convert_task_transitions_common_publish_keys_different_values(self):
converter = workflows_base.WorkflowConverter()
task_spec = OrderedMap([
('publish', OrderedMap([
('good_data', '{{ _.good }}'),
('common_key_1', '{{ _.common_data }}'),
('common_key_2', '{{ _.different_data_1_1 }}'),
('common_key_3', '{{ _.different_data_1_2 }}'),
])),
('publish-on-error', OrderedMap([
('bad_data', '{{ _.bad }}'),
('common_key_1', '{{ _.common_data }}'),
('common_key_2', '{{ _.different_data_2_1 }}'),
('common_key_3', '{{ _.different_data_2_2 }}'),
])),
('on-complete', [
OrderedMap([('do_thing_sometimes', '{{ _.d }}')]),
'do_thing_always'
]),
])
expr_converter = jinja.JinjaExpressionConverter()
with self.assertRaises(NotImplementedError) as ctx_m:
converter.convert_task_transitions("tsk_name", task_spec, expr_converter, set())
rgx = re.compile(r"Task 'tsk_name' contains one or more keys "
r"\((?:common_key_2, common_key_3|common_key_3, common_key_2)\) "
r"in both publish and publish-on-error dictionaries that have different "
r"values\. Please either remove the common keys, or ensure that the "
r"values of any common keys are the same\.")
self.assertRegex(str(ctx_m.exception), rgx)
def test_convert_task_transitions_common_publish_keys_same_values(self):
converter = workflows_base.WorkflowConverter()
task_spec = OrderedMap([
('publish', OrderedMap([
('good_data', '{{ _.good }}'),
('common_data_1', '{{ _.common_1 }}'),
('common_data_2', '{{ _.common_2 }}'),
])),
('publish-on-error', OrderedMap([
('bad_data', '{{ _.bad }}'),
('common_data_1', '{{ _.common_1 }}'),
('common_data_2', '{{ _.common_2 }}'),
])),
('on-complete', [
'do_thing_always'
]),
])
expr_converter = jinja.JinjaExpressionConverter()
result = converter.convert_task_transitions("task_name", task_spec, expr_converter, set())
self.assertDictEqual(result, OrderedMap([
('next', [
OrderedMap([
('publish', [
{'good_data': '{{ ctx().good }}'},
{'common_data_1': '{{ ctx().common_1 }}'},
{'common_data_2': '{{ ctx().common_2 }}'},
{'bad_data': '{{ ctx().bad }}'},
]),
('do', [
'do_thing_always',
]),
]),
]),
]))
def test_convert_action_mistral_builtin_actions(self):
converter = workflows_base.WorkflowConverter()
self.assertEqual(converter.convert_action('std.fail'), 'fail')
self.assertEqual(converter.convert_action('std.http'), 'core.http')
self.assertEqual(converter.convert_action('std.mistral_http'), 'core.http')
self.assertEqual(converter.convert_action('std.noop'), 'core.noop')
def test_convert_action_others(self):
converter = workflows_base.WorkflowConverter()
# Test non-built-in actions
self.assertEqual(converter.convert_action('basil.exposition'), 'basil.exposition')
self.assertEqual(converter.convert_action('vanessakensington'), 'vanessakensington')
self.assertEqual(converter.convert_action('foxxy_cleopatra'), 'foxxy_cleopatra')
self.assertEqual(converter.convert_action('number 3'), 'number 3')
def test_dict_to_list(self):
converter = workflows_base.WorkflowConverter()
d = OrderedMap([('key1', 'value1'),
('key2', 'value2'),
('key3', 'value3')])
result = converter.dict_to_list(d)
self.assertEqual(result, [{'key1': 'value1'},
{'key2': 'value2'},
{'key3': 'value3'}])
def test_invalid_retry_continue_on_expression(self):
retry = {
'count': 30,
'delay': 5,
'continue-on': 'one fish, two fish, red fish, blue fish',
}
converter = workflows_base.WorkflowConverter()
with self.assertRaises(NotImplementedError):
converter.convert_retry(retry, 'retry_task_continue_on')
def test_invalid_retry_break_on_expression(self):
retry = {
'count': 30,
'delay': 5,
'break-on': 'and the cat in the hat knows a lot about that',
}
converter = workflows_base.WorkflowConverter()
with self.assertRaises(NotImplementedError):
converter.convert_retry(retry, 'retry_task_break_on')
def test_convert_with_items_expr_list_unrecognized_expression(self):
wi_list = [
'a in <% [0, 1, 2] %>',
'BLARGETH', # bad syntax
'c in <% $.all_the_things %>',
]
converter = workflows_base.WorkflowConverter()
with self.assertRaises(NotImplementedError):
converter.convert_with_items_expr(wi_list, yql.YaqlExpressionConverter)
def test_convert_with_items_expr_list(self):
wi_list = [
'a in <% [0, 1, 2] %>',
'b in [3, 4, 5]',
'c in <% $.all_the_things %>',
]
converter = workflows_base.WorkflowConverter()
actual = converter.convert_with_items_expr(wi_list, yql.YaqlExpressionConverter)
expected = "a, b, c in <% zip([0, 1, 2], [3, 4, 5], ctx().all_the_things) %>"
self.assertEqual(expected, actual)
def test_convert_with_items_expr_list_one_element(self):
# Check that with-items expression lists with a single element don't
# get put into a zip() expression
wi_list = [
'a in <% [0, 1, 2] %>',
]
converter = workflows_base.WorkflowConverter()
actual = converter.convert_with_items_expr(wi_list, yql.YaqlExpressionConverter)
expected = "a in <% [0, 1, 2] %>"
self.assertEqual(expected, actual)
def test_retry_continue_and_break_on_different_expression_types(self):
retry = {
'count': 30,
'delay': 5,
'continue-on': '<% $.foo = "continue" %>',
'break-on': '{{ _.foo = "break" }}',
}
converter = workflows_base.WorkflowConverter()
with self.assertRaises(NotImplementedError):
converter.convert_retry(retry, 'different_expression_types')
def test_convert_workflow_unsupported_attributes(self):
mistral_wf = OrderedMap([
('version', '1.0'),
('output-on-error', OrderedMap([
('stdout', "{{ _.stdout }}"),
('stderr', "<% $.stderr %>"),
])),
])
converter = workflows_base.WorkflowConverter()
with self.assertRaises(NotImplementedError):
converter.convert(mistral_wf)
def test_convert_with_items(self):
wi = {
'with-items': 'b in <% [3, 4, 5] %>',
}
converter = workflows_base.WorkflowConverter()
actual = converter.convert_with_items(wi, yql.YaqlExpressionConverter)
# This should NOT have a concurrency key
expected = {
'items': 'b in <% [3, 4, 5] %>',
}
self.assertEqual(expected, actual)
def test_simple_retry(self):
retry = {
'count': 30,
'delay': 5,
}
converter = workflows_base.WorkflowConverter()
actual = converter.convert_retry(retry, 'retry_task')
expected = OrderedMap([
('count', 30),
('delay', 5),
])
self.assertEqual(expected, actual)
def test_convert_with_items_concurrency_jinja(self):
wi = {
'with-items': 'b in <% [3, 4, 5] %>',
'concurrency': '{{ _.count }}',
}
converter = workflows_base.WorkflowConverter()
actual = converter.convert_with_items(wi, yql.YaqlExpressionConverter)
# This must have a concurrency key
expected = {
'items': 'b in <% [3, 4, 5] %>',
'concurrency': '{{ ctx().count }}',
}
self.assertEqual(expected, actual)
def test_convert_action_unsupported_mistral_builtin_actions(self):
converter = workflows_base.WorkflowConverter()
with self.assertRaises(NotImplementedError):
converter.convert_action('std.echo')
with self.assertRaises(NotImplementedError):
converter.convert_action('std.email')
with self.assertRaises(NotImplementedError):
converter.convert_action('std.javascript')
with self.assertRaises(NotImplementedError):
converter.convert_action('std.js')
with self.assertRaises(NotImplementedError):
converter.convert_action('std.ssh')
def test_convert_workflow_unsupported_types(self):
converter = workflows_base.WorkflowConverter()
with self.assertRaises(NotImplementedError):
converter.convert(OrderedMap([
('version', '1.0'),
('type', 'reverse'),
]))
with self.assertRaises(NotImplementedError):
converter.convert(OrderedMap([
('version', '1.0'),
('type', 'junk'),
]))
def test_retry_break_on(self):
retry = {
'count': 30,
'delay': 5,
'break-on': '<% $.foo = "break" %>',
}
converter = workflows_base.WorkflowConverter()
actual = converter.convert_retry(retry, 'retry_task_break_on')
expected = OrderedMap([
('count', 30),
('delay', 5),
('when', '<% failed() and not (ctx().foo = "break") %>'),
])
self.assertEqual(expected, actual)
def test_retry_continue_on(self):
retry = {
'count': 30,
'delay': 5,
'continue-on': '<% $.foo = "continue" %>',
}
converter = workflows_base.WorkflowConverter()
actual = converter.convert_retry(retry, 'retry_task_continue_on')
expected = OrderedMap([
('count', 30),
('delay', 5),
('when', '<% succeeded() and (ctx().foo = "continue") %>'),
])
self.assertEqual(expected, actual)
def test_group_task_transitions(self):
converter = workflows_base.WorkflowConverter()
transitions_list = [
"simple transition string",
{"key": "expr"},
"another simple transition string",
{"key2": "expression"},
{"key3": "expr"},
]
simple, expr = converter.group_task_transitions(transitions_list)
self.assertEqual(simple, ["simple transition string",
"another simple transition string"])
expected = OrderedMap([("expr", ["key", "key3"]),
("expression", ["key2"])])
self.assertEqual(expr, expected)
def test_convert_tasks_join(self):
converter = workflows_base.WorkflowConverter()
expr_converter = yql.YaqlExpressionConverter()
mistral_tasks = OrderedMap([
('jinja_task', OrderedMap([
('action', 'mypack.actionname'),
('join', 'all'),
]))
])
result = converter.convert_tasks(mistral_tasks, expr_converter, set())
self.assertEqual(result, OrderedMap([
('jinja_task', OrderedMap([
('action', 'mypack.actionname'),
('join', 'all'),
]))
]))
def test_default_task_transition_map(self):
converter = workflows_base.WorkflowConverter()
result = converter.default_task_transition_map()
self.assertEqual(result, OrderedMap([
('on-success', OrderedMap([
('publish', OrderedMap()),
('orquesta_expr', 'succeeded()'),
])),
('on-error', OrderedMap([
('publish', OrderedMap()),
('orquesta_expr', 'failed()'),
])),
('on-complete', OrderedMap([
('publish', OrderedMap()),
('orquesta_expr', None),
])),
]))
def test_convert_task_transition_simple_no_publish(self):
converter = workflows_base.WorkflowConverter()
transitions = ['a', 'b', 'c']
publish = None
orquesta_expr = 'succeeded()'
expr_converter = jinja.JinjaExpressionConverter()
result = converter.convert_task_transition_simple(transitions,
publish,
orquesta_expr,
expr_converter)
expected = OrderedMap([
('when', '{{ succeeded() }}'),
('do', ['a', 'b', 'c']),
])
self.assertEqual(result, expected)
def test_convert_task_transition_simple_no_transitions(self):
converter = workflows_base.WorkflowConverter()
transitions = None
publish = OrderedMap([('key_plain', 'data'),
('key_expr', '{{ _.test }}')])
orquesta_expr = 'succeeded()'
expr_converter = jinja.JinjaExpressionConverter()
result = converter.convert_task_transition_simple(transitions,
publish,
orquesta_expr,
expr_converter)
expected = OrderedMap([
('when', '{{ succeeded() }}'),
('publish', [
{'key_plain': 'data'},
{'key_expr': '{{ ctx().test }}'}
]),
])
self.assertEqual(result, expected)
def test_convert_task_transition_expr_no_orquesta_expr(self):
converter = workflows_base.WorkflowConverter()
expression_list = OrderedMap([('<% $.test %>', ['task1', 'task3']),
('{{ _.other }}', ['task2'])])
orquesta_expr = None
result = converter.convert_task_transition_expr('task_name',
expression_list,
{},
orquesta_expr)
expected = [
OrderedMap([
('when', '<% ctx().test %>'),
('do', ['task1', 'task3']),
]),
OrderedMap([
('when', '{{ ctx().other }}'),
('do', ['task2']),
]),
]
self.assertEqual(result, expected)
def convert_file(self, filename, expr_type=None):
# parse the Mistral workflow from file
mistral_wf_data, mistral_wf_data_ruamel = yaml_utils.read_yaml(
filename)
# validate the Mistral workflow before we start
mistral_wf_spec = mistral_workflow.instantiate(mistral_wf_data)
self.validate_workflow_spec(mistral_wf_spec)
# convert Mistral -> Orquesta
mistral_wf = mistral_wf_data_ruamel[mistral_wf_spec.name]
workflow_converter = workflows_base.WorkflowConverter()
orquesta_wf_data_ruamel = workflow_converter.convert(
mistral_wf, expr_type, force=self.args.force)
orquesta_wf_data_str = yaml_utils.obj_to_yaml(orquesta_wf_data_ruamel)
orquesta_wf_data = yaml_utils.yaml_to_obj(orquesta_wf_data_str)
# validate we've generated a proper Orquesta workflow
orquesta_wf_spec = native_v1_models.instantiate(orquesta_wf_data)
if not self.args.force:
self.validate_workflow_spec(orquesta_wf_spec)
# write out the new Orquesta workflow to a YAML string
return yaml_utils.obj_to_yaml(orquesta_wf_data_ruamel)
