def test_convert_with_items_expr_matching_regex_str(self): wi_str = 'b in <% [3, 4, 5] %>' converter = workflows_base.WorkflowConverter() actual = converter.convert_with_items_expr(wi_str, yql.YaqlExpressionConverter) expected = 'b in <% [3, 4, 5] %>' self.assertEqual(expected, actual) wi_str = 'i in <% $.items %>' converter = workflows_base.WorkflowConverter() actual = converter.convert_with_items_expr(wi_str, yql.YaqlExpressionConverter) expected = 'i in <% ctx().items %>' self.assertEqual(expected, actual)
def test_convert_tasks_yaql(self): converter = workflows_base.WorkflowConverter() expr_converter = yql.YaqlExpressionConverter() mistral_tasks = OrderedMap([ ('jinja_task', OrderedMap([ ('action', 'mypack.actionname'), ('input', OrderedMap([ ('cmd', '<% $.test %>'), ])), ('on-success', ['next_task']), ])) ]) result = converter.convert_tasks(mistral_tasks, expr_converter, set()) self.assertEqual(result, OrderedMap([ ('jinja_task', OrderedMap([ ('action', 'mypack.actionname'), ('input', OrderedMap([ ('cmd', '<% ctx().test %>'), ])), ('next', [ OrderedMap([ ('when', '<% succeeded() %>'), ('do', [ 'next_task', ]) ]), ]), ])) ]))
def test_convert_task_transition_simple_yaql(self): converter = workflows_base.WorkflowConverter() transitions = ['a', 'b', 'c'] publish = OrderedMap([('key_plain', 'data'), ('key_expr', '{{ _.test }}')]) orquesta_expr = 'succeeded()' expr_converter = yql.YaqlExpressionConverter() result = converter.convert_task_transition_simple(transitions, publish, orquesta_expr, expr_converter) # note: only the orquesta_expr is converted using YAQL, the expressions # in the rest are converted independently because they may each # be a different type expected = OrderedMap([ ('when', '<% succeeded() %>'), ('publish', [ {'key_plain': 'data'}, {'key_expr': '{{ ctx().test }}'} ]), ('do', ['a', 'b', 'c']), ]) self.assertEqual(result, expected)
def test_convert_tasks_unsupported_attributes(self): converter = workflows_base.WorkflowConverter() expr_converter = yql.YaqlExpressionConverter() with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('keep-result', True)) converter.convert_tasks(mistral_tasks, expr_converter, set()) with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('pause-before', True)) converter.convert_tasks(mistral_tasks, expr_converter, set()) with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('safe-rerun', True)) converter.convert_tasks(mistral_tasks, expr_converter, set()) with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('target', 'some-node')) converter.convert_tasks(mistral_tasks, expr_converter, set()) with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('timeout', 60)) converter.convert_tasks(mistral_tasks, expr_converter, set()) with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('wait-after', 60)) converter.convert_tasks(mistral_tasks, expr_converter, set()) with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('wait-before', 60)) converter.convert_tasks(mistral_tasks, expr_converter, set()) with self.assertRaises(NotImplementedError): mistral_tasks = self._create_task(('workflow', 'someworkflowname')) converter.convert_tasks(mistral_tasks, expr_converter, set())
def test_convert_task_transitions_common_publish_keys_different_values(self): converter = workflows_base.WorkflowConverter() task_spec = OrderedMap([ ('publish', OrderedMap([ ('good_data', '{{ _.good }}'), ('common_key_1', '{{ _.common_data }}'), ('common_key_2', '{{ _.different_data_1_1 }}'), ('common_key_3', '{{ _.different_data_1_2 }}'), ])), ('publish-on-error', OrderedMap([ ('bad_data', '{{ _.bad }}'), ('common_key_1', '{{ _.common_data }}'), ('common_key_2', '{{ _.different_data_2_1 }}'), ('common_key_3', '{{ _.different_data_2_2 }}'), ])), ('on-complete', [ OrderedMap([('do_thing_sometimes', '{{ _.d }}')]), 'do_thing_always' ]), ]) expr_converter = jinja.JinjaExpressionConverter() with self.assertRaises(NotImplementedError) as ctx_m: converter.convert_task_transitions("tsk_name", task_spec, expr_converter, set()) rgx = re.compile(r"Task 'tsk_name' contains one or more keys " r"\((?:common_key_2, common_key_3|common_key_3, common_key_2)\) " r"in both publish and publish-on-error dictionaries that have different " r"values\. Please either remove the common keys, or ensure that the " r"values of any common keys are the same\.") self.assertRegex(str(ctx_m.exception), rgx)
def test_convert_task_transitions_common_publish_keys_same_values(self): converter = workflows_base.WorkflowConverter() task_spec = OrderedMap([ ('publish', OrderedMap([ ('good_data', '{{ _.good }}'), ('common_data_1', '{{ _.common_1 }}'), ('common_data_2', '{{ _.common_2 }}'), ])), ('publish-on-error', OrderedMap([ ('bad_data', '{{ _.bad }}'), ('common_data_1', '{{ _.common_1 }}'), ('common_data_2', '{{ _.common_2 }}'), ])), ('on-complete', [ 'do_thing_always' ]), ]) expr_converter = jinja.JinjaExpressionConverter() result = converter.convert_task_transitions("task_name", task_spec, expr_converter, set()) self.assertDictEqual(result, OrderedMap([ ('next', [ OrderedMap([ ('publish', [ {'good_data': '{{ ctx().good }}'}, {'common_data_1': '{{ ctx().common_1 }}'}, {'common_data_2': '{{ ctx().common_2 }}'}, {'bad_data': '{{ ctx().bad }}'}, ]), ('do', [ 'do_thing_always', ]), ]), ]), ]))
def test_convert_action_mistral_builtin_actions(self): converter = workflows_base.WorkflowConverter() self.assertEqual(converter.convert_action('std.fail'), 'fail') self.assertEqual(converter.convert_action('std.http'), 'core.http') self.assertEqual(converter.convert_action('std.mistral_http'), 'core.http') self.assertEqual(converter.convert_action('std.noop'), 'core.noop')
def test_convert_action_others(self): converter = workflows_base.WorkflowConverter() # Test non-built-in actions self.assertEqual(converter.convert_action('basil.exposition'), 'basil.exposition') self.assertEqual(converter.convert_action('vanessakensington'), 'vanessakensington') self.assertEqual(converter.convert_action('foxxy_cleopatra'), 'foxxy_cleopatra') self.assertEqual(converter.convert_action('number 3'), 'number 3')
def test_dict_to_list(self): converter = workflows_base.WorkflowConverter() d = OrderedMap([('key1', 'value1'), ('key2', 'value2'), ('key3', 'value3')]) result = converter.dict_to_list(d) self.assertEqual(result, [{'key1': 'value1'}, {'key2': 'value2'}, {'key3': 'value3'}])
def test_invalid_retry_continue_on_expression(self): retry = { 'count': 30, 'delay': 5, 'continue-on': 'one fish, two fish, red fish, blue fish', } converter = workflows_base.WorkflowConverter() with self.assertRaises(NotImplementedError): converter.convert_retry(retry, 'retry_task_continue_on')
def test_invalid_retry_break_on_expression(self): retry = { 'count': 30, 'delay': 5, 'break-on': 'and the cat in the hat knows a lot about that', } converter = workflows_base.WorkflowConverter() with self.assertRaises(NotImplementedError): converter.convert_retry(retry, 'retry_task_break_on')
def test_convert_with_items_expr_list_unrecognized_expression(self): wi_list = [ 'a in <% [0, 1, 2] %>', 'BLARGETH', # bad syntax 'c in <% $.all_the_things %>', ] converter = workflows_base.WorkflowConverter() with self.assertRaises(NotImplementedError): converter.convert_with_items_expr(wi_list, yql.YaqlExpressionConverter)
def test_convert_with_items_expr_list(self): wi_list = [ 'a in <% [0, 1, 2] %>', 'b in [3, 4, 5]', 'c in <% $.all_the_things %>', ] converter = workflows_base.WorkflowConverter() actual = converter.convert_with_items_expr(wi_list, yql.YaqlExpressionConverter) expected = "a, b, c in <% zip([0, 1, 2], [3, 4, 5], ctx().all_the_things) %>" self.assertEqual(expected, actual)
def test_convert_with_items_expr_list_one_element(self): # Check that with-items expression lists with a single element don't # get put into a zip() expression wi_list = [ 'a in <% [0, 1, 2] %>', ] converter = workflows_base.WorkflowConverter() actual = converter.convert_with_items_expr(wi_list, yql.YaqlExpressionConverter) expected = "a in <% [0, 1, 2] %>" self.assertEqual(expected, actual)
def test_retry_continue_and_break_on_different_expression_types(self): retry = { 'count': 30, 'delay': 5, 'continue-on': '<% $.foo = "continue" %>', 'break-on': '{{ _.foo = "break" }}', } converter = workflows_base.WorkflowConverter() with self.assertRaises(NotImplementedError): converter.convert_retry(retry, 'different_expression_types')
def test_convert_workflow_unsupported_attributes(self): mistral_wf = OrderedMap([ ('version', '1.0'), ('output-on-error', OrderedMap([ ('stdout', "{{ _.stdout }}"), ('stderr', "<% $.stderr %>"), ])), ]) converter = workflows_base.WorkflowConverter() with self.assertRaises(NotImplementedError): converter.convert(mistral_wf)
def test_convert_with_items(self): wi = { 'with-items': 'b in <% [3, 4, 5] %>', } converter = workflows_base.WorkflowConverter() actual = converter.convert_with_items(wi, yql.YaqlExpressionConverter) # This should NOT have a concurrency key expected = { 'items': 'b in <% [3, 4, 5] %>', } self.assertEqual(expected, actual)
def test_simple_retry(self): retry = { 'count': 30, 'delay': 5, } converter = workflows_base.WorkflowConverter() actual = converter.convert_retry(retry, 'retry_task') expected = OrderedMap([ ('count', 30), ('delay', 5), ]) self.assertEqual(expected, actual)
def test_convert_with_items_concurrency_jinja(self): wi = { 'with-items': 'b in <% [3, 4, 5] %>', 'concurrency': '{{ _.count }}', } converter = workflows_base.WorkflowConverter() actual = converter.convert_with_items(wi, yql.YaqlExpressionConverter) # This must have a concurrency key expected = { 'items': 'b in <% [3, 4, 5] %>', 'concurrency': '{{ ctx().count }}', } self.assertEqual(expected, actual)
def test_convert_action_unsupported_mistral_builtin_actions(self): converter = workflows_base.WorkflowConverter() with self.assertRaises(NotImplementedError): converter.convert_action('std.echo') with self.assertRaises(NotImplementedError): converter.convert_action('std.email') with self.assertRaises(NotImplementedError): converter.convert_action('std.javascript') with self.assertRaises(NotImplementedError): converter.convert_action('std.js') with self.assertRaises(NotImplementedError): converter.convert_action('std.ssh')
def test_convert_workflow_unsupported_types(self): converter = workflows_base.WorkflowConverter() with self.assertRaises(NotImplementedError): converter.convert(OrderedMap([ ('version', '1.0'), ('type', 'reverse'), ])) with self.assertRaises(NotImplementedError): converter.convert(OrderedMap([ ('version', '1.0'), ('type', 'junk'), ]))
def test_retry_break_on(self): retry = { 'count': 30, 'delay': 5, 'break-on': '<% $.foo = "break" %>', } converter = workflows_base.WorkflowConverter() actual = converter.convert_retry(retry, 'retry_task_break_on') expected = OrderedMap([ ('count', 30), ('delay', 5), ('when', '<% failed() and not (ctx().foo = "break") %>'), ]) self.assertEqual(expected, actual)
def test_retry_continue_on(self): retry = { 'count': 30, 'delay': 5, 'continue-on': '<% $.foo = "continue" %>', } converter = workflows_base.WorkflowConverter() actual = converter.convert_retry(retry, 'retry_task_continue_on') expected = OrderedMap([ ('count', 30), ('delay', 5), ('when', '<% succeeded() and (ctx().foo = "continue") %>'), ]) self.assertEqual(expected, actual)
def test_group_task_transitions(self): converter = workflows_base.WorkflowConverter() transitions_list = [ "simple transition string", {"key": "expr"}, "another simple transition string", {"key2": "expression"}, {"key3": "expr"}, ] simple, expr = converter.group_task_transitions(transitions_list) self.assertEqual(simple, ["simple transition string", "another simple transition string"]) expected = OrderedMap([("expr", ["key", "key3"]), ("expression", ["key2"])]) self.assertEqual(expr, expected)
def test_convert_tasks_join(self): converter = workflows_base.WorkflowConverter() expr_converter = yql.YaqlExpressionConverter() mistral_tasks = OrderedMap([ ('jinja_task', OrderedMap([ ('action', 'mypack.actionname'), ('join', 'all'), ])) ]) result = converter.convert_tasks(mistral_tasks, expr_converter, set()) self.assertEqual(result, OrderedMap([ ('jinja_task', OrderedMap([ ('action', 'mypack.actionname'), ('join', 'all'), ])) ]))
def test_default_task_transition_map(self): converter = workflows_base.WorkflowConverter() result = converter.default_task_transition_map() self.assertEqual(result, OrderedMap([ ('on-success', OrderedMap([ ('publish', OrderedMap()), ('orquesta_expr', 'succeeded()'), ])), ('on-error', OrderedMap([ ('publish', OrderedMap()), ('orquesta_expr', 'failed()'), ])), ('on-complete', OrderedMap([ ('publish', OrderedMap()), ('orquesta_expr', None), ])), ]))
def test_convert_task_transition_simple_no_publish(self): converter = workflows_base.WorkflowConverter() transitions = ['a', 'b', 'c'] publish = None orquesta_expr = 'succeeded()' expr_converter = jinja.JinjaExpressionConverter() result = converter.convert_task_transition_simple(transitions, publish, orquesta_expr, expr_converter) expected = OrderedMap([ ('when', '{{ succeeded() }}'), ('do', ['a', 'b', 'c']), ]) self.assertEqual(result, expected)
def test_convert_task_transition_simple_no_transitions(self): converter = workflows_base.WorkflowConverter() transitions = None publish = OrderedMap([('key_plain', 'data'), ('key_expr', '{{ _.test }}')]) orquesta_expr = 'succeeded()' expr_converter = jinja.JinjaExpressionConverter() result = converter.convert_task_transition_simple(transitions, publish, orquesta_expr, expr_converter) expected = OrderedMap([ ('when', '{{ succeeded() }}'), ('publish', [ {'key_plain': 'data'}, {'key_expr': '{{ ctx().test }}'} ]), ]) self.assertEqual(result, expected)
def test_convert_task_transition_expr_no_orquesta_expr(self): converter = workflows_base.WorkflowConverter() expression_list = OrderedMap([('<% $.test %>', ['task1', 'task3']), ('{{ _.other }}', ['task2'])]) orquesta_expr = None result = converter.convert_task_transition_expr('task_name', expression_list, {}, orquesta_expr) expected = [ OrderedMap([ ('when', '<% ctx().test %>'), ('do', ['task1', 'task3']), ]), OrderedMap([ ('when', '{{ ctx().other }}'), ('do', ['task2']), ]), ] self.assertEqual(result, expected)
def convert_file(self, filename, expr_type=None): # parse the Mistral workflow from file mistral_wf_data, mistral_wf_data_ruamel = yaml_utils.read_yaml( filename) # validate the Mistral workflow before we start mistral_wf_spec = mistral_workflow.instantiate(mistral_wf_data) self.validate_workflow_spec(mistral_wf_spec) # convert Mistral -> Orquesta mistral_wf = mistral_wf_data_ruamel[mistral_wf_spec.name] workflow_converter = workflows_base.WorkflowConverter() orquesta_wf_data_ruamel = workflow_converter.convert( mistral_wf, expr_type, force=self.args.force) orquesta_wf_data_str = yaml_utils.obj_to_yaml(orquesta_wf_data_ruamel) orquesta_wf_data = yaml_utils.yaml_to_obj(orquesta_wf_data_str) # validate we've generated a proper Orquesta workflow orquesta_wf_spec = native_v1_models.instantiate(orquesta_wf_data) if not self.args.force: self.validate_workflow_spec(orquesta_wf_spec) # write out the new Orquesta workflow to a YAML string return yaml_utils.obj_to_yaml(orquesta_wf_data_ruamel)