def test_liminal_config_snapshot(self, find_config_files_mock, get_airflow_dir_mock, path_exists_mock): subliminal = { "name": "my_subliminal_test", "type": "sub", "variables": { "var": 1, "var-2": True }, "pipelines": [ {"name": "mypipe1", "param": "{{var}}"}, {"name": "mypipe2", "param": "{{var-2 }}"} ] } expected = {'name': 'my_subliminal_test', 'type': 'sub', 'service_defaults': {'description': 'add defaults parameters for all services'}, 'task_defaults': { 'description': 'add defaults parameters for all tasks separate by task type'}, 'pipeline_defaults': { 'description': 'add defaults parameters for all pipelines', 'before_tasks': [{'task': 'start', 'type': 'job_start'}], 'after_tasks': [{'task': 'end', 'type': 'job_end'}]}, 'variables': {'var': 1, 'var-2': True}, 'pipelines': [ {'name': 'mypipe1', 'param': '1', 'description': 'add defaults parameters for all pipelines', 'tasks': [{'task': 'start', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}, {'name': 'mypipe2', 'param': 'True', 'description': 'add defaults parameters for all pipelines', 'tasks': [{'task': 'start', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}], 'services': []} find_config_files_mock.return_value = { "my_subliminal_test": subliminal } get_airflow_dir_mock.return_value = "/tmp" path_exists_mock.return_value = True with mock.patch("builtins.open", mock.mock_open()) as m: with mock.patch("yaml.dump") as ydm: config_util = ConfigUtil("") config_util.safe_load(is_render_variables=True) config_util.snapshot_final_liminal_configs() m.assert_called_once_with( os.path.join('/tmp', '../liminal_config_files/my_subliminal_test.yml'), 'w') ydm.assert_called_once_with(expected, m.return_value, default_flow_style=False)
def register_dags(configs_path): """ Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs. """ logging.info(f'Registering DAGs from path: {configs_path}') config_util = ConfigUtil(configs_path) # TODO - change is_render_variable to False when runtime resolving is available configs = config_util.safe_load(is_render_variables=True) if os.getenv('POD_NAMESPACE') != "jenkins": config_util.snapshot_final_liminal_configs() dags = [] logging.info(f'found {len(configs)} liminal configs in path: {configs_path}') for config in configs: name = config['name'] if 'name' in config else None try: if not name: raise ValueError('liminal.yml missing field `name`') logging.info(f"Registering DAGs for {name}") owner = config.get('owner') trigger_rule = 'all_success' if 'always_run' in config and config['always_run']: trigger_rule = 'all_done' executors = __initialize_executors(config) for pipeline in config['pipelines']: default_args = __default_args(pipeline) dag = __initialize_dag(default_args, pipeline, owner) parent = None for task in pipeline['tasks']: task_type = task['type'] task_instance = get_task_class(task_type)( task_id=task['task'], dag=dag, parent=parent, trigger_rule=trigger_rule, liminal_config=config, pipeline_config=pipeline, task_config=task, executor=executors.get(task.get('executor')) ) parent = task_instance.apply_task_to_dag() logging.info(f'registered DAG {dag.dag_id}: {dag.tasks}') dags.append((pipeline['pipeline'], dag)) except Exception: logging.error(f'Failed to register DAGs for {name}') traceback.print_exc() return dags
def build_liminal_apps(path): """ Build images for liminal apps in path. """ config_util = ConfigUtil(path) configs = config_util.safe_load(is_render_variables=True, soft_merge=True) for liminal_config in configs: base_path = os.path.dirname( files_util.resolve_pipeline_source_file(liminal_config['name'])) if 'pipelines' in liminal_config: for pipeline in liminal_config['pipelines']: for task in pipeline['tasks']: task_name = task['task'] if 'source' in task: task_type = task['type'] builder_class = __get_task_build_class(task_type) if builder_class: __build_image(base_path, task, builder_class) else: raise ValueError(f'No such task type: {task_type}') else: logging.info( f'No source configured for task {task_name}, skipping build..' ) if 'services' in liminal_config: for service in liminal_config['services']: service_type = service['type'] builder_class = __get_service_build_class(service_type) if builder_class: __build_image(base_path, service, builder_class) else: raise ValueError(f'No such service type: {service_type}')
def test_soft_merge_load(self, find_config_files_mock): subliminal = { "name": "my_name", "type": "sub", "super": "my_super" } find_config_files_mock.return_value = {"my_subliminal_test": subliminal} config_util = ConfigUtil("") self.assertEqual([subliminal], config_util.safe_load(is_render_variables=True, soft_merge=True))
def test_soft_merge_load(self, find_config_files_mock): subliminal = { 'name': 'my_name', 'type': 'sub', 'super': 'my_super' } find_config_files_mock.return_value = {'my_subliminal_test': subliminal} config_util = ConfigUtil('') self.assertEqual([subliminal], config_util.safe_load(is_render_variables=True, soft_merge=True))
def test_safe_load(self, find_config_files_mock): subliminal = { "name": "my_subliminal_test", "type": "sub", "super": "my_superliminal_test", "pipelines": [ {"name": "mypipe1", "param": "constant"}, {"name": "mypipe2", "param": "constant"} ], "pipeline_defaults": { "param1": "param1_value" }, "task_defaults": { "job_start": { "task_sub_def": "task_sub_def_value" } } } superliminal = { "name": "my_superliminal_test", "type": "super", "super": "super_superliminal", "pipeline_defaults": { "param2": "param2super_value", "param3": "param3super_value" }, "task_defaults": { "job_start": { "task_def1": "task_def1_value", "task_def2": { "task_def2_1": "task_def2_1_value", } } } } super_superliminal = { "name": "super_superliminal", "type": "super", "pipeline_defaults": { "param2": "param2super_value", "param3": "param3hyper_value", "param4": "param4hyper_value" } } expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}], 'name': 'my_subliminal_test', 'pipeline_defaults': {'param1': 'param1_value'}, 'pipelines': [{'description': 'add defaults parameters for all pipelines', 'name': 'mypipe1', 'param': 'constant', 'param1': 'param1_value', 'param2': 'param2super_value', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'task': 'start', 'task_def1': 'task_def1_value', 'task_def2': {'task_def2_1': 'task_def2_1_value'}, 'task_sub_def': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}, {'description': 'add defaults parameters for all pipelines', 'name': 'mypipe2', 'param': 'constant', 'param1': 'param1_value', 'param2': 'param2super_value', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'task': 'start', 'task_def1': 'task_def1_value', 'task_def2': {'task_def2_1': 'task_def2_1_value'}, 'task_sub_def': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}], 'service_defaults': {'description': 'add defaults parameters for all ' 'services'}, 'services': [], 'super': 'my_superliminal_test', 'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}}, 'type': 'sub'}] find_config_files_mock.return_value = { "my_subliminal_test": subliminal, "my_superliminal_test": superliminal, "super_superliminal": super_superliminal } config_util = ConfigUtil("") self.assertEqual(expected, config_util.safe_load(is_render_variables=True)) # validate cache self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load_with_variables(self, find_config_files_mock): subliminal = { "name": "my_subliminal_test", "type": "sub", "super": "my_superliminal_test", "variables": { "var": "simple case", "var-2": "-case", "var_2": "_case", "image": "prod image", "a": "{{env}}1", "b": "{{a}}2", "c": "{{a}}{{b}}2" }, "pipelines": [ {"name": "mypipe1", "param": "{{var}}", "tasks": [ {'task': 'sub_tasks', 'type': 'dummy'}, ]}, {"name": "mypipe2", "param": "{{var-2 }}", "tasks": [ {'task': 'sub_tasks', 'type': 'dummy'}, ]} ], "pipeline_defaults": { "param1": "{{var-2}}" }, "task_defaults": { "job_start": { "task_def1:": "task_sub_def_value" } }, "services": [ { "name": "my_python_server", "type": "python_server", "image": "{{image}}" }, { "name": "my_python_server_for_stg", "type": "python_server", "image": "{{default_image}}" } ]} superliminal = { "name": "my_superliminal_test", "type": "super", "variables": { "var-2": "override", "var3": "super_var", "default_image": "default_image_value", "image": "default_image_value" }, "super": "super_superliminal", "pipeline_defaults": { "param2": "{{pipe-var}}", "param3": "param3super_value", "before_tasks": [ {'task': 'second_task', 'type': 'dummy'}, ] }, "task_defaults": { "pipeline": { "path": "{{var-2}}", "task_def1": "task_def1_value", "task_def2": { "task_def2_1": "task_def2_1_value", } } } } super_superliminal = { "name": "super_superliminal", "type": "super", "variables": { "default_image": "def_default_image_value" }, "pipeline_defaults": { "global_conf": "{{var3}}", "param2": "param2super_value", "param3": "param3hyper_value", "param4": "param4hyper_value", "after_tasks": [ {'task': 'before_last_task', 'type': 'dummy'}, ] } } expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}], 'name': 'my_subliminal_test', 'pipeline_defaults': {'param1': '-case'}, 'pipelines': [{'description': 'add defaults parameters for all pipelines', 'global_conf': 'super_var', 'name': 'mypipe1', 'param': 'simple case', 'param1': '-case', 'param2': '{{pipe-var}}', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'task': 'start', 'task_def1:': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'second_task', 'type': 'dummy'}, {'task': 'sub_tasks', 'type': 'dummy'}, {'task': 'before_last_task', 'type': 'dummy'}, {'task': 'end', 'type': 'job_end'}]}, {'description': 'add defaults parameters for all pipelines', 'global_conf': 'super_var', 'name': 'mypipe2', 'param': '-case', 'param1': '-case', 'param2': '{{pipe-var}}', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'task': 'start', 'task_def1:': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'second_task', 'type': 'dummy'}, {'task': 'sub_tasks', 'type': 'dummy'}, {'task': 'before_last_task', 'type': 'dummy'}, {'task': 'end', 'type': 'job_end'}]}], 'service_defaults': {'description': 'add defaults parameters for all ' 'services'}, 'services': [{'description': 'add defaults parameters for all services', 'image': 'prod image', 'name': 'my_python_server', 'type': 'python_server'}, {'description': 'add defaults parameters for all services', 'image': 'default_image_value', 'name': 'my_python_server_for_stg', 'type': 'python_server'}], 'super': 'my_superliminal_test', 'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}}, 'type': 'sub', 'variables': {'a': 'myenv1', 'b': 'myenv12', 'c': 'myenv1myenv122', 'image': 'prod image', 'var': 'simple case', 'var-2': '-case', 'var_2': '_case'}}] find_config_files_mock.return_value = { "my_subliminal_test": subliminal, "my_superliminal_test": superliminal, "super_superliminal": super_superliminal } config_util = ConfigUtil("") self.assertEqual(expected, config_util.safe_load(is_render_variables=True)) # validate cache self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load_with_variables(self, find_config_files_mock): subliminal = { 'name': 'my_subliminal_test', 'type': 'sub', 'super': 'my_superliminal_test', 'variables': { 'var': 'simple case', 'var-2': '-case', 'var_2': '_case', 'image': 'prod image', 'a': '{{env}}1', 'b': '{{a}}2', 'c': '{{a}}{{b}}2' }, 'pipelines': [ {'name': 'mypipe1', 'param': '{{var}}', 'tasks': [ {'task': 'sub_tasks', 'type': 'dummy'}, ]}, {'name': 'mypipe2', 'param': '{{var-2 }}', 'tasks': [ {'task': 'sub_tasks', 'type': 'dummy'}, ]} ], 'pipeline_defaults': { 'param1': '{{var-2}}' }, 'task_defaults': { 'job_start': { 'task_def1:': 'task_sub_def_value' } }, 'services': [ { 'name': 'my_python_server', 'type': 'python_server', 'image': '{{image}}' }, { 'name': 'my_python_server_for_stg', 'type': 'python_server', 'image': '{{default_image}}' } ]} superliminal = { 'name': 'my_superliminal_test', 'type': 'super', 'variables': { 'var-2': 'override', 'var3': 'super_var', 'default_image': 'default_image_value', 'image': 'default_image_value' }, 'super': 'super_superliminal', 'pipeline_defaults': { 'param2': '{{pipe-var}}', 'param3': 'param3super_value', 'before_tasks': [ {'task': 'second_task', 'type': 'dummy'}, ] }, 'task_defaults': { 'pipeline': { 'path': '{{var-2}}', 'task_def1': 'task_def1_value', 'task_def2': { 'task_def2_1': 'task_def2_1_value', } } } } super_superliminal = { 'name': 'super_superliminal', 'type': 'super', 'variables': { 'default_image': 'def_default_image_value' }, 'pipeline_defaults': { 'global_conf': '{{var3}}', 'param2': 'param2super_value', 'param3': 'param3hyper_value', 'param4': 'param4hyper_value', 'after_tasks': [ {'task': 'before_last_task', 'type': 'dummy'}, ] } } expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}, {'executor': 'airflow_executor', 'type': 'airflow'}], 'name': 'my_subliminal_test', 'pipeline_defaults': {'param1': '-case'}, 'pipelines': [{'description': 'add defaults parameters for all pipelines', 'global_conf': 'super_var', 'name': 'mypipe1', 'param': 'simple case', 'param1': '-case', 'param2': '{{pipe-var}}', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'executor': 'airflow_executor', 'task': 'start', 'task_def1:': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'second_task', 'type': 'dummy'}, {'task': 'sub_tasks', 'type': 'dummy'}, {'task': 'before_last_task', 'type': 'dummy'}, {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'}]}, {'description': 'add defaults parameters for all pipelines', 'global_conf': 'super_var', 'name': 'mypipe2', 'param': '-case', 'param1': '-case', 'param2': '{{pipe-var}}', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'executor': 'airflow_executor', 'task': 'start', 'task_def1:': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'second_task', 'type': 'dummy'}, {'task': 'sub_tasks', 'type': 'dummy'}, {'task': 'before_last_task', 'type': 'dummy'}, {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'}]}], 'service_defaults': {'description': 'add defaults parameters for all ' 'services'}, 'images': [], 'services': [{'description': 'add defaults parameters for all services', 'image': 'prod image', 'name': 'my_python_server', 'type': 'python_server'}, {'description': 'add defaults parameters for all services', 'image': 'default_image_value', 'name': 'my_python_server_for_stg', 'type': 'python_server'}], 'super': 'my_superliminal_test', 'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}}, 'type': 'sub', 'variables': {'a': 'myenv1', 'b': 'myenv12', 'c': 'myenv1myenv122', 'image': 'prod image', 'var': 'simple case', 'var-2': '-case', 'var_2': '_case'}}] find_config_files_mock.return_value = { 'my_subliminal_test': subliminal, 'my_superliminal_test': superliminal, 'super_superliminal': super_superliminal } config_util = ConfigUtil('') self.assertEqual(expected, config_util.safe_load(is_render_variables=True)) # validate cache self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load(self, find_config_files_mock): subliminal = { 'name': 'my_subliminal_test', 'type': 'sub', 'super': 'my_superliminal_test', 'images': [{ 'image': 'my_image' }], 'pipelines': [ {'name': 'mypipe1', 'param': 'constant'}, {'name': 'mypipe2', 'param': 'constant'} ], 'pipeline_defaults': { 'param1': 'param1_value' }, 'task_defaults': { 'job_start': { 'task_sub_def': 'task_sub_def_value' } } } superliminal = { 'name': 'my_superliminal_test', 'type': 'super', 'super': 'super_superliminal', 'images': [{ 'image': 'my_image', 'source': '.' }], 'pipeline_defaults': { 'param2': 'param2super_value', 'param3': 'param3super_value' }, 'task_defaults': { 'job_start': { 'task_def1': 'task_def1_value', 'task_def2': { 'task_def2_1': 'task_def2_1_value', } } } } super_superliminal = { 'name': 'super_superliminal', 'type': 'super', 'pipeline_defaults': { 'param2': 'param2super_value', 'param3': 'param3hyper_value', 'param4': 'param4hyper_value' } } expected = [{ 'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}], 'name': 'my_subliminal_test', 'pipeline_defaults': {'param1': 'param1_value'}, 'pipelines': [{'description': 'add defaults parameters for all pipelines', 'name': 'mypipe1', 'param': 'constant', 'param1': 'param1_value', 'param2': 'param2super_value', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'task': 'start', 'task_def1': 'task_def1_value', 'task_def2': {'task_def2_1': 'task_def2_1_value'}, 'task_sub_def': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}, {'description': 'add defaults parameters for all pipelines', 'name': 'mypipe2', 'param': 'constant', 'param1': 'param1_value', 'param2': 'param2super_value', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'task': 'start', 'task_def1': 'task_def1_value', 'task_def2': {'task_def2_1': 'task_def2_1_value'}, 'task_sub_def': 'task_sub_def_value', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}], 'service_defaults': {'description': 'add defaults parameters for all ' 'services'}, 'images': [{'image': 'my_image', 'source': '.'}], 'services': [], 'super': 'my_superliminal_test', 'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}}, 'type': 'sub' }] expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}, {'executor': 'airflow_executor', 'type': 'airflow'}], 'images': [{'image': 'my_image', 'source': '.'}], 'name': 'my_subliminal_test', 'pipeline_defaults': {'param1': 'param1_value'}, 'pipelines': [{'description': 'add defaults parameters for all pipelines', 'name': 'mypipe1', 'param': 'constant', 'param1': 'param1_value', 'param2': 'param2super_value', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'executor': 'airflow_executor', 'task': 'start', 'task_def1': 'task_def1_value', 'task_def2': {'task_def2_1': 'task_def2_1_value'}, 'task_sub_def': 'task_sub_def_value', 'type': 'job_start'}, {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'}]}, {'description': 'add defaults parameters for all pipelines', 'name': 'mypipe2', 'param': 'constant', 'param1': 'param1_value', 'param2': 'param2super_value', 'param3': 'param3super_value', 'param4': 'param4hyper_value', 'tasks': [{'executor': 'airflow_executor', 'task': 'start', 'task_def1': 'task_def1_value', 'task_def2': {'task_def2_1': 'task_def2_1_value'}, 'task_sub_def': 'task_sub_def_value', 'type': 'job_start'}, {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'}]}], 'service_defaults': {'description': 'add defaults parameters for all ' 'services'}, 'services': [], 'super': 'my_superliminal_test', 'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}}, 'type': 'sub'}] find_config_files_mock.return_value = { 'my_subliminal_test': subliminal, 'my_superliminal_test': superliminal, 'super_superliminal': super_superliminal } config_util = ConfigUtil('') self.assertEqual(expected, config_util.safe_load(is_render_variables=True)) # validate cache self.assertEqual(expected, config_util.loaded_subliminals)