def register_dags(configs_path): """ Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs. """ logging.info(f'Registering DAGs from path: {configs_path}') config_util = ConfigUtil(configs_path) # TODO - change is_render_variable to False when runtime resolving is available configs = config_util.safe_load(is_render_variables=True) if os.getenv('POD_NAMESPACE') != "jenkins": config_util.snapshot_final_liminal_configs() dags = [] logging.info(f'found {len(configs)} liminal configs in path: {configs_path}') for config in configs: name = config['name'] if 'name' in config else None try: if not name: raise ValueError('liminal.yml missing field `name`') logging.info(f"Registering DAGs for {name}") owner = config.get('owner') trigger_rule = 'all_success' if 'always_run' in config and config['always_run']: trigger_rule = 'all_done' executors = __initialize_executors(config) for pipeline in config['pipelines']: default_args = __default_args(pipeline) dag = __initialize_dag(default_args, pipeline, owner) parent = None for task in pipeline['tasks']: task_type = task['type'] task_instance = get_task_class(task_type)( task_id=task['task'], dag=dag, parent=parent, trigger_rule=trigger_rule, liminal_config=config, pipeline_config=pipeline, task_config=task, executor=executors.get(task.get('executor')) ) parent = task_instance.apply_task_to_dag() logging.info(f'registered DAG {dag.dag_id}: {dag.tasks}') dags.append((pipeline['pipeline'], dag)) except Exception: logging.error(f'Failed to register DAGs for {name}') traceback.print_exc() return dags
def test_liminal_config_snapshot(self, find_config_files_mock, get_airflow_dir_mock, path_exists_mock): subliminal = { 'name': 'my_subliminal_test', 'type': 'sub', 'variables': { 'var': 1, 'var-2': True }, 'pipelines': [ {'name': 'mypipe1', 'param': '{{var}}'}, {'name': 'mypipe2', 'param': '{{var-2 }}'} ] } expected = {'name': 'my_subliminal_test', 'type': 'sub', 'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}, {'executor': 'airflow_executor', 'type': 'airflow'}], 'service_defaults': {'description': 'add defaults parameters for all services'}, 'task_defaults': { 'description': 'add defaults parameters for all tasks separate by task type', 'python': {'executor': 'default_k8s'}, 'spark': {'executor': 'default_k8s'}, 'job_end': {'executor': 'airflow_executor'}, 'job_start': {'executor': 'airflow_executor'}}, 'pipeline_defaults': { 'description': 'add defaults parameters for all pipelines', 'before_tasks': [{'task': 'start', 'type': 'job_start'}], 'after_tasks': [{'task': 'end', 'type': 'job_end'}]}, 'variables': {'var': 1, 'var-2': True}, 'pipelines': [ {'name': 'mypipe1', 'param': '1', 'description': 'add defaults parameters for all pipelines', 'tasks': [{'task': 'start', 'type': 'job_start', 'executor': 'airflow_executor'}, {'task': 'end', 'type': 'job_end', 'executor': 'airflow_executor'}]}, {'name': 'mypipe2', 'param': 'True', 'description': 'add defaults parameters for all pipelines', 'tasks': [{'task': 'start', 'type': 'job_start', 'executor': 'airflow_executor'}, {'task': 'end', 'type': 'job_end', 'executor': 'airflow_executor'}]}], 'images': [], 'services': []} find_config_files_mock.return_value = { 'my_subliminal_test': subliminal } get_airflow_dir_mock.return_value = '/tmp' path_exists_mock.return_value = True with mock.patch('builtins.open', mock.mock_open()) as m: with mock.patch('yaml.dump') as ydm: config_util = ConfigUtil('') config_util.safe_load(is_render_variables=True) config_util.snapshot_final_liminal_configs() m.assert_called_once_with( os.path.join('/tmp', '../liminal_config_files/my_subliminal_test.yml'), 'w') ydm.assert_called_once_with(expected, m.return_value, default_flow_style=False)
def test_liminal_config_snapshot(self, find_config_files_mock, get_airflow_dir_mock, path_exists_mock): subliminal = { "name": "my_subliminal_test", "type": "sub", "variables": { "var": 1, "var-2": True }, "pipelines": [ {"name": "mypipe1", "param": "{{var}}"}, {"name": "mypipe2", "param": "{{var-2 }}"} ] } expected = {'name': 'my_subliminal_test', 'type': 'sub', 'service_defaults': {'description': 'add defaults parameters for all services'}, 'task_defaults': { 'description': 'add defaults parameters for all tasks separate by task type'}, 'pipeline_defaults': { 'description': 'add defaults parameters for all pipelines', 'before_tasks': [{'task': 'start', 'type': 'job_start'}], 'after_tasks': [{'task': 'end', 'type': 'job_end'}]}, 'variables': {'var': 1, 'var-2': True}, 'pipelines': [ {'name': 'mypipe1', 'param': '1', 'description': 'add defaults parameters for all pipelines', 'tasks': [{'task': 'start', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}, {'name': 'mypipe2', 'param': 'True', 'description': 'add defaults parameters for all pipelines', 'tasks': [{'task': 'start', 'type': 'job_start'}, {'task': 'end', 'type': 'job_end'}]}], 'services': []} find_config_files_mock.return_value = { "my_subliminal_test": subliminal } get_airflow_dir_mock.return_value = "/tmp" path_exists_mock.return_value = True with mock.patch("builtins.open", mock.mock_open()) as m: with mock.patch("yaml.dump") as ydm: config_util = ConfigUtil("") config_util.safe_load(is_render_variables=True) config_util.snapshot_final_liminal_configs() m.assert_called_once_with( os.path.join('/tmp', '../liminal_config_files/my_subliminal_test.yml'), 'w') ydm.assert_called_once_with(expected, m.return_value, default_flow_style=False)