def test_get_superliminal(self, find_config_files_mock):
base = {'name': 'base',
'pipeline_defaults': {'after_tasks': [{'task': 'end', 'type': 'job_end'}],
'before_tasks': [{'task': 'start', 'type': 'job_start'}],
'description': 'add defaults parameters for all '
'pipelines'},
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'task_defaults': {'description': 'add defaults parameters for all tasks '
'separate by task type'},
'type': 'super'}
subliminal = {
"name": "subliminal_test",
"type": "sub"
}
find_config_files_mock.return_value = {
"subliminal_test": subliminal
}
config_util = ConfigUtil("")
self.assertEqual(base,
config_util._ConfigUtil__get_superliminal(subliminal))
self.assertEqual({},
config_util._ConfigUtil__get_superliminal(base))
liminal = {
"name": "subliminal_test",
"type": "sub",
"super": "my_superliminal"
}
with self.assertRaises(FileNotFoundError):
config_util._ConfigUtil__get_superliminal(liminal)
def test_safe_load(self, find_config_files_mock):
subliminal = {
"name": "my_subliminal_test",
"type": "sub",
"super": "my_superliminal_test",
"pipelines": [
{"name": "mypipe1", "param": "constant"},
{"name": "mypipe2", "param": "constant"}
],
"pipeline_defaults": {
"param1": "param1_value"
},
"task_defaults": {
"job_start": {
"task_sub_def": "task_sub_def_value"
}
}
}
superliminal = {
"name": "my_superliminal_test",
"type": "super",
"super": "super_superliminal",
"pipeline_defaults": {
"param2": "param2super_value",
"param3": "param3super_value"
},
"task_defaults": {
"job_start": {
"task_def1": "task_def1_value",
"task_def2": {
"task_def2_1": "task_def2_1_value",
}
}
}
}
super_superliminal = {
"name": "super_superliminal",
"type": "super",
"pipeline_defaults": {
"param2": "param2super_value",
"param3": "param3hyper_value",
"param4": "param4hyper_value"
}
}
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': 'param1_value'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe1',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe2',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'services': [],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
'type': 'sub'}]
find_config_files_mock.return_value = {
"my_subliminal_test": subliminal,
"my_superliminal_test": superliminal,
"super_superliminal": super_superliminal
}
config_util = ConfigUtil("")
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load_with_variables(self, find_config_files_mock):
subliminal = {
"name": "my_subliminal_test",
"type": "sub",
"super": "my_superliminal_test",
"variables": {
"var": "simple case",
"var-2": "-case",
"var_2": "_case",
"image": "prod image",
"a": "{{env}}1",
"b": "{{a}}2",
"c": "{{a}}{{b}}2"
},
"pipelines": [
{"name": "mypipe1", "param": "{{var}}",
"tasks": [
{'task': 'sub_tasks',
'type': 'dummy'},
]},
{"name": "mypipe2", "param": "{{var-2 }}", "tasks": [
{'task': 'sub_tasks',
'type': 'dummy'},
]}
],
"pipeline_defaults": {
"param1": "{{var-2}}"
},
"task_defaults": {
"job_start": {
"task_def1:": "task_sub_def_value"
}
},
"services": [
{
"name": "my_python_server",
"type": "python_server",
"image": "{{image}}"
},
{
"name": "my_python_server_for_stg",
"type": "python_server",
"image": "{{default_image}}"
}
]}
superliminal = {
"name": "my_superliminal_test",
"type": "super",
"variables": {
"var-2": "override",
"var3": "super_var",
"default_image": "default_image_value",
"image": "default_image_value"
},
"super": "super_superliminal",
"pipeline_defaults": {
"param2": "{{pipe-var}}",
"param3": "param3super_value",
"before_tasks": [
{'task': 'second_task', 'type': 'dummy'},
]
},
"task_defaults": {
"pipeline": {
"path": "{{var-2}}",
"task_def1": "task_def1_value",
"task_def2": {
"task_def2_1": "task_def2_1_value",
}
}
}
}
super_superliminal = {
"name": "super_superliminal",
"type": "super",
"variables": {
"default_image": "def_default_image_value"
},
"pipeline_defaults": {
"global_conf": "{{var3}}",
"param2": "param2super_value",
"param3": "param3hyper_value",
"param4": "param4hyper_value",
"after_tasks": [
{'task': 'before_last_task', 'type': 'dummy'},
]
}
}
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': '-case'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe1',
'param': 'simple case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'task': 'end', 'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe2',
'param': '-case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'task': 'end', 'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'services': [{'description': 'add defaults parameters for all services',
'image': 'prod image',
'name': 'my_python_server',
'type': 'python_server'},
{'description': 'add defaults parameters for all services',
'image': 'default_image_value',
'name': 'my_python_server_for_stg',
'type': 'python_server'}],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
'type': 'sub',
'variables': {'a': 'myenv1',
'b': 'myenv12',
'c': 'myenv1myenv122',
'image': 'prod image',
'var': 'simple case',
'var-2': '-case',
'var_2': '_case'}}]
find_config_files_mock.return_value = {
"my_subliminal_test": subliminal,
"my_superliminal_test": superliminal,
"super_superliminal": super_superliminal
}
config_util = ConfigUtil("")
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
def register_dags(configs_path):
"""
Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs.
"""
logging.info(f'Registering DAGs from path: {configs_path}')
config_util = ConfigUtil(configs_path)
# TODO - change is_render_variable to False when runtime resolving is available
configs = config_util.safe_load(is_render_variables=True)
if os.getenv('POD_NAMESPACE') != "jenkins":
config_util.snapshot_final_liminal_configs()
dags = []
logging.info(
f'found {len(configs)} liminal configs in path: {configs_path}')
for config in configs:
name = config['name'] if 'name' in config else None
try:
if not name:
raise ValueError('liminal.yml missing field `name`')
logging.info(f"Registering DAGs for {name}")
owner = config.get('owner')
trigger_rule = 'all_success'
if 'always_run' in config and config['always_run']:
trigger_rule = 'all_done'
executors = __initialize_executors(config)
default_executor = airflow.AirflowExecutor("default_executor",
liminal_config=config,
executor_config={})
for pipeline in config['pipelines']:
default_args = __default_args(pipeline)
dag = __initialize_dag(default_args, pipeline, owner)
parent = None
for task in pipeline['tasks']:
task_type = task['type']
task_instance = get_task_class(task_type)(
task_id=task['task'],
dag=dag,
parent=parent,
trigger_rule=trigger_rule,
liminal_config=config,
pipeline_config=pipeline,
task_config=task)
executor_id = task.get('executor')
if executor_id:
executor = executors[executor_id]
else:
logging.info(
f"Did not find `executor` in ${task['task']} config."
f" Using the default executor (${type(default_executor)})"
f" instead.")
executor = default_executor
parent = executor.apply_task_to_dag(task=task_instance)
logging.info(f'registered DAG {dag.dag_id}: {dag.tasks}')
dags.append((pipeline['pipeline'], dag))
except Exception:
logging.error(f'Failed to register DAGs for {name}')
traceback.print_exc()
return dags
def test_safe_load_with_variables(self, find_config_files_mock):
subliminal = {
'name': 'my_subliminal_test',
'type': 'sub',
'super': 'my_superliminal_test',
'variables': {
'var': 'simple case',
'var-2': '-case',
'var_2': '_case',
'image': 'prod image',
'a': '{{env}}1',
'b': '{{a}}2',
'c': '{{a}}{{b}}2'
},
'pipelines': [
{'name': 'mypipe1', 'param': '{{var}}',
'tasks': [
{'task': 'sub_tasks',
'type': 'dummy'},
]},
{'name': 'mypipe2', 'param': '{{var-2 }}', 'tasks': [
{'task': 'sub_tasks',
'type': 'dummy'},
]}
],
'pipeline_defaults': {
'param1': '{{var-2}}'
},
'task_defaults': {
'job_start': {
'task_def1:': 'task_sub_def_value'
}
},
'services': [
{
'name': 'my_python_server',
'type': 'python_server',
'image': '{{image}}'
},
{
'name': 'my_python_server_for_stg',
'type': 'python_server',
'image': '{{default_image}}'
}
]}
superliminal = {
'name': 'my_superliminal_test',
'type': 'super',
'variables': {
'var-2': 'override',
'var3': 'super_var',
'default_image': 'default_image_value',
'image': 'default_image_value'
},
'super': 'super_superliminal',
'pipeline_defaults': {
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'before_tasks': [
{'task': 'second_task', 'type': 'dummy'},
]
},
'task_defaults': {
'pipeline': {
'path': '{{var-2}}',
'task_def1': 'task_def1_value',
'task_def2': {
'task_def2_1': 'task_def2_1_value',
}
}
}
}
super_superliminal = {
'name': 'super_superliminal',
'type': 'super',
'variables': {
'default_image': 'def_default_image_value'
},
'pipeline_defaults': {
'global_conf': '{{var3}}',
'param2': 'param2super_value',
'param3': 'param3hyper_value',
'param4': 'param4hyper_value',
'after_tasks': [
{'task': 'before_last_task', 'type': 'dummy'},
]
}
}
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
{'executor': 'airflow_executor', 'type': 'airflow'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': '-case'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe1',
'param': 'simple case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe2',
'param': '-case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'images': [],
'services': [{'description': 'add defaults parameters for all services',
'image': 'prod image',
'name': 'my_python_server',
'type': 'python_server'},
{'description': 'add defaults parameters for all services',
'image': 'default_image_value',
'name': 'my_python_server_for_stg',
'type': 'python_server'}],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
'type': 'sub',
'variables': {'a': 'myenv1',
'b': 'myenv12',
'c': 'myenv1myenv122',
'image': 'prod image',
'var': 'simple case',
'var-2': '-case',
'var_2': '_case'}}]
find_config_files_mock.return_value = {
'my_subliminal_test': subliminal,
'my_superliminal_test': superliminal,
'super_superliminal': super_superliminal
}
config_util = ConfigUtil('')
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load(self, find_config_files_mock):
subliminal = {
'name': 'my_subliminal_test',
'type': 'sub',
'super': 'my_superliminal_test',
'images': [{
'image': 'my_image'
}],
'pipelines': [
{'name': 'mypipe1', 'param': 'constant'},
{'name': 'mypipe2', 'param': 'constant'}
],
'pipeline_defaults': {
'param1': 'param1_value'
},
'task_defaults': {
'job_start': {
'task_sub_def': 'task_sub_def_value'
}
}
}
superliminal = {
'name': 'my_superliminal_test',
'type': 'super',
'super': 'super_superliminal',
'images': [{
'image': 'my_image',
'source': '.'
}],
'pipeline_defaults': {
'param2': 'param2super_value',
'param3': 'param3super_value'
},
'task_defaults': {
'job_start': {
'task_def1': 'task_def1_value',
'task_def2': {
'task_def2_1': 'task_def2_1_value',
}
}
}
}
super_superliminal = {
'name': 'super_superliminal',
'type': 'super',
'pipeline_defaults': {
'param2': 'param2super_value',
'param3': 'param3hyper_value',
'param4': 'param4hyper_value'
}
}
expected = [{
'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': 'param1_value'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe1',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe2',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'images': [{'image': 'my_image', 'source': '.'}],
'services': [],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
'type': 'sub'
}]
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
{'executor': 'airflow_executor', 'type': 'airflow'}],
'images': [{'image': 'my_image', 'source': '.'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': 'param1_value'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe1',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe2',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'services': [],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
'type': 'sub'}]
find_config_files_mock.return_value = {
'my_subliminal_test': subliminal,
'my_superliminal_test': superliminal,
'super_superliminal': super_superliminal
}
config_util = ConfigUtil('')
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
