def test_liminal_config_snapshot(self, find_config_files_mock,
get_airflow_dir_mock, path_exists_mock):
subliminal = {
"name": "my_subliminal_test",
"type": "sub",
"variables": {
"var": 1,
"var-2": True
},
"pipelines": [
{"name": "mypipe1", "param": "{{var}}"},
{"name": "mypipe2", "param": "{{var-2 }}"}
]
}
expected = {'name': 'my_subliminal_test', 'type': 'sub',
'service_defaults': {'description': 'add defaults parameters for all services'},
'task_defaults': {
'description': 'add defaults parameters for all tasks separate by task type'},
'pipeline_defaults': {
'description': 'add defaults parameters for all pipelines',
'before_tasks': [{'task': 'start', 'type': 'job_start'}],
'after_tasks': [{'task': 'end', 'type': 'job_end'}]},
'variables': {'var': 1, 'var-2': True}, 'pipelines': [
{'name': 'mypipe1', 'param': '1',
'description': 'add defaults parameters for all pipelines',
'tasks': [{'task': 'start', 'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]},
{'name': 'mypipe2', 'param': 'True',
'description': 'add defaults parameters for all pipelines',
'tasks': [{'task': 'start', 'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]}], 'services': []}
find_config_files_mock.return_value = {
"my_subliminal_test": subliminal
}
get_airflow_dir_mock.return_value = "/tmp"
path_exists_mock.return_value = True
with mock.patch("builtins.open", mock.mock_open()) as m:
with mock.patch("yaml.dump") as ydm:
config_util = ConfigUtil("")
config_util.safe_load(is_render_variables=True)
config_util.snapshot_final_liminal_configs()
m.assert_called_once_with(
os.path.join('/tmp', '../liminal_config_files/my_subliminal_test.yml'), 'w')
ydm.assert_called_once_with(expected, m.return_value, default_flow_style=False)
def register_dags(configs_path):
"""
Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs.
"""
logging.info(f'Registering DAGs from path: {configs_path}')
config_util = ConfigUtil(configs_path)
# TODO - change is_render_variable to False when runtime resolving is available
configs = config_util.safe_load(is_render_variables=True)
if os.getenv('POD_NAMESPACE') != "jenkins":
config_util.snapshot_final_liminal_configs()
dags = []
logging.info(f'found {len(configs)} liminal configs in path: {configs_path}')
for config in configs:
name = config['name'] if 'name' in config else None
try:
if not name:
raise ValueError('liminal.yml missing field `name`')
logging.info(f"Registering DAGs for {name}")
owner = config.get('owner')
trigger_rule = 'all_success'
if 'always_run' in config and config['always_run']:
trigger_rule = 'all_done'
executors = __initialize_executors(config)
for pipeline in config['pipelines']:
default_args = __default_args(pipeline)
dag = __initialize_dag(default_args, pipeline, owner)
parent = None
for task in pipeline['tasks']:
task_type = task['type']
task_instance = get_task_class(task_type)(
task_id=task['task'],
dag=dag,
parent=parent,
trigger_rule=trigger_rule,
liminal_config=config,
pipeline_config=pipeline,
task_config=task,
executor=executors.get(task.get('executor'))
)
parent = task_instance.apply_task_to_dag()
logging.info(f'registered DAG {dag.dag_id}: {dag.tasks}')
dags.append((pipeline['pipeline'], dag))
except Exception:
logging.error(f'Failed to register DAGs for {name}')
traceback.print_exc()
return dags
def build_liminal_apps(path):
"""
Build images for liminal apps in path.
"""
config_util = ConfigUtil(path)
configs = config_util.safe_load(is_render_variables=True, soft_merge=True)
for liminal_config in configs:
base_path = os.path.dirname(
files_util.resolve_pipeline_source_file(liminal_config['name']))
if 'pipelines' in liminal_config:
for pipeline in liminal_config['pipelines']:
for task in pipeline['tasks']:
task_name = task['task']
if 'source' in task:
task_type = task['type']
builder_class = __get_task_build_class(task_type)
if builder_class:
__build_image(base_path, task, builder_class)
else:
raise ValueError(f'No such task type: {task_type}')
else:
logging.info(
f'No source configured for task {task_name}, skipping build..'
)
if 'services' in liminal_config:
for service in liminal_config['services']:
service_type = service['type']
builder_class = __get_service_build_class(service_type)
if builder_class:
__build_image(base_path, service, builder_class)
else:
raise ValueError(f'No such service type: {service_type}')
def test_soft_merge_load(self, find_config_files_mock):
subliminal = {
"name": "my_name",
"type": "sub",
"super": "my_super"
}
find_config_files_mock.return_value = {"my_subliminal_test": subliminal}
config_util = ConfigUtil("")
self.assertEqual([subliminal],
config_util.safe_load(is_render_variables=True, soft_merge=True))
def test_soft_merge_load(self, find_config_files_mock):
subliminal = {
'name': 'my_name',
'type': 'sub',
'super': 'my_super'
}
find_config_files_mock.return_value = {'my_subliminal_test': subliminal}
config_util = ConfigUtil('')
self.assertEqual([subliminal],
config_util.safe_load(is_render_variables=True, soft_merge=True))
def test_safe_load(self, find_config_files_mock):
subliminal = {
"name": "my_subliminal_test",
"type": "sub",
"super": "my_superliminal_test",
"pipelines": [
{"name": "mypipe1", "param": "constant"},
{"name": "mypipe2", "param": "constant"}
],
"pipeline_defaults": {
"param1": "param1_value"
},
"task_defaults": {
"job_start": {
"task_sub_def": "task_sub_def_value"
}
}
}
superliminal = {
"name": "my_superliminal_test",
"type": "super",
"super": "super_superliminal",
"pipeline_defaults": {
"param2": "param2super_value",
"param3": "param3super_value"
},
"task_defaults": {
"job_start": {
"task_def1": "task_def1_value",
"task_def2": {
"task_def2_1": "task_def2_1_value",
}
}
}
}
super_superliminal = {
"name": "super_superliminal",
"type": "super",
"pipeline_defaults": {
"param2": "param2super_value",
"param3": "param3hyper_value",
"param4": "param4hyper_value"
}
}
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': 'param1_value'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe1',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe2',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'services': [],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
'type': 'sub'}]
find_config_files_mock.return_value = {
"my_subliminal_test": subliminal,
"my_superliminal_test": superliminal,
"super_superliminal": super_superliminal
}
config_util = ConfigUtil("")
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load_with_variables(self, find_config_files_mock):
subliminal = {
"name": "my_subliminal_test",
"type": "sub",
"super": "my_superliminal_test",
"variables": {
"var": "simple case",
"var-2": "-case",
"var_2": "_case",
"image": "prod image",
"a": "{{env}}1",
"b": "{{a}}2",
"c": "{{a}}{{b}}2"
},
"pipelines": [
{"name": "mypipe1", "param": "{{var}}",
"tasks": [
{'task': 'sub_tasks',
'type': 'dummy'},
]},
{"name": "mypipe2", "param": "{{var-2 }}", "tasks": [
{'task': 'sub_tasks',
'type': 'dummy'},
]}
],
"pipeline_defaults": {
"param1": "{{var-2}}"
},
"task_defaults": {
"job_start": {
"task_def1:": "task_sub_def_value"
}
},
"services": [
{
"name": "my_python_server",
"type": "python_server",
"image": "{{image}}"
},
{
"name": "my_python_server_for_stg",
"type": "python_server",
"image": "{{default_image}}"
}
]}
superliminal = {
"name": "my_superliminal_test",
"type": "super",
"variables": {
"var-2": "override",
"var3": "super_var",
"default_image": "default_image_value",
"image": "default_image_value"
},
"super": "super_superliminal",
"pipeline_defaults": {
"param2": "{{pipe-var}}",
"param3": "param3super_value",
"before_tasks": [
{'task': 'second_task', 'type': 'dummy'},
]
},
"task_defaults": {
"pipeline": {
"path": "{{var-2}}",
"task_def1": "task_def1_value",
"task_def2": {
"task_def2_1": "task_def2_1_value",
}
}
}
}
super_superliminal = {
"name": "super_superliminal",
"type": "super",
"variables": {
"default_image": "def_default_image_value"
},
"pipeline_defaults": {
"global_conf": "{{var3}}",
"param2": "param2super_value",
"param3": "param3hyper_value",
"param4": "param4hyper_value",
"after_tasks": [
{'task': 'before_last_task', 'type': 'dummy'},
]
}
}
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': '-case'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe1',
'param': 'simple case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'task': 'end', 'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe2',
'param': '-case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'task': 'end', 'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'services': [{'description': 'add defaults parameters for all services',
'image': 'prod image',
'name': 'my_python_server',
'type': 'python_server'},
{'description': 'add defaults parameters for all services',
'image': 'default_image_value',
'name': 'my_python_server_for_stg',
'type': 'python_server'}],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
'type': 'sub',
'variables': {'a': 'myenv1',
'b': 'myenv12',
'c': 'myenv1myenv122',
'image': 'prod image',
'var': 'simple case',
'var-2': '-case',
'var_2': '_case'}}]
find_config_files_mock.return_value = {
"my_subliminal_test": subliminal,
"my_superliminal_test": superliminal,
"super_superliminal": super_superliminal
}
config_util = ConfigUtil("")
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load_with_variables(self, find_config_files_mock):
subliminal = {
'name': 'my_subliminal_test',
'type': 'sub',
'super': 'my_superliminal_test',
'variables': {
'var': 'simple case',
'var-2': '-case',
'var_2': '_case',
'image': 'prod image',
'a': '{{env}}1',
'b': '{{a}}2',
'c': '{{a}}{{b}}2'
},
'pipelines': [
{'name': 'mypipe1', 'param': '{{var}}',
'tasks': [
{'task': 'sub_tasks',
'type': 'dummy'},
]},
{'name': 'mypipe2', 'param': '{{var-2 }}', 'tasks': [
{'task': 'sub_tasks',
'type': 'dummy'},
]}
],
'pipeline_defaults': {
'param1': '{{var-2}}'
},
'task_defaults': {
'job_start': {
'task_def1:': 'task_sub_def_value'
}
},
'services': [
{
'name': 'my_python_server',
'type': 'python_server',
'image': '{{image}}'
},
{
'name': 'my_python_server_for_stg',
'type': 'python_server',
'image': '{{default_image}}'
}
]}
superliminal = {
'name': 'my_superliminal_test',
'type': 'super',
'variables': {
'var-2': 'override',
'var3': 'super_var',
'default_image': 'default_image_value',
'image': 'default_image_value'
},
'super': 'super_superliminal',
'pipeline_defaults': {
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'before_tasks': [
{'task': 'second_task', 'type': 'dummy'},
]
},
'task_defaults': {
'pipeline': {
'path': '{{var-2}}',
'task_def1': 'task_def1_value',
'task_def2': {
'task_def2_1': 'task_def2_1_value',
}
}
}
}
super_superliminal = {
'name': 'super_superliminal',
'type': 'super',
'variables': {
'default_image': 'def_default_image_value'
},
'pipeline_defaults': {
'global_conf': '{{var3}}',
'param2': 'param2super_value',
'param3': 'param3hyper_value',
'param4': 'param4hyper_value',
'after_tasks': [
{'task': 'before_last_task', 'type': 'dummy'},
]
}
}
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
{'executor': 'airflow_executor', 'type': 'airflow'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': '-case'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe1',
'param': 'simple case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'global_conf': 'super_var',
'name': 'mypipe2',
'param': '-case',
'param1': '-case',
'param2': '{{pipe-var}}',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1:': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'second_task', 'type': 'dummy'},
{'task': 'sub_tasks', 'type': 'dummy'},
{'task': 'before_last_task', 'type': 'dummy'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'images': [],
'services': [{'description': 'add defaults parameters for all services',
'image': 'prod image',
'name': 'my_python_server',
'type': 'python_server'},
{'description': 'add defaults parameters for all services',
'image': 'default_image_value',
'name': 'my_python_server_for_stg',
'type': 'python_server'}],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
'type': 'sub',
'variables': {'a': 'myenv1',
'b': 'myenv12',
'c': 'myenv1myenv122',
'image': 'prod image',
'var': 'simple case',
'var-2': '-case',
'var_2': '_case'}}]
find_config_files_mock.return_value = {
'my_subliminal_test': subliminal,
'my_superliminal_test': superliminal,
'super_superliminal': super_superliminal
}
config_util = ConfigUtil('')
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
def test_safe_load(self, find_config_files_mock):
subliminal = {
'name': 'my_subliminal_test',
'type': 'sub',
'super': 'my_superliminal_test',
'images': [{
'image': 'my_image'
}],
'pipelines': [
{'name': 'mypipe1', 'param': 'constant'},
{'name': 'mypipe2', 'param': 'constant'}
],
'pipeline_defaults': {
'param1': 'param1_value'
},
'task_defaults': {
'job_start': {
'task_sub_def': 'task_sub_def_value'
}
}
}
superliminal = {
'name': 'my_superliminal_test',
'type': 'super',
'super': 'super_superliminal',
'images': [{
'image': 'my_image',
'source': '.'
}],
'pipeline_defaults': {
'param2': 'param2super_value',
'param3': 'param3super_value'
},
'task_defaults': {
'job_start': {
'task_def1': 'task_def1_value',
'task_def2': {
'task_def2_1': 'task_def2_1_value',
}
}
}
}
super_superliminal = {
'name': 'super_superliminal',
'type': 'super',
'pipeline_defaults': {
'param2': 'param2super_value',
'param3': 'param3hyper_value',
'param4': 'param4hyper_value'
}
}
expected = [{
'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': 'param1_value'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe1',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe2',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'task': 'end', 'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'images': [{'image': 'my_image', 'source': '.'}],
'services': [],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
'type': 'sub'
}]
expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
{'executor': 'airflow_executor', 'type': 'airflow'}],
'images': [{'image': 'my_image', 'source': '.'}],
'name': 'my_subliminal_test',
'pipeline_defaults': {'param1': 'param1_value'},
'pipelines': [{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe1',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]},
{'description': 'add defaults parameters for all pipelines',
'name': 'mypipe2',
'param': 'constant',
'param1': 'param1_value',
'param2': 'param2super_value',
'param3': 'param3super_value',
'param4': 'param4hyper_value',
'tasks': [{'executor': 'airflow_executor',
'task': 'start',
'task_def1': 'task_def1_value',
'task_def2': {'task_def2_1': 'task_def2_1_value'},
'task_sub_def': 'task_sub_def_value',
'type': 'job_start'},
{'executor': 'airflow_executor',
'task': 'end',
'type': 'job_end'}]}],
'service_defaults': {'description': 'add defaults parameters for all '
'services'},
'services': [],
'super': 'my_superliminal_test',
'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
'type': 'sub'}]
find_config_files_mock.return_value = {
'my_subliminal_test': subliminal,
'my_superliminal_test': superliminal,
'super_superliminal': super_superliminal
}
config_util = ConfigUtil('')
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)