def build_liminal_apps(path):
"""
Build images for liminal apps in path.
"""
config_files = files_util.find_config_files(path)
for config_file in config_files:
print(f'Building artifacts for file: {config_file}')
base_path = os.path.dirname(config_file)
with open(config_file) as stream:
liminal_config = yaml.safe_load(stream)
for pipeline in liminal_config['pipelines']:
for task in pipeline['tasks']:
task_name = task['task']
if 'source' in task:
task_type = task['type']
builder_class = __get_task_build_class(task_type)
if builder_class:
__build_image(base_path, task, builder_class)
else:
raise ValueError(f'No such task type: {task_type}')
else:
print(f'No source configured for task {task_name}, skipping build..')
for service in liminal_config['services']:
service_type = service['type']
builder_class = __get_service_build_class(service_type)
if builder_class:
__build_image(base_path, service, builder_class)
else:
raise ValueError(f'No such service type: {service_type}')
def register_dags(configs_path):
"""
Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs.
"""
logging.info(f'Registering DAG from path: {configs_path}')
config_files = files_util.find_config_files(configs_path)
dags = []
logging.info(f'found {len(config_files)} in path: {configs_path}')
for config_file in config_files:
logging.info(f'Registering DAG for file: {config_file}')
with open(config_file) as stream:
config = yaml.safe_load(stream)
for pipeline in config['pipelines']:
pipeline_name = pipeline['pipeline']
default_args = {k: v for k, v in pipeline.items()}
override_args = {
'start_date':
datetime.combine(pipeline['start_date'],
datetime.min.time()),
__DEPENDS_ON_PAST:
default_args[__DEPENDS_ON_PAST]
if __DEPENDS_ON_PAST in default_args else False,
}
default_args.update(override_args)
dag = DAG(dag_id=pipeline_name,
default_args=default_args,
dagrun_timeout=timedelta(
minutes=pipeline['timeout_minutes']),
catchup=False)
job_start_task = JobStartTask(dag, config, pipeline, {}, None,
'all_success')
parent = job_start_task.apply_task_to_dag()
trigger_rule = 'all_success'
if 'always_run' in config and config['always_run']:
trigger_rule = 'all_done'
for task in pipeline['tasks']:
task_type = task['type']
task_instance = get_task_class(task_type)(
dag, config, pipeline, task,
parent if parent else None, trigger_rule)
parent = task_instance.apply_task_to_dag()
job_end_task = JobEndTask(dag, config, pipeline, {}, parent,
'all_done')
job_end_task.apply_task_to_dag()
logging.info(f'registered DAG {dag.dag_id}: {dag.tasks}')
globals()[pipeline_name] = dag
dags.append(dag)
return dags