def get_partial():
with open('pipeline-features.yaml') as f:
tasks = yaml.safe_load(f)
meta = {'extract_product': False, 'extract_upstream': False}
spec = DAGSpec({'tasks': tasks, 'meta': meta})
return spec.to_dag()
def init_dag_from_partial(cls, partial):
"""Initialize partial returned by get_partial()
"""
if isinstance(partial, (str, Path)):
with open(partial) as f:
tasks = yaml.safe_load(f)
# cannot extract upstream because this is an incomplete DAG
meta = {'extract_product': False, 'extract_upstream': False}
spec = DAGSpec(
{
'tasks': tasks,
'meta': meta
},
parent_path=Path(partial).parent,
)
return spec.to_dag()
elif isinstance(partial, DAG):
return partial
else:
raise TypeError(f'Expected {cls.__name__}.get_partial() to '
'return a str, pathlib.Path or ploomber.DAG, '
f'got {type(partial).__name__}')