def test_add_task_from_scaffold(backup_test_pkg, tmp_directory):
yaml = """
meta:
source_loader:
module: test_pkg
extract_product: True
tasks:
- source: notebook.ipynb
- source: notebook.py
- source: test_pkg.functions.my_new_function
"""
Path('pipeline.yaml').write_text(yaml)
spec, path_to_spec = scaffold.load_dag()
scaffold.add(spec, path_to_spec)
code = Path(backup_test_pkg, 'functions.py').read_text()
module = ast.parse(code)
function_names = {
element.name
for element in module.body if hasattr(element, 'name')
}
assert 'my_new_function' in function_names
assert Path(backup_test_pkg, 'notebook.ipynb').exists()
assert Path(backup_test_pkg, 'notebook.py').exists()
def test_add_task_from_scaffold(backup_test_pkg, tmp_directory):
yaml = """
meta:
source_loader:
module: test_pkg
extract_product: True
tasks:
- source: notebook.ipynb
- source: notebook.py
- source: test_pkg.functions.my_new_function
"""
Path('pipeline.yaml').write_text(yaml)
# FIXME: this will fail because TaskSpec validates that the
# dotted path actually exists. I think the cleanest solution
# is to add a special class method for DAGSpec that allows the lazy
# load to skip validating the last attribute...
spec, _, path_to_spec = scaffold.load_dag()
scaffold.add(spec, path_to_spec)
code = Path(backup_test_pkg, 'functions.py').read_text()
module = ast.parse(code)
function_names = {
element.name
for element in module.body if hasattr(element, 'name')
}
assert 'my_new_function' in function_names
assert Path(backup_test_pkg, 'notebook.ipynb').exists()
assert Path(backup_test_pkg, 'notebook.py').exists()
def scaffold(conda, package, entry_point, empty):
"""Create new projects (if no pipeline.yaml exists) or add missings tasks
"""
template = '-e/--entry-point is not compatible with the {flag} flag'
if entry_point and conda:
raise click.ClickException(template.format(flag='--conda'))
if entry_point and package:
raise click.ClickException(template.format(flag='--package'))
if entry_point and empty:
raise click.ClickException(template.format(flag='--empty'))
# try to load a dag by looking in default places
if not entry_point:
loaded = _scaffold.load_dag()
else:
try:
loaded = DAGSpec(entry_point, lazy_import=True), Path(entry_point)
except Exception as e:
raise click.ClickException(e) from e
if loaded:
# add scaffold tasks
spec, path_to_spec = loaded
_scaffold.add(spec, path_to_spec)
else:
scaffold_project.cli(project_path=None,
conda=conda,
package=package,
empty=empty)
def test_add_task_when_using_import_tasks_from(tmp_directory):
spec = """
meta:
import_tasks_from: subdir/tasks.yaml
extract_product: True
tasks: []
"""
tasks = """
- source: notebook.py
"""
Path('pipeline.yaml').write_text(spec)
subdir = Path('subdir')
subdir.mkdir()
(subdir / 'tasks.yaml').write_text(tasks)
spec, path_to_spec = scaffold.load_dag()
scaffold.add(spec, path_to_spec)
assert (subdir / 'notebook.py').exists()
def scaffold(conda, package, entry_point, empty):
"""Create new projects (if no pipeline.yaml exists) or add missings tasks
"""
template = '-e/--entry-point is not compatible with the {flag} flag'
if entry_point and conda:
err = template.format(flag='--conda')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_conda_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)
if entry_point and package:
err = template.format(flag='--package')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_package_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)
if entry_point and empty:
err = template.format(flag='--empty')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_empty_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)
# try to load a dag by looking in default places
if entry_point is None:
loaded = _scaffold.load_dag()
else:
try:
loaded = (
DAGSpec(entry_point, lazy_import='skip'),
Path(entry_point).parent,
Path(entry_point),
)
except Exception as e:
telemetry.log_api("scaffold_error",
metadata={
'type': 'dag_load_failed',
'exception': e,
'argv': sys.argv
})
raise click.ClickException(e) from e
if loaded:
# existing pipeline, add tasks
spec, _, path_to_spec = loaded
_scaffold.add(spec, path_to_spec)
telemetry.log_api("ploomber_scaffold",
dag=loaded,
metadata={
'type': 'add_task',
'argv': sys.argv
})
else:
# no pipeline, create base project
telemetry.log_api("ploomber_scaffold",
metadata={
'type': 'base_project',
'argv': sys.argv
})
scaffold_project.cli(project_path=None,
conda=conda,
package=package,
empty=empty)