def test_with_pip(tmp_directory, package):
scaffold.cli(project_path='myproj', conda=False, package=package)
os.chdir('myproj')
readme = Path('README.md').read_text()
assert 'Requires [Miniconda]' not in readme
assert 'source {path-to-venv}/bin/activate' in readme
def scaffold(conda, package, entry_point, empty):
"""Create new projects (if no pipeline.yaml exists) or add missings tasks
"""
template = '-e/--entry-point is not compatible with the {flag} flag'
if entry_point and conda:
raise click.ClickException(template.format(flag='--conda'))
if entry_point and package:
raise click.ClickException(template.format(flag='--package'))
if entry_point and empty:
raise click.ClickException(template.format(flag='--empty'))
# try to load a dag by looking in default places
if not entry_point:
loaded = _scaffold.load_dag()
else:
try:
loaded = DAGSpec(entry_point, lazy_import=True), Path(entry_point)
except Exception as e:
raise click.ClickException(e) from e
if loaded:
# add scaffold tasks
spec, path_to_spec = loaded
_scaffold.add(spec, path_to_spec)
else:
scaffold_project.cli(project_path=None,
conda=conda,
package=package,
empty=empty)
def scaffold():
"""Create new projects and add template tasks
"""
if Path('pipeline.yaml').exists():
_scaffold.add()
else:
scaffold_project.cli(project_path=None)
def setup_env(request, tmp_path_factory):
"""
Configures environment. This takes a while, to re-use existing conda env
call: pytest --cache-env
"""
tmp_target = tmp_path_factory.mktemp('session-wide-tmp-simple')
old = os.getcwd()
os.chdir(tmp_target)
scaffold.cli(project_path='my_simple_project', conda=False, package=False)
os.chdir('my_simple_project')
egg_info = Path('src', 'package_name.egg-info')
if egg_info.exists():
shutil.rmtree(egg_info)
if request.config.getoption("--cache-env"):
print('Using cached env...')
else:
install.main(use_lock=False)
# versioneer depends on this
run("""
git init
git config user.email "*****@*****.**"
git config user.name "Your Name"
git add --all
git commit -m "my first commit"
""")
yield tmp_target
os.chdir(old)
def test_with_conda(tmp_directory, package):
scaffold.cli(project_path='myproj', conda=True, package=package)
os.chdir('myproj')
readme = Path('README.md').read_text()
conda_msg = ('# activate conda environment\n' 'conda activate myproj')
assert 'Requires [Miniconda]' in readme
assert conda_msg in readme
def test_output_message(tmp_directory, capsys, conda, package, deps,
expected_pipeline):
scaffold.cli(project_path='myproj', conda=conda, package=package)
captured = capsys.readouterr()
assert f'Pipeline at: {expected_pipeline}' in captured.out
assert f'Add dependencies to {deps}' in captured.out
def test_folder_layout(tmp_directory, conda):
scaffold.cli(project_path='myproj', conda=conda, package=False)
os.chdir('myproj')
readme = Path('README.md').read_text()
assert Path('pipeline.yaml').is_file()
assert not Path('tests').exists()
assert '## Testing\n\n```sh\npytest\n```' not in readme
def test_rejects_dashes_if_package(tmp_directory):
with pytest.raises(ScaffoldError) as excinfo:
scaffold.cli(project_path='my-project',
conda=False,
package=True,
empty=True)
assert 'is not a valid name' in str(excinfo.value)
def test_empty_no_package(tmp_directory):
scaffold.cli(project_path='myproj', conda=False, package=False, empty=True)
expected = (
'tasks:\n # Add tasks here...\n\n # Example\n # - '
'source: path/to/script.py\n # product: products/report.ipynb\n')
assert not Path('myproj', 'tasks').exists()
assert not Path('myproj', 'scripts').exists()
assert Path('myproj', 'pipeline.yaml').read_text() == expected
def test_doesnt_show_instructions_if_name_passed(monkeypatch, tmp_directory):
mock = Mock()
monkeypatch.setattr(scaffold, '_echo_instructions', mock)
scaffold.cli(project_path='my-project',
conda=False,
package=False,
empty=True)
mock.assert_not_called()
def test_determines_what_reqs_file_to_use_if_conda_not_installed(
monkeypatch, tmp_directory):
monkeypatch.setattr(scaffold.shutil, 'which', lambda _: None)
scaffold.cli(project_path='my-project',
conda=None,
package=False,
empty=True)
assert Path('my-project', 'requirements.txt').exists()
assert not Path('my-project', 'environment.yml').exists()
def test_shows_instructions_if_incorrect_name_passed(monkeypatch,
tmp_directory):
mock = Mock()
monkeypatch.setattr(scaffold, '_get_instructions', mock)
with pytest.raises(ScaffoldError):
scaffold.cli(project_path='my project',
conda=False,
package=False,
empty=True)
mock.assert_called_once()
def test_empty_package(tmp_directory):
scaffold.cli(project_path='myproj', conda=False, package=True, empty=True)
expected = ('meta:\n # paths in task sources (e.g., scripts/fit.py)'
' are relative to src/package_name\n source_loader:\n '
'module: package_name\n\ntasks:\n # Add tasks here...\n\n '
'# Example\n # - source: scripts/fit.py\n # product: '
'products/report.ipynb\n')
pkg_root = Path('myproj', 'src', 'myproj')
assert not Path(pkg_root, 'tasks').exists()
assert not Path(pkg_root, 'scripts').exists()
assert Path(pkg_root, 'pipeline.yaml').read_text() == expected
def test_conda(tmp_directory):
"""
Instead of running the same tests with conda (takes too long), we check the
environments are equivalent
"""
scaffold.cli(project_path='with_conda', conda=True, package=True)
scaffold.cli(project_path='with_pip', conda=False, package=True)
def _get_deps_from_env(name):
env = yaml.safe_load(Path('with_conda', name).read_text())
return set(env['dependencies'][-1]['pip'])
def _get_deps_from_reqs(name):
lines = Path('with_pip', name).read_text().splitlines()
return set(line for line in lines if not line.startswith('#'))
env = _get_deps_from_env('environment.yml')
env_dev = _get_deps_from_env('environment.dev.yml')
req = _get_deps_from_reqs('requirements.txt')
req_dev = _get_deps_from_reqs('requirements.dev.txt')
assert env == req
assert env_dev == req_dev
def scaffold(conda, package, entry_point, empty):
"""Create new projects (if no pipeline.yaml exists) or add missings tasks
"""
template = '-e/--entry-point is not compatible with the {flag} flag'
if entry_point and conda:
err = template.format(flag='--conda')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_conda_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)
if entry_point and package:
err = template.format(flag='--package')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_package_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)
if entry_point and empty:
err = template.format(flag='--empty')
telemetry.log_api("scaffold_error",
metadata={
'type': 'entry_and_empty_flag',
'exception': err,
'argv': sys.argv
})
raise click.ClickException(err)
# try to load a dag by looking in default places
if entry_point is None:
loaded = _scaffold.load_dag()
else:
try:
loaded = (
DAGSpec(entry_point, lazy_import='skip'),
Path(entry_point).parent,
Path(entry_point),
)
except Exception as e:
telemetry.log_api("scaffold_error",
metadata={
'type': 'dag_load_failed',
'exception': e,
'argv': sys.argv
})
raise click.ClickException(e) from e
if loaded:
# existing pipeline, add tasks
spec, _, path_to_spec = loaded
_scaffold.add(spec, path_to_spec)
telemetry.log_api("ploomber_scaffold",
dag=loaded,
metadata={
'type': 'add_task',
'argv': sys.argv
})
else:
# no pipeline, create base project
telemetry.log_api("ploomber_scaffold",
metadata={
'type': 'base_project',
'argv': sys.argv
})
scaffold_project.cli(project_path=None,
conda=conda,
package=package,
empty=empty)
def test_check_layout(tmp_directory, conda):
scaffold.cli(project_path='myproj', conda=conda, package=True)
os.chdir('myproj')
assert Path('src/myproj/pipeline.yaml').is_file()
def test_accepts_dashes_if_non_package(tmp_directory, name, package):
scaffold.cli(project_path=name, conda=False, package=package, empty=True)
