def test_with_pip(tmp_directory, package): scaffold.cli(project_path='myproj', conda=False, package=package) os.chdir('myproj') readme = Path('README.md').read_text() assert 'Requires [Miniconda]' not in readme assert 'source {path-to-venv}/bin/activate' in readme
def scaffold(conda, package, entry_point, empty): """Create new projects (if no pipeline.yaml exists) or add missings tasks """ template = '-e/--entry-point is not compatible with the {flag} flag' if entry_point and conda: raise click.ClickException(template.format(flag='--conda')) if entry_point and package: raise click.ClickException(template.format(flag='--package')) if entry_point and empty: raise click.ClickException(template.format(flag='--empty')) # try to load a dag by looking in default places if not entry_point: loaded = _scaffold.load_dag() else: try: loaded = DAGSpec(entry_point, lazy_import=True), Path(entry_point) except Exception as e: raise click.ClickException(e) from e if loaded: # add scaffold tasks spec, path_to_spec = loaded _scaffold.add(spec, path_to_spec) else: scaffold_project.cli(project_path=None, conda=conda, package=package, empty=empty)
def scaffold(): """Create new projects and add template tasks """ if Path('pipeline.yaml').exists(): _scaffold.add() else: scaffold_project.cli(project_path=None)
def setup_env(request, tmp_path_factory): """ Configures environment. This takes a while, to re-use existing conda env call: pytest --cache-env """ tmp_target = tmp_path_factory.mktemp('session-wide-tmp-simple') old = os.getcwd() os.chdir(tmp_target) scaffold.cli(project_path='my_simple_project', conda=False, package=False) os.chdir('my_simple_project') egg_info = Path('src', 'package_name.egg-info') if egg_info.exists(): shutil.rmtree(egg_info) if request.config.getoption("--cache-env"): print('Using cached env...') else: install.main(use_lock=False) # versioneer depends on this run(""" git init git config user.email "*****@*****.**" git config user.name "Your Name" git add --all git commit -m "my first commit" """) yield tmp_target os.chdir(old)
def test_with_conda(tmp_directory, package): scaffold.cli(project_path='myproj', conda=True, package=package) os.chdir('myproj') readme = Path('README.md').read_text() conda_msg = ('# activate conda environment\n' 'conda activate myproj') assert 'Requires [Miniconda]' in readme assert conda_msg in readme
def test_output_message(tmp_directory, capsys, conda, package, deps, expected_pipeline): scaffold.cli(project_path='myproj', conda=conda, package=package) captured = capsys.readouterr() assert f'Pipeline at: {expected_pipeline}' in captured.out assert f'Add dependencies to {deps}' in captured.out
def test_folder_layout(tmp_directory, conda): scaffold.cli(project_path='myproj', conda=conda, package=False) os.chdir('myproj') readme = Path('README.md').read_text() assert Path('pipeline.yaml').is_file() assert not Path('tests').exists() assert '## Testing\n\n```sh\npytest\n```' not in readme
def test_rejects_dashes_if_package(tmp_directory): with pytest.raises(ScaffoldError) as excinfo: scaffold.cli(project_path='my-project', conda=False, package=True, empty=True) assert 'is not a valid name' in str(excinfo.value)
def test_empty_no_package(tmp_directory): scaffold.cli(project_path='myproj', conda=False, package=False, empty=True) expected = ( 'tasks:\n # Add tasks here...\n\n # Example\n # - ' 'source: path/to/script.py\n # product: products/report.ipynb\n') assert not Path('myproj', 'tasks').exists() assert not Path('myproj', 'scripts').exists() assert Path('myproj', 'pipeline.yaml').read_text() == expected
def test_doesnt_show_instructions_if_name_passed(monkeypatch, tmp_directory): mock = Mock() monkeypatch.setattr(scaffold, '_echo_instructions', mock) scaffold.cli(project_path='my-project', conda=False, package=False, empty=True) mock.assert_not_called()
def test_determines_what_reqs_file_to_use_if_conda_not_installed( monkeypatch, tmp_directory): monkeypatch.setattr(scaffold.shutil, 'which', lambda _: None) scaffold.cli(project_path='my-project', conda=None, package=False, empty=True) assert Path('my-project', 'requirements.txt').exists() assert not Path('my-project', 'environment.yml').exists()
def test_shows_instructions_if_incorrect_name_passed(monkeypatch, tmp_directory): mock = Mock() monkeypatch.setattr(scaffold, '_get_instructions', mock) with pytest.raises(ScaffoldError): scaffold.cli(project_path='my project', conda=False, package=False, empty=True) mock.assert_called_once()
def test_empty_package(tmp_directory): scaffold.cli(project_path='myproj', conda=False, package=True, empty=True) expected = ('meta:\n # paths in task sources (e.g., scripts/fit.py)' ' are relative to src/package_name\n source_loader:\n ' 'module: package_name\n\ntasks:\n # Add tasks here...\n\n ' '# Example\n # - source: scripts/fit.py\n # product: ' 'products/report.ipynb\n') pkg_root = Path('myproj', 'src', 'myproj') assert not Path(pkg_root, 'tasks').exists() assert not Path(pkg_root, 'scripts').exists() assert Path(pkg_root, 'pipeline.yaml').read_text() == expected
def test_conda(tmp_directory): """ Instead of running the same tests with conda (takes too long), we check the environments are equivalent """ scaffold.cli(project_path='with_conda', conda=True, package=True) scaffold.cli(project_path='with_pip', conda=False, package=True) def _get_deps_from_env(name): env = yaml.safe_load(Path('with_conda', name).read_text()) return set(env['dependencies'][-1]['pip']) def _get_deps_from_reqs(name): lines = Path('with_pip', name).read_text().splitlines() return set(line for line in lines if not line.startswith('#')) env = _get_deps_from_env('environment.yml') env_dev = _get_deps_from_env('environment.dev.yml') req = _get_deps_from_reqs('requirements.txt') req_dev = _get_deps_from_reqs('requirements.dev.txt') assert env == req assert env_dev == req_dev
def scaffold(conda, package, entry_point, empty): """Create new projects (if no pipeline.yaml exists) or add missings tasks """ template = '-e/--entry-point is not compatible with the {flag} flag' if entry_point and conda: err = template.format(flag='--conda') telemetry.log_api("scaffold_error", metadata={ 'type': 'entry_and_conda_flag', 'exception': err, 'argv': sys.argv }) raise click.ClickException(err) if entry_point and package: err = template.format(flag='--package') telemetry.log_api("scaffold_error", metadata={ 'type': 'entry_and_package_flag', 'exception': err, 'argv': sys.argv }) raise click.ClickException(err) if entry_point and empty: err = template.format(flag='--empty') telemetry.log_api("scaffold_error", metadata={ 'type': 'entry_and_empty_flag', 'exception': err, 'argv': sys.argv }) raise click.ClickException(err) # try to load a dag by looking in default places if entry_point is None: loaded = _scaffold.load_dag() else: try: loaded = ( DAGSpec(entry_point, lazy_import='skip'), Path(entry_point).parent, Path(entry_point), ) except Exception as e: telemetry.log_api("scaffold_error", metadata={ 'type': 'dag_load_failed', 'exception': e, 'argv': sys.argv }) raise click.ClickException(e) from e if loaded: # existing pipeline, add tasks spec, _, path_to_spec = loaded _scaffold.add(spec, path_to_spec) telemetry.log_api("ploomber_scaffold", dag=loaded, metadata={ 'type': 'add_task', 'argv': sys.argv }) else: # no pipeline, create base project telemetry.log_api("ploomber_scaffold", metadata={ 'type': 'base_project', 'argv': sys.argv }) scaffold_project.cli(project_path=None, conda=conda, package=package, empty=empty)
def test_check_layout(tmp_directory, conda): scaffold.cli(project_path='myproj', conda=conda, package=True) os.chdir('myproj') assert Path('src/myproj/pipeline.yaml').is_file()
def test_accepts_dashes_if_non_package(tmp_directory, name, package): scaffold.cli(project_path=name, conda=False, package=package, empty=True)