def data(tmpdir_factory, _cookiecutter_config_file, docker_compose):
p = tmpdir_factory.mktemp("data")
compose = p.join("docker-compose.yaml")
compose.write(docker_compose)
snakefile = p.join("Snakefile")
SNAKEFILE.copy(snakefile)
cluster_config = p.join("cluster-config.yaml")
CLUSTERCONFIG.copy(cluster_config)
template = os.path.join(os.path.abspath(os.path.dirname(__file__)),
os.pardir)
output_factory = tmpdir_factory.mktemp
c = Cookies(template, output_factory, _cookiecutter_config_file)
c._new_output_dir = lambda: str(p.join("slurm"))
c.bake(
extra_context={
'partition': 'normal',
'output': 'logs/slurm-%j.out',
'error': 'logs/slurm-%j.err'
})
# Advanced setting
c = Cookies(template, output_factory, _cookiecutter_config_file)
c._new_output_dir = lambda: str(p.join("slurm-advanced"))
c.bake(
extra_context={
'partition': 'normal',
'output': 'logs/slurm-%j.out',
'error': 'logs/slurm-%j.err',
'submit_script': 'slurm-submit-advanced.py'
})
return p
def _cookie_factory(sbatch_defaults=_sbatch_defaults,
advanced="no",
cluster_name=None,
cluster_config=None,
yamlconfig=_yamlconfig_default):
cookie_template = pjoin(os.path.abspath(pytest.dname), os.pardir)
output_factory = tmpdir_factory.mktemp
c = Cookies(cookie_template, output_factory, _cookiecutter_config_file)
c._new_output_dir = lambda: str(datadir)
extra_context = {
"sbatch_defaults": sbatch_defaults,
"advanced_argument_conversion": advanced
}
if cluster_name is not None:
extra_context["cluster_name"] = cluster_name
if cluster_config is not None:
extra_context["cluster_config"] = cluster_config
c.bake(extra_context=extra_context)
config = datadir.join("slurm").join("config.yaml")
config_d = dict([
tuple(line.split(":")) for line in config.read().split("\n")
if line != ""
])
config_d.update(**yamlconfig)
config.write("\n".join(f"{k}: {v}" for k, v in config_d.items()))
def test_project_id_hook(cookies: Cookies) -> None:
wrong_ids = [
"qwe/qwe",
"qwe?qwe",
"qwe!qwe",
"qwe.qwe",
"qwe%qwe",
"qwe-qwe",
"-qwe",
"qwe-",
"qwe-qwe",
"123",
"1 23",
"1qwe23",
]
correct_ids = [
"qwe",
"q",
"qwe_qwe",
"_qwe",
"qwe_",
"qwe123",
"qwe_123",
"qwe" * 20,
]
for id_ in wrong_ids:
result = cookies.bake(extra_context={"project_id": id_})
assert result.exit_code != 0, id_
assert isinstance(result.exception, FailedHookException)
for id_ in correct_ids:
result = cookies.bake(extra_context={"project_id": id_})
assert result.exit_code == 0, id_
def test_homepage_context(context: Dict[str, Any], expected: str,
cookies: Cookies) -> None:
"""Default homepage is generated from repository URL."""
result = cookies.bake(extra_context=context)
actual = result.context["project_homepage"]
assert actual == expected
def test_bake_project_and_run_tests(cookies: Cookies):
result = cookies.bake()
project_path_str = str(result.project)
with utils.use_cwd(project_path_str):
with utils.use_poetry():
utils.check_call("poetry run pytest")
def test_project_config_with_comments(cookies: Cookies, preserve_comments: str) -> None:
result = cookies.bake(
extra_context={
"project_dir": "project-with-comments",
"preserve Neuro Flow template hints": preserve_comments,
}
)
assert result.exit_code == 0
comment_sign = "#"
with inside_dir(str(result.project_path)):
live_file_content = Path(".neuro/live.yml").read_text()
project_file_content = Path(".neuro/project.yml").read_text()
l_com_exists = comment_sign in live_file_content
p_com_exists = comment_sign in project_file_content
if preserve_comments == "yes":
assert l_com_exists, ".neuro/live.yml file does not contain comments"
assert p_com_exists, ".neuro/project.yml file does not contain comments"
elif preserve_comments == "no":
assert not l_com_exists, ".neuro/live.yml file contains comments"
assert not p_com_exists, ".neuro/project.yml file contains comments"
else:
raise RuntimeError(
f"invalid value '{preserve_comments}' for 'preserve_comments' arg. "
" Only 'yes' and 'no' are allowed."
)
def test_prettier_format(cookies: Cookies) -> None:
"""Generated files must pass Prettier format checker."""
result = cookies.bake(extra_context={})
proj_dir = result.project_path
process = run_command(command="prettier --check .", work_dir=proj_dir)
assert process.returncode == 0, process.stderr.decode("utf-8")
def test_existing_paths(context: Dict[str, Any], paths: List[str],
cookies: Cookies) -> None:
"""Check that specific paths exist after scaffolding."""
result = cookies.bake(extra_context=context)
for path in paths:
file_path = result.project_path / path
assert file_path.exists()
def test_removed_paths(context: Dict[str, Any], paths: List[str],
cookies: Cookies) -> None:
"""Check that specific paths are removed after scaffolding."""
result = cookies.bake(extra_context=context)
for path in paths:
remove_path = result.project_path / path
assert not remove_path.exists()
def test_bake_project_with_default_options_works(cookies: Cookies):
result = cookies.bake()
assert result.exception is None
assert result.exit_code == 0
assert result.project.isdir()
assert result.project.listdir(), "Output dir should contain some files"
def test_flakehell_passes(cookies: Cookies, context: Dict[str, str],
context_override: Dict[str, str]) -> None:
"""Generated project should pass flakehell."""
result = cookies.bake(extra_context={**context, **context_override})
try:
# The black step is executed by the post hooks
# we need to run everything in the same step so that flakehell uses the
# virtualenv.
sh.bash(
"-c",
"virtualenv -p `which python3.7` env; "
"source env/bin/activate; "
"pip install pip-tools; "
"pip-compile -U --allow-unsafe setup.py; "
"pip-compile -U --allow-unsafe requirements-dev.in "
" --output-file requirements-dev.txt; "
"pip install -r requirements-dev.txt; "
"pip install -e .; "
"black --exclude env .; "
"flakehell lint src tests",
_cwd=str(result.project),
)
except sh.ErrorReturnCode as error:
pytest.fail(error.stdout.decode())
def test_project_dir_hook(cookies: Cookies) -> None:
result = cookies.bake(extra_context={"project_dir": "myproject"})
assert result.exit_code == 0
result = cookies.bake(extra_context={"project_dir": "my-project"})
assert result.exit_code == 0
result = cookies.bake(extra_context={"project_dir": "my?project"})
assert result.exit_code != 0
if sys.platform == "win32":
# Unfortunately, pre_gen hook is called before cookiecutter copies the template
# into the TMP dir for rendering.
# This will not hurt the user,
# but the error message will also include a traceback
assert isinstance(result.exception, OSError)
else:
assert isinstance(result.exception, FailedHookException)
result = cookies.bake(extra_context={"project_dir": "t" * 256})
assert result.exit_code != 0
def test_yamllint_passes(cookies: Cookies, context: Dict[str, str],
context_override: Dict[str, str]) -> None:
"""Generated project pass yamllint."""
result = cookies.bake(extra_context={**context, **context_override})
try:
sh.yamllint(".", _cwd=str(result.project))
except sh.ErrorReturnCode as error:
pytest.fail(error.stdout.decode())
def test_badges_separate_lines(context: Dict[str, Any],
cookies: Cookies) -> None:
"""Readme files must have all badge links on separate lines."""
result = cookies.bake(extra_context=context)
readme = result.project_path / "README.md"
regex = re.compile(r"img\.shields\.io")
for line in readme.read_text().split("\n"):
assert len(regex.findall(line)) < 2
def test_black_is_able_to_correct_files(
cookies: Cookies, context: Dict[str, str],
context_override: Dict[str, str]) -> None:
"""Black is run on the post hooks, make sure it runs without problem."""
result = cookies.bake(extra_context={**context, **context_override})
try:
sh.black("--exclude", "migrations", _cwd=str(result.project))
except sh.ErrorReturnCode as error:
pytest.fail(error.stdout.decode())
def bake_in_temp_dir(cookies: Cookies, *args, **kwargs):
"""
Delete the temporal directory that is created when executing the tests
:param cookies: pytest_cookies.Cookies,
cookie to be baked and its temporal files will be removed
"""
result = cookies.bake(*args, **kwargs)
try:
yield result
finally:
rmtree(str(result.project))
def data(tmpdir_factory, _cookiecutter_config_file):
p = tmpdir_factory.mktemp("data")
snakefile = p.join("Snakefile")
SNAKEFILE.copy(snakefile)
cluster_config = p.join("cluster-config.yaml")
CLUSTERCONFIG.copy(cluster_config)
template = os.path.join(os.path.abspath(os.path.dirname(__file__)),
os.pardir)
output_factory = tmpdir_factory.mktemp
defaults = "partition=normal output=logs/slurm-%j.out error=logs/slurm-%j.err"
c = Cookies(template, output_factory, _cookiecutter_config_file)
c._new_output_dir = lambda: str(p.join("slurm"))
c.bake(extra_context={"sbatch_defaults": defaults})
# Advanced setting
c = Cookies(template, output_factory, _cookiecutter_config_file)
c._new_output_dir = lambda: str(p.join("slurm-advanced"))
c.bake(extra_context={
"sbatch_defaults": defaults,
"advanced_argument_conversion": "yes",
})
return p
def get_cli(cookies: Cookies, context: Dict[str, str]) -> ModuleType:
result: Result = cookies.bake(extra_context=context)
project_path, project_slug, project_dir = project_info(result)
module_path: str = os.path.join(project_dir, 'cli.py')
module_name: str = '.'.join([project_slug, 'cli'])
spec: ModuleSpec = util.spec_from_file_location(module_name, module_path)
cli: ModuleType = util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(cli) # type: ignore
return cli
def test_text_existence(
context: Dict[str, Any],
paths: List[str],
text: str,
exist: bool,
cookies: Cookies,
) -> None:
"""Check for existence of text in files."""
result = cookies.bake(extra_context=context)
for path in paths:
text_exists = text in (result.project_path / path).read_text()
assert text_exists == exist
def bake_in_temp_dir(cookies: Cookies, *args: Any,
**kwargs: Dict[str, str]) -> Result:
"""
Delete the temporal directory that is created when executing the tests
:param cookies: pytest_cookies.Cookies,
cookie to be baked and its temporal files will be removed
"""
result = cookies.bake(*args, **kwargs)
# print('=' * 80 + '\n', "Result info:", repr(result), '\n' + ('=' * 80))
try:
yield result
finally:
rmtree(str(result.project))
def test_mkdocs_build(cookies: Cookies) -> None:
"""Mkdocs must be able to build documentation for baked project."""
result = cookies.bake(extra_context={})
expected = 0
process = run_command(command="poetry install",
work_dir=result.project_path)
# Poetry prints errors to stdout instead of stderr.
assert process.returncode == expected, process.stdout.decode("utf-8")
process = run_command(command="poetry run mkdocs build",
work_dir=result.project_path)
assert process.returncode == expected, process.stderr.decode("utf-8")
def test_project_generation_without_external_hooks(
cookies: Cookies, context: Dict[str, str],
context_override: Dict[str, str]) -> None:
"""Test that project is generated and fully rendered."""
result = cookies.bake(extra_context={**context, **context_override})
assert result.exit_code == 0
assert result.exception is None
assert result.project.basename == context_override.get(
"project_slug", context.get("project_slug", None))
assert result.project.isdir()
paths = build_files_list(str(result.project))
assert paths
check_paths(paths)
def test_pytest_test(cookies: Cookies) -> None:
"""Generated files must pass Pytest unit tests."""
result = cookies.bake(extra_context={})
expected = 0
process = run_command(command="poetry install",
work_dir=result.project_path)
# Poetry prints errors to stdout instead of stderr.
assert process.returncode == expected, process.stdout.decode("utf-8")
process = run_command(command="poetry run pytest",
work_dir=result.project_path)
# Pytest prints errors to stdout instead of stderr.
assert process.returncode == expected, process.stdout.decode("utf-8")
def test_project_description(cookies: Cookies) -> None:
descriptions = [
# " ",
"Descrition!",
"123",
"https://github.com/neuro-inc/cookiecutter-neuro-project/",
]
for descr in descriptions:
result = cookies.bake(extra_context={"project_description": descr})
assert result.exit_code == 0, descr
with inside_dir(str(result.project_path)):
readme_content = Path("README.md").read_text()
if descr:
assert "## Project description" in readme_content
assert descr in readme_content
def bake_in_temp_dir(cookies: Cookies, *args: Any, **kwargs: Any) -> Result:
"""Delete the temporal directory that is created when executing the tests.
Args:
cookies: A cookie to be baked and its temporal files will be removed.
*args: Variable length argument list to be passed to the bake command.
**kwargs: Arbitrary keyword arguments to be passed to the bake command.
Yields:
The baked cookie.
"""
result = cookies.bake(*args, **kwargs)
try:
yield result
finally:
rmtree(str(result.project_path))
def test_generated_package_is_installable(cookies: Cookies,
context: Dict[str, str]) -> None:
"""Generated project should be able to build the documentation."""
result = cookies.bake(extra_context={**context})
try:
result = sh.bash(
"-c",
"virtualenv -p `which python3.7` env; "
"source env/bin/activate; "
"pip install -e .; "
"python -c 'import my_test_project'",
_cwd=str(result.project),
)
except sh.ErrorReturnCode as error:
pytest.fail(error.stderr.decode())
def test_mkdocs_build_valid_site(cookies: Cookies, context: Dict[str,
str]) -> None:
"""Generated project should be able to build the documentation."""
result = cookies.bake(extra_context={**context})
try:
result = sh.bash(
"-c",
"virtualenv -p `which python3.7` env; "
"source env/bin/activate; "
"pip install pip-tools; "
"pip-compile --allow-unsafe; "
"pip-compile docs/requirements.in --output-file docs/requirements.txt; "
"pip install -r docs/requirements.txt; "
"pip install -e .; "
"mkdocs build",
_cwd=str(result.project),
)
except sh.ErrorReturnCode as error:
pytest.fail(error.stderr.decode())
def test_neuro_flow_live(cookies: Cookies, preserve_comments: str) -> None:
result = cookies.bake(
extra_context={
"project_dir": "test-project",
"project_id": "awesome_project",
"preserve Neuro Flow template hints": preserve_comments,
}
)
with inside_dir(str(result.project_path)):
proc = exec("neuro-flow --show-traceback ps")
assert "JOB" in proc.stdout, proc
proc = exec("neuro-flow --show-traceback status train", assert_exit_code=False)
assert "is not running" in proc.stdout, proc
proc = exec("neuro-flow --show-traceback run --dry-run train")
assert "neuro run" in proc.stdout, proc
assert "--tag=project:awesome-project" in proc.stdout, proc
proc = exec("neuro-flow --show-traceback run --dry-run remote_debug")
assert "neuro run" in proc.stdout, proc
assert "--tag=project:awesome-project" in proc.stdout, proc
def test_run_flake8(cookies: Cookies) -> None:
result = cookies.bake(extra_context={"project_dir": "flake8-compat"})
with inside_dir(str(result.project_path)):
subprocess.check_call(["flake8"])
def test_project_tree(cookies: Cookies) -> None:
result = cookies.bake(extra_context={"project_dir": "test-project"})
assert result.exit_code == 0
assert result.exception is None
assert result.project_path.name == "test-project"