Esempio n. 1
0
def data(tmpdir_factory, _cookiecutter_config_file, docker_compose):
    p = tmpdir_factory.mktemp("data")
    compose = p.join("docker-compose.yaml")
    compose.write(docker_compose)
    snakefile = p.join("Snakefile")
    SNAKEFILE.copy(snakefile)
    cluster_config = p.join("cluster-config.yaml")
    CLUSTERCONFIG.copy(cluster_config)
    template = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                            os.pardir)
    output_factory = tmpdir_factory.mktemp
    c = Cookies(template, output_factory, _cookiecutter_config_file)
    c._new_output_dir = lambda: str(p.join("slurm"))
    c.bake(
        extra_context={
            'partition': 'normal',
            'output': 'logs/slurm-%j.out',
            'error': 'logs/slurm-%j.err'
        })
    # Advanced setting
    c = Cookies(template, output_factory, _cookiecutter_config_file)
    c._new_output_dir = lambda: str(p.join("slurm-advanced"))
    c.bake(
        extra_context={
            'partition': 'normal',
            'output': 'logs/slurm-%j.out',
            'error': 'logs/slurm-%j.err',
            'submit_script': 'slurm-submit-advanced.py'
        })
    return p
Esempio n. 2
0
 def _cookie_factory(sbatch_defaults=_sbatch_defaults,
                     advanced="no",
                     cluster_name=None,
                     cluster_config=None,
                     yamlconfig=_yamlconfig_default):
     cookie_template = pjoin(os.path.abspath(pytest.dname), os.pardir)
     output_factory = tmpdir_factory.mktemp
     c = Cookies(cookie_template, output_factory, _cookiecutter_config_file)
     c._new_output_dir = lambda: str(datadir)
     extra_context = {
         "sbatch_defaults": sbatch_defaults,
         "advanced_argument_conversion": advanced
     }
     if cluster_name is not None:
         extra_context["cluster_name"] = cluster_name
     if cluster_config is not None:
         extra_context["cluster_config"] = cluster_config
     c.bake(extra_context=extra_context)
     config = datadir.join("slurm").join("config.yaml")
     config_d = dict([
         tuple(line.split(":")) for line in config.read().split("\n")
         if line != ""
     ])
     config_d.update(**yamlconfig)
     config.write("\n".join(f"{k}: {v}" for k, v in config_d.items()))
def test_project_id_hook(cookies: Cookies) -> None:
    wrong_ids = [
        "qwe/qwe",
        "qwe?qwe",
        "qwe!qwe",
        "qwe.qwe",
        "qwe%qwe",
        "qwe-qwe",
        "-qwe",
        "qwe-",
        "qwe-qwe",
        "123",
        "1 23",
        "1qwe23",
    ]
    correct_ids = [
        "qwe",
        "q",
        "qwe_qwe",
        "_qwe",
        "qwe_",
        "qwe123",
        "qwe_123",
        "qwe" * 20,
    ]
    for id_ in wrong_ids:
        result = cookies.bake(extra_context={"project_id": id_})
        assert result.exit_code != 0, id_
        assert isinstance(result.exception, FailedHookException)
    for id_ in correct_ids:
        result = cookies.bake(extra_context={"project_id": id_})
        assert result.exit_code == 0, id_
def test_homepage_context(context: Dict[str, Any], expected: str,
                          cookies: Cookies) -> None:
    """Default homepage is generated from repository URL."""
    result = cookies.bake(extra_context=context)

    actual = result.context["project_homepage"]
    assert actual == expected
def test_bake_project_and_run_tests(cookies: Cookies):
    result = cookies.bake()
    project_path_str = str(result.project)

    with utils.use_cwd(project_path_str):
        with utils.use_poetry():
            utils.check_call("poetry run pytest")
def test_project_config_with_comments(cookies: Cookies, preserve_comments: str) -> None:
    result = cookies.bake(
        extra_context={
            "project_dir": "project-with-comments",
            "preserve Neuro Flow template hints": preserve_comments,
        }
    )
    assert result.exit_code == 0
    comment_sign = "#"
    with inside_dir(str(result.project_path)):
        live_file_content = Path(".neuro/live.yml").read_text()
        project_file_content = Path(".neuro/project.yml").read_text()
        l_com_exists = comment_sign in live_file_content
        p_com_exists = comment_sign in project_file_content
        if preserve_comments == "yes":
            assert l_com_exists, ".neuro/live.yml file does not contain comments"
            assert p_com_exists, ".neuro/project.yml file does not contain comments"
        elif preserve_comments == "no":
            assert not l_com_exists, ".neuro/live.yml file contains comments"
            assert not p_com_exists, ".neuro/project.yml file contains comments"
        else:
            raise RuntimeError(
                f"invalid value '{preserve_comments}' for 'preserve_comments' arg. "
                " Only 'yes' and 'no' are allowed."
            )
def test_prettier_format(cookies: Cookies) -> None:
    """Generated files must pass Prettier format checker."""
    result = cookies.bake(extra_context={})
    proj_dir = result.project_path

    process = run_command(command="prettier --check .", work_dir=proj_dir)
    assert process.returncode == 0, process.stderr.decode("utf-8")
def test_existing_paths(context: Dict[str, Any], paths: List[str],
                        cookies: Cookies) -> None:
    """Check that specific paths exist after scaffolding."""
    result = cookies.bake(extra_context=context)
    for path in paths:
        file_path = result.project_path / path
        assert file_path.exists()
def test_removed_paths(context: Dict[str, Any], paths: List[str],
                       cookies: Cookies) -> None:
    """Check that specific paths are removed after scaffolding."""
    result = cookies.bake(extra_context=context)
    for path in paths:
        remove_path = result.project_path / path
        assert not remove_path.exists()
def test_bake_project_with_default_options_works(cookies: Cookies):
    result = cookies.bake()

    assert result.exception is None
    assert result.exit_code == 0
    assert result.project.isdir()
    assert result.project.listdir(), "Output dir should contain some files"
def test_flakehell_passes(cookies: Cookies, context: Dict[str, str],
                          context_override: Dict[str, str]) -> None:
    """Generated project should pass flakehell."""
    result = cookies.bake(extra_context={**context, **context_override})

    try:
        # The black step is executed by the post hooks
        # we need to run everything in the same step so that flakehell uses the
        # virtualenv.
        sh.bash(
            "-c",
            "virtualenv -p `which python3.7` env; "
            "source env/bin/activate; "
            "pip install pip-tools; "
            "pip-compile -U --allow-unsafe setup.py; "
            "pip-compile -U --allow-unsafe requirements-dev.in "
            "   --output-file requirements-dev.txt; "
            "pip install -r requirements-dev.txt; "
            "pip install -e .; "
            "black --exclude env .; "
            "flakehell lint src tests",
            _cwd=str(result.project),
        )
    except sh.ErrorReturnCode as error:
        pytest.fail(error.stdout.decode())
def test_project_dir_hook(cookies: Cookies) -> None:
    result = cookies.bake(extra_context={"project_dir": "myproject"})
    assert result.exit_code == 0
    result = cookies.bake(extra_context={"project_dir": "my-project"})
    assert result.exit_code == 0
    result = cookies.bake(extra_context={"project_dir": "my?project"})
    assert result.exit_code != 0
    if sys.platform == "win32":
        # Unfortunately, pre_gen hook is called before cookiecutter copies the template
        #  into the TMP dir for rendering.
        # This will not hurt the user,
        #  but the error message will also include a traceback
        assert isinstance(result.exception, OSError)
    else:
        assert isinstance(result.exception, FailedHookException)
    result = cookies.bake(extra_context={"project_dir": "t" * 256})
    assert result.exit_code != 0
def test_yamllint_passes(cookies: Cookies, context: Dict[str, str],
                         context_override: Dict[str, str]) -> None:
    """Generated project pass yamllint."""
    result = cookies.bake(extra_context={**context, **context_override})

    try:
        sh.yamllint(".", _cwd=str(result.project))
    except sh.ErrorReturnCode as error:
        pytest.fail(error.stdout.decode())
def test_badges_separate_lines(context: Dict[str, Any],
                               cookies: Cookies) -> None:
    """Readme files must have all badge links on separate lines."""
    result = cookies.bake(extra_context=context)
    readme = result.project_path / "README.md"

    regex = re.compile(r"img\.shields\.io")
    for line in readme.read_text().split("\n"):
        assert len(regex.findall(line)) < 2
def test_black_is_able_to_correct_files(
        cookies: Cookies, context: Dict[str, str],
        context_override: Dict[str, str]) -> None:
    """Black is run on the post hooks, make sure it runs without problem."""
    result = cookies.bake(extra_context={**context, **context_override})

    try:
        sh.black("--exclude", "migrations", _cwd=str(result.project))
    except sh.ErrorReturnCode as error:
        pytest.fail(error.stdout.decode())
def bake_in_temp_dir(cookies: Cookies, *args, **kwargs):
    """
    Delete the temporal directory that is created when executing the tests
    :param cookies: pytest_cookies.Cookies,
        cookie to be baked and its temporal files will be removed
    """
    result = cookies.bake(*args, **kwargs)
    try:
        yield result
    finally:
        rmtree(str(result.project))
Esempio n. 17
0
def data(tmpdir_factory, _cookiecutter_config_file):
    p = tmpdir_factory.mktemp("data")
    snakefile = p.join("Snakefile")
    SNAKEFILE.copy(snakefile)
    cluster_config = p.join("cluster-config.yaml")
    CLUSTERCONFIG.copy(cluster_config)
    template = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                            os.pardir)
    output_factory = tmpdir_factory.mktemp
    defaults = "partition=normal output=logs/slurm-%j.out error=logs/slurm-%j.err"
    c = Cookies(template, output_factory, _cookiecutter_config_file)
    c._new_output_dir = lambda: str(p.join("slurm"))
    c.bake(extra_context={"sbatch_defaults": defaults})
    # Advanced setting
    c = Cookies(template, output_factory, _cookiecutter_config_file)
    c._new_output_dir = lambda: str(p.join("slurm-advanced"))
    c.bake(extra_context={
        "sbatch_defaults": defaults,
        "advanced_argument_conversion": "yes",
    })
    return p
def get_cli(cookies: Cookies, context: Dict[str, str]) -> ModuleType:
    result: Result = cookies.bake(extra_context=context)
    project_path, project_slug, project_dir = project_info(result)
    module_path: str = os.path.join(project_dir, 'cli.py')
    module_name: str = '.'.join([project_slug, 'cli'])
    spec: ModuleSpec = util.spec_from_file_location(module_name, module_path)
    cli: ModuleType = util.module_from_spec(spec)

    assert spec.loader is not None
    spec.loader.exec_module(cli)  # type: ignore

    return cli
def test_text_existence(
    context: Dict[str, Any],
    paths: List[str],
    text: str,
    exist: bool,
    cookies: Cookies,
) -> None:
    """Check for existence of text in files."""
    result = cookies.bake(extra_context=context)
    for path in paths:
        text_exists = text in (result.project_path / path).read_text()
        assert text_exists == exist
def bake_in_temp_dir(cookies: Cookies, *args: Any,
                     **kwargs: Dict[str, str]) -> Result:
    """
    Delete the temporal directory that is created when executing the tests
    :param cookies: pytest_cookies.Cookies,
        cookie to be baked and its temporal files will be removed
    """
    result = cookies.bake(*args, **kwargs)
    # print('=' * 80 + '\n', "Result info:", repr(result), '\n' + ('=' * 80))
    try:
        yield result
    finally:
        rmtree(str(result.project))
def test_mkdocs_build(cookies: Cookies) -> None:
    """Mkdocs must be able to build documentation for baked project."""
    result = cookies.bake(extra_context={})
    expected = 0

    process = run_command(command="poetry install",
                          work_dir=result.project_path)
    # Poetry prints errors to stdout instead of stderr.
    assert process.returncode == expected, process.stdout.decode("utf-8")

    process = run_command(command="poetry run mkdocs build",
                          work_dir=result.project_path)
    assert process.returncode == expected, process.stderr.decode("utf-8")
def test_project_generation_without_external_hooks(
        cookies: Cookies, context: Dict[str, str],
        context_override: Dict[str, str]) -> None:
    """Test that project is generated and fully rendered."""
    result = cookies.bake(extra_context={**context, **context_override})

    assert result.exit_code == 0
    assert result.exception is None
    assert result.project.basename == context_override.get(
        "project_slug", context.get("project_slug", None))
    assert result.project.isdir()
    paths = build_files_list(str(result.project))
    assert paths
    check_paths(paths)
def test_pytest_test(cookies: Cookies) -> None:
    """Generated files must pass Pytest unit tests."""
    result = cookies.bake(extra_context={})
    expected = 0

    process = run_command(command="poetry install",
                          work_dir=result.project_path)
    # Poetry prints errors to stdout instead of stderr.
    assert process.returncode == expected, process.stdout.decode("utf-8")

    process = run_command(command="poetry run pytest",
                          work_dir=result.project_path)
    # Pytest prints errors to stdout instead of stderr.
    assert process.returncode == expected, process.stdout.decode("utf-8")
def test_project_description(cookies: Cookies) -> None:
    descriptions = [
        # " ",
        "Descrition!",
        "123",
        "https://github.com/neuro-inc/cookiecutter-neuro-project/",
    ]
    for descr in descriptions:
        result = cookies.bake(extra_context={"project_description": descr})
        assert result.exit_code == 0, descr
        with inside_dir(str(result.project_path)):
            readme_content = Path("README.md").read_text()
            if descr:
                assert "## Project description" in readme_content
                assert descr in readme_content
Esempio n. 25
0
def bake_in_temp_dir(cookies: Cookies, *args: Any, **kwargs: Any) -> Result:
    """Delete the temporal directory that is created when executing the tests.

    Args:
        cookies: A cookie to be baked and its temporal files will be removed.
        *args: Variable length argument list to be passed to the bake command.
        **kwargs: Arbitrary keyword arguments to be passed to the bake command.

    Yields:
        The baked cookie.
    """
    result = cookies.bake(*args, **kwargs)
    try:
        yield result
    finally:
        rmtree(str(result.project_path))
def test_generated_package_is_installable(cookies: Cookies,
                                          context: Dict[str, str]) -> None:
    """Generated project should be able to build the documentation."""
    result = cookies.bake(extra_context={**context})

    try:
        result = sh.bash(
            "-c",
            "virtualenv -p `which python3.7` env; "
            "source env/bin/activate; "
            "pip install -e .; "
            "python -c 'import my_test_project'",
            _cwd=str(result.project),
        )
    except sh.ErrorReturnCode as error:
        pytest.fail(error.stderr.decode())
def test_mkdocs_build_valid_site(cookies: Cookies, context: Dict[str,
                                                                 str]) -> None:
    """Generated project should be able to build the documentation."""
    result = cookies.bake(extra_context={**context})

    try:
        result = sh.bash(
            "-c",
            "virtualenv -p `which python3.7` env; "
            "source env/bin/activate; "
            "pip install pip-tools; "
            "pip-compile --allow-unsafe; "
            "pip-compile docs/requirements.in --output-file docs/requirements.txt; "
            "pip install -r docs/requirements.txt; "
            "pip install -e .; "
            "mkdocs build",
            _cwd=str(result.project),
        )
    except sh.ErrorReturnCode as error:
        pytest.fail(error.stderr.decode())
Esempio n. 28
0
def test_neuro_flow_live(cookies: Cookies, preserve_comments: str) -> None:
    result = cookies.bake(
        extra_context={
            "project_dir": "test-project",
            "project_id": "awesome_project",
            "preserve Neuro Flow template hints": preserve_comments,
        }
    )
    with inside_dir(str(result.project_path)):
        proc = exec("neuro-flow --show-traceback ps")
        assert "JOB" in proc.stdout, proc

        proc = exec("neuro-flow --show-traceback status train", assert_exit_code=False)
        assert "is not running" in proc.stdout, proc

        proc = exec("neuro-flow --show-traceback run --dry-run train")
        assert "neuro run" in proc.stdout, proc
        assert "--tag=project:awesome-project" in proc.stdout, proc

        proc = exec("neuro-flow --show-traceback run --dry-run remote_debug")
        assert "neuro run" in proc.stdout, proc
        assert "--tag=project:awesome-project" in proc.stdout, proc
def test_run_flake8(cookies: Cookies) -> None:
    result = cookies.bake(extra_context={"project_dir": "flake8-compat"})
    with inside_dir(str(result.project_path)):
        subprocess.check_call(["flake8"])
def test_project_tree(cookies: Cookies) -> None:
    result = cookies.bake(extra_context={"project_dir": "test-project"})
    assert result.exit_code == 0
    assert result.exception is None
    assert result.project_path.name == "test-project"