def _parse_pyproject_toml(platform: str,
include_dev_dependencies: bool) -> LockSpecification:
specs: List[str] = []
deps = PyProject.get().senv.dependencies
if include_dev_dependencies:
deps.update(PyProject.get().senv.dev_dependencies)
for depname, depattrs in deps.items():
conda_dep_name = normalize_pypi_name(depname)
if isinstance(depattrs, Mapping):
poetry_version_spec = depattrs["version"]
# TODO: support additional features such as markers for things like sys_platform, platform_system
elif isinstance(depattrs, str):
poetry_version_spec = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}")
conda_version = poetry_version_to_conda_version(poetry_version_spec)
spec = to_match_spec(conda_dep_name, conda_version)
if conda_dep_name == "python":
specs.insert(0, spec)
else:
specs.append(spec)
return LockSpecification(specs=specs,
channels=PyProject.get().senv.conda_channels,
platform=platform)
def pyproject_callback(pyproject_file: Path = typer.Option(Path(".") /
"pyproject.toml",
"-f",
"--pyproject-file",
exists=True)):
PyProject.read_toml(pyproject_file)
chdir(PyProject.get().config_path.parent)
def test_lock_based_on_tested_includes_pinned_dependencies(
temp_appdirs_pyproject, cli_runner, appdirs_env_lock_path):
with cd(temp_appdirs_pyproject.parent):
click_line = next(
l for l in appdirs_env_lock_path.read_text().splitlines()
if "click" in l)
result = cli_runner.invoke(
app,
[
"package",
"-f",
str(temp_appdirs_pyproject),
"lock",
"--platforms",
"linux-64",
"--based-on-tested-lock-file",
str(appdirs_env_lock_path.resolve()),
],
catch_exceptions=False,
)
assert result.exit_code == 0
assert PyProject.get().senv.package.conda_lock_path.exists()
assert click_line in PyProject.get(
).senv.package.conda_lock_path.read_text()
def test_senv_overrides_poetry():
config_dict = {
"tool": {
"poetry": {
"version": "poetry1",
"description": "poetry2",
"name": "poetry3",
},
"senv": {
"version": "senv1",
"description": "senv2",
"name": "senv3",
},
}
}
config = PyProject(**config_dict)
assert config.version == "senv1"
assert config.senv.description == "senv2"
assert config.package_name == "senv3"
# without senv, it should use the poetry information
del config_dict["tool"]["senv"]
config = PyProject(**config_dict)
assert config.version == "poetry1"
assert config.senv.description == "poetry2"
assert config.package_name == "poetry3"
def test_pyproject_to_conda_creates_recipe_right_params():
PyProject.read_toml(SIMPLE_PYPROJECT_TOML)
recipe = pyproject_to_meta()
# it should ignore the dev-environments
assert recipe.package.name == "test_name"
assert recipe.package.version == "0.1.0"
assert "python" in recipe.requirements.host[0]
assert "3.7.0" in recipe.requirements.host[0]
def pyproject_to_conda_env_dict() -> Dict:
channels = PyProject.get().senv.conda_channels
dependencies = _get_dependencies_from_pyproject(
include_dev_dependencies=True)
return dict(name=PyProject.get().env.name,
channels=channels,
dependencies=dependencies)
def test_pyproject_to_conda_dev_env_dict_generates_env_with_dev_deps():
PyProject.read_toml(SIMPLE_PYPROJECT_TOML)
env_dict = pyproject_to_conda_env_dict()
dep_names = [d.split(" ")[0] for d in env_dict["dependencies"]]
assert len(env_dict["dependencies"]) == 8
# it should not ignore the dev-environments
assert "python" in dep_names
assert "ensureconda" in dep_names
assert "appdirs" in dep_names
assert "pyinstaller" in dep_names
assert "pytest" in dep_names
def test_env_name_default_to_package_name():
config_dict: Dict[str, Any] = {
"tool": {
"senv": {
"name": "p_name",
},
}
}
config = PyProject(**config_dict)
assert config.env.name == "p_name"
config_dict["tool"]["senv"]["env"] = {"name": "env_name"}
config2 = PyProject(**config_dict)
assert config2.env.name == "env_name"
def test_pyproject_to_conda_creates_recipe_with_deps():
PyProject.read_toml(SIMPLE_PYPROJECT_TOML)
recipe = pyproject_to_meta()
deps = recipe.requirements.run
dep_names = [d.split(" ")[0] for d in deps]
# it should ignore the dev-environments
assert len(deps) == 6
assert "python" in dep_names
assert "ensureconda" in dep_names
assert "click" in dep_names
assert "tomlkit" in dep_names
assert "appdirs" in dep_names
assert "conda-lock" in dep_names
def test_conda_build_dir_can_not_be_in_project(tmp_path):
tmp_toml = tmp_path / "tmp_toml"
config_dict = {
"tool": {
"senv": {
"name": "senv3",
"package": {
"conda-build-path": "./dist",
},
},
}
}
tmp_toml.write_text(toml.dumps(config_dict))
with cd(tmp_path), pytest.raises(SenvBadConfiguration):
PyProject.read_toml(tmp_toml)
def sync(build_system: BuildSystem = typer.Option(get_default_env_build_system)
):
c = PyProject.get()
if build_system == BuildSystem.POETRY:
with cd(c.config_path.parent):
subprocess.check_call(
[c.poetry_path, "install", "--remove-untracked"])
elif build_system == BuildSystem.CONDA:
if not c.env.conda_lock_path.exists():
log.info("No lock file found, locking environment now")
lock(build_system=build_system, platforms=get_conda_platforms())
with c.env.platform_conda_lock as lock_file:
result = subprocess.run([
str(c.conda_path),
"create",
"--file",
str(lock_file.resolve()),
"--yes",
"--name",
c.env.name,
])
if result.returncode != 0:
raise typer.Abort("Failed syncing environment")
else:
raise NotImplementedError()
def test_remove_config_key_removes_it_from_file(temp_pyproject, cli_runner):
cli_runner.invoke(
app,
[
"config",
"set",
"env.build-system",
"poetry",
"-f",
str(temp_pyproject),
],
)
cli_runner.invoke(
app,
[
"config",
"remove",
"env.build-system",
"-f",
str(temp_pyproject),
],
)
assert (PyProject.read_toml(temp_pyproject).senv.env.build_system ==
BuildSystem.CONDA)
def update(
build_system: BuildSystem = typer.Option(get_default_env_build_system),
platforms: List[str] = typer.Option(
get_conda_platforms,
case_sensitive=False,
help="conda platforms, for example osx-64 or linux-64",
),
):
if build_system == BuildSystem.POETRY:
with cd(PyProject.get().config_path.parent):
subprocess.check_call([PyProject.get().poetry_path, "update"])
elif build_system == BuildSystem.CONDA:
lock(build_system=build_system, platforms=platforms)
sync(build_system=build_system)
else:
raise NotImplementedError()
def _build_temp_pyproject(pyproject_path: Path):
temp_path = tmp_path / "pyproject.toml"
copyfile(pyproject_path, temp_path)
c = PyProject.read_toml(temp_path)
project = tmp_path / c.package_name.replace("-", "_") / "main.py"
project.parent.mkdir(parents=True, exist_ok=True)
project.write_text("print('hello world')")
return temp_path
def test_config_build_system_has_to_be_enum():
config_dict = {
"tool": {
"senv": {
"name": "test_name",
"env": {
"build-system": "poetry"
}
}
}
}
config = PyProject(**config_dict)
assert config.senv.env.build_system == BuildSystem.POETRY
config_dict["tool"]["senv"]["env"]["build-system"] = "no_build_system"
with pytest.raises(ValueError):
PyProject(**config_dict)
def run(
ctx: typer.Context,
build_system: BuildSystem = typer.Option(get_default_env_build_system),
):
if build_system == BuildSystem.POETRY:
with cd(PyProject.get().config_path.parent):
subprocess.check_call(["poetry", "run"] + ctx.args)
elif build_system == BuildSystem.CONDA:
subprocess.check_call([
"conda",
"run",
"-n",
PyProject.get().env.name,
"--no-capture-output",
"--live-stream",
] + ctx.args)
else:
raise NotImplementedError()
def test_set_config_add_value_to_pyproject(temp_pyproject, cli_runner):
result = cli_runner.invoke(
app,
[
"config",
"-f",
str(temp_pyproject),
"set",
"env.conda-lock-platforms",
"linux-64",
],
catch_exceptions=False,
)
assert result.exit_code == 0
PyProject.read_toml(temp_pyproject)
assert PyProject.get().senv.env.conda_lock_platforms == {"linux-64"}
def test_set_config_with_wrong_value_does_not_change_pyproject(
temp_pyproject, cli_runner):
original_config = PyProject.read_toml(temp_pyproject).dict()
cli_runner.invoke(
app,
[
"-f",
str(temp_pyproject),
"config",
"set",
"conda-path",
"none_existing_path",
],
catch_exceptions=False,
)
new_config = PyProject.read_toml(temp_pyproject).dict()
assert new_config == original_config
def publish_locked_package(
build_system: BuildSystem = typer.Option(get_default_package_build_system),
repository_url: Optional[str] = None,
username: str = typer.Option(
..., "--username", "-u", envvar="SENV_PUBLISHER_USERNAME"
),
password: str = typer.Option(
..., "--password", "-p", envvar="SENV_PUBLISHER_PASSWORD"
),
lock_file: Path = typer.Option(
lambda: PyProject.get().senv.package.conda_lock_path,
"--lock-file",
"-l",
exists=True,
),
yes: bool = build_yes_option(),
):
c: PyProject = PyProject.get()
with auto_confirm_yes(yes):
if build_system == BuildSystem.POETRY:
raise NotImplementedError("publish locked ")
elif build_system == BuildSystem.CONDA:
with cd_tmp_dir() as tmp_dir:
meta_path = tmp_dir / "conda.recipe" / "meta.yaml"
temp_lock_path = meta_path.parent / "package_locked_file.lock.json"
meta_path.parent.mkdir(parents=True)
shutil.copyfile(
str(lock_file.absolute()), str(temp_lock_path.absolute())
)
locked_package_to_recipe_yaml(temp_lock_path, meta_path)
build_conda_package_from_recipe(meta_path.absolute())
with cd(meta_path.parent):
repository_url = repository_url or c.senv.package.conda_publish_url
publish_conda(
username,
password,
repository_url,
package_name=c.package_name_locked,
)
else:
raise NotImplementedError()
def set_new_setting_value(
key: AllowedConfigKeys = typer.Argument(...),
value: str = typer.Argument(
None,
help=
"Value of the setting. For multi value setting like the conda-platforms,"
" separate them with a comma ','",
),
):
set_config_value_to_pyproject(PyProject.get().config_path, key, value)
def test_lock_appdirs_simple_does_not_include_fake_dependencies(
temp_appdirs_pyproject, cli_runner):
with cd(temp_appdirs_pyproject.parent):
result = cli_runner.invoke(
app,
[
"package",
"-f",
str(temp_appdirs_pyproject),
"lock",
"--platforms",
"linux-64",
],
catch_exceptions=False,
)
assert result.exit_code == 0
assert PyProject.get().senv.package.conda_lock_path.exists()
assert "click" not in PyProject.get(
).senv.package.conda_lock_path.read_text()
def lock_app(
build_system: BuildSystem = typer.Option(get_default_package_build_system),
platforms: List[str] = typer.Option(
get_conda_platforms,
case_sensitive=False,
help="conda platforms, for example osx-64 and/or linux-64",
),
based_on_tested_lock_file: Optional[Path] = based_on_tested_lock_file_option,
conda_channels: Optional[List[str]] = typer.Option(
get_conda_channels,
),
output: Path = typer.Option(
lambda: PyProject.get().senv.package.conda_lock_path, "--output", "-o"
),
):
c = PyProject.get()
platforms = platforms
if build_system == BuildSystem.POETRY:
raise NotImplementedError()
elif build_system == BuildSystem.CONDA:
output.parent.mkdir(exist_ok=True, parents=True)
if based_on_tested_lock_file is None:
combined_lock = generate_combined_conda_lock_file(
platforms,
dict(
name=c.package_name,
channels=conda_channels,
dependencies={c.package_name: f"=={c.version}"},
),
)
output.write_text(combined_lock.json(indent=2))
else:
combined_lock = generate_app_lock_file_based_on_tested_lock_path(
lock_path=based_on_tested_lock_file,
conda_channels=conda_channels,
platforms=platforms,
)
output.write_text(combined_lock.json(indent=2))
log.info(f"Package lock file generated in {output.resolve()}")
else:
raise NotImplementedError()
def build_package(
build_system: BuildSystem = typer.Option(get_default_package_build_system),
python_version: Optional[str] = None,
):
# todo add progress bar
if build_system == BuildSystem.POETRY:
with cd(PyProject.get().config_path.parent):
subprocess.check_call([PyProject.get().poetry_path, "build"])
elif build_system == BuildSystem.CONDA:
with tmp_env():
meta_path = (
PyProject.get().config_path.parent / "conda.recipe" / "meta.yaml"
)
pyproject_to_recipe_yaml(
python_version=python_version,
output=meta_path,
)
build_conda_package_from_recipe(meta_path, python_version)
else:
raise NotImplementedError()
def build_conda_package_from_recipe(meta_path: Path,
python_version: Optional[str] = None):
set_conda_build_path()
if which("conda-mambabuild") is None:
_install_package_dependencies()
args = ["conda-mambabuild", "--build-only", "--override-channels"]
for c in PyProject.get().senv.conda_channels:
args += ["--channel", c]
if python_version:
args.extend(["--python", python_version])
result = subprocess.run(args + [str(meta_path.parent)])
if result.returncode != 0:
raise typer.Abort("Failed building conda package")
def pyproject_to_meta(
*,
python_version: Optional[str] = None,
) -> CondaMeta:
"""
:param python_version: python version used to create the conda meta file
"""
dependencies = _get_dependencies_from_pyproject(
include_dev_dependencies=False)
python_version = _populate_python_version(python_version, dependencies)
if python_version != PyProject.get().python_version:
log.warning(
"Python version in the pyproject.toml is different than the one provided"
)
if python_version is None:
raise SenvInvalidPythonVersion(
f"No python version provided or defined in {PyProject.get().config_path}"
)
c: PyProject = PyProject.get()
license = c.senv.license if c.senv.license != "Proprietary" else "INTERNAL"
entry_points = [
f"{name} = {module}" for name, module in c.senv.scripts.items()
]
return CondaMeta(
package=_Package(name=c.package_name, version=c.version),
source=_Source(path=c.config_path.parent.resolve()),
build=_Build(entry_points=entry_points),
requirements=_Requirements(host=[python_version, "pip", "poetry"],
run=dependencies),
about=_About(
home=c.senv.homepage,
license=license,
description=c.senv.description,
doc_url=c.senv.documentation,
),
extra=_Extra(maintainers=c.senv.authors),
)
def _validate_toml(toml):
try:
c = PyProject(**toml)
c._config_path = PyProject.get().config_path
c.validate_fields()
except ValidationError as e:
log.error(str(e))
raise typer.Abort()
def test_env_locks_builds_the_lock_files_in_the_configured_directory(
temp_pyproject, cli_runner
):
lock_file = temp_pyproject.parent / "my_lock_folder"
PyProject.read_toml(temp_pyproject)
set_new_setting_value(AllowedConfigKeys.CONDA_ENV_LOCK_PATH, str(lock_file))
result = cli_runner.invoke(
app,
[
"env",
"-f",
str(temp_pyproject),
"lock",
"--platforms",
"osx-64",
],
catch_exceptions=False,
)
assert result.exit_code == 0
assert lock_file.exists()
combined_lock = CombinedCondaLock.parse_file(lock_file)
assert set(combined_lock.platform_tar_links.keys()) == {"osx-64"}
def test_config_conda_and_poetry_path_have_to_be_executable(key):
config_dict = {
"tool": {
"senv": {
key: str(Path(__file__)),
}
}
}
try:
PyProject(**config_dict)
pytest.fail("config should raise exception as path is not executable")
except ValueError as e:
assert "not executable" in str(e).lower()
assert "not found" not in str(e).lower()
def test_config_conda_and_poetry_path_have_to_exists(key):
config_dict = {
"tool": {
"senv": {
key: str(Path("/no/real/path")),
}
}
}
try:
PyProject(**config_dict)
pytest.fail("config should raise exception as paths do not exists")
except ValueError as e:
assert "not found" in str(e).lower()
assert "not executable" not in str(e).lower()
def _add_app_lockfile_metadata(lockfile: Path):
lock_content = lockfile.read_text()
if "@EXPLICIT" not in lock_content:
raise SenvxMalformedAppLockFile("No @EXPLICIT found in lock file")
lock_header, tars = lock_content.split("@EXPLICIT", 1)
c = PyProject.get()
metadata = LockFileMetaData(
package_name=c.package_name,
entry_points=list(c.senv.scripts.keys()),
)
meta_json = (
"\n".join([f"# {l}"
for l in metadata.json(indent=2).splitlines()]) + "\n")
lockfile.write_text(lock_header + "# @METADATA_INIT\n" + meta_json +
"# @METADATA_END\n" + "@EXPLICIT\n" + tars)
