def _parse_pyproject_toml(platform: str,
include_dev_dependencies: bool) -> LockSpecification:
specs: List[str] = []
deps = PyProject.get().senv.dependencies
if include_dev_dependencies:
deps.update(PyProject.get().senv.dev_dependencies)
for depname, depattrs in deps.items():
conda_dep_name = normalize_pypi_name(depname)
if isinstance(depattrs, Mapping):
poetry_version_spec = depattrs["version"]
# TODO: support additional features such as markers for things like sys_platform, platform_system
elif isinstance(depattrs, str):
poetry_version_spec = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}")
conda_version = poetry_version_to_conda_version(poetry_version_spec)
spec = to_match_spec(conda_dep_name, conda_version)
if conda_dep_name == "python":
specs.insert(0, spec)
else:
specs.append(spec)
return LockSpecification(specs=specs,
channels=PyProject.get().senv.conda_channels,
platform=platform)
def test_lock_based_on_tested_includes_pinned_dependencies(
temp_appdirs_pyproject, cli_runner, appdirs_env_lock_path):
with cd(temp_appdirs_pyproject.parent):
click_line = next(
l for l in appdirs_env_lock_path.read_text().splitlines()
if "click" in l)
result = cli_runner.invoke(
app,
[
"package",
"-f",
str(temp_appdirs_pyproject),
"lock",
"--platforms",
"linux-64",
"--based-on-tested-lock-file",
str(appdirs_env_lock_path.resolve()),
],
catch_exceptions=False,
)
assert result.exit_code == 0
assert PyProject.get().senv.package.conda_lock_path.exists()
assert click_line in PyProject.get(
).senv.package.conda_lock_path.read_text()
def pyproject_to_conda_env_dict() -> Dict:
channels = PyProject.get().senv.conda_channels
dependencies = _get_dependencies_from_pyproject(
include_dev_dependencies=True)
return dict(name=PyProject.get().env.name,
channels=channels,
dependencies=dependencies)
def sync(build_system: BuildSystem = typer.Option(get_default_env_build_system)
):
c = PyProject.get()
if build_system == BuildSystem.POETRY:
with cd(c.config_path.parent):
subprocess.check_call(
[c.poetry_path, "install", "--remove-untracked"])
elif build_system == BuildSystem.CONDA:
if not c.env.conda_lock_path.exists():
log.info("No lock file found, locking environment now")
lock(build_system=build_system, platforms=get_conda_platforms())
with c.env.platform_conda_lock as lock_file:
result = subprocess.run([
str(c.conda_path),
"create",
"--file",
str(lock_file.resolve()),
"--yes",
"--name",
c.env.name,
])
if result.returncode != 0:
raise typer.Abort("Failed syncing environment")
else:
raise NotImplementedError()
def pyproject_callback(pyproject_file: Path = typer.Option(Path(".") /
"pyproject.toml",
"-f",
"--pyproject-file",
exists=True)):
PyProject.read_toml(pyproject_file)
chdir(PyProject.get().config_path.parent)
def _validate_toml(toml):
try:
c = PyProject(**toml)
c._config_path = PyProject.get().config_path
c.validate_fields()
except ValidationError as e:
log.error(str(e))
raise typer.Abort()
def update(
build_system: BuildSystem = typer.Option(get_default_env_build_system),
platforms: List[str] = typer.Option(
get_conda_platforms,
case_sensitive=False,
help="conda platforms, for example osx-64 or linux-64",
),
):
if build_system == BuildSystem.POETRY:
with cd(PyProject.get().config_path.parent):
subprocess.check_call([PyProject.get().poetry_path, "update"])
elif build_system == BuildSystem.CONDA:
lock(build_system=build_system, platforms=platforms)
sync(build_system=build_system)
else:
raise NotImplementedError()
def run(
ctx: typer.Context,
build_system: BuildSystem = typer.Option(get_default_env_build_system),
):
if build_system == BuildSystem.POETRY:
with cd(PyProject.get().config_path.parent):
subprocess.check_call(["poetry", "run"] + ctx.args)
elif build_system == BuildSystem.CONDA:
subprocess.check_call([
"conda",
"run",
"-n",
PyProject.get().env.name,
"--no-capture-output",
"--live-stream",
] + ctx.args)
else:
raise NotImplementedError()
def set_new_setting_value(
key: AllowedConfigKeys = typer.Argument(...),
value: str = typer.Argument(
None,
help=
"Value of the setting. For multi value setting like the conda-platforms,"
" separate them with a comma ','",
),
):
set_config_value_to_pyproject(PyProject.get().config_path, key, value)
def lock_app(
build_system: BuildSystem = typer.Option(get_default_package_build_system),
platforms: List[str] = typer.Option(
get_conda_platforms,
case_sensitive=False,
help="conda platforms, for example osx-64 and/or linux-64",
),
based_on_tested_lock_file: Optional[Path] = based_on_tested_lock_file_option,
conda_channels: Optional[List[str]] = typer.Option(
get_conda_channels,
),
output: Path = typer.Option(
lambda: PyProject.get().senv.package.conda_lock_path, "--output", "-o"
),
):
c = PyProject.get()
platforms = platforms
if build_system == BuildSystem.POETRY:
raise NotImplementedError()
elif build_system == BuildSystem.CONDA:
output.parent.mkdir(exist_ok=True, parents=True)
if based_on_tested_lock_file is None:
combined_lock = generate_combined_conda_lock_file(
platforms,
dict(
name=c.package_name,
channels=conda_channels,
dependencies={c.package_name: f"=={c.version}"},
),
)
output.write_text(combined_lock.json(indent=2))
else:
combined_lock = generate_app_lock_file_based_on_tested_lock_path(
lock_path=based_on_tested_lock_file,
conda_channels=conda_channels,
platforms=platforms,
)
output.write_text(combined_lock.json(indent=2))
log.info(f"Package lock file generated in {output.resolve()}")
else:
raise NotImplementedError()
def test_lock_appdirs_simple_does_not_include_fake_dependencies(
temp_appdirs_pyproject, cli_runner):
with cd(temp_appdirs_pyproject.parent):
result = cli_runner.invoke(
app,
[
"package",
"-f",
str(temp_appdirs_pyproject),
"lock",
"--platforms",
"linux-64",
],
catch_exceptions=False,
)
assert result.exit_code == 0
assert PyProject.get().senv.package.conda_lock_path.exists()
assert "click" not in PyProject.get(
).senv.package.conda_lock_path.read_text()
def publish_locked_package(
build_system: BuildSystem = typer.Option(get_default_package_build_system),
repository_url: Optional[str] = None,
username: str = typer.Option(
..., "--username", "-u", envvar="SENV_PUBLISHER_USERNAME"
),
password: str = typer.Option(
..., "--password", "-p", envvar="SENV_PUBLISHER_PASSWORD"
),
lock_file: Path = typer.Option(
lambda: PyProject.get().senv.package.conda_lock_path,
"--lock-file",
"-l",
exists=True,
),
yes: bool = build_yes_option(),
):
c: PyProject = PyProject.get()
with auto_confirm_yes(yes):
if build_system == BuildSystem.POETRY:
raise NotImplementedError("publish locked ")
elif build_system == BuildSystem.CONDA:
with cd_tmp_dir() as tmp_dir:
meta_path = tmp_dir / "conda.recipe" / "meta.yaml"
temp_lock_path = meta_path.parent / "package_locked_file.lock.json"
meta_path.parent.mkdir(parents=True)
shutil.copyfile(
str(lock_file.absolute()), str(temp_lock_path.absolute())
)
locked_package_to_recipe_yaml(temp_lock_path, meta_path)
build_conda_package_from_recipe(meta_path.absolute())
with cd(meta_path.parent):
repository_url = repository_url or c.senv.package.conda_publish_url
publish_conda(
username,
password,
repository_url,
package_name=c.package_name_locked,
)
else:
raise NotImplementedError()
def build_package(
build_system: BuildSystem = typer.Option(get_default_package_build_system),
python_version: Optional[str] = None,
):
# todo add progress bar
if build_system == BuildSystem.POETRY:
with cd(PyProject.get().config_path.parent):
subprocess.check_call([PyProject.get().poetry_path, "build"])
elif build_system == BuildSystem.CONDA:
with tmp_env():
meta_path = (
PyProject.get().config_path.parent / "conda.recipe" / "meta.yaml"
)
pyproject_to_recipe_yaml(
python_version=python_version,
output=meta_path,
)
build_conda_package_from_recipe(meta_path, python_version)
else:
raise NotImplementedError()
def pyproject_to_meta(
*,
python_version: Optional[str] = None,
) -> CondaMeta:
"""
:param python_version: python version used to create the conda meta file
"""
dependencies = _get_dependencies_from_pyproject(
include_dev_dependencies=False)
python_version = _populate_python_version(python_version, dependencies)
if python_version != PyProject.get().python_version:
log.warning(
"Python version in the pyproject.toml is different than the one provided"
)
if python_version is None:
raise SenvInvalidPythonVersion(
f"No python version provided or defined in {PyProject.get().config_path}"
)
c: PyProject = PyProject.get()
license = c.senv.license if c.senv.license != "Proprietary" else "INTERNAL"
entry_points = [
f"{name} = {module}" for name, module in c.senv.scripts.items()
]
return CondaMeta(
package=_Package(name=c.package_name, version=c.version),
source=_Source(path=c.config_path.parent.resolve()),
build=_Build(entry_points=entry_points),
requirements=_Requirements(host=[python_version, "pip", "poetry"],
run=dependencies),
about=_About(
home=c.senv.homepage,
license=license,
description=c.senv.description,
doc_url=c.senv.documentation,
),
extra=_Extra(maintainers=c.senv.authors),
)
def build_conda_package_from_recipe(meta_path: Path,
python_version: Optional[str] = None):
set_conda_build_path()
if which("conda-mambabuild") is None:
_install_package_dependencies()
args = ["conda-mambabuild", "--build-only", "--override-channels"]
for c in PyProject.get().senv.conda_channels:
args += ["--channel", c]
if python_version:
args.extend(["--python", python_version])
result = subprocess.run(args + [str(meta_path.parent)])
if result.returncode != 0:
raise typer.Abort("Failed building conda package")
def _add_app_lockfile_metadata(lockfile: Path):
lock_content = lockfile.read_text()
if "@EXPLICIT" not in lock_content:
raise SenvxMalformedAppLockFile("No @EXPLICIT found in lock file")
lock_header, tars = lock_content.split("@EXPLICIT", 1)
c = PyProject.get()
metadata = LockFileMetaData(
package_name=c.package_name,
entry_points=list(c.senv.scripts.keys()),
)
meta_json = (
"\n".join([f"# {l}"
for l in metadata.json(indent=2).splitlines()]) + "\n")
lockfile.write_text(lock_header + "# @METADATA_INIT\n" + meta_json +
"# @METADATA_END\n" + "@EXPLICIT\n" + tars)
def appdirs_env_lock_path(temp_appdirs_pyproject) -> Path:
with cd(temp_appdirs_pyproject.parent):
# env lock first to get the env locks that we will use to tests our code
result = CliRunner().invoke(
app,
[
"env",
"-f",
str(temp_appdirs_pyproject),
"lock",
"--platforms",
"linux-64",
],
catch_exceptions=False,
)
assert result.exit_code == 0
yield PyProject.get().senv.env.conda_lock_path
def test_set_config_add_value_to_pyproject(temp_pyproject, cli_runner):
result = cli_runner.invoke(
app,
[
"config",
"-f",
str(temp_pyproject),
"set",
"env.conda-lock-platforms",
"linux-64",
],
catch_exceptions=False,
)
assert result.exit_code == 0
PyProject.read_toml(temp_pyproject)
assert PyProject.get().senv.env.conda_lock_platforms == {"linux-64"}
def combine_conda_lock_files(directory: Path,
platforms: List[str]) -> "CombinedCondaLock":
platform_tar_links = {}
for platform in platforms:
lock_file = directory / f"conda-{platform}.lock"
lock_text = lock_file.read_text()
clean_lock_test = lock_text.split("@EXPLICIT", 1)[1].strip()
tar_links = [line.strip() for line in clean_lock_test.splitlines()]
platform_tar_links[platform] = tar_links
c = PyProject.get()
metadata = LockFileMetaData(
package_name=c.package_name,
entry_points=list(c.senv.scripts.keys()),
version=c.version,
)
return CombinedCondaLock(metadata=metadata,
platform_tar_links=platform_tar_links)
def generate_app_lock_file_based_on_tested_lock_path(
lock_path: Path, conda_channels: List[str],
platforms: List[str]) -> CombinedCondaLock:
platforms_set = set(platforms)
c = PyProject.get()
direct_dependencies_name = {
normalize_pypi_name(d).lower()
for d in c.senv.dependencies.keys()
}
# always include python even if it is not in the dependencies
direct_dependencies_name.add("python")
combined_lock = CombinedCondaLock.parse_file(lock_path)
combined_lock_platforms_set = set(combined_lock.platform_tar_links.keys())
if not platforms_set.issubset(combined_lock_platforms_set):
raise SenvNotAllPlatformsInBaseLockFile(
platforms_set.difference(combined_lock_platforms_set))
with cd_tmp_dir() as tmp_dir:
with ProcessPoolExecutor() as executor:
for platform in platforms_set:
tar_urls = combined_lock.platform_tar_links[platform]
# add the current package
dependencies = {
c.package_name: f"=={c.version}",
}
# pin version for all direct dependencies
for line in tar_urls:
channel, dep = line.rsplit("/", 1)
name, version, _ = dep.rsplit("-", 2)
if name.lower() in direct_dependencies_name:
dependencies[name] = f"=={version}"
yaml_path = create_env_yaml(
channels=conda_channels,
output=Path(tmp_dir) / "env.yaml",
dependencies=dependencies,
)
executor.submit(
run_lock,
[yaml_path],
conda_exe=str(c.conda_path.resolve()),
platforms=[platform],
)
return combine_conda_lock_files(tmp_dir, list(platforms))
def test_lock_appdirs_simple_includes_metadata(temp_appdirs_pyproject,
cli_runner):
with cd(temp_appdirs_pyproject.parent):
cli_runner.invoke(
app,
[
"package",
"-f",
str(temp_appdirs_pyproject),
"lock",
"--platforms",
"osx-64",
],
catch_exceptions=False,
)
conda_lock = CombinedCondaLock.parse_file(
PyProject.get().senv.package.conda_lock_path)
assert conda_lock.metadata.package_name == "appdirs"
assert conda_lock.metadata.entry_points == []
def locked_package_to_recipe_yaml(lock_file: Path, output: Path):
c: PyProject = PyProject.get()
license_ = c.senv.license if c.senv.license != "Proprietary" else "INTERNAL"
meta = CondaMeta(
package=_Package(name=c.package_name_locked, version=c.version),
build=_Build(
script="echo 'This package is not installable through conda,"
" use senvx to install it'"),
source=_Source(path=lock_file.absolute()),
requirements=_Requirements(host=[], run=[]),
about=_About(
home=c.senv.homepage,
license=license_,
description=c.senv.description,
doc_url=c.senv.documentation,
),
extra=_Extra(maintainers=c.senv.authors),
)
meta_to_recipe_yaml(meta=meta, output=output)
def publish_package(
build_system: BuildSystem = typer.Option(get_default_package_build_system),
python_version: Optional[str] = None,
build: bool = typer.Option(False, "--build", "-b"),
repository_url: Optional[str] = None,
username: str = typer.Option(
..., "--username", "-u", envvar="SENV_PUBLISHER_USERNAME"
),
password: str = typer.Option(
..., "--password", "-p", envvar="SENV_PUBLISHER_PASSWORD"
),
yes: bool = build_yes_option(),
):
with auto_confirm_yes(yes):
if build:
build_package(build_system=build_system, python_version=python_version)
if build_system == BuildSystem.POETRY:
with cd(PyProject.get().config_path.parent):
repository_url = (
repository_url
or PyProject.get().senv.package.poetry_publish_repository
)
if repository_url is not None:
subprocess.check_call(
[
PyProject.get().poetry_path,
"config",
f"repositories.senv_{PyProject.get().package_name}",
repository_url,
]
)
args = [PyProject.get().poetry_path, "publish"]
if username and password:
args += ["--username", username, "--password", password]
subprocess.check_call(args)
elif build_system == BuildSystem.CONDA:
with cd(PyProject.get().config_path.parent):
repository_url = (
repository_url or PyProject.get().senv.package.conda_publish_url
)
if repository_url is None:
# todo add logic to publish to conda-forge
raise NotImplementedError(
"repository_url is required to publish a conda environment. "
)
publish_conda(username, password, repository_url)
else:
raise NotImplementedError()
def create_env_yaml(
*,
dependencies: Dict[str, Any],
output: Path,
name: Optional[str] = None,
channels: Optional[List[str]] = None,
) -> Path:
if channels is None:
channels = []
c = PyProject.get()
yaml_dict = dict(
name=name or c.package_name,
channels=channels if channels is not None else c.senv.conda_channels,
dependencies=dependencies,
)
recipe_dir = output.parent
recipe_dir.mkdir(parents=True, exist_ok=True)
_yaml_safe_dump(yaml_dict, output)
return output
def lock(
build_system: BuildSystem = typer.Option(get_default_env_build_system),
platforms: List[str] = typer.Option(
get_conda_platforms,
case_sensitive=False,
help="conda platforms, for example osx-64 or linux-64",
),
):
c = PyProject.get()
if build_system == BuildSystem.POETRY:
with cd(c.config_path.parent):
subprocess.check_call([c.poetry_path, "lock"])
elif build_system == BuildSystem.CONDA:
c.env.conda_lock_path.parent.mkdir(exist_ok=True, parents=True)
combined_lock = generate_combined_conda_lock_file(
platforms,
pyproject_to_conda_env_dict(),
)
c.env.conda_lock_path.write_text(combined_lock.json(indent=2))
else:
raise NotImplementedError()
def generate_combined_conda_lock_file(platforms: List[str],
env_dict: Dict) -> "CombinedCondaLock":
c = PyProject.get()
with NamedTemporaryFile(mode="w+") as f, cd_tmp_dir(
) as tmp_dir, MySpinner("Building lock files...") as status:
status.start()
yaml.safe_dump(env_dict, f)
processes = []
with ProcessPoolExecutor() as executor:
for platform in platforms:
p = executor.submit(
run_lock,
[Path(f.name)],
conda_exe=str(c.conda_path.resolve()),
platforms=[platform],
channel_overrides=env_dict["channels"],
kinds=["explicit"],
)
processes.append(p)
status.writeln("combining lock files...")
return combine_conda_lock_files(tmp_dir, platforms)
def test_env_locks_builds_the_lock_files_in_default_env_lock_files(
temp_pyproject, cli_runner
):
with cd(temp_pyproject.parent):
result = cli_runner.invoke(
app,
[
"env",
"-f",
str(temp_pyproject),
"lock",
"--platforms",
"linux-64",
],
catch_exceptions=False,
)
assert result.exit_code == 0
lock_file = PyProject.get().senv.env.__fields__["conda_lock_path"].default
assert lock_file.exists()
combined_lock = CombinedCondaLock.parse_file(lock_file)
assert set(combined_lock.platform_tar_links.keys()) == {"linux-64"}
def shell(build_system: BuildSystem = typer.Option(
get_default_env_build_system)):
c = PyProject.get()
# conda activate does not work using the conda executable path (I am not sure why)
# force adding the conda executable to the path and then call it
environ[
"PATH"] = f"{c.conda_path.parent}{os.path.pathsep}{environ.get('PATH')}"
if build_system == BuildSystem.POETRY:
cwd = os.getcwd()
with cd(c.config_path.parent):
with spawn_shell(command="poetry shell", cwd=cwd):
pass
elif build_system == BuildSystem.CONDA:
with spawn_shell(
command=
f"{shlex.quote(str(c.conda_path.name))} activate {c.env.name}",
):
pass
else:
raise NotImplementedError()
environ["PATH"] = os.path.pathsep.join(
environ.get("PATH").split(os.path.pathsep)[1:])
def pyproject_to_env_app_yaml(
*,
app_name: Optional[str] = None,
channels: Optional[List[str]] = None,
output: Path = Path("app_environment.yaml"),
) -> Path:
"""
Generates a basic yaml with only it's current version as the dependency
In order to use it, the package has to be published
:param app_name: the name of the app,
by default it will use the name of the package in pyproject.toml
:param channels: the conda channels needed for the env,
by default using the channels defined in pyproject.toml
:param output: where to save the yaml
:return: output
"""
c = PyProject.get()
return create_env_yaml(
name=app_name,
channels=channels,
dependencies={c.package_name: f"=={c.version}"},
output=output,
)
def publish_conda(
username: str,
password: str,
repository_url: str,
package_name: Optional[str] = None,
):
c = PyProject.get()
conda_dist = c.senv.package.conda_build_path
files_to_upload = list(
conda_dist.glob(
f"*/{package_name or c.package_name}-{c.version}*.tar.bz2"))
if len(files_to_upload) == 0:
log.warning(f'No files found to upload in "{conda_dist}",'
" you need to build the package before uploading it")
raise typer.Abort()
# todo we might need to be more specific here
if repository_url.endswith("anaconda.org"):
return publish_conda_to_anaconda_org(username, password,
files_to_upload)
for tar_path in files_to_upload:
package = tar_path.name
arch = tar_path.parent.name
dest = f"{repository_url}/{arch}/{package}"
resp = requests.head(dest)
if resp.status_code != 404:
log.warning("Object already exists not reuploading...")
else:
subprocess.check_call([
"curl",
f"-u{username}:{password}",
"-T",
str(tar_path.resolve()),
dest,
], )
