def test_aggregate_lock_specs():
gpu_spec = LockSpecification(
specs=["pytorch"],
channels=["pytorch", "conda-forge"],
platform="linux-64",
)
base_spec = LockSpecification(
specs=["python =3.7"],
channels=["conda-forge"],
platform="linux-64",
)
assert (aggregate_lock_specs([gpu_spec,
base_spec]).env_hash() == LockSpecification(
specs=["pytorch", "python =3.7"],
channels=["pytorch", "conda-forge"],
platform="linux-64",
).env_hash())
assert (aggregate_lock_specs([base_spec,
gpu_spec]).env_hash() == LockSpecification(
specs=["pytorch", "python =3.7"],
channels=["conda-forge"],
platform="linux-64",
).env_hash())
def test_aggregate_lock_specs():
gpu_spec = LockSpecification(
dependencies=[_make_spec("pytorch")],
channels=["pytorch", "conda-forge"],
platforms=["linux-64"],
sources=[pathlib.Path("ml-stuff.yml")],
)
base_spec = LockSpecification(
dependencies=[_make_spec("python", "=3.7")],
channels=["conda-forge"],
platforms=["linux-64"],
sources=[pathlib.Path("base-env.yml")],
)
# NB: content hash explicitly does not depend on the source file names
assert (aggregate_lock_specs(
[gpu_spec, base_spec]).content_hash() == LockSpecification(
dependencies=[_make_spec("pytorch"),
_make_spec("python", "=3.7")],
channels=["pytorch", "conda-forge"],
platforms=["linux-64"],
sources=[],
).content_hash())
assert (aggregate_lock_specs(
[base_spec, gpu_spec]).content_hash() != LockSpecification(
dependencies=[_make_spec("pytorch"),
_make_spec("python", "=3.7")],
channels=["conda-forge"],
platforms=["linux-64"],
sources=[],
).content_hash())
def parse_poetry_pyproject_toml(
pyproject_toml: pathlib.Path, platform: str, include_dev_dependencies: bool
) -> LockSpecification:
contents = toml.load(pyproject_toml)
specs: List[str] = []
dependency_sections = ["dependencies"]
if include_dev_dependencies:
dependency_sections.append("dev-dependencies")
for key in dependency_sections:
deps = get_in(["tool", "poetry", key], contents, {})
for depname, depattrs in deps.items():
conda_dep_name = normalize_pypi_name(depname)
if isinstance(depattrs, collections.Mapping):
poetry_version_spec = depattrs["version"]
# TODO: support additional features such as markers for things like sys_platform, platform_system
elif isinstance(depattrs, str):
poetry_version_spec = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}"
)
conda_version = poetry_version_to_conda_version(poetry_version_spec)
spec = to_match_spec(conda_dep_name, conda_version)
if conda_dep_name == "python":
specs.insert(0, spec)
else:
specs.append(spec)
channels = get_in(["tool", "conda-lock", "channels"], contents, [])
return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_poetry_pyproject_toml(pyproject_toml: pathlib.Path,
platform: str) -> LockSpecification:
contents = toml.load(pyproject_toml)
specs: List[str] = []
for key in ["dependencies", "dev-dependencies"]:
deps = contents.get("tool", {}).get("poetry", {}).get(key, {})
for depname, depattrs in deps.items():
conda_dep_name = normalize_pypi_name(depname)
if isinstance(depattrs, collections.Mapping):
poetry_version_spec = depattrs["version"]
# TODO: support additional features such as markerts for things like sys_platform, platform_system
elif isinstance(depattrs, str):
poetry_version_spec = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}"
)
conda_version = poetry_version_to_conda_version(
poetry_version_spec)
if conda_version:
spec = f"{conda_dep_name}[version{conda_version}]"
else:
spec = f"{conda_dep_name}"
if conda_dep_name == "python":
specs.insert(0, spec)
else:
specs.append(spec)
channels = contents.get("tool", {}).get("conda-lock",
{}).get("channels", [])
return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_meta_yaml_file(meta_yaml_file: pathlib.Path,
platform: str) -> LockSpecification:
"""Parse a simple meta-yaml file for dependencies.
* This does not support multi-output files and will ignore all lines with selectors
"""
if not meta_yaml_file.exists():
raise FileNotFoundError(f"{meta_yaml_file} not found")
with meta_yaml_file.open("r") as fo:
filtered_recipe = "\n".join(
filter_platform_selectors(fo.read(), platform=platform))
t = jinja2.Template(filtered_recipe, undefined=UndefinedNeverFail)
rendered = t.render()
meta_yaml_data = yaml.safe_load(rendered)
channels = meta_yaml_data.get("extra", {}).get("channels", [])
specs = []
def add_spec(spec):
if spec is None:
return
specs.append(spec)
for s in meta_yaml_data.get("requirements", {}).get("host", []):
add_spec(s)
for s in meta_yaml_data.get("requirements", {}).get("run", []):
add_spec(s)
for s in meta_yaml_data.get("test", {}).get("requires", []):
add_spec(s)
return LockSpecification(specs=specs, channels=channels, platform=platform)
def specification_with_dependencies(
path: pathlib.Path, toml_contents: Mapping[str, Any],
dependencies: List[Dependency]) -> LockSpecification:
for depname, depattrs in get_in(["tool", "conda-lock", "dependencies"],
toml_contents, {}).items():
if isinstance(depattrs, str):
conda_version = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}")
dependencies.append(
VersionedDependency(
name=depname,
version=conda_version,
manager="conda",
optional=False,
category="main",
extras=[],
))
return LockSpecification(
dependencies,
channels=get_in(["tool", "conda-lock", "channels"], toml_contents, []),
platforms=get_in(["tool", "conda-lock", "platforms"], toml_contents,
[]),
sources=[path],
)
def test_default_virtual_package_input_hash_stability():
from conda_lock.virtual_package import default_virtual_package_repodata
vpr = default_virtual_package_repodata()
expected = {
"linux-64":
"93c22a62ca75ed0fd7649a6c9fbac611fd42a694465841b141c91aa2d4edf1b3",
"linux-aarch64":
"e1115c4d229438be0bd3e79c3734afb1f2fb8db42cf0c20c0e2ede5405e97e25",
"linux-ppc64le":
"d980051789ba7e6374c0833bf615b060bc0c5dfa63907eb4f11ac85f4dbb80da",
"osx-64":
"8e2e62ea8061892d10606e9a10f05f4c7358c798e5a2d390b1206568bf9338a2",
"osx-arm64":
"00eb1bef60572765717bba1fd86da4527f3b69bd40eb51cd0b60cdc89c27f5a6",
"win-64":
"d97edec84c3f450ac23bd2fbac57f77c0b0bffd5313114c1fa8c28c4df8ead6e",
}
spec = LockSpecification(
dependencies=[],
channels=[],
platforms=list(expected.keys()),
sources=[],
virtual_package_repo=vpr,
)
assert spec.content_hash() == expected
def parse_flit_pyproject_toml(pyproject_toml: pathlib.Path, platform: str,
include_dev_dependencies: bool):
contents = toml.load(pyproject_toml)
requirements = get_in(["tool", "flit", "metadata", "requires"], contents,
[])
if include_dev_dependencies:
requirements += get_in(
["tool", "flit", "metadata", "requires-extra", "test"], contents,
[])
requirements += get_in(
["tool", "flit", "metadata", "requires-extra", "dev"], contents,
[])
dependency_sections = ["tool"]
if include_dev_dependencies:
dependency_sections += ["dev-dependencies"]
specs = [python_requirement_to_conda_spec(req) for req in requirements]
conda_deps = get_in(["tool", "conda-lock", "dependencies"], contents, {})
specs.extend(parse_conda_dependencies(conda_deps))
channels = get_in(["tool", "conda-lock", "channels"], contents, [])
return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_environment_file(environment_file: pathlib.Path,
platform: str) -> LockSpecification:
if not environment_file.exists():
raise FileNotFoundError(f"{environment_file} not found")
with environment_file.open("r") as fo:
filtered_content = "\n".join(
filter_platform_selectors(fo.read(), platform=platform))
env_yaml_data = yaml.safe_load(filtered_content)
# TODO: we basically ignore most of the fields for now.
# notable pip deps are just skipped below
specs = env_yaml_data["dependencies"]
channels = env_yaml_data.get("channels", [])
# Split out any sub spec sections from the dependencies mapping
mapping_specs = [x for x in specs if not isinstance(x, str)]
specs = [x for x in specs if isinstance(x, str)]
# Print a warning if there are pip specs in the dependencies
for mapping_spec in mapping_specs:
if "pip" in mapping_spec:
print(
("Warning, found pip deps not included in the lock file! You'll need to install "
"them separately"),
file=sys.stderr,
)
return LockSpecification(specs=specs, channels=channels, platform=platform)
def _parse_pyproject_toml(platform: str,
include_dev_dependencies: bool) -> LockSpecification:
specs: List[str] = []
deps = PyProject.get().senv.dependencies
if include_dev_dependencies:
deps.update(PyProject.get().senv.dev_dependencies)
for depname, depattrs in deps.items():
conda_dep_name = normalize_pypi_name(depname)
if isinstance(depattrs, Mapping):
poetry_version_spec = depattrs["version"]
# TODO: support additional features such as markers for things like sys_platform, platform_system
elif isinstance(depattrs, str):
poetry_version_spec = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}")
conda_version = poetry_version_to_conda_version(poetry_version_spec)
spec = to_match_spec(conda_dep_name, conda_version)
if conda_dep_name == "python":
specs.insert(0, spec)
else:
specs.append(spec)
return LockSpecification(specs=specs,
channels=PyProject.get().senv.conda_channels,
platform=platform)
def _parse_meta_yaml_file_for_platform(
meta_yaml_file: pathlib.Path,
platform: str,
) -> LockSpecification:
"""Parse a simple meta-yaml file for dependencies, assuming the target platform.
* This does not support multi-output files and will ignore all lines with selectors other than platform
"""
if not meta_yaml_file.exists():
raise FileNotFoundError(f"{meta_yaml_file} not found")
with meta_yaml_file.open("r") as fo:
filtered_recipe = "\n".join(
filter_platform_selectors(fo.read(), platform=platform)
)
t = jinja2.Template(filtered_recipe, undefined=UndefinedNeverFail)
rendered = t.render()
meta_yaml_data = yaml.safe_load(rendered)
channels = get_in(["extra", "channels"], meta_yaml_data, [])
depenencies: List[Dependency] = []
def add_spec(spec: str, category: str):
if spec is None:
return
# TODO: This does not parse conda requirements with build strings
dep = parse_python_requirement(
spec,
manager="conda",
optional=category != "main",
category=category,
normalize_name=False,
)
dep.selectors.platform = [platform]
depenencies.append(dep)
def add_requirements_from_recipe_or_output(yaml_data):
for s in get_in(["requirements", "host"], yaml_data, []):
add_spec(s, "main")
for s in get_in(["requirements", "run"], yaml_data, []):
add_spec(s, "main")
for s in get_in(["test", "requires"], yaml_data, []):
add_spec(s, "dev")
add_requirements_from_recipe_or_output(meta_yaml_data)
for output in get_in(["outputs"], meta_yaml_data, []):
add_requirements_from_recipe_or_output(output)
return LockSpecification(
dependencies=depenencies,
channels=channels,
platforms=[platform],
sources=[meta_yaml_file],
)
def test_virtual_package_input_hash_stability():
from conda_lock.virtual_package import virtual_package_repo_from_specification
test_dir = TEST_DIR.joinpath("test-cuda")
spec = test_dir / "virtual-packages-old-glibc.yaml"
vpr = virtual_package_repo_from_specification(spec)
spec = LockSpecification(
dependencies=[],
channels=[],
platforms=["linux-64"],
sources=[],
virtual_package_repo=vpr,
)
expected = "d8d0e556f97aed2eaa05fe9728b5a1c91c1b532d3eed409474e8a9b85b633a26"
assert spec.content_hash() == {"linux-64": expected}
def aggregate_lock_specs(
lock_specs: List[LockSpecification]) -> LockSpecification:
# union the dependencies
specs = list(
set(chain.from_iterable([lock_spec.specs
for lock_spec in lock_specs])))
# pick the first non-empty channel
channels: List[str] = next(
(lock_spec.channels for lock_spec in lock_specs if lock_spec.channels),
[])
# pick the first non-empty platform
platform = next((lock_spec.platform
for lock_spec in lock_specs if lock_spec.platform), "")
return LockSpecification(specs=specs, channels=channels, platform=platform)
def test_poetry_version_parsing_constraints(package, version, url_pattern):
_conda_exe = determine_conda_executable("conda", no_mamba=True)
spec = LockSpecification(
specs=[
to_match_spec(package, poetry_version_to_conda_version(version))
],
channels=["conda-forge"],
platform="linux-64",
)
lockfile_contents = create_lockfile_from_spec(conda=_conda_exe,
channels=spec.channels,
spec=spec)
for line in lockfile_contents:
if url_pattern in line:
break
else:
raise ValueError(f"could not find {package} {version}")
def test_poetry_version_parsing_constraints(package, version, url_pattern,
capsys):
_conda_exe = determine_conda_executable(None,
mamba=False,
micromamba=False)
vpr = default_virtual_package_repodata()
with vpr, capsys.disabled():
with tempfile.NamedTemporaryFile(dir=".") as tf:
spec = LockSpecification(
dependencies=[
VersionedDependency(
name=package,
version=poetry_version_to_conda_version(version),
manager="conda",
optional=False,
category="main",
extras=[],
)
],
channels=["conda-forge"],
platforms=["linux-64"],
# NB: this file must exist for relative path resolution to work
# in create_lockfile_from_spec
sources=[pathlib.Path(tf.name)],
virtual_package_repo=vpr,
)
lockfile_contents = create_lockfile_from_spec(
conda=_conda_exe,
spec=spec,
lockfile_path=pathlib.Path(DEFAULT_LOCKFILE_NAME),
)
python = next(p for p in lockfile_contents.package
if p.name == "python")
assert url_pattern in python.url
def parse_environment_file(environment_file: pathlib.Path,
pip_support: bool = False) -> LockSpecification:
"""
Parse dependencies from a conda environment specification
Parameters
----------
environment_file :
Path to environment.yml
pip_support :
Emit dependencies in pip section of environment.yml. If False, print a
warning and ignore pip dependencies.
"""
dependencies: List[Dependency] = []
if not environment_file.exists():
raise FileNotFoundError(f"{environment_file} not found")
with environment_file.open("r") as fo:
content = fo.read()
filtered_content = "\n".join(
filter_platform_selectors(content, platform=None))
assert yaml.safe_load(filtered_content) == yaml.safe_load(
content), "selectors are temporarily gone"
env_yaml_data = yaml.safe_load(filtered_content)
specs = env_yaml_data["dependencies"]
channels = env_yaml_data.get("channels", [])
# These extension fields are nonstandard
platforms = env_yaml_data.get("platforms", [])
category = env_yaml_data.get("category") or "main"
# Split out any sub spec sections from the dependencies mapping
mapping_specs = [x for x in specs if not isinstance(x, str)]
specs = [x for x in specs if isinstance(x, str)]
for spec in specs:
from ..vendor.conda.models.match_spec import MatchSpec
ms = MatchSpec(spec)
dependencies.append(
VersionedDependency(
name=ms.name,
version=ms.get("version", ""),
manager="conda",
optional=category != "main",
category=category,
extras=[],
build=ms.get("build"),
))
for mapping_spec in mapping_specs:
if "pip" in mapping_spec:
if pip_support:
for spec in mapping_spec["pip"]:
if re.match(r"^-e .*$", spec):
print(
(f"Warning: editable pip dep '{spec}' will not be included in the lock file. "
"You will need to install it separately."),
file=sys.stderr,
)
continue
dependencies.append(
parse_python_requirement(
spec,
manager="pip",
optional=category != "main",
category=category,
))
# ensure pip is in target env
dependencies.append(
parse_python_requirement("pip", manager="conda"))
else:
print(
("Warning: found pip deps, but conda-lock was installed without pypi support. "
"pip dependencies will not be included in the lock file. Either install them "
"separately, or install conda-lock with `-E pip_support`."
),
file=sys.stderr,
)
return LockSpecification(
dependencies=dependencies,
channels=channels,
platforms=platforms,
sources=[environment_file],
)