def test_default_virtual_package_input_hash_stability(): from conda_lock.virtual_package import default_virtual_package_repodata vpr = default_virtual_package_repodata() expected = { "linux-64": "93c22a62ca75ed0fd7649a6c9fbac611fd42a694465841b141c91aa2d4edf1b3", "linux-aarch64": "e1115c4d229438be0bd3e79c3734afb1f2fb8db42cf0c20c0e2ede5405e97e25", "linux-ppc64le": "d980051789ba7e6374c0833bf615b060bc0c5dfa63907eb4f11ac85f4dbb80da", "osx-64": "8e2e62ea8061892d10606e9a10f05f4c7358c798e5a2d390b1206568bf9338a2", "osx-arm64": "00eb1bef60572765717bba1fd86da4527f3b69bd40eb51cd0b60cdc89c27f5a6", "win-64": "d97edec84c3f450ac23bd2fbac57f77c0b0bffd5313114c1fa8c28c4df8ead6e", } spec = LockSpecification( dependencies=[], channels=[], platforms=list(expected.keys()), sources=[], virtual_package_repo=vpr, ) assert spec.content_hash() == expected
def test_aggregate_lock_specs(): gpu_spec = LockSpecification( specs=["pytorch"], channels=["pytorch", "conda-forge"], platform="linux-64", ) base_spec = LockSpecification( specs=["python =3.7"], channels=["conda-forge"], platform="linux-64", ) assert (aggregate_lock_specs([gpu_spec, base_spec]).env_hash() == LockSpecification( specs=["pytorch", "python =3.7"], channels=["pytorch", "conda-forge"], platform="linux-64", ).env_hash()) assert (aggregate_lock_specs([base_spec, gpu_spec]).env_hash() == LockSpecification( specs=["pytorch", "python =3.7"], channels=["conda-forge"], platform="linux-64", ).env_hash())
def test_aggregate_lock_specs(): gpu_spec = LockSpecification( dependencies=[_make_spec("pytorch")], channels=["pytorch", "conda-forge"], platforms=["linux-64"], sources=[pathlib.Path("ml-stuff.yml")], ) base_spec = LockSpecification( dependencies=[_make_spec("python", "=3.7")], channels=["conda-forge"], platforms=["linux-64"], sources=[pathlib.Path("base-env.yml")], ) # NB: content hash explicitly does not depend on the source file names assert (aggregate_lock_specs( [gpu_spec, base_spec]).content_hash() == LockSpecification( dependencies=[_make_spec("pytorch"), _make_spec("python", "=3.7")], channels=["pytorch", "conda-forge"], platforms=["linux-64"], sources=[], ).content_hash()) assert (aggregate_lock_specs( [base_spec, gpu_spec]).content_hash() != LockSpecification( dependencies=[_make_spec("pytorch"), _make_spec("python", "=3.7")], channels=["conda-forge"], platforms=["linux-64"], sources=[], ).content_hash())
def specification_with_dependencies( path: pathlib.Path, toml_contents: Mapping[str, Any], dependencies: List[Dependency]) -> LockSpecification: for depname, depattrs in get_in(["tool", "conda-lock", "dependencies"], toml_contents, {}).items(): if isinstance(depattrs, str): conda_version = depattrs else: raise TypeError( f"Unsupported type for dependency: {depname}: {depattrs:r}") dependencies.append( VersionedDependency( name=depname, version=conda_version, manager="conda", optional=False, category="main", extras=[], )) return LockSpecification( dependencies, channels=get_in(["tool", "conda-lock", "channels"], toml_contents, []), platforms=get_in(["tool", "conda-lock", "platforms"], toml_contents, []), sources=[path], )
def parse_meta_yaml_file(meta_yaml_file: pathlib.Path, platform: str) -> LockSpecification: """Parse a simple meta-yaml file for dependencies. * This does not support multi-output files and will ignore all lines with selectors """ if not meta_yaml_file.exists(): raise FileNotFoundError(f"{meta_yaml_file} not found") with meta_yaml_file.open("r") as fo: filtered_recipe = "\n".join( filter_platform_selectors(fo.read(), platform=platform)) t = jinja2.Template(filtered_recipe, undefined=UndefinedNeverFail) rendered = t.render() meta_yaml_data = yaml.safe_load(rendered) channels = meta_yaml_data.get("extra", {}).get("channels", []) specs = [] def add_spec(spec): if spec is None: return specs.append(spec) for s in meta_yaml_data.get("requirements", {}).get("host", []): add_spec(s) for s in meta_yaml_data.get("requirements", {}).get("run", []): add_spec(s) for s in meta_yaml_data.get("test", {}).get("requires", []): add_spec(s) return LockSpecification(specs=specs, channels=channels, platform=platform)
def _parse_pyproject_toml(platform: str, include_dev_dependencies: bool) -> LockSpecification: specs: List[str] = [] deps = PyProject.get().senv.dependencies if include_dev_dependencies: deps.update(PyProject.get().senv.dev_dependencies) for depname, depattrs in deps.items(): conda_dep_name = normalize_pypi_name(depname) if isinstance(depattrs, Mapping): poetry_version_spec = depattrs["version"] # TODO: support additional features such as markers for things like sys_platform, platform_system elif isinstance(depattrs, str): poetry_version_spec = depattrs else: raise TypeError( f"Unsupported type for dependency: {depname}: {depattrs:r}") conda_version = poetry_version_to_conda_version(poetry_version_spec) spec = to_match_spec(conda_dep_name, conda_version) if conda_dep_name == "python": specs.insert(0, spec) else: specs.append(spec) return LockSpecification(specs=specs, channels=PyProject.get().senv.conda_channels, platform=platform)
def test_virtual_package_input_hash_stability(): from conda_lock.virtual_package import virtual_package_repo_from_specification test_dir = TEST_DIR.joinpath("test-cuda") spec = test_dir / "virtual-packages-old-glibc.yaml" vpr = virtual_package_repo_from_specification(spec) spec = LockSpecification( dependencies=[], channels=[], platforms=["linux-64"], sources=[], virtual_package_repo=vpr, ) expected = "d8d0e556f97aed2eaa05fe9728b5a1c91c1b532d3eed409474e8a9b85b633a26" assert spec.content_hash() == {"linux-64": expected}
def parse_poetry_pyproject_toml(pyproject_toml: pathlib.Path, platform: str) -> LockSpecification: contents = toml.load(pyproject_toml) specs: List[str] = [] for key in ["dependencies", "dev-dependencies"]: deps = contents.get("tool", {}).get("poetry", {}).get(key, {}) for depname, depattrs in deps.items(): conda_dep_name = normalize_pypi_name(depname) if isinstance(depattrs, collections.Mapping): poetry_version_spec = depattrs["version"] # TODO: support additional features such as markerts for things like sys_platform, platform_system elif isinstance(depattrs, str): poetry_version_spec = depattrs else: raise TypeError( f"Unsupported type for dependency: {depname}: {depattrs:r}" ) conda_version = poetry_version_to_conda_version( poetry_version_spec) if conda_version: spec = f"{conda_dep_name}[version{conda_version}]" else: spec = f"{conda_dep_name}" if conda_dep_name == "python": specs.insert(0, spec) else: specs.append(spec) channels = contents.get("tool", {}).get("conda-lock", {}).get("channels", []) return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_flit_pyproject_toml(pyproject_toml: pathlib.Path, platform: str, include_dev_dependencies: bool): contents = toml.load(pyproject_toml) requirements = get_in(["tool", "flit", "metadata", "requires"], contents, []) if include_dev_dependencies: requirements += get_in( ["tool", "flit", "metadata", "requires-extra", "test"], contents, []) requirements += get_in( ["tool", "flit", "metadata", "requires-extra", "dev"], contents, []) dependency_sections = ["tool"] if include_dev_dependencies: dependency_sections += ["dev-dependencies"] specs = [python_requirement_to_conda_spec(req) for req in requirements] conda_deps = get_in(["tool", "conda-lock", "dependencies"], contents, {}) specs.extend(parse_conda_dependencies(conda_deps)) channels = get_in(["tool", "conda-lock", "channels"], contents, []) return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_poetry_pyproject_toml( pyproject_toml: pathlib.Path, platform: str, include_dev_dependencies: bool ) -> LockSpecification: contents = toml.load(pyproject_toml) specs: List[str] = [] dependency_sections = ["dependencies"] if include_dev_dependencies: dependency_sections.append("dev-dependencies") for key in dependency_sections: deps = get_in(["tool", "poetry", key], contents, {}) for depname, depattrs in deps.items(): conda_dep_name = normalize_pypi_name(depname) if isinstance(depattrs, collections.Mapping): poetry_version_spec = depattrs["version"] # TODO: support additional features such as markers for things like sys_platform, platform_system elif isinstance(depattrs, str): poetry_version_spec = depattrs else: raise TypeError( f"Unsupported type for dependency: {depname}: {depattrs:r}" ) conda_version = poetry_version_to_conda_version(poetry_version_spec) spec = to_match_spec(conda_dep_name, conda_version) if conda_dep_name == "python": specs.insert(0, spec) else: specs.append(spec) channels = get_in(["tool", "conda-lock", "channels"], contents, []) return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_environment_file(environment_file: pathlib.Path, platform: str) -> LockSpecification: if not environment_file.exists(): raise FileNotFoundError(f"{environment_file} not found") with environment_file.open("r") as fo: filtered_content = "\n".join( filter_platform_selectors(fo.read(), platform=platform)) env_yaml_data = yaml.safe_load(filtered_content) # TODO: we basically ignore most of the fields for now. # notable pip deps are just skipped below specs = env_yaml_data["dependencies"] channels = env_yaml_data.get("channels", []) # Split out any sub spec sections from the dependencies mapping mapping_specs = [x for x in specs if not isinstance(x, str)] specs = [x for x in specs if isinstance(x, str)] # Print a warning if there are pip specs in the dependencies for mapping_spec in mapping_specs: if "pip" in mapping_spec: print( ("Warning, found pip deps not included in the lock file! You'll need to install " "them separately"), file=sys.stderr, ) return LockSpecification(specs=specs, channels=channels, platform=platform)
def _parse_meta_yaml_file_for_platform( meta_yaml_file: pathlib.Path, platform: str, ) -> LockSpecification: """Parse a simple meta-yaml file for dependencies, assuming the target platform. * This does not support multi-output files and will ignore all lines with selectors other than platform """ if not meta_yaml_file.exists(): raise FileNotFoundError(f"{meta_yaml_file} not found") with meta_yaml_file.open("r") as fo: filtered_recipe = "\n".join( filter_platform_selectors(fo.read(), platform=platform) ) t = jinja2.Template(filtered_recipe, undefined=UndefinedNeverFail) rendered = t.render() meta_yaml_data = yaml.safe_load(rendered) channels = get_in(["extra", "channels"], meta_yaml_data, []) depenencies: List[Dependency] = [] def add_spec(spec: str, category: str): if spec is None: return # TODO: This does not parse conda requirements with build strings dep = parse_python_requirement( spec, manager="conda", optional=category != "main", category=category, normalize_name=False, ) dep.selectors.platform = [platform] depenencies.append(dep) def add_requirements_from_recipe_or_output(yaml_data): for s in get_in(["requirements", "host"], yaml_data, []): add_spec(s, "main") for s in get_in(["requirements", "run"], yaml_data, []): add_spec(s, "main") for s in get_in(["test", "requires"], yaml_data, []): add_spec(s, "dev") add_requirements_from_recipe_or_output(meta_yaml_data) for output in get_in(["outputs"], meta_yaml_data, []): add_requirements_from_recipe_or_output(output) return LockSpecification( dependencies=depenencies, channels=channels, platforms=[platform], sources=[meta_yaml_file], )
def aggregate_lock_specs( lock_specs: List[LockSpecification]) -> LockSpecification: # union the dependencies specs = list( set(chain.from_iterable([lock_spec.specs for lock_spec in lock_specs]))) # pick the first non-empty channel channels: List[str] = next( (lock_spec.channels for lock_spec in lock_specs if lock_spec.channels), []) # pick the first non-empty platform platform = next((lock_spec.platform for lock_spec in lock_specs if lock_spec.platform), "") return LockSpecification(specs=specs, channels=channels, platform=platform)
def test_poetry_version_parsing_constraints(package, version, url_pattern): _conda_exe = determine_conda_executable("conda", no_mamba=True) spec = LockSpecification( specs=[ to_match_spec(package, poetry_version_to_conda_version(version)) ], channels=["conda-forge"], platform="linux-64", ) lockfile_contents = create_lockfile_from_spec(conda=_conda_exe, channels=spec.channels, spec=spec) for line in lockfile_contents: if url_pattern in line: break else: raise ValueError(f"could not find {package} {version}")
def create_lockfile_from_spec( *, conda: PathLike, spec: LockSpecification, platforms: List[str] = [], lockfile_path: pathlib.Path, update_spec: Optional[UpdateSpecification] = None, ) -> Lockfile: """ Solve or update specification """ assert spec.virtual_package_repo is not None virtual_package_channel = spec.virtual_package_repo.channel_url locked: Dict[Tuple[str, str, str], LockedDependency] = {} for platform in platforms or spec.platforms: deps = _solve_for_arch( conda=conda, spec=spec, platform=platform, channels=[*spec.channels, virtual_package_channel], update_spec=update_spec, ) for dep in deps: locked[(dep.manager, dep.name, dep.platform)] = dep return Lockfile( package=[locked[k] for k in locked], metadata=LockMeta( content_hash=spec.content_hash(), channels=spec.channels, platforms=spec.platforms, sources=[ relative_path(lockfile_path.parent, source) for source in spec.sources ], ), )
def test_poetry_version_parsing_constraints(package, version, url_pattern, capsys): _conda_exe = determine_conda_executable(None, mamba=False, micromamba=False) vpr = default_virtual_package_repodata() with vpr, capsys.disabled(): with tempfile.NamedTemporaryFile(dir=".") as tf: spec = LockSpecification( dependencies=[ VersionedDependency( name=package, version=poetry_version_to_conda_version(version), manager="conda", optional=False, category="main", extras=[], ) ], channels=["conda-forge"], platforms=["linux-64"], # NB: this file must exist for relative path resolution to work # in create_lockfile_from_spec sources=[pathlib.Path(tf.name)], virtual_package_repo=vpr, ) lockfile_contents = create_lockfile_from_spec( conda=_conda_exe, spec=spec, lockfile_path=pathlib.Path(DEFAULT_LOCKFILE_NAME), ) python = next(p for p in lockfile_contents.package if p.name == "python") assert url_pattern in python.url
def parse_environment_file(environment_file: pathlib.Path, pip_support: bool = False) -> LockSpecification: """ Parse dependencies from a conda environment specification Parameters ---------- environment_file : Path to environment.yml pip_support : Emit dependencies in pip section of environment.yml. If False, print a warning and ignore pip dependencies. """ dependencies: List[Dependency] = [] if not environment_file.exists(): raise FileNotFoundError(f"{environment_file} not found") with environment_file.open("r") as fo: content = fo.read() filtered_content = "\n".join( filter_platform_selectors(content, platform=None)) assert yaml.safe_load(filtered_content) == yaml.safe_load( content), "selectors are temporarily gone" env_yaml_data = yaml.safe_load(filtered_content) specs = env_yaml_data["dependencies"] channels = env_yaml_data.get("channels", []) # These extension fields are nonstandard platforms = env_yaml_data.get("platforms", []) category = env_yaml_data.get("category") or "main" # Split out any sub spec sections from the dependencies mapping mapping_specs = [x for x in specs if not isinstance(x, str)] specs = [x for x in specs if isinstance(x, str)] for spec in specs: from ..vendor.conda.models.match_spec import MatchSpec ms = MatchSpec(spec) dependencies.append( VersionedDependency( name=ms.name, version=ms.get("version", ""), manager="conda", optional=category != "main", category=category, extras=[], build=ms.get("build"), )) for mapping_spec in mapping_specs: if "pip" in mapping_spec: if pip_support: for spec in mapping_spec["pip"]: if re.match(r"^-e .*$", spec): print( (f"Warning: editable pip dep '{spec}' will not be included in the lock file. " "You will need to install it separately."), file=sys.stderr, ) continue dependencies.append( parse_python_requirement( spec, manager="pip", optional=category != "main", category=category, )) # ensure pip is in target env dependencies.append( parse_python_requirement("pip", manager="conda")) else: print( ("Warning: found pip deps, but conda-lock was installed without pypi support. " "pip dependencies will not be included in the lock file. Either install them " "separately, or install conda-lock with `-E pip_support`." ), file=sys.stderr, ) return LockSpecification( dependencies=dependencies, channels=channels, platforms=platforms, sources=[environment_file], )