def add_requirements_from_recipe_or_output(yaml_data):
for s in get_in(["requirements", "host"], yaml_data, []):
add_spec(s, "main")
for s in get_in(["requirements", "run"], yaml_data, []):
add_spec(s, "main")
for s in get_in(["test", "requires"], yaml_data, []):
add_spec(s, "dev")
def parse_poetry_pyproject_toml(
pyproject_toml: pathlib.Path, platform: str, include_dev_dependencies: bool
) -> LockSpecification:
contents = toml.load(pyproject_toml)
specs: List[str] = []
dependency_sections = ["dependencies"]
if include_dev_dependencies:
dependency_sections.append("dev-dependencies")
for key in dependency_sections:
deps = get_in(["tool", "poetry", key], contents, {})
for depname, depattrs in deps.items():
conda_dep_name = normalize_pypi_name(depname)
if isinstance(depattrs, collections.Mapping):
poetry_version_spec = depattrs["version"]
# TODO: support additional features such as markers for things like sys_platform, platform_system
elif isinstance(depattrs, str):
poetry_version_spec = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}"
)
conda_version = poetry_version_to_conda_version(poetry_version_spec)
spec = to_match_spec(conda_dep_name, conda_version)
if conda_dep_name == "python":
specs.insert(0, spec)
else:
specs.append(spec)
channels = get_in(["tool", "conda-lock", "channels"], contents, [])
return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_requirements_pyproject_toml(
pyproject_toml_path: pathlib.Path,
contents: Mapping[str, Any],
prefix: Sequence[str],
main_tag: str,
optional_tag: str,
dev_tags: AbstractSet[str] = {"dev", "test"},
):
"""
PEP621 and flit
"""
dependencies: List[Dependency] = []
sections = {(*prefix, main_tag): "main"}
for extra in dev_tags:
sections[(*prefix, optional_tag, extra)] = "dev"
for extra in set(get_in([*prefix, optional_tag], contents,
{}).keys()).difference(dev_tags):
sections[(*prefix, optional_tag, extra)] = extra
for path, category in sections.items():
for dep in get_in(list(path), contents, []):
dependencies.append(
parse_python_requirement(dep,
manager="conda",
category=category,
optional=category != "main"))
return specification_with_dependencies(pyproject_toml_path, contents,
dependencies)
def parse_pyproject_toml(pyproject_toml: pathlib.Path, ) -> LockSpecification:
contents = toml.load(pyproject_toml)
build_system = get_in(["build-system", "build-backend"], contents)
pep_621_probe = get_in(["project", "dependencies"], contents)
parse = parse_poetry_pyproject_toml
if pep_621_probe is not None:
parse = partial(
parse_requirements_pyproject_toml,
prefix=("project", ),
main_tag="dependencies",
optional_tag="optional-dependencies",
)
elif build_system.startswith("poetry"):
parse = parse_poetry_pyproject_toml
elif build_system.startswith("flit"):
parse = partial(
parse_requirements_pyproject_toml,
prefix=("tool", "flit", "metadata"),
main_tag="requires",
optional_tag="requires-extra",
)
else:
import warnings
warnings.warn(
"Could not detect build-system in pyproject.toml. Assuming poetry"
)
return parse(pyproject_toml, contents)
def specification_with_dependencies(
path: pathlib.Path, toml_contents: Mapping[str, Any],
dependencies: List[Dependency]) -> LockSpecification:
for depname, depattrs in get_in(["tool", "conda-lock", "dependencies"],
toml_contents, {}).items():
if isinstance(depattrs, str):
conda_version = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs:r}")
dependencies.append(
VersionedDependency(
name=depname,
version=conda_version,
manager="conda",
optional=False,
category="main",
extras=[],
))
return LockSpecification(
dependencies,
channels=get_in(["tool", "conda-lock", "channels"], toml_contents, []),
platforms=get_in(["tool", "conda-lock", "platforms"], toml_contents,
[]),
sources=[path],
)
def parse_flit_pyproject_toml(pyproject_toml: pathlib.Path, platform: str,
include_dev_dependencies: bool):
contents = toml.load(pyproject_toml)
requirements = get_in(["tool", "flit", "metadata", "requires"], contents,
[])
if include_dev_dependencies:
requirements += get_in(
["tool", "flit", "metadata", "requires-extra", "test"], contents,
[])
requirements += get_in(
["tool", "flit", "metadata", "requires-extra", "dev"], contents,
[])
dependency_sections = ["tool"]
if include_dev_dependencies:
dependency_sections += ["dev-dependencies"]
specs = [python_requirement_to_conda_spec(req) for req in requirements]
conda_deps = get_in(["tool", "conda-lock", "dependencies"], contents, {})
specs.extend(parse_conda_dependencies(conda_deps))
channels = get_in(["tool", "conda-lock", "channels"], contents, [])
return LockSpecification(specs=specs, channels=channels, platform=platform)
def add_requirements_from_recipe_or_output(yaml_data):
for s in get_in(["requirements", "host"], yaml_data, []):
add_spec(s)
for s in get_in(["requirements", "run"], yaml_data, []):
add_spec(s)
if include_dev_dependencies:
for s in get_in(["test", "requires"], yaml_data, []):
add_spec(s)
def _parse_meta_yaml_file_for_platform(
meta_yaml_file: pathlib.Path,
platform: str,
) -> LockSpecification:
"""Parse a simple meta-yaml file for dependencies, assuming the target platform.
* This does not support multi-output files and will ignore all lines with selectors other than platform
"""
if not meta_yaml_file.exists():
raise FileNotFoundError(f"{meta_yaml_file} not found")
with meta_yaml_file.open("r") as fo:
filtered_recipe = "\n".join(
filter_platform_selectors(fo.read(), platform=platform)
)
t = jinja2.Template(filtered_recipe, undefined=UndefinedNeverFail)
rendered = t.render()
meta_yaml_data = yaml.safe_load(rendered)
channels = get_in(["extra", "channels"], meta_yaml_data, [])
depenencies: List[Dependency] = []
def add_spec(spec: str, category: str):
if spec is None:
return
# TODO: This does not parse conda requirements with build strings
dep = parse_python_requirement(
spec,
manager="conda",
optional=category != "main",
category=category,
normalize_name=False,
)
dep.selectors.platform = [platform]
depenencies.append(dep)
def add_requirements_from_recipe_or_output(yaml_data):
for s in get_in(["requirements", "host"], yaml_data, []):
add_spec(s, "main")
for s in get_in(["requirements", "run"], yaml_data, []):
add_spec(s, "main")
for s in get_in(["test", "requires"], yaml_data, []):
add_spec(s, "dev")
add_requirements_from_recipe_or_output(meta_yaml_data)
for output in get_in(["outputs"], meta_yaml_data, []):
add_requirements_from_recipe_or_output(output)
return LockSpecification(
dependencies=depenencies,
channels=channels,
platforms=[platform],
sources=[meta_yaml_file],
)
def parse_meta_yaml_file(meta_yaml_file: pathlib.Path, platform: str,
include_dev_dependencies: bool) -> LockSpecification:
"""Parse a simple meta-yaml file for dependencies.
* This does not support multi-output files and will ignore all lines with selectors
"""
if not meta_yaml_file.exists():
raise FileNotFoundError(f"{meta_yaml_file} not found")
with meta_yaml_file.open("r") as fo:
filtered_recipe = "\n".join(
filter_platform_selectors(fo.read(), platform=platform))
t = jinja2.Template(filtered_recipe, undefined=UndefinedNeverFail)
rendered = t.render()
meta_yaml_data = yaml.safe_load(rendered)
channels = get_in(["extra", "channels"], meta_yaml_data, [])
specs = []
def add_spec(spec):
if spec is None:
return
specs.append(spec)
def add_requirements_from_recipe_or_output(yaml_data):
for s in get_in(["requirements", "host"], yaml_data, []):
add_spec(s)
for s in get_in(["requirements", "run"], yaml_data, []):
add_spec(s)
if include_dev_dependencies:
for s in get_in(["test", "requires"], yaml_data, []):
add_spec(s)
add_requirements_from_recipe_or_output(meta_yaml_data)
for output in get_in(["outputs"], meta_yaml_data, []):
add_requirements_from_recipe_or_output(output)
return LockSpecification(specs=specs, channels=channels, platform=platform)
def parse_pyproject_toml(pyproject_toml: pathlib.Path, platform: str,
include_dev_dependencies: bool):
contents = toml.load(pyproject_toml)
build_system = get_in(["build-system", "build-backend"], contents)
parse = parse_poetry_pyproject_toml
if build_system.startswith("poetry"):
parse = parse_poetry_pyproject_toml
elif build_system.startswith("flit"):
parse = parse_flit_pyproject_toml
else:
import warnings
warnings.warn(
"Could not detect build-system in pyproject.toml. Assuming poetry"
)
return parse(pyproject_toml, platform, include_dev_dependencies)
def parse_poetry_pyproject_toml(
path: pathlib.Path,
contents: Mapping[str, Any],
) -> LockSpecification:
"""
Parse dependencies from a poetry pyproject.toml file
Each dependency is assigned a category depending on which section it appears in:
* dependencies in [tool.poetry.dependencies] have category main
* dependencies in [tool.poetry.dev-dependencies] have category dev
* dependencies in each `key` of [tool.poetry.extras] have category `key`
* By default, dependency names are translated to the conda equivalent, with two exceptions:
- If a dependency has `source = "pypi"`, it is treated as a pip dependency (by name)
- If a dependency has a url, it is treated as a direct pip dependency (by url)
* markers are not supported
"""
dependencies: List[Dependency] = []
categories = {"dependencies": "main", "dev-dependencies": "dev"}
dep_to_extra = {}
for category, deps in get_in(["tool", "poetry", "extras"], contents,
{}).items():
for dep in deps:
dep_to_extra[dep] = category
for section, default_category in categories.items():
for depname, depattrs in get_in(["tool", "poetry", section], contents,
{}).items():
category = dep_to_extra.get(depname) or default_category
optional = category != "main"
manager: Literal["conda", "pip"] = "conda"
url = None
extras = []
if isinstance(depattrs, collections.abc.Mapping):
poetry_version_spec = depattrs.get("version", None)
url = depattrs.get("url", None)
optional = depattrs.get("optional", False)
extras = depattrs.get("extras", [])
# If a dependency is explicitly marked as sourced from pypi,
# or is a URL dependency, delegate to the pip section
if (depattrs.get("source", None) == "pypi"
or poetry_version_spec is None):
manager = "pip"
# TODO: support additional features such as markers for things like sys_platform, platform_system
elif isinstance(depattrs, str):
poetry_version_spec = depattrs
else:
raise TypeError(
f"Unsupported type for dependency: {depname}: {depattrs}")
if manager == "conda":
name = normalize_pypi_name(depname)
version = poetry_version_to_conda_version(poetry_version_spec)
else:
name = depname
version = poetry_version_spec
if version is None:
if url is None:
raise ValueError(
f"dependency {depname} has neither version nor url")
url, hashes = urldefrag(url)
dependencies.append(
URLDependency(
name=name,
url=url,
hashes=[hashes],
manager=manager,
optional=optional,
category=category,
extras=extras,
))
else:
dependencies.append(
VersionedDependency(
name=name,
version=version,
manager=manager,
optional=optional,
category=category,
extras=extras,
))
return specification_with_dependencies(path, contents, dependencies)
