def __call__(
self,
requirements_relpath: str = "requirements.txt",
*,
module_mapping: Optional[Mapping[str, Iterable[str]]] = None,
type_stubs_module_mapping: Optional[Mapping[str, Iterable[str]]] = None,
) -> None:
"""
:param requirements_relpath: The relpath from this BUILD file to the requirements file.
Defaults to a `requirements.txt` file sibling to the BUILD file.
:param module_mapping: a mapping of requirement names to a list of the modules they provide.
For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the
requirement name as the default module, e.g. "Django" will default to
`modules=["django"]`.
"""
req_file_tgt = self._parse_context.create_object(
"_python_requirements_file",
name=requirements_relpath.replace(os.path.sep, "_"),
sources=[requirements_relpath],
)
requirements_dep = f":{req_file_tgt.name}"
normalized_module_mapping = normalize_module_mapping(module_mapping)
normalized_type_stubs_module_mapping = normalize_module_mapping(type_stubs_module_mapping)
req_file = Path(get_buildroot(), self._parse_context.rel_path, requirements_relpath)
requirements = parse_requirements_file(
req_file.read_text(), rel_path=str(req_file.relative_to(get_buildroot()))
)
grouped_requirements = groupby(requirements, lambda parsed_req: parsed_req.project_name)
for project_name, parsed_reqs_ in grouped_requirements:
normalized_proj_name = canonicalize_project_name(project_name)
req_module_mapping = (
{normalized_proj_name: normalized_module_mapping[normalized_proj_name]}
if normalized_proj_name in normalized_module_mapping
else {}
)
req_stubs_mapping = (
{normalized_proj_name: normalized_type_stubs_module_mapping[normalized_proj_name]}
if normalized_proj_name in normalized_type_stubs_module_mapping
else {}
)
self._parse_context.create_object(
"python_requirement_library",
name=project_name,
requirements=list(parsed_reqs_),
module_mapping=req_module_mapping,
type_stubs_module_mapping=req_stubs_mapping,
dependencies=[requirements_dep],
)
def __call__(
self,
*,
source: str = "requirements.txt",
module_mapping: Mapping[str, Iterable[str]] | None = None,
type_stubs_module_mapping: Mapping[str, Iterable[str]] | None = None,
overrides: OVERRIDES_TYPE = None,
) -> None:
"""
:param module_mapping: a mapping of requirement names to a list of the modules they provide.
For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the
requirement name as the default module, e.g. "Django" will default to
`modules=["django"]`.
"""
req_file_tgt = self._parse_context.create_object(
"_python_requirements_file",
name=source.replace(os.path.sep, "_"),
sources=[source],
)
requirements_dep = f":{req_file_tgt.name}"
normalized_module_mapping = normalize_module_mapping(module_mapping)
normalized_type_stubs_module_mapping = normalize_module_mapping(
type_stubs_module_mapping)
req_file = Path(get_buildroot(), self._parse_context.rel_path, source)
requirements = parse_requirements_file(
req_file.read_text(),
rel_path=str(req_file.relative_to(get_buildroot())))
dependencies_overrides = flatten_overrides_to_dependency_field(
overrides,
macro_name="python_requirements",
build_file_dir=self._parse_context.rel_path)
grouped_requirements = groupby(
requirements, lambda parsed_req: parsed_req.project_name)
for project_name, parsed_reqs_ in grouped_requirements:
normalized_proj_name = canonicalize_project_name(project_name)
self._parse_context.create_object(
"python_requirement",
name=project_name,
requirements=list(parsed_reqs_),
modules=normalized_module_mapping.get(normalized_proj_name),
type_stub_modules=normalized_type_stubs_module_mapping.get(
normalized_proj_name),
dependencies=[
requirements_dep,
*dependencies_overrides.get(normalized_proj_name, []),
],
)
def test_parse_requirements_file() -> None:
content = dedent("""\
# Comment.
--find-links=https://duckduckgo.com
ansicolors>=1.18.0
Django==3.2 ; python_version>'3'
Un-Normalized-PROJECT # Inline comment.
pip@ git+https://github.com/pypa/pip.git
""")
assert set(parse_requirements_file(content, rel_path="foo.txt")) == {
Requirement.parse("ansicolors>=1.18.0"),
Requirement.parse("Django==3.2 ; python_version>'3'"),
Requirement.parse("Un-Normalized-PROJECT"),
Requirement.parse("pip@ git+https://github.com/pypa/pip.git"),
}
async def determine_global_requirement_constraints(
python_setup: PythonSetup, ) -> GlobalRequirementConstraints:
if not python_setup.requirement_constraints:
return GlobalRequirementConstraints()
constraints_file_contents = await Get(
DigestContents,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
"the option `[python].requirement_constraints`",
),
)
return GlobalRequirementConstraints(
parse_requirements_file(
constraints_file_contents[0].content.decode(),
rel_path=constraints_file_contents[0].path,
))
def __call__(
self,
requirements_relpath: str = "requirements.txt",
*,
module_mapping: Optional[Mapping[str, Iterable[str]]] = None,
) -> None:
"""
:param requirements_relpath: The relpath from this BUILD file to the requirements file.
Defaults to a `requirements.txt` file sibling to the BUILD file.
:param module_mapping: a mapping of requirement names to a list of the modules they provide.
For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the
requirement name as the default module, e.g. "Django" will default to
`modules=["django"]`.
"""
req_file_tgt = self._parse_context.create_object(
"_python_requirements_file",
name=requirements_relpath.replace(os.path.sep, "_"),
sources=[requirements_relpath],
)
requirements_dep = f":{req_file_tgt.name}"
req_file = Path(get_buildroot(), self._parse_context.rel_path, requirements_relpath)
requirements = parse_requirements_file(
req_file.read_text(), rel_path=str(req_file.relative_to(get_buildroot()))
)
for parsed_req in requirements:
req_module_mapping = (
{parsed_req.project_name: module_mapping[parsed_req.project_name]}
if module_mapping and parsed_req.project_name in module_mapping
else None
)
self._parse_context.create_object(
"python_requirement_library",
name=parsed_req.project_name,
requirements=[parsed_req],
module_mapping=req_module_mapping,
dependencies=[requirements_dep],
)
def test_parse_requirements_file() -> None:
content = dedent(
r"""\
# Comment.
--find-links=https://duckduckgo.com
-r more_reqs.txt
ansicolors>=1.18.0
Django==3.2 ; python_version>'3'
Un-Normalized-PROJECT # Inline comment.
pip@ git+https://github.com/pypa/pip.git
setuptools==54.1.2; python_version >= "3.6" \
--hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \
--hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff
wheel==1.2.3 --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b
"""
)
assert set(parse_requirements_file(content, rel_path="foo.txt")) == {
PipRequirement.parse("ansicolors>=1.18.0"),
PipRequirement.parse("Django==3.2 ; python_version>'3'"),
PipRequirement.parse("Un-Normalized-PROJECT"),
PipRequirement.parse("pip@ git+https://github.com/pypa/pip.git"),
PipRequirement.parse("setuptools==54.1.2; python_version >= '3.6'"),
PipRequirement.parse("wheel==1.2.3"),
}
async def pex_from_targets(
request: PexFromTargetsRequest,
python_setup: PythonSetup,
constraints_file: MaybeConstraintsFile,
) -> PexRequest:
if request.direct_deps_only:
targets = await Get(Targets, Addresses(request.addresses))
direct_deps = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets
)
all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets))
else:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(request.addresses)
)
all_targets = transitive_targets.closure
input_digests = []
if request.additional_sources:
input_digests.append(request.additional_sources)
if request.include_source_files:
prepared_sources = await Get(
StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)
)
input_digests.append(prepared_sources.stripped_source_files.snapshot.digest)
merged_input_digest = await Get(Digest, MergeDigests(input_digests))
if request.hardcoded_interpreter_constraints:
interpreter_constraints = request.hardcoded_interpreter_constraints
else:
calculated_constraints = PexInterpreterConstraints.create_from_targets(
all_targets, python_setup
)
# If there are no targets, we fall back to the global constraints. This is relevant,
# for example, when running `./pants repl` with no specs.
interpreter_constraints = calculated_constraints or PexInterpreterConstraints(
python_setup.interpreter_constraints
)
exact_reqs = PexRequirements.create_from_requirement_fields(
(
tgt[PythonRequirementsField]
for tgt in all_targets
if tgt.has_field(PythonRequirementsField)
),
additional_requirements=request.additional_requirements,
)
requirements = exact_reqs
repository_pex: Pex | None = None
description = request.description
if constraints_file.path:
constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest)
constraints_file_reqs = set(
parse_requirements_file(
constraints_file_contents[0].content.decode(),
rel_path=constraints_file.path,
)
)
# In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let
# packaging canonicalize for us.
# See: https://www.python.org/dev/peps/pep-0503/#normalized-names
url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch'
name_reqs = set() # E.g., foobar>=1.2.3
name_req_projects = set()
for req_str in exact_reqs:
req = Requirement.parse(req_str)
if req.url: # type: ignore[attr-defined]
url_reqs.add(req)
else:
name_reqs.add(req)
name_req_projects.add(canonicalize_project_name(req.project_name))
constraint_file_projects = {
canonicalize_project_name(req.project_name) for req in constraints_file_reqs
}
# Constraints files must only contain name reqs, not URL reqs (those are already
# constrained by their very nature). See https://github.com/pypa/pip/issues/8210.
unconstrained_projects = name_req_projects - constraint_file_projects
if unconstrained_projects:
constraints_descr = (
f"constraints file {constraints_file.path}"
if python_setup.requirement_constraints
else f"_python_constraints target {python_setup.requirement_constraints_target}"
)
logger.warning(
f"The {constraints_descr} does not contain entries for the following "
f"requirements: {', '.join(unconstrained_projects)}"
)
if python_setup.resolve_all_constraints:
if unconstrained_projects:
logger.warning(
"Ignoring `[python_setup].resolve_all_constraints` option because constraints "
"file does not cover all requirements."
)
else:
# To get a full set of requirements we must add the URL requirements to the
# constraints file, since the latter cannot contain URL requirements.
# NB: We can only add the URL requirements we know about here, i.e., those that
# are transitive deps of the targets in play. There may be others in the repo.
# So we may end up creating a few different repository pexes, each with identical
# name requirements but different subsets of URL requirements. Fortunately since
# all these repository pexes will have identical pinned versions of everything,
# this is not a correctness issue, only a performance one.
# TODO: Address this as part of providing proper lockfile support. However we
# generate lockfiles, they must be able to include URL requirements.
all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)}
repository_pex = await Get(
Pex,
PexRequest(
description=f"Resolving {python_setup.requirement_constraints}",
output_filename="repository.pex",
internal_only=request.internal_only,
requirements=PexRequirements(all_constraints),
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
additional_args=["-vvv"],
),
)
elif (
python_setup.resolve_all_constraints
and python_setup.resolve_all_constraints_was_set_explicitly()
):
raise ValueError(
"[python-setup].resolve_all_constraints is enabled, so either "
"[python-setup].requirement_constraints or "
"[python-setup].requirement_constraints_target must also be provided."
)
return PexRequest(
output_filename=request.output_filename,
internal_only=request.internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
main=request.main,
sources=merged_input_digest,
additional_inputs=request.additional_inputs,
repository_pex=repository_pex,
additional_args=request.additional_args,
description=description,
)
async def generate_from_python_requirement(
request: GenerateFromPythonRequirementsRequest, python_setup: PythonSetup
) -> GeneratedTargets:
generator = request.generator
requirements_rel_path = generator[PythonRequirementsSourceField].value
requirements_full_path = generator[PythonRequirementsSourceField].file_path
overrides = {
canonicalize_project_name(k): v
for k, v in request.require_unparametrized_overrides().items()
}
file_tgt = TargetGeneratorSourcesHelperTarget(
{TargetGeneratorSourcesHelperSourcesField.alias: [requirements_rel_path]},
Address(
generator.address.spec_path,
target_name=generator.address.target_name,
relative_file_path=requirements_rel_path,
),
)
digest_contents = await Get(
DigestContents,
PathGlobs(
[requirements_full_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"{generator}'s field `{PythonRequirementsSourceField.alias}`",
),
)
requirements = parse_requirements_file(
digest_contents[0].content.decode(), rel_path=requirements_full_path
)
grouped_requirements = itertools.groupby(
requirements, lambda parsed_req: parsed_req.project_name
)
# Validate the resolve is legal.
generator[PythonRequirementResolveField].normalized_value(python_setup)
module_mapping = generator[ModuleMappingField].value
stubs_mapping = generator[TypeStubsModuleMappingField].value
inherited_fields = {
field.alias: field.value
for field in request.generator.field_values.values()
if isinstance(field, (*COMMON_TARGET_FIELDS, PythonRequirementResolveField))
}
def generate_tgt(
project_name: str, parsed_reqs: Iterable[PipRequirement]
) -> PythonRequirementTarget:
normalized_proj_name = canonicalize_project_name(project_name)
tgt_overrides = overrides.pop(normalized_proj_name, {})
if Dependencies.alias in tgt_overrides:
tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
file_tgt.address.spec
]
return PythonRequirementTarget(
{
**inherited_fields,
PythonRequirementsField.alias: list(parsed_reqs),
PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
normalized_proj_name
),
# This may get overridden by `tgt_overrides`, which will have already added in
# the file tgt.
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
generator.address.create_generated(project_name),
)
result = tuple(
generate_tgt(project_name, parsed_reqs_)
for project_name, parsed_reqs_ in grouped_requirements
) + (file_tgt,)
if overrides:
raise InvalidFieldException(
f"Unused key in the `overrides` field for {request.generator.address}: "
f"{sorted(overrides)}"
)
return GeneratedTargets(generator, result)
def validate_python_requirements(path: str, contents: bytes) -> None:
for _ in parse_requirements_file(contents.decode(), rel_path=path):
pass
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest:
if request.direct_deps_only:
targets = await Get(Targets, Addresses(request.addresses))
direct_deps = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets
)
all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets))
else:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(request.addresses)
)
all_targets = transitive_targets.closure
input_digests = []
if request.additional_sources:
input_digests.append(request.additional_sources)
if request.include_source_files:
prepared_sources = await Get(
StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)
)
input_digests.append(prepared_sources.stripped_source_files.snapshot.digest)
merged_input_digest = await Get(Digest, MergeDigests(input_digests))
if request.hardcoded_interpreter_constraints:
interpreter_constraints = request.hardcoded_interpreter_constraints
else:
calculated_constraints = PexInterpreterConstraints.create_from_targets(
all_targets, python_setup
)
# If there are no targets, we fall back to the global constraints. This is relevant,
# for example, when running `./pants repl` with no specs.
interpreter_constraints = calculated_constraints or PexInterpreterConstraints(
python_setup.interpreter_constraints
)
exact_reqs = PexRequirements.create_from_requirement_fields(
(
tgt[PythonRequirementsField]
for tgt in all_targets
if tgt.has_field(PythonRequirementsField)
),
additional_requirements=request.additional_requirements,
)
requirements = exact_reqs
description = request.description
if python_setup.requirement_constraints:
# In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let
# packaging canonicalize for us.
# See: https://www.python.org/dev/peps/pep-0503/#normalized-names
exact_req_projects = {
canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs
}
constraints_file_contents = await Get(
DigestContents,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--python-setup-requirement-constraints`",
),
)
constraints_file_reqs = set(
parse_requirements_file(
constraints_file_contents[0].content.decode(),
rel_path=python_setup.requirement_constraints,
)
)
constraint_file_projects = {
canonicalize_project_name(req.project_name) for req in constraints_file_reqs
}
unconstrained_projects = exact_req_projects - constraint_file_projects
if unconstrained_projects:
logger.warning(
f"The constraints file {python_setup.requirement_constraints} does not contain "
f"entries for the following requirements: {', '.join(unconstrained_projects)}"
)
if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or (
python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES
and request.internal_only
):
if unconstrained_projects:
logger.warning(
"Ignoring resolve_all_constraints setting in [python_setup] scope "
"because constraints file does not cover all requirements."
)
else:
requirements = PexRequirements(str(req) for req in constraints_file_reqs)
description = description or f"Resolving {python_setup.requirement_constraints}"
elif (
python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER
and python_setup.resolve_all_constraints_was_set_explicitly()
):
raise ValueError(
f"[python-setup].resolve_all_constraints is set to "
f"{python_setup.resolve_all_constraints.value}, so "
f"[python-setup].requirement_constraints must also be provided."
)
return PexRequest(
output_filename=request.output_filename,
internal_only=request.internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
entry_point=request.entry_point,
sources=merged_input_digest,
additional_inputs=request.additional_inputs,
additional_args=request.additional_args,
description=description,
)
async def pex_from_targets(
request: PexFromTargetsRequest,
python_setup: PythonSetup,
constraints_file: MaybeConstraintsFile,
) -> PexRequest:
if request.direct_deps_only:
targets = await Get(Targets, Addresses(request.addresses))
direct_deps = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies)))
for tgt in targets)
all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets))
else:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(request.addresses))
all_targets = transitive_targets.closure
input_digests = []
if request.additional_sources:
input_digests.append(request.additional_sources)
if request.include_source_files:
prepared_sources = await Get(StrippedPythonSourceFiles,
PythonSourceFilesRequest(all_targets))
input_digests.append(
prepared_sources.stripped_source_files.snapshot.digest)
merged_input_digest = await Get(Digest, MergeDigests(input_digests))
if request.hardcoded_interpreter_constraints:
interpreter_constraints = request.hardcoded_interpreter_constraints
else:
calculated_constraints = PexInterpreterConstraints.create_from_targets(
all_targets, python_setup)
# If there are no targets, we fall back to the global constraints. This is relevant,
# for example, when running `./pants repl` with no specs.
interpreter_constraints = calculated_constraints or PexInterpreterConstraints(
python_setup.interpreter_constraints)
exact_reqs = PexRequirements.create_from_requirement_fields(
(tgt[PythonRequirementsField]
for tgt in all_targets if tgt.has_field(PythonRequirementsField)),
additional_requirements=request.additional_requirements,
)
requirements = exact_reqs
repository_pex: Pex | None = None
description = request.description
if constraints_file.path:
constraints_file_contents = await Get(DigestContents, Digest,
constraints_file.digest)
constraints_file_reqs = set(
parse_requirements_file(
constraints_file_contents[0].content.decode(),
rel_path=constraints_file.path,
))
# In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let
# packaging canonicalize for us.
# See: https://www.python.org/dev/peps/pep-0503/#normalized-names
exact_req_projects = {
canonicalize_project_name(Requirement.parse(req).project_name)
for req in exact_reqs
}
constraint_file_projects = {
canonicalize_project_name(req.project_name)
for req in constraints_file_reqs
}
unconstrained_projects = exact_req_projects - constraint_file_projects
if unconstrained_projects:
constraints_descr = (
f"constraints file {constraints_file.path}"
if python_setup.requirement_constraints else
f"_python_constraints target {python_setup.requirement_constraints_target}"
)
logger.warning(
f"The {constraints_descr} does not contain entries for the following "
f"requirements: {', '.join(unconstrained_projects)}")
if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or (
python_setup.resolve_all_constraints
== ResolveAllConstraintsOption.NONDEPLOYABLES
and request.internal_only):
if unconstrained_projects:
logger.warning(
"Ignoring `[python_setup].resolve_all_constraints` option because constraints "
"file does not cover all requirements.")
else:
repository_pex = await Get(
Pex,
PexRequest(
description=
f"Resolving {python_setup.requirement_constraints}",
output_filename="repository.pex",
internal_only=request.internal_only,
requirements=PexRequirements(
str(req) for req in constraints_file_reqs),
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
),
)
elif (python_setup.resolve_all_constraints !=
ResolveAllConstraintsOption.NEVER
and python_setup.resolve_all_constraints_was_set_explicitly()):
raise ValueError(
"[python-setup].resolve_all_constraints is set to "
f"{python_setup.resolve_all_constraints.value}, so "
"either [python-setup].requirement_constraints or "
"[python-setup].requirement_constraints_target must also be provided."
)
return PexRequest(
output_filename=request.output_filename,
internal_only=request.internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
main=request.main,
sources=merged_input_digest,
additional_inputs=request.additional_inputs,
repository_pex=repository_pex,
additional_args=request.additional_args,
description=description,
)
async def _setup_constraints_repository_pex(
request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup
) -> _ConstraintsRepositoryPex:
# NB: it isn't safe to resolve against the whole constraints file if
# platforms are in use. See https://github.com/pantsbuild/pants/issues/12222.
if not python_setup.resolve_all_constraints or request.platforms:
return _ConstraintsRepositoryPex(None)
constraints_path = python_setup.requirement_constraints
assert constraints_path is not None
constraints_file_contents = await Get(
DigestContents,
PathGlobs(
[constraints_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `[python].requirement_constraints`",
),
)
constraints_file_reqs = set(
parse_requirements_file(
constraints_file_contents[0].content.decode(), rel_path=constraints_path
)
)
# In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let
# packaging canonicalize for us.
# See: https://www.python.org/dev/peps/pep-0503/#normalized-names
url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch'
name_reqs = set() # E.g., foobar>=1.2.3
name_req_projects = set()
for req_str in request.requirements.req_strings:
req = PipRequirement.parse(req_str)
if req.url:
url_reqs.add(req)
else:
name_reqs.add(req)
name_req_projects.add(canonicalize_project_name(req.project_name))
constraint_file_projects = {
canonicalize_project_name(req.project_name) for req in constraints_file_reqs
}
# Constraints files must only contain name reqs, not URL reqs (those are already
# constrained by their very nature). See https://github.com/pypa/pip/issues/8210.
unconstrained_projects = name_req_projects - constraint_file_projects
if unconstrained_projects:
logger.warning(
f"The constraints file {constraints_path} does not contain "
f"entries for the following requirements: {', '.join(unconstrained_projects)}.\n\n"
f"Ignoring `[python_setup].resolve_all_constraints` option."
)
return _ConstraintsRepositoryPex(None)
# To get a full set of requirements we must add the URL requirements to the
# constraints file, since the latter cannot contain URL requirements.
# NB: We can only add the URL requirements we know about here, i.e., those that
# are transitive deps of the targets in play. There may be others in the repo.
# So we may end up creating a few different repository pexes, each with identical
# name requirements but different subsets of URL requirements. Fortunately since
# all these repository pexes will have identical pinned versions of everything,
# this is not a correctness issue, only a performance one.
all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)}
repository_pex = await Get(
Pex,
PexRequest(
description=f"Resolving {constraints_path}",
output_filename="repository.pex",
internal_only=request.internal_only,
requirements=PexRequirements(
all_constraints,
apply_constraints=True,
# TODO: See PexRequirements docs.
is_all_constraints_resolve=True,
),
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
),
)
return _ConstraintsRepositoryPex(repository_pex)
async def load_lockfile(
request: LoadedLockfileRequest,
python_setup: PythonSetup,
) -> LoadedLockfile:
lockfile = request.lockfile
if isinstance(lockfile, Lockfile):
synthetic_lock = False
lockfile_path = lockfile.file_path
lockfile_digest = await Get(
Digest,
PathGlobs(
[lockfile_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=lockfile.file_path_description_of_origin,
),
)
_digest_contents = await Get(DigestContents, Digest, lockfile_digest)
lock_bytes = _digest_contents[0].content
else:
synthetic_lock = True
_fc = lockfile.file_content
lockfile_path, lock_bytes = (_fc.path, _fc.content)
lockfile_digest = await Get(Digest, CreateDigest([_fc]))
is_pex_native = is_probably_pex_json_lockfile(lock_bytes)
if is_pex_native:
header_delimiter = "//"
lockfile_digest = await Get(
Digest,
CreateDigest(
[FileContent(lockfile_path, _strip_comments_from_pex_json_lockfile(lock_bytes))]
),
)
requirement_estimate = _pex_lockfile_requirement_count(lock_bytes)
constraints_strings = None
else:
header_delimiter = "#"
lock_string = lock_bytes.decode()
# Note: this is a very naive heuristic. It will overcount because entries often
# have >1 line due to `--hash`.
requirement_estimate = len(lock_string.splitlines())
constraints_strings = FrozenOrderedSet(
str(req) for req in parse_requirements_file(lock_string, rel_path=lockfile_path)
)
metadata: PythonLockfileMetadata | None = None
if should_validate_metadata(lockfile, python_setup):
metadata = PythonLockfileMetadata.from_lockfile(
lock_bytes,
**(dict() if synthetic_lock else dict(lockfile_path=lockfile_path)),
resolve_name=lockfile.resolve_name,
delimeter=header_delimiter,
)
return LoadedLockfile(
lockfile_digest,
lockfile_path,
metadata,
requirement_estimate,
is_pex_native,
constraints_strings,
original_lockfile=lockfile,
)