def test_default_module_mapping_is_normalized() -> None: for k in DEFAULT_MODULE_MAPPING: assert k == canonicalize_project_name( k ), "Please update `DEFAULT_MODULE_MAPPING` to use canonical project names" for k in DEFAULT_TYPE_STUB_MODULE_MAPPING: assert k == canonicalize_project_name( k ), "Please update `DEFAULT_TYPE_STUB_MODULE_MAPPING` to use canonical project names"
def normalize_module_mapping( mapping: Mapping[str, Iterable[str]] | None ) -> FrozenDict[str, tuple[str, ...]]: return FrozenDict({ canonicalize_project_name(k): tuple(v) for k, v in (mapping or {}).items() })
def generate_tgt( project_name: str, parsed_reqs: Iterable[PipRequirement] ) -> PythonRequirementTarget: normalized_proj_name = canonicalize_project_name(project_name) tgt_overrides = overrides.pop(normalized_proj_name, {}) if Dependencies.alias in tgt_overrides: tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [ file_tgt.address.spec ] return PythonRequirementTarget( { **inherited_fields, PythonRequirementsField.alias: list(parsed_reqs), PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name), PythonRequirementTypeStubModulesField.alias: stubs_mapping.get( normalized_proj_name ), # This may get overridden by `tgt_overrides`, which will have already added in # the file tgt. Dependencies.alias: [file_tgt.address.spec], **tgt_overrides, }, generator.address.create_generated(project_name), )
def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget: if info.get("extras"): raw_req += f"[{','.join(info['extras'])}]" raw_req += info.get("version", "") if info.get("markers"): raw_req += f";{info['markers']}" parsed_req = PipRequirement.parse(raw_req) normalized_proj_name = canonicalize_project_name( parsed_req.project_name) tgt_overrides = overrides.pop(normalized_proj_name, {}) if Dependencies.alias in tgt_overrides: tgt_overrides[Dependencies.alias] = list( tgt_overrides[Dependencies.alias]) + [file_tgt.address.spec] return PythonRequirementTarget( { **inherited_fields, PythonRequirementsField.alias: [parsed_req], PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name), PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(normalized_proj_name), # This may get overridden by `tgt_overrides`, which will have already added in # the file tgt. Dependencies.alias: [file_tgt.address.spec], **tgt_overrides, }, generator.address.create_generated(parsed_req.project_name), )
def __call__( self, requirements_relpath: str = "requirements.txt", *, module_mapping: Optional[Mapping[str, Iterable[str]]] = None, type_stubs_module_mapping: Optional[Mapping[str, Iterable[str]]] = None, ) -> None: """ :param requirements_relpath: The relpath from this BUILD file to the requirements file. Defaults to a `requirements.txt` file sibling to the BUILD file. :param module_mapping: a mapping of requirement names to a list of the modules they provide. For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the requirement name as the default module, e.g. "Django" will default to `modules=["django"]`. """ req_file_tgt = self._parse_context.create_object( "_python_requirements_file", name=requirements_relpath.replace(os.path.sep, "_"), sources=[requirements_relpath], ) requirements_dep = f":{req_file_tgt.name}" normalized_module_mapping = normalize_module_mapping(module_mapping) normalized_type_stubs_module_mapping = normalize_module_mapping(type_stubs_module_mapping) req_file = Path(get_buildroot(), self._parse_context.rel_path, requirements_relpath) requirements = parse_requirements_file( req_file.read_text(), rel_path=str(req_file.relative_to(get_buildroot())) ) grouped_requirements = groupby(requirements, lambda parsed_req: parsed_req.project_name) for project_name, parsed_reqs_ in grouped_requirements: normalized_proj_name = canonicalize_project_name(project_name) req_module_mapping = ( {normalized_proj_name: normalized_module_mapping[normalized_proj_name]} if normalized_proj_name in normalized_module_mapping else {} ) req_stubs_mapping = ( {normalized_proj_name: normalized_type_stubs_module_mapping[normalized_proj_name]} if normalized_proj_name in normalized_type_stubs_module_mapping else {} ) self._parse_context.create_object( "python_requirement_library", name=project_name, requirements=list(parsed_reqs_), module_mapping=req_module_mapping, type_stubs_module_mapping=req_stubs_mapping, dependencies=[requirements_dep], )
def __call__( self, *, source: str = "requirements.txt", module_mapping: Mapping[str, Iterable[str]] | None = None, type_stubs_module_mapping: Mapping[str, Iterable[str]] | None = None, overrides: OVERRIDES_TYPE = None, ) -> None: """ :param module_mapping: a mapping of requirement names to a list of the modules they provide. For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the requirement name as the default module, e.g. "Django" will default to `modules=["django"]`. """ req_file_tgt = self._parse_context.create_object( "_python_requirements_file", name=source.replace(os.path.sep, "_"), sources=[source], ) requirements_dep = f":{req_file_tgt.name}" normalized_module_mapping = normalize_module_mapping(module_mapping) normalized_type_stubs_module_mapping = normalize_module_mapping( type_stubs_module_mapping) req_file = Path(get_buildroot(), self._parse_context.rel_path, source) requirements = parse_requirements_file( req_file.read_text(), rel_path=str(req_file.relative_to(get_buildroot()))) dependencies_overrides = flatten_overrides_to_dependency_field( overrides, macro_name="python_requirements", build_file_dir=self._parse_context.rel_path) grouped_requirements = groupby( requirements, lambda parsed_req: parsed_req.project_name) for project_name, parsed_reqs_ in grouped_requirements: normalized_proj_name = canonicalize_project_name(project_name) self._parse_context.create_object( "python_requirement", name=project_name, requirements=list(parsed_reqs_), modules=normalized_module_mapping.get(normalized_proj_name), type_stub_modules=normalized_type_stubs_module_mapping.get( normalized_proj_name), dependencies=[ requirements_dep, *dependencies_overrides.get(normalized_proj_name, []), ], )
def __call__( self, pyproject_toml_relpath: str = "pyproject.toml", *, module_mapping: Optional[Mapping[str, Iterable[str]]] = None, type_stubs_module_mapping: Optional[Mapping[str, Iterable[str]]] = None, ) -> None: """ :param pyproject_toml_relpath: The relpath from this BUILD file to the requirements file. Defaults to a `requirements.txt` file sibling to the BUILD file. :param module_mapping: a mapping of requirement names to a list of the modules they provide. For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the requirement name as the default module, e.g. "Django" will default to `modules=["django"]`. """ req_file_tgt = self._parse_context.create_object( "_python_requirements_file", name=pyproject_toml_relpath.replace(os.path.sep, "_"), sources=[pyproject_toml_relpath], ) requirements_dep = f":{req_file_tgt.name}" normalized_module_mapping = normalize_module_mapping(module_mapping) normalized_type_stubs_module_mapping = normalize_module_mapping( type_stubs_module_mapping) requirements = parse_pyproject_toml( PyProjectToml.create(self._parse_context, pyproject_toml_relpath)) for parsed_req in requirements: normalized_proj_name = canonicalize_project_name( parsed_req.project_name) req_module_mapping = ({ normalized_proj_name: normalized_module_mapping[normalized_proj_name] } if normalized_proj_name in normalized_module_mapping else {}) req_stubs_mapping = ({ normalized_proj_name: normalized_type_stubs_module_mapping[normalized_proj_name] } if normalized_proj_name in normalized_type_stubs_module_mapping else {}) self._parse_context.create_object( "python_requirement_library", name=parsed_req.project_name, requirements=[parsed_req], module_mapping=req_module_mapping, type_stubs_module_mapping=req_stubs_mapping, dependencies=[requirements_dep], )
def __call__( self, *, source: str = "pyproject.toml", module_mapping: Mapping[str, Iterable[str]] | None = None, type_stubs_module_mapping: Mapping[str, Iterable[str]] | None = None, overrides: OVERRIDES_TYPE = None, ) -> None: """ :param module_mapping: a mapping of requirement names to a list of the modules they provide. For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the requirement name as the default module, e.g. "Django" will default to `modules=["django"]`. """ req_file_tgt = self._parse_context.create_object( TargetGeneratorSourcesHelperTarget.alias, name=source.replace(os.path.sep, "_"), sources=[source], ) requirements_dep = f":{req_file_tgt.name}" normalized_module_mapping = normalize_module_mapping(module_mapping) normalized_type_stubs_module_mapping = normalize_module_mapping( type_stubs_module_mapping) dependencies_overrides = flatten_overrides_to_dependency_field( overrides, macro_name="python_requirements", build_file_dir=self._parse_context.rel_path) requirements = parse_pyproject_toml( PyProjectToml.deprecated_macro_create(self._parse_context, source)) for parsed_req in requirements: normalized_proj_name = canonicalize_project_name( parsed_req.project_name) self._parse_context.create_object( "python_requirement", name=parsed_req.project_name, requirements=[parsed_req], modules=normalized_module_mapping.get(normalized_proj_name), type_stub_modules=normalized_type_stubs_module_mapping.get( normalized_proj_name), dependencies=[ requirements_dep, *dependencies_overrides.get(normalized_proj_name, []), ], )
def assert_pants_requirement( rule_runner: RuleRunner, build_file_entry: str, *, expected_target_name: str, expected_dist: str = "pantsbuild.pants", expected_module: str = "pants", ) -> None: rule_runner.add_to_build_file("3rdparty/python", f"{build_file_entry}\n") target = rule_runner.get_target( Address("3rdparty/python", target_name=expected_target_name)) assert isinstance(target, PythonRequirementLibrary) assert target[PythonRequirementsField].value == ( Requirement.parse(f"{expected_dist}=={pants_version()}"), ) module_mapping = target[ModuleMappingField].value assert isinstance(module_mapping, FrozenDict) assert module_mapping.get( canonicalize_project_name(expected_dist)) == (expected_module, )
def validate_pytest_cov_included(self) -> None: for s in self.extra_requirements: try: req = Requirement.parse(s).project_name except Exception as e: raise ValueError( format_invalid_requirement_string_error( s, e, description_of_origin="`[pytest].extra_requirements`")) if canonicalize_project_name(req) == "pytest-cov": return raise ValueError( "You set `[test].use_coverage`, but `[pytest].extra_requirements` is missing " "`pytest-cov`, which is needed to collect coverage data.\n\nThis happens when " "overriding the `extra_requirements` option. Please either explicitly add back " "`pytest-cov` or use `extra_requirements.add` to keep Pants's default, rather than " "overriding it. Run `./pants help-advanced pytest` to see the default version of " f"`pytest-cov` and see {doc_url('options#list-values')} for more on adding vs. " "overriding list options.")
def validate_pytest_cov_included(self) -> None: for s in self.extra_requirements: try: req = PipRequirement.parse(s).project_name except Exception as e: raise ValueError( f"Invalid requirement '{s}' in `[pytest].extra_requirements`: {e}" ) if canonicalize_project_name(req) == "pytest-cov": return raise ValueError( softwrap(f""" You set `[test].use_coverage`, but `[pytest].extra_requirements` is missing `pytest-cov`, which is needed to collect coverage data. This happens when overriding the `extra_requirements` option. Please either explicitly add back `pytest-cov` or use `extra_requirements.add` to keep Pants's default, rather than overriding it. Run `{bin_name()} help-advanced pytest` to see the default version of `pytest-cov` and see {doc_url('options#list-values')} for more on adding vs. overriding list options. """))
def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget: normalized_proj_name = canonicalize_project_name(parsed_req.project_name) tgt_overrides = overrides.pop(normalized_proj_name, {}) if Dependencies.alias in tgt_overrides: tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [ file_tgt.address.spec ] return PythonRequirementTarget( { **request.template, PythonRequirementsField.alias: [parsed_req], PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name), PythonRequirementTypeStubModulesField.alias: stubs_mapping.get( normalized_proj_name ), # This may get overridden by `tgt_overrides`, which will have already added in # the file tgt. Dependencies.alias: [file_tgt.address.spec], **tgt_overrides, }, request.template_address.create_generated(parsed_req.project_name), union_membership, )
def __call__( self, requirements_relpath: str = "Pipfile.lock", module_mapping: Optional[Mapping[str, Iterable[str]]] = None, type_stubs_module_mapping: Optional[Mapping[str, Iterable[str]]] = None, pipfile_target: Optional[str] = None, ) -> None: """ :param requirements_relpath: The relpath from this BUILD file to the requirements file. Defaults to a `Pipfile.lock` file sibling to the BUILD file. :param module_mapping: a mapping of requirement names to a list of the modules they provide. For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the requirement name as the default module, e.g. "Django" will default to `modules=["django"]`. :param pipfile_target: a `_python_requirements_file` target to provide for cache invalidation if the requirements_relpath value is not in the current rel_path """ requirements_path = Path(get_buildroot(), self._parse_context.rel_path, requirements_relpath) lock_info = json.loads(requirements_path.read_text()) if pipfile_target: requirements_dep = pipfile_target else: requirements_file_target_name = requirements_relpath self._parse_context.create_object( "_python_requirements_file", name=requirements_file_target_name, sources=[requirements_relpath], ) requirements_dep = f":{requirements_file_target_name}" normalized_module_mapping = normalize_module_mapping(module_mapping) normalized_type_stubs_module_mapping = normalize_module_mapping( type_stubs_module_mapping) requirements = { **lock_info.get("default", {}), **lock_info.get("develop", {}) } for req, info in requirements.items(): extras = [x for x in info.get("extras", [])] extras_str = f"[{','.join(extras)}]" if extras else "" req_str = f"{req}{extras_str}{info.get('version','')}" if info.get("markers"): req_str += f";{info['markers']}" parsed_req = Requirement.parse(req_str) normalized_proj_name = canonicalize_project_name( parsed_req.project_name) req_module_mapping = ({ normalized_proj_name: normalized_module_mapping[normalized_proj_name] } if normalized_proj_name in normalized_module_mapping else {}) req_stubs_mapping = ({ normalized_proj_name: normalized_type_stubs_module_mapping[normalized_proj_name] } if normalized_proj_name in normalized_type_stubs_module_mapping else {}) self._parse_context.create_object( "python_requirement_library", name=parsed_req.project_name, requirements=[parsed_req], dependencies=[requirements_dep], module_mapping=req_module_mapping, type_stubs_module_mapping=req_stubs_mapping, )
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: transitive_targets = await Get(TransitiveTargets, Addresses, request.addresses) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets) ) input_digests.append(prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( ( tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility) ), python_setup, ) exact_reqs = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, ) requirements = exact_reqs description = request.description if python_setup.requirement_constraints: # In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraints_file_contents = await Get( DigestContents, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin="the option `--python-setup-requirement-constraints`", ), ) constraints_file_reqs = set( parse_requirements(next(iter(constraints_file_contents)).content.decode()) ) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {python_setup.requirement_constraints} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only ): if unconstrained_projects: logger.warning( "Ignoring resolve_all_constraints setting in [python_setup] scope " "because constraints file does not cover all requirements." ) else: requirements = PexRequirements(str(req) for req in constraints_file_reqs) description = description or f"Resolving {python_setup.requirement_constraints}" elif ( python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( f"[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " f"[python-setup].requirement_constraints must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=description, )
async def pex_from_targets( request: PexFromTargetsRequest, python_setup: PythonSetup, constraints_file: MaybeConstraintsFile, ) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets ) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses) ) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets) ) input_digests.append(prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup ) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints ) exact_reqs = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, ) requirements = exact_reqs repository_pex: Pex | None = None description = request.description if constraints_file.path: constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_file.path, ) ) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch' name_reqs = set() # E.g., foobar>=1.2.3 name_req_projects = set() for req_str in exact_reqs: req = Requirement.parse(req_str) if req.url: # type: ignore[attr-defined] url_reqs.add(req) else: name_reqs.add(req) name_req_projects.add(canonicalize_project_name(req.project_name)) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. unconstrained_projects = name_req_projects - constraint_file_projects if unconstrained_projects: constraints_descr = ( f"constraints file {constraints_file.path}" if python_setup.requirement_constraints else f"_python_constraints target {python_setup.requirement_constraints_target}" ) logger.warning( f"The {constraints_descr} does not contain entries for the following " f"requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints: if unconstrained_projects: logger.warning( "Ignoring `[python_setup].resolve_all_constraints` option because constraints " "file does not cover all requirements." ) else: # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. # NB: We can only add the URL requirements we know about here, i.e., those that # are transitive deps of the targets in play. There may be others in the repo. # So we may end up creating a few different repository pexes, each with identical # name requirements but different subsets of URL requirements. Fortunately since # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. # TODO: Address this as part of providing proper lockfile support. However we # generate lockfiles, they must be able to include URL requirements. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} repository_pex = await Get( Pex, PexRequest( description=f"Resolving {python_setup.requirement_constraints}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements(all_constraints), interpreter_constraints=interpreter_constraints, platforms=request.platforms, additional_args=["-vvv"], ), ) elif ( python_setup.resolve_all_constraints and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( "[python-setup].resolve_all_constraints is enabled, so either " "[python-setup].requirement_constraints or " "[python-setup].requirement_constraints_target must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_input_digest, additional_inputs=request.additional_inputs, repository_pex=repository_pex, additional_args=request.additional_args, description=description, )
def __call__( self, *, source: str = "Pipfile.lock", module_mapping: Mapping[str, Iterable[str]] | None = None, type_stubs_module_mapping: Mapping[str, Iterable[str]] | None = None, pipfile_target: str | None = None, overrides: OVERRIDES_TYPE = None, ) -> None: """ :param module_mapping: a mapping of requirement names to a list of the modules they provide. For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the requirement name as the default module, e.g. "Django" will default to `modules=["django"]`. :param pipfile_target: a `_python_requirements_file` target to provide for cache invalidation if the requirements_relpath value is not in the current rel_path """ requirements_path = Path(get_buildroot(), self._parse_context.rel_path, source) lock_info = json.loads(requirements_path.read_text()) if pipfile_target: requirements_dep = pipfile_target else: requirements_file_target_name = source self._parse_context.create_object( "_python_requirements_file", name=requirements_file_target_name, sources=[source], ) requirements_dep = f":{requirements_file_target_name}" normalized_module_mapping = normalize_module_mapping(module_mapping) normalized_type_stubs_module_mapping = normalize_module_mapping( type_stubs_module_mapping) dependencies_overrides = flatten_overrides_to_dependency_field( overrides, macro_name="python_requirements", build_file_dir=self._parse_context.rel_path) requirements = { **lock_info.get("default", {}), **lock_info.get("develop", {}) } for req, info in requirements.items(): extras = [x for x in info.get("extras", [])] extras_str = f"[{','.join(extras)}]" if extras else "" req_str = f"{req}{extras_str}{info.get('version','')}" if info.get("markers"): req_str += f";{info['markers']}" parsed_req = PipRequirement.parse(req_str) normalized_proj_name = canonicalize_project_name( parsed_req.project_name) self._parse_context.create_object( "python_requirement", name=parsed_req.project_name, requirements=[parsed_req], dependencies=[ requirements_dep, *dependencies_overrides.get(normalized_proj_name, []), ], modules=normalized_module_mapping.get(normalized_proj_name), type_stub_modules=normalized_type_stubs_module_mapping.get( normalized_proj_name), )
async def generate_from_python_requirement( request: GenerateFromPythonRequirementsRequest, python_setup: PythonSetup ) -> GeneratedTargets: generator = request.generator requirements_rel_path = generator[PythonRequirementsSourceField].value requirements_full_path = generator[PythonRequirementsSourceField].file_path overrides = { canonicalize_project_name(k): v for k, v in request.require_unparametrized_overrides().items() } file_tgt = TargetGeneratorSourcesHelperTarget( {TargetGeneratorSourcesHelperSourcesField.alias: [requirements_rel_path]}, Address( generator.address.spec_path, target_name=generator.address.target_name, relative_file_path=requirements_rel_path, ), ) digest_contents = await Get( DigestContents, PathGlobs( [requirements_full_path], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin=f"{generator}'s field `{PythonRequirementsSourceField.alias}`", ), ) requirements = parse_requirements_file( digest_contents[0].content.decode(), rel_path=requirements_full_path ) grouped_requirements = itertools.groupby( requirements, lambda parsed_req: parsed_req.project_name ) # Validate the resolve is legal. generator[PythonRequirementResolveField].normalized_value(python_setup) module_mapping = generator[ModuleMappingField].value stubs_mapping = generator[TypeStubsModuleMappingField].value inherited_fields = { field.alias: field.value for field in request.generator.field_values.values() if isinstance(field, (*COMMON_TARGET_FIELDS, PythonRequirementResolveField)) } def generate_tgt( project_name: str, parsed_reqs: Iterable[PipRequirement] ) -> PythonRequirementTarget: normalized_proj_name = canonicalize_project_name(project_name) tgt_overrides = overrides.pop(normalized_proj_name, {}) if Dependencies.alias in tgt_overrides: tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [ file_tgt.address.spec ] return PythonRequirementTarget( { **inherited_fields, PythonRequirementsField.alias: list(parsed_reqs), PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name), PythonRequirementTypeStubModulesField.alias: stubs_mapping.get( normalized_proj_name ), # This may get overridden by `tgt_overrides`, which will have already added in # the file tgt. Dependencies.alias: [file_tgt.address.spec], **tgt_overrides, }, generator.address.create_generated(project_name), ) result = tuple( generate_tgt(project_name, parsed_reqs_) for project_name, parsed_reqs_ in grouped_requirements ) + (file_tgt,) if overrides: raise InvalidFieldException( f"Unused key in the `overrides` field for {request.generator.address}: " f"{sorted(overrides)}" ) return GeneratedTargets(generator, result)
async def _setup_constraints_repository_pex( request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup ) -> _ConstraintsRepositoryPex: # NB: it isn't safe to resolve against the whole constraints file if # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222. if not python_setup.resolve_all_constraints or request.platforms: return _ConstraintsRepositoryPex(None) constraints_path = python_setup.requirement_constraints assert constraints_path is not None constraints_file_contents = await Get( DigestContents, PathGlobs( [constraints_path], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `[python].requirement_constraints`", ), ) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_path ) ) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch' name_reqs = set() # E.g., foobar>=1.2.3 name_req_projects = set() for req_str in request.requirements.req_strings: req = PipRequirement.parse(req_str) if req.url: url_reqs.add(req) else: name_reqs.add(req) name_req_projects.add(canonicalize_project_name(req.project_name)) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. unconstrained_projects = name_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {constraints_path} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}.\n\n" f"Ignoring `[python_setup].resolve_all_constraints` option." ) return _ConstraintsRepositoryPex(None) # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. # NB: We can only add the URL requirements we know about here, i.e., those that # are transitive deps of the targets in play. There may be others in the repo. # So we may end up creating a few different repository pexes, each with identical # name requirements but different subsets of URL requirements. Fortunately since # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} repository_pex = await Get( Pex, PexRequest( description=f"Resolving {constraints_path}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( all_constraints, apply_constraints=True, # TODO: See PexRequirements docs. is_all_constraints_resolve=True, ), interpreter_constraints=request.interpreter_constraints, platforms=request.platforms, additional_args=request.additional_lockfile_args, ), ) return _ConstraintsRepositoryPex(repository_pex)
async def generate_from_python_requirement( request: GenerateFromPoetryRequirementsRequest, build_root: BuildRoot, union_membership: UnionMembership, ) -> GeneratedTargets: generator = request.generator pyproject_rel_path = generator[PoetryRequirementsSourceField].value pyproject_full_path = generator[PoetryRequirementsSourceField].file_path overrides = { canonicalize_project_name(k): v for k, v in request.require_unparametrized_overrides().items() } file_tgt = TargetGeneratorSourcesHelperTarget( {TargetGeneratorSourcesHelperSourcesField.alias: pyproject_rel_path}, Address( request.template_address.spec_path, target_name=request.template_address.target_name, relative_file_path=pyproject_rel_path, ), union_membership, ) digest_contents = await Get( DigestContents, PathGlobs( [pyproject_full_path], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin=f"{generator}'s field `{PoetryRequirementsSourceField.alias}`", ), ) requirements = parse_pyproject_toml( PyProjectToml( build_root=PurePath(build_root.path), toml_relpath=PurePath(pyproject_full_path), toml_contents=digest_contents[0].content.decode(), ) ) module_mapping = generator[ModuleMappingField].value stubs_mapping = generator[TypeStubsModuleMappingField].value def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget: normalized_proj_name = canonicalize_project_name(parsed_req.project_name) tgt_overrides = overrides.pop(normalized_proj_name, {}) if Dependencies.alias in tgt_overrides: tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [ file_tgt.address.spec ] return PythonRequirementTarget( { **request.template, PythonRequirementsField.alias: [parsed_req], PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name), PythonRequirementTypeStubModulesField.alias: stubs_mapping.get( normalized_proj_name ), # This may get overridden by `tgt_overrides`, which will have already added in # the file tgt. Dependencies.alias: [file_tgt.address.spec], **tgt_overrides, }, request.template_address.create_generated(parsed_req.project_name), union_membership, ) result = tuple(generate_tgt(requirement) for requirement in requirements) + (file_tgt,) if overrides: raise InvalidFieldException( softwrap( f""" Unused key in the `overrides` field for {request.template_address}: {sorted(overrides)} """ ) ) return GeneratedTargets(generator, result)
async def map_third_party_modules_to_addresses( all_python_tgts: AllPythonTargets, python_setup: PythonSetup, ) -> ThirdPartyPythonModuleMapping: resolves_to_modules_to_providers: DefaultDict[ ResolveName, DefaultDict[str, list[ModuleProvider]] ] = defaultdict(lambda: defaultdict(list)) for tgt in all_python_tgts.third_party: resolve = tgt[PythonRequirementResolveField].normalized_value(python_setup) def add_modules(modules: Iterable[str], *, type_stub: bool = False) -> None: for module in modules: resolves_to_modules_to_providers[resolve][module].append( ModuleProvider( tgt.address, ModuleProviderType.TYPE_STUB if type_stub else ModuleProviderType.IMPL, ) ) explicit_modules = tgt.get(PythonRequirementModulesField).value if explicit_modules: add_modules(explicit_modules) continue explicit_stub_modules = tgt.get(PythonRequirementTypeStubModulesField).value if explicit_stub_modules: add_modules(explicit_stub_modules, type_stub=True) continue # Else, fall back to defaults. for req in tgt[PythonRequirementsField].value: # NB: We don't use `canonicalize_project_name()` for the fallback value because we # want to preserve `.` in the module name. See # https://www.python.org/dev/peps/pep-0503/#normalized-names. proj_name = canonicalize_project_name(req.project_name) fallback_value = req.project_name.strip().lower().replace("-", "_") in_stubs_map = proj_name in DEFAULT_TYPE_STUB_MODULE_MAPPING starts_with_prefix = fallback_value.startswith(("types_", "stubs_")) ends_with_prefix = fallback_value.endswith(("_types", "_stubs")) if proj_name not in DEFAULT_MODULE_MAPPING and ( in_stubs_map or starts_with_prefix or ends_with_prefix ): if in_stubs_map: stub_modules = DEFAULT_TYPE_STUB_MODULE_MAPPING[proj_name] else: stub_modules = ( fallback_value[6:] if starts_with_prefix else fallback_value[:-6], ) add_modules(stub_modules, type_stub=True) else: add_modules(DEFAULT_MODULE_MAPPING.get(proj_name, (fallback_value,))) return ThirdPartyPythonModuleMapping( ( resolve, FrozenDict( (mod, tuple(sorted(providers))) for mod, providers in sorted(mapping.items()) ), ) for resolve, mapping in sorted(resolves_to_modules_to_providers.items()) )
async def map_third_party_modules_to_addresses( ) -> ThirdPartyPythonModuleMapping: all_targets = await Get(Targets, AddressSpecs([DescendantAddresses("")])) modules_to_addresses: dict[str, Address] = {} modules_to_stub_addresses: dict[str, Address] = {} modules_with_multiple_owners: DefaultDict[str, set[Address]] = defaultdict(set) for tgt in all_targets: if not tgt.has_field(PythonRequirementsField): continue module_map = { **DEFAULT_MODULE_MAPPING, **tgt.get(ModuleMappingField).value } stubs_module_map = { **DEFAULT_TYPE_STUB_MODULE_MAPPING, **tgt.get(TypeStubsModuleMappingField).value, } for req in tgt[PythonRequirementsField].value: # NB: We don't use `canonicalize_project_name()` for the fallback value because we # want to preserve `.` in the module name. See # https://www.python.org/dev/peps/pep-0503/#normalized-names. proj_name = canonicalize_project_name(req.project_name) fallback_value = req.project_name.strip().lower().replace("-", "_") # Handle if it's a type stub. in_stubs_map = proj_name in stubs_module_map starts_with_prefix = fallback_value.startswith( ("types_", "stubs_")) ends_with_prefix = fallback_value.endswith(("_types", "_stubs")) if proj_name not in module_map and (in_stubs_map or starts_with_prefix or ends_with_prefix): if in_stubs_map: modules = stubs_module_map[proj_name] else: modules = (fallback_value[6:] if starts_with_prefix else fallback_value[:-6], ) for module in modules: if module in modules_with_multiple_owners: modules_with_multiple_owners[module].add(tgt.address) elif module in modules_to_stub_addresses: modules_with_multiple_owners[module].update( {modules_to_stub_addresses[module], tgt.address}) else: modules_to_stub_addresses[module] = tgt.address # Else it's a normal requirement. else: modules = module_map.get(proj_name, (fallback_value, )) for module in modules: if module in modules_with_multiple_owners: modules_with_multiple_owners[module].add(tgt.address) elif module in modules_to_addresses: modules_with_multiple_owners[module].update( {modules_to_addresses[module], tgt.address}) else: modules_to_addresses[module] = tgt.address # Remove modules with ambiguous owners. for module in modules_with_multiple_owners: if module in modules_to_addresses: modules_to_addresses.pop(module) if module in modules_to_stub_addresses: modules_to_stub_addresses.pop(module) merged_mapping: DefaultDict[str, list[Address]] = defaultdict(list) for k, v in modules_to_addresses.items(): merged_mapping[k].append(v) for k, v in modules_to_stub_addresses.items(): merged_mapping[k].append(v) return ThirdPartyPythonModuleMapping( mapping=FrozenDict( (k, tuple(sorted(v))) for k, v in sorted(merged_mapping.items())), ambiguous_modules=FrozenDict( (k, tuple(sorted(v))) for k, v in sorted(modules_with_multiple_owners.items())), )
async def pex_from_targets( request: PexFromTargetsRequest, python_setup: PythonSetup, constraints_file: MaybeConstraintsFile, ) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses)) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get(StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)) input_digests.append( prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints) exact_reqs = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) requirements = exact_reqs repository_pex: Pex | None = None description = request.description if constraints_file.path: constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_file.path, )) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: constraints_descr = ( f"constraints file {constraints_file.path}" if python_setup.requirement_constraints else f"_python_constraints target {python_setup.requirement_constraints_target}" ) logger.warning( f"The {constraints_descr} does not contain entries for the following " f"requirements: {', '.join(unconstrained_projects)}") if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only): if unconstrained_projects: logger.warning( "Ignoring `[python_setup].resolve_all_constraints` option because constraints " "file does not cover all requirements.") else: repository_pex = await Get( Pex, PexRequest( description= f"Resolving {python_setup.requirement_constraints}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( str(req) for req in constraints_file_reqs), interpreter_constraints=interpreter_constraints, platforms=request.platforms, ), ) elif (python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly()): raise ValueError( "[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " "either [python-setup].requirement_constraints or " "[python-setup].requirement_constraints_target must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_input_digest, additional_inputs=request.additional_inputs, repository_pex=repository_pex, additional_args=request.additional_args, description=description, )
async def _setup_constraints_repository_pex( constraints_request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup, global_requirement_constraints: GlobalRequirementConstraints, ) -> OptionalPexRequest: request = constraints_request.repository_pex_request if not python_setup.resolve_all_constraints: return OptionalPexRequest(None) constraints_path = python_setup.requirement_constraints assert constraints_path is not None transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) requirements = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in transitive_targets.closure if tgt.has_field(PythonRequirementsField)), constraints_strings=(str(constraint) for constraint in global_requirement_constraints), ) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names url_reqs = set( ) # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch' name_reqs = set() # E.g., foobar>=1.2.3 name_req_projects = set() constraints_file_reqs = set(global_requirement_constraints) for req_str in requirements.req_strings: req = PipRequirement.parse(req_str) if req.url: url_reqs.add(req) else: name_reqs.add(req) name_req_projects.add(canonicalize_project_name(req.project_name)) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. unconstrained_projects = name_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( softwrap(f""" The constraints file {constraints_path} does not contain entries for the following requirements: {', '.join(unconstrained_projects)}. Ignoring `[python_setup].resolve_all_constraints` option. """)) return OptionalPexRequest(None) interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest, request.to_interpreter_constraints_request(), ) # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. # NB: We can only add the URL requirements we know about here, i.e., those that # are transitive deps of the targets in play. There may be others in the repo. # So we may end up creating a few different repository pexes, each with identical # name requirements but different subsets of URL requirements. Fortunately since # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} repository_pex = PexRequest( description=f"Resolving {constraints_path}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( all_constraints, constraints_strings=( str(constraint) for constraint in global_requirement_constraints), ), # Monolithic PEXes like the repository PEX should always use the Packed layout. layout=PexLayout.PACKED, interpreter_constraints=interpreter_constraints, platforms=request.platforms, complete_platforms=request.complete_platforms, additional_args=request.additional_lockfile_args, ) return OptionalPexRequest(repository_pex)
async def generate_from_pipenv_requirement( request: GenerateFromPipenvRequirementsRequest, python_setup: PythonSetup) -> GeneratedTargets: generator = request.generator lock_rel_path = generator[PipenvSourceField].value lock_full_path = generator[PipenvSourceField].file_path overrides = { canonicalize_project_name(k): v for k, v in request.require_unparametrized_overrides().items() } file_tgt = TargetGeneratorSourcesHelperTarget( {TargetGeneratorSourcesHelperSourcesField.alias: [lock_rel_path]}, Address( generator.address.spec_path, target_name=generator.address.target_name, relative_file_path=lock_rel_path, ), ) digest_contents = await Get( DigestContents, PathGlobs( [lock_full_path], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin= f"{generator}'s field `{PipenvSourceField.alias}`", ), ) lock_info = json.loads(digest_contents[0].content) # Validate the resolve is legal. generator[PythonRequirementResolveField].normalized_value(python_setup) module_mapping = generator[ModuleMappingField].value stubs_mapping = generator[TypeStubsModuleMappingField].value inherited_fields = { field.alias: field.value for field in request.generator.field_values.values() if isinstance(field, (*COMMON_TARGET_FIELDS, PythonRequirementResolveField)) } def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget: if info.get("extras"): raw_req += f"[{','.join(info['extras'])}]" raw_req += info.get("version", "") if info.get("markers"): raw_req += f";{info['markers']}" parsed_req = PipRequirement.parse(raw_req) normalized_proj_name = canonicalize_project_name( parsed_req.project_name) tgt_overrides = overrides.pop(normalized_proj_name, {}) if Dependencies.alias in tgt_overrides: tgt_overrides[Dependencies.alias] = list( tgt_overrides[Dependencies.alias]) + [file_tgt.address.spec] return PythonRequirementTarget( { **inherited_fields, PythonRequirementsField.alias: [parsed_req], PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name), PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(normalized_proj_name), # This may get overridden by `tgt_overrides`, which will have already added in # the file tgt. Dependencies.alias: [file_tgt.address.spec], **tgt_overrides, }, generator.address.create_generated(parsed_req.project_name), ) result = tuple( generate_tgt(req, info) for req, info in { **lock_info.get("default", {}), **lock_info.get("develop", {}) }.items()) + (file_tgt, ) if overrides: raise InvalidFieldException( f"Unused key in the `overrides` field for {request.generator.address}: " f"{sorted(overrides)}") return GeneratedTargets(generator, result)
async def map_third_party_modules_to_addresses( all_python_tgts: AllPythonTargets, ) -> ThirdPartyPythonModuleMapping: modules_to_addresses: dict[str, Address] = {} modules_to_stub_addresses: dict[str, Address] = {} modules_with_multiple_owners: DefaultDict[str, set[Address]] = defaultdict(set) def add_modules(modules: tuple[str, ...], address: Address) -> None: for module in modules: if module in modules_with_multiple_owners: modules_with_multiple_owners[module].add(address) elif module in modules_to_addresses: modules_with_multiple_owners[module].update( {modules_to_addresses[module], address}) else: modules_to_addresses[module] = address def add_stub_modules(modules: tuple[str, ...], address: Address) -> None: for module in modules: if module in modules_with_multiple_owners: modules_with_multiple_owners[module].add(address) elif module in modules_to_stub_addresses: modules_with_multiple_owners[module].update( {modules_to_stub_addresses[module], address}) else: modules_to_stub_addresses[module] = address for tgt in all_python_tgts.third_party: explicit_modules = tgt.get(PythonRequirementModulesField).value if explicit_modules: add_modules(explicit_modules, tgt.address) continue explicit_stub_modules = tgt.get( PythonRequirementTypeStubModulesField).value if explicit_stub_modules: add_stub_modules(explicit_stub_modules, tgt.address) continue # Else, fall back to defaults. for req in tgt[PythonRequirementsField].value: # NB: We don't use `canonicalize_project_name()` for the fallback value because we # want to preserve `.` in the module name. See # https://www.python.org/dev/peps/pep-0503/#normalized-names. proj_name = canonicalize_project_name(req.project_name) fallback_value = req.project_name.strip().lower().replace("-", "_") in_stubs_map = proj_name in DEFAULT_TYPE_STUB_MODULE_MAPPING starts_with_prefix = fallback_value.startswith( ("types_", "stubs_")) ends_with_prefix = fallback_value.endswith(("_types", "_stubs")) if proj_name not in DEFAULT_MODULE_MAPPING and ( in_stubs_map or starts_with_prefix or ends_with_prefix): if in_stubs_map: stub_modules = DEFAULT_TYPE_STUB_MODULE_MAPPING[proj_name] else: stub_modules = (fallback_value[6:] if starts_with_prefix else fallback_value[:-6], ) add_stub_modules(stub_modules, tgt.address) else: add_modules( DEFAULT_MODULE_MAPPING.get(proj_name, (fallback_value, )), tgt.address) # Remove modules with ambiguous owners. for module in modules_with_multiple_owners: if module in modules_to_addresses: modules_to_addresses.pop(module) if module in modules_to_stub_addresses: modules_to_stub_addresses.pop(module) merged_mapping: DefaultDict[str, list[Address]] = defaultdict(list) for k, v in modules_to_addresses.items(): merged_mapping[k].append(v) for k, v in modules_to_stub_addresses.items(): merged_mapping[k].append(v) return ThirdPartyPythonModuleMapping( mapping=FrozenDict( (k, tuple(sorted(v))) for k, v in sorted(merged_mapping.items())), ambiguous_modules=FrozenDict( (k, tuple(sorted(v))) for k, v in sorted(modules_with_multiple_owners.items())), )