async def setup_user_lockfile_requests( requested: RequestedPythonUserResolveNames, all_targets: AllTargets, python_setup: PythonSetup) -> UserGenerateLockfiles: if not (python_setup.enable_resolves and python_setup.resolves_generate_lockfiles): return UserGenerateLockfiles() resolve_to_requirements_fields = defaultdict(set) for tgt in all_targets: if not tgt.has_fields( (PythonRequirementResolveField, PythonRequirementsField)): continue resolve = tgt[PythonRequirementResolveField].normalized_value( python_setup) resolve_to_requirements_fields[resolve].add( tgt[PythonRequirementsField]) return UserGenerateLockfiles( GeneratePythonLockfile( requirements=PexRequirements.create_from_requirement_fields( resolve_to_requirements_fields[resolve], constraints_strings=(), ).req_strings, interpreter_constraints=InterpreterConstraints( python_setup.resolves_to_interpreter_constraints.get( resolve, python_setup.interpreter_constraints)), resolve_name=resolve, lockfile_dest=python_setup.resolves[resolve], use_pex=python_setup.generate_lockfiles_with_pex, ) for resolve in requested)
async def determine_requirement_strings_in_closure( request: _PexRequirementsRequest, global_requirement_constraints: GlobalRequirementConstraints ) -> PexRequirements: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) return PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in transitive_targets.closure if tgt.has_field(PythonRequirementsField) ), constraints_strings=(str(constraint) for constraint in global_requirement_constraints), )
async def pylint_first_party_plugins( pylint: Pylint) -> PylintFirstPartyPlugins: if not pylint.source_plugins: return PylintFirstPartyPlugins(FrozenOrderedSet(), FrozenOrderedSet(), EMPTY_DIGEST) plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, pylint.source_plugins) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)) requirements_fields: OrderedSet[PythonRequirementsField] = OrderedSet() interpreter_constraints_fields: OrderedSet[ InterpreterConstraintsField] = OrderedSet() for tgt in transitive_targets.closure: if tgt.has_field(PythonRequirementsField): requirements_fields.add(tgt[PythonRequirementsField]) if tgt.has_field(InterpreterConstraintsField): interpreter_constraints_fields.add( tgt[InterpreterConstraintsField]) # NB: Pylint source plugins must be explicitly loaded via PYTHONPATH (i.e. PEX_EXTRA_SYS_PATH). # The value must point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) stripped_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure)) prefixed_sources = await Get( Digest, AddPrefix(stripped_sources.stripped_source_files.snapshot.digest, PylintFirstPartyPlugins.PREFIX), ) return PylintFirstPartyPlugins( requirement_strings=PexRequirements.create_from_requirement_fields( requirements_fields, constraints_strings=(), ).req_strings, interpreter_constraints_fields=FrozenOrderedSet( interpreter_constraints_fields), sources_digest=prefixed_sources, )
async def mypy_first_party_plugins(mypy: MyPy, ) -> MyPyFirstPartyPlugins: if not mypy.source_plugins: return MyPyFirstPartyPlugins(FrozenOrderedSet(), EMPTY_DIGEST, ()) plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)) requirements = PexRequirements.create_from_requirement_fields( (plugin_tgt[PythonRequirementsField] for plugin_tgt in transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField)), constraints_strings=(), ) sources = await Get(PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure)) return MyPyFirstPartyPlugins( requirement_strings=requirements.req_strings, sources_digest=sources.source_files.snapshot.digest, source_roots=sources.source_roots, )
async def get_requirements( dep_owner: DependencyOwner, union_membership: UnionMembership, setup_py_generation: SetupPyGeneration, ) -> ExportedTargetRequirements: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([dep_owner.exported_target.target.address]), ) ownable_tgts = [ tgt for tgt in transitive_targets.closure if is_ownable_target(tgt, union_membership) ] owners = await MultiGet(Get(ExportedTarget, OwnedDependency(tgt)) for tgt in ownable_tgts) owned_by_us: set[Target] = set() owned_by_others: set[Target] = set() for tgt, owner in zip(ownable_tgts, owners): (owned_by_us if owner == dep_owner.exported_target else owned_by_others).add(tgt) # Get all 3rdparty deps of our owned deps. # # Note that we need only consider requirements that are direct dependencies of our owned deps: # If T depends on R indirectly, then it must be via some direct deps U1, U2, ... For each such U, # if U is in the owned deps then we'll pick up R through U. And if U is not in the owned deps # then it's owned by an exported target ET, and so R will be in the requirements for ET, and we # will require ET. direct_deps_tgts = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in owned_by_us ) transitive_excludes: FrozenOrderedSet[Target] = FrozenOrderedSet() uneval_trans_excl = [ tgt.get(Dependencies).unevaluated_transitive_excludes for tgt in transitive_targets.closure ] if uneval_trans_excl: nested_trans_excl = await MultiGet( Get(Targets, UnparsedAddressInputs, unparsed) for unparsed in uneval_trans_excl ) transitive_excludes = FrozenOrderedSet( itertools.chain.from_iterable(excludes for excludes in nested_trans_excl) ) direct_deps_chained = FrozenOrderedSet(itertools.chain.from_iterable(direct_deps_tgts)) direct_deps_with_excl = direct_deps_chained.difference(transitive_excludes) req_strs = list( PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in direct_deps_with_excl if tgt.has_field(PythonRequirementsField) ), constraints_strings=(), ).req_strings ) # Add the requirements on any exported targets on which we depend. kwargs_for_exported_targets_we_depend_on = await MultiGet( Get(SetupKwargs, OwnedDependency(tgt)) for tgt in owned_by_others ) req_strs.extend( f"{kwargs.name}{setup_py_generation.first_party_dependency_version(kwargs.version)}" for kwargs in set(kwargs_for_exported_targets_we_depend_on) ) return ExportedTargetRequirements(req_strs)
async def _setup_constraints_repository_pex( constraints_request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup, global_requirement_constraints: GlobalRequirementConstraints, ) -> OptionalPexRequest: request = constraints_request.repository_pex_request # NB: it isn't safe to resolve against the whole constraints file if # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222. if not python_setup.resolve_all_constraints or request.platforms or request.complete_platforms: return OptionalPexRequest(None) constraints_path = python_setup.requirement_constraints assert constraints_path is not None transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) requirements = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in transitive_targets.closure if tgt.has_field(PythonRequirementsField) ), constraints_strings=(str(constraint) for constraint in global_requirement_constraints), ) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch' name_reqs = set() # E.g., foobar>=1.2.3 name_req_projects = set() constraints_file_reqs = set(global_requirement_constraints) for req_str in requirements.req_strings: req = PipRequirement.parse(req_str) if req.url: url_reqs.add(req) else: name_reqs.add(req) name_req_projects.add(canonicalize_project_name(req.project_name)) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. unconstrained_projects = name_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {constraints_path} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}.\n\n" f"Ignoring `[python_setup].resolve_all_constraints` option." ) return OptionalPexRequest(None) interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest, request.to_interpreter_constraints_request(), ) # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. # NB: We can only add the URL requirements we know about here, i.e., those that # are transitive deps of the targets in play. There may be others in the repo. # So we may end up creating a few different repository pexes, each with identical # name requirements but different subsets of URL requirements. Fortunately since # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} repository_pex = PexRequest( description=f"Resolving {constraints_path}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( all_constraints, constraints_strings=(str(constraint) for constraint in global_requirement_constraints), # TODO: See PexRequirements docs. is_all_constraints_resolve=True, ), interpreter_constraints=interpreter_constraints, platforms=request.platforms, complete_platforms=request.complete_platforms, additional_args=request.additional_lockfile_args, ) return OptionalPexRequest(repository_pex)