def assert_resolved(*, entry_point: Optional[str], expected: Optional[str]) -> None: addr = Address("src/python/project") rule_runner.create_file("src/python/project/app.py") rule_runner.create_file("src/python/project/f2.py") ep_field = PexEntryPointField(entry_point, address=addr) result = rule_runner.request(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field)]) assert result.val == expected
async def inject_pex_binary_entry_point_dependency( request: InjectPexBinaryEntryPointDependency, python_infer_subsystem: PythonInferSubsystem) -> InjectedDependencies: if not python_infer_subsystem.entry_points: return InjectedDependencies() original_tgt = await Get(WrappedTarget, Address, request.dependencies_field.address) explicitly_provided_deps, entry_point = await MultiGet( Get(ExplicitlyProvidedDependencies, DependenciesRequest(original_tgt.target[Dependencies])), Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest( original_tgt.target[PexEntryPointField]), ), ) if entry_point.val is None: return InjectedDependencies() owners = await Get(PythonModuleOwners, PythonModule(entry_point.val.module)) address = original_tgt.target.address explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference( owners.ambiguous, address, import_reference="module", context= (f"The pex_binary target {address} has the field " f"`entry_point={repr(original_tgt.target[PexEntryPointField].value.spec)}`, which " f"maps to the Python module `{entry_point.val.module}`"), ) maybe_disambiguated = explicitly_provided_deps.disambiguated_via_ignores( owners.ambiguous) unambiguous_owners = owners.unambiguous or ( (maybe_disambiguated, ) if maybe_disambiguated else ()) return InjectedDependencies(unambiguous_owners)
async def package_pex_binary( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage: resolved_entry_point = await Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point)) output_filename = field_set.output_path.value_or_default(field_set.address, file_ending="pex") two_step_pex = await Get( TwoStepPex, TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, # TODO(John Sirois): Support ConsoleScript in PexBinary targets: # https://github.com/pantsbuild/pants/issues/11619 main=resolved_entry_point.val, platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), output_filename=output_filename, additional_args=field_set.generate_additional_args( pex_binary_defaults), )), ) return BuiltPackage(two_step_pex.pex.digest, (BuiltPackageArtifact(output_filename), ))
def assert_resolved(*, entry_point: str | None, expected: EntryPoint | None, is_file: bool) -> None: addr = Address("src/python/project") rule_runner.create_file("src/python/project/app.py") rule_runner.create_file("src/python/project/f2.py") ep_field = PexEntryPointField(entry_point, addr) result = rule_runner.request(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field)]) assert result.val == expected assert result.file_name_used == is_file
async def package_pex_binary( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, union_membership: UnionMembership, ) -> BuiltPackage: resolved_entry_point, transitive_targets = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. file_tgts = targets_with_sources_types([FileSourceField], transitive_targets.dependencies, union_membership) if file_tgts: files_addresses = sorted(tgt.address.spec for tgt in file_tgts) logger.warning( softwrap(f""" The `pex_binary` target {field_set.address} transitively depends on the below `files` targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` are not able to load files within the binary itself; instead, they read from the current working directory. Instead, use `resources` targets or wrap this `pex_binary` in an `archive`. See {doc_url('resources')}. Files targets dependencies: {files_addresses} """)) output_filename = field_set.output_path.value_or_default(file_ending="pex") complete_platforms = await Get(CompletePlatforms, PexCompletePlatformsField, field_set.complete_platforms) pex = await Get( Pex, PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, main=resolved_entry_point.val or field_set.script.value, platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), complete_platforms=complete_platforms, output_filename=output_filename, layout=PexLayout(field_set.layout.value), additional_args=field_set.generate_additional_args( pex_binary_defaults), include_requirements=field_set.include_requirements.value, include_local_dists=True, ), ) return BuiltPackage(pex.digest, (BuiltPackageArtifact(output_filename), ))
async def inject_pex_binary_entry_point_dependency( request: InjectPexBinaryEntryPointDependency, python_infer_subsystem: PythonInferSubsystem, python_setup: PythonSetup, ) -> InjectedDependencies: if not python_infer_subsystem.entry_points: return InjectedDependencies() original_tgt = await Get( WrappedTarget, WrappedTargetRequest( request.dependencies_field.address, description_of_origin="<infallible>" ), ) entry_point_field = original_tgt.target.get(PexEntryPointField) if entry_point_field.value is None: return InjectedDependencies() explicitly_provided_deps, entry_point = await MultiGet( Get(ExplicitlyProvidedDependencies, DependenciesRequest(original_tgt.target[Dependencies])), Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(entry_point_field)), ) if entry_point.val is None: return InjectedDependencies() owners = await Get( PythonModuleOwners, PythonModuleOwnersRequest( entry_point.val.module, resolve=original_tgt.target[PythonResolveField].normalized_value(python_setup), ), ) address = original_tgt.target.address explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference( owners.ambiguous, address, # If the entry point was specified as a file, like `app.py`, we know the module must # live in the pex_binary's directory or subdirectory, so the owners must be ancestors. owners_must_be_ancestors=entry_point.file_name_used, import_reference="module", context=softwrap( f""" The pex_binary target {address} has the field `entry_point={repr(entry_point_field.value.spec)}`, which maps to the Python module `{entry_point.val.module}` """ ), ) maybe_disambiguated = explicitly_provided_deps.disambiguated( owners.ambiguous, owners_must_be_ancestors=entry_point.file_name_used ) unambiguous_owners = owners.unambiguous or ( (maybe_disambiguated,) if maybe_disambiguated else () ) return InjectedDependencies(unambiguous_owners)
def assert_resolved( *, entry_point: Optional[str], source: Optional[str] = None, expected: Optional[str] ) -> None: addr = Address("src/python/project") rule_runner.create_file("src/python/project/app.py") rule_runner.create_file("src/python/project/f2.py") ep_field = PexEntryPointField(entry_point, address=addr) sources = DeprecatedPexBinarySources([source] if source else None, address=addr) result = rule_runner.request( ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field, sources)] ) assert result.val == expected
async def inject_pex_binary_entry_point_dependency( request: InjectPexBinaryEntryPointDependency, python_infer_subsystem: PythonInferSubsystem ) -> InjectedDependencies: if not python_infer_subsystem.entry_points: return InjectedDependencies() original_tgt = await Get(WrappedTarget, Address, request.dependencies_field.address) entry_point = await Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(original_tgt.target[PexEntryPointField]), ) if entry_point.val is None: return InjectedDependencies() module, _, _func = entry_point.val.partition(":") owners = await Get(PythonModuleOwners, PythonModule(module)) return InjectedDependencies(owners)
async def package_pex_binary( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, union_membership: UnionMembership, ) -> BuiltPackage: resolved_entry_point, transitive_targets = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. files_tgts = targets_with_sources_types([FilesSources], transitive_targets.dependencies, union_membership) if files_tgts: files_addresses = sorted(tgt.address.spec for tgt in files_tgts) logger.warning( f"The pex_binary target {field_set.address} transitively depends on the below files " "targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` " "are not able to load files within the binary itself; instead, they read from the " "current working directory." "\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. See " f"{bracketed_docs_url('resources')}." f"\n\nFiles targets dependencies: {files_addresses}") output_filename = field_set.output_path.value_or_default(field_set.address, file_ending="pex") two_step_pex = await Get( TwoStepPex, TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, # TODO(John Sirois): Support ConsoleScript in PexBinary targets: # https://github.com/pantsbuild/pants/issues/11619 main=resolved_entry_point.val, platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), output_filename=output_filename, additional_args=field_set.generate_additional_args( pex_binary_defaults), )), ) return BuiltPackage(two_step_pex.pex.digest, (BuiltPackageArtifact(output_filename), ))
async def inject_pex_binary_entry_point_dependency( request: InjectPexBinaryEntryPointDependency, python_infer_subsystem: PythonInferSubsystem) -> InjectedDependencies: if not python_infer_subsystem.entry_points: return InjectedDependencies() original_tgt = await Get(WrappedTarget, Address, request.dependencies_field.address) entry_point = await Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest( original_tgt.target[PexEntryPointField], original_tgt.target[DeprecatedPexBinarySources]), ) if entry_point.val is None: return InjectedDependencies() module, _, _func = entry_point.val.partition(":") owners = await Get(PythonModuleOwners, PythonModule(module)) # TODO: remove the check for == self once the `sources` field is removed. return InjectedDependencies(owner for owner in owners if owner != request.dependencies_field.address)
async def package_pex_binary( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage: resolved_entry_point = await Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point)) output_filename = field_set.output_path.value_or_default(field_set.address, file_ending="pex") two_step_pex = await Get( TwoStepPex, TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, entry_point=resolved_entry_point.val, platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), output_filename=output_filename, additional_args=field_set.generate_additional_args( pex_binary_defaults), )), ) return BuiltPackage(two_step_pex.pex.digest, (BuiltPackageArtifact(output_filename), ))
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, python_setup: PythonSetup, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in local_dists_get. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest( [field_set.address], output_filename=f"{field_set.address.target_name}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, resolve_and_lockfile=field_set.resolve.resolve_and_lockfile( python_setup), additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) pex_get = Get(Pex, PexRequest, requirements_pex_request) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, sources=sources, ), ) merged_digest = await Get( Digest, MergeDigests([ pex.digest, local_dists.pex.digest, local_dists.remaining_sources.source_files.snapshot.digest, ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def find_putative_targets( req: PutativePythonTargetsRequest, all_owned_sources: AllOwnedSources, python_setup: PythonSetup, ) -> PutativeTargets: # Find library/test/test_util targets. all_py_files_globs: PathGlobs = req.search_paths.path_globs("*.py") all_py_files = await Get(Paths, PathGlobs, all_py_files_globs) unowned_py_files = set(all_py_files.files) - set(all_owned_sources) classified_unowned_py_files = classify_source_files(unowned_py_files) pts = [] for tgt_type, paths in classified_unowned_py_files.items(): for dirname, filenames in group_by_dir(paths).items(): if issubclass(tgt_type, PythonTestsGeneratorTarget): name = "tests" kwargs = {"name": name} elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget): name = "test_utils" kwargs = {"name": name} else: name = os.path.basename(dirname) kwargs = {} if ( python_setup.tailor_ignore_solitary_init_files and tgt_type == PythonSourcesGeneratorTarget and filenames == {"__init__.py"} ): continue pts.append( PutativeTarget.for_target_type( tgt_type, dirname, name, sorted(filenames), kwargs=kwargs ) ) if python_setup.tailor_requirements_targets: # Find requirements files. all_requirements_files_globs: PathGlobs = req.search_paths.path_globs("*requirements*.txt") all_requirements_files = await Get(Paths, PathGlobs, all_requirements_files_globs) unowned_requirements_files = set(all_requirements_files.files) - set(all_owned_sources) for req_file in unowned_requirements_files: path, name = os.path.split(req_file) pts.append( PutativeTarget( path=path, # python_requirements is a macro and doesn't take a name argument, but the # PutativeTarget still needs a name for display purposes. name=name, type_alias="python_requirements", triggering_sources=[req_file], owned_sources=[req_file], addressable=False, kwargs={} if name == "requirements.txt" else {"requirements_relpath": name}, ) ) if python_setup.tailor_pex_binary_targets: # Find binary targets. # Get all files whose content indicates that they are entry points. digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs) entry_points = [ file_content.path for file_content in digest_contents if is_entry_point(file_content.content) ] # Get the modules for these entry points. src_roots = await Get( SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points) ) module_to_entry_point = {} for entry_point in entry_points: entry_point_path = PurePath(entry_point) src_root = src_roots.path_to_root[entry_point_path] stripped_entry_point = entry_point_path.relative_to(src_root.path) module = PythonModule.create_from_stripped_path(stripped_entry_point) module_to_entry_point[module.module] = entry_point # Get existing binary targets for these entry points. entry_point_dirs = {os.path.dirname(entry_point) for entry_point in entry_points} possible_existing_binary_targets = await Get( UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in entry_point_dirs) ) possible_existing_binary_entry_points = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField])) for t in possible_existing_binary_targets if t.has_field(PexEntryPointField) ) possible_existing_entry_point_modules = { rep.val.module for rep in possible_existing_binary_entry_points if rep.val } unowned_entry_point_modules = ( module_to_entry_point.keys() - possible_existing_entry_point_modules ) # Generate new targets for entry points that don't already have one. for entry_point_module in unowned_entry_point_modules: entry_point = module_to_entry_point[entry_point_module] path, fname = os.path.split(entry_point) name = os.path.splitext(fname)[0] pts.append( PutativeTarget.for_target_type( target_type=PexBinary, path=path, name=name, triggering_sources=tuple(), kwargs={"name": name, "entry_point": fname}, ) ) return PutativeTargets(pts)
async def resolve_python_distribution_entry_points( request: ResolvePythonDistributionEntryPointsRequest, ) -> ResolvedPythonDistributionEntryPoints: if request.entry_points_field: if request.entry_points_field.value is None: return ResolvedPythonDistributionEntryPoints() address = request.entry_points_field.address all_entry_points = cast(_EntryPointsDictType, request.entry_points_field.value) elif request.provides_field: address = request.provides_field.address provides_field_value = cast( _EntryPointsDictType, request.provides_field.value.kwargs.get("entry_points") or {} ) if provides_field_value: all_entry_points = provides_field_value else: return ResolvedPythonDistributionEntryPoints() else: return ResolvedPythonDistributionEntryPoints() classified_entry_points = list(_classify_entry_points(all_entry_points)) # Pick out all target addresses up front, so we can use MultiGet later. # # This calls for a bit of trickery however (using the "y_by_x" mapping dicts), so we keep track # of which address belongs to which entry point. I.e. the `address_by_ref` and # `binary_entry_point_by_address` variables. target_refs = [ entry_point_str for is_target, _, _, entry_point_str in classified_entry_points if is_target ] # Intermediate step, as Get(Targets) returns a deduplicated set.. which breaks in case of # multiple input refs that maps to the same target. target_addresses = await Get( Addresses, UnparsedAddressInputs( target_refs, owning_address=address, description_of_origin="TODO(#14468)", ), ) address_by_ref = dict(zip(target_refs, target_addresses)) targets = await Get(Targets, Addresses, target_addresses) # Check that we only have targets with a pex entry_point field. for target in targets: if not target.has_field(PexEntryPointField): raise InvalidEntryPoint( softwrap( f""" All target addresses in the entry_points field must be for pex_binary targets, but the target {address} includes the value {target.address}, which has the target type {target.alias}. Alternatively, you can use a module like "project.app:main". See {doc_url('python-distributions')}. """ ) ) binary_entry_points = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(target[PexEntryPointField]), ) for target in targets ) binary_entry_point_by_address = { target.address: entry_point for target, entry_point in zip(targets, binary_entry_points) } entry_points: DefaultDict[str, Dict[str, PythonDistributionEntryPoint]] = defaultdict(dict) # Parse refs/replace with resolved pex entry point, and validate console entry points have function. for is_target, category, name, ref in classified_entry_points: owner: Optional[Address] = None if is_target: owner = address_by_ref[ref] entry_point = binary_entry_point_by_address[owner].val if entry_point is None: logger.warning( softwrap( f""" The entry point {name} in {category} references a pex_binary target {ref} which does not set `entry_point`. Skipping. """ ) ) continue else: entry_point = EntryPoint.parse(ref, f"{name} for {address} {category}") if category in ["console_scripts", "gui_scripts"] and not entry_point.function: url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point" raise InvalidEntryPoint( dedent( f"""\ Every entry point in `{category}` for {address} must end in the format `:my_func`, but {name} set it to {entry_point.spec!r}. For example, set `entry_points={{"{category}": {{"{name}": "{entry_point.module}:main}} }}`. See {url}. """ ) ) entry_points[category][name] = PythonDistributionEntryPoint(entry_point, owner) return ResolvedPythonDistributionEntryPoints( FrozenDict( {category: FrozenDict(entry_points) for category, entry_points in entry_points.items()} ) )
async def find_putative_targets( req: PutativePythonTargetsRequest, all_owned_sources: AllOwnedSources, python_setup: PythonSetup, ) -> PutativeTargets: pts = [] if python_setup.tailor_source_targets: # Find library/test/test_util targets. all_py_files_globs: PathGlobs = req.path_globs("*.py", "*.pyi") all_py_files = await Get(Paths, PathGlobs, all_py_files_globs) unowned_py_files = set(all_py_files.files) - set(all_owned_sources) classified_unowned_py_files = classify_source_files(unowned_py_files) for tgt_type, paths in classified_unowned_py_files.items(): for dirname, filenames in group_by_dir(paths).items(): name: str | None if issubclass(tgt_type, PythonTestsGeneratorTarget): name = "tests" elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget): name = "test_utils" else: name = None if (python_setup.tailor_ignore_solitary_init_files and tgt_type == PythonSourcesGeneratorTarget and filenames == {"__init__.py"}): continue pts.append( PutativeTarget.for_target_type( tgt_type, path=dirname, name=name, triggering_sources=sorted(filenames))) if python_setup.tailor_requirements_targets: # Find requirements files. ( all_requirements_files, all_pipenv_lockfile_files, all_pyproject_toml_contents, ) = await MultiGet( Get(DigestContents, PathGlobs, req.path_globs("*requirements*.txt")), Get(DigestContents, PathGlobs, req.path_globs("Pipfile.lock")), Get(DigestContents, PathGlobs, req.path_globs("pyproject.toml")), ) def add_req_targets(files: Iterable[FileContent], alias: str, target_name: str) -> None: contents = {i.path: i.content for i in files} unowned_files = set(contents) - set(all_owned_sources) for fp in unowned_files: path, name = os.path.split(fp) try: validate(fp, contents[fp], alias) except Exception as e: logger.warning( f"An error occurred when validating `{fp}`: {e}.\n\n" "You'll need to create targets for its contents manually.\n" "To silence this error in future, see " "https://www.pantsbuild.org/docs/reference-tailor#section-ignore-paths \n" ) continue pts.append( PutativeTarget( path=path, name=target_name, type_alias=alias, triggering_sources=[fp], owned_sources=[name], kwargs=({} if alias != "python_requirements" or name == "requirements.txt" else { "source": name }), )) def validate(path: str, contents: bytes, alias: str) -> None: if alias == "python_requirements": return validate_python_requirements(path, contents) elif alias == "pipenv_requirements": return validate_pipenv_requirements(contents) elif alias == "poetry_requirements": return validate_poetry_requirements(contents) def validate_python_requirements(path: str, contents: bytes) -> None: for _ in parse_requirements_file(contents.decode(), rel_path=path): pass def validate_pipenv_requirements(contents: bytes) -> None: parse_pipenv_requirements(contents) def validate_poetry_requirements(contents: bytes) -> None: p = PyProjectToml(PurePath(), PurePath(), contents.decode()) parse_pyproject_toml(p) add_req_targets(all_requirements_files, "python_requirements", "reqs") add_req_targets(all_pipenv_lockfile_files, "pipenv_requirements", "pipenv") add_req_targets( { fc for fc in all_pyproject_toml_contents if b"[tool.poetry" in fc.content }, "poetry_requirements", "poetry", ) if python_setup.tailor_pex_binary_targets: # Find binary targets. # Get all files whose content indicates that they are entry points or are __main__.py files. digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs) all_main_py = await Get(Paths, PathGlobs, req.path_globs("__main__.py")) entry_points = [ file_content.path for file_content in digest_contents if is_entry_point(file_content.content) ] + list(all_main_py.files) # Get the modules for these entry points. src_roots = await Get(SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points)) module_to_entry_point = {} for entry_point in entry_points: entry_point_path = PurePath(entry_point) src_root = src_roots.path_to_root[entry_point_path] stripped_entry_point = entry_point_path.relative_to(src_root.path) module = module_from_stripped_path(stripped_entry_point) module_to_entry_point[module] = entry_point # Get existing binary targets for these entry points. entry_point_dirs = { os.path.dirname(entry_point) for entry_point in entry_points } possible_existing_binary_targets = await Get( UnexpandedTargets, RawSpecs( ancestor_globs=tuple( AncestorGlobSpec(d) for d in entry_point_dirs), description_of_origin="the `pex_binary` tailor rule", ), ) possible_existing_binary_entry_points = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField])) for t in possible_existing_binary_targets if t.has_field(PexEntryPointField)) possible_existing_entry_point_modules = { rep.val.module for rep in possible_existing_binary_entry_points if rep.val } unowned_entry_point_modules = (module_to_entry_point.keys() - possible_existing_entry_point_modules) # Generate new targets for entry points that don't already have one. for entry_point_module in unowned_entry_point_modules: entry_point = module_to_entry_point[entry_point_module] path, fname = os.path.split(entry_point) name = os.path.splitext(fname)[0] pts.append( PutativeTarget.for_target_type( target_type=PexBinary, path=path, name=name, triggering_sources=tuple(), kwargs={"entry_point": fname}, )) return PutativeTargets(pts)
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment) -> RunRequest: run_in_sandbox = field_set.run_in_sandbox.value entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) addresses = [field_set.address] interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest(addresses)) pex_filename = (field_set.address.generated_name.replace(".", "_") if field_set.address.generated_name else field_set.address.target_name) pex_get = Get( Pex, PexFromTargetsRequest( [field_set.address], output_filename=f"{pex_filename}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=interpreter_constraints, sources=sources, ), ) input_digests = [ pex.digest, local_dists.pex.digest, # Note regarding inline mode: You might think that the sources don't need to be copied # into the chroot when using inline sources. But they do, because some of them might be # codegenned, and those won't exist in the inline source tree. Rather than incurring the # complexity of figuring out here which sources were codegenned, we copy everything. # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline # sources will take precedence and their copies in the chroot will be ignored. local_dists.remaining_sources.source_files.snapshot.digest, ] merged_digest = await Get(Digest, MergeDigests(input_digests)) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] # The order here is important: we want the in-repo sources to take precedence over their # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode). source_roots = [ *([] if run_in_sandbox else sources.source_roots), *chrooted_source_roots, ] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([field_set.address], internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request. interpreter_constraints, additional_args=field_set.generate_additional_args( pex_binary_defaults), internal_only=True, # Note that the entry point file is not in the PEX itself. It's loaded by setting # `PEX_EXTRA_SYS_PATH`. # TODO(John Sirois): Support ConsoleScript in PexBinary targets: # https://github.com/pantsbuild/pants/issues/11619 main=entry_point.val, ), ) requirements, sources, runner_pex = await MultiGet(requirements_request, sources_request, runner_pex_request) merged_digest = await Get( Digest, MergeDigests([ requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) args = pex_env.create_argv(in_chroot(runner_pex.name), python=runner_pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=runner_pex.python is not None), "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot: exported_target = request.exported_target exported_addr = exported_target.target.address owned_deps, transitive_targets = await MultiGet( Get(OwnedDependencies, DependencyOwner(exported_target)), Get(TransitiveTargets, TransitiveTargetsRequest([exported_target.target.address])), ) # files() targets aren't owned by a single exported target - they aren't code, so # we allow them to be in multiple dists. This is helpful for, e.g., embedding # a standard license file in a dist. files_targets = (tgt for tgt in transitive_targets.closure if tgt.has_field(FilesSources)) targets = Targets(itertools.chain((od.target for od in owned_deps), files_targets)) sources, requirements = await MultiGet( Get(SetupPySources, SetupPySourcesRequest(targets, py2=request.py2)), Get(ExportedTargetRequirements, DependencyOwner(exported_target)), ) # Generate the kwargs for the setup() call. In addition to using the kwargs that are either # explicitly provided or generated via a user's plugin, we add additional kwargs based on the # resolved requirements and sources. target = exported_target.target resolved_setup_kwargs = await Get(SetupKwargs, ExportedTarget, exported_target) setup_kwargs = resolved_setup_kwargs.kwargs.copy() # NB: We are careful to not overwrite these values, but we also don't expect them to have been # set. The user must have have gone out of their way to use a `SetupKwargs` plugin, and to have # specified `SetupKwargs(_allow_banned_keys=True)`. setup_kwargs.update( { "package_dir": {"": CHROOT_SOURCE_ROOT, **setup_kwargs.get("package_dir", {})}, "packages": (*sources.packages, *(setup_kwargs.get("packages", []))), "namespace_packages": ( *sources.namespace_packages, *setup_kwargs.get("namespace_packages", []), ), "package_data": {**dict(sources.package_data), **setup_kwargs.get("package_data", {})}, "install_requires": (*requirements, *setup_kwargs.get("install_requires", [])), } ) # Add any `pex_binary` targets from `setup_py().with_binaries()` to the dist's entry points. key_to_binary_spec = exported_target.provides.binaries binaries = await Get( Targets, UnparsedAddressInputs(key_to_binary_spec.values(), owning_address=target.address) ) entry_point_requests = [] for binary in binaries: if not binary.has_field(PexEntryPointField): raise InvalidEntryPoint( "Expected addresses to `pex_binary` targets in `.with_binaries()` for the " f"`provides` field for {exported_addr}, but found {binary.address} with target " f"type {binary.alias}." ) entry_point = binary[PexEntryPointField].value url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point" if not entry_point: raise InvalidEntryPoint( "Every `pex_binary` used in `.with_binaries()` for the `provides` field for " f"{exported_addr} must explicitly set the `entry_point` field, but " f"{binary.address} left the field off. Set `entry_point` to either " f"`app.py:func` or the longhand `path.to.app:func`. See {url}." ) if ":" not in entry_point: raise InvalidEntryPoint( "Every `pex_binary` used in `with_binaries()` for the `provides()` field for " f"{exported_addr} must end in the format `:my_func` for the `entry_point` field, " f"but {binary.address} set it to {repr(entry_point)}. For example, set " f"`entry_point='{entry_point}:main'. See {url}." ) entry_point_requests.append(ResolvePexEntryPointRequest(binary[PexEntryPointField])) binary_entry_points = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest, request) for request in entry_point_requests ) for key, binary_entry_point in zip(key_to_binary_spec.keys(), binary_entry_points): entry_points = setup_kwargs.setdefault("entry_points", {}) console_scripts = entry_points.setdefault("console_scripts", []) console_scripts.append(f"{key}={binary_entry_point.val}") # Generate the setup script. setup_py_content = SETUP_BOILERPLATE.format( target_address_spec=target.address.spec, setup_kwargs_str=distutils_repr(setup_kwargs), ).encode() files_to_create = [ FileContent("setup.py", setup_py_content), FileContent("MANIFEST.in", "include *.py".encode()), ] extra_files_digest, src_digest = await MultiGet( Get(Digest, CreateDigest(files_to_create)), # Nest the sources under the src/ prefix. Get(Digest, AddPrefix(sources.digest, CHROOT_SOURCE_ROOT)), ) chroot_digest = await Get(Digest, MergeDigests((src_digest, extra_files_digest))) return SetupPyChroot(chroot_digest, FinalizedSetupKwargs(setup_kwargs, address=target.address))