def assert_resolved(*, entry_point: Optional[str], expected: Optional[str]) -> None:
addr = Address("src/python/project")
rule_runner.create_file("src/python/project/app.py")
rule_runner.create_file("src/python/project/f2.py")
ep_field = PexEntryPointField(entry_point, address=addr)
result = rule_runner.request(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field)])
assert result.val == expected
async def inject_pex_binary_entry_point_dependency(
request: InjectPexBinaryEntryPointDependency,
python_infer_subsystem: PythonInferSubsystem) -> InjectedDependencies:
if not python_infer_subsystem.entry_points:
return InjectedDependencies()
original_tgt = await Get(WrappedTarget, Address,
request.dependencies_field.address)
explicitly_provided_deps, entry_point = await MultiGet(
Get(ExplicitlyProvidedDependencies,
DependenciesRequest(original_tgt.target[Dependencies])),
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(
original_tgt.target[PexEntryPointField]),
),
)
if entry_point.val is None:
return InjectedDependencies()
owners = await Get(PythonModuleOwners,
PythonModule(entry_point.val.module))
address = original_tgt.target.address
explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
owners.ambiguous,
address,
import_reference="module",
context=
(f"The pex_binary target {address} has the field "
f"`entry_point={repr(original_tgt.target[PexEntryPointField].value.spec)}`, which "
f"maps to the Python module `{entry_point.val.module}`"),
)
maybe_disambiguated = explicitly_provided_deps.disambiguated_via_ignores(
owners.ambiguous)
unambiguous_owners = owners.unambiguous or (
(maybe_disambiguated, ) if maybe_disambiguated else ())
return InjectedDependencies(unambiguous_owners)
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage:
resolved_entry_point = await Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point))
output_filename = field_set.output_path.value_or_default(field_set.address,
file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
def assert_resolved(*, entry_point: str | None,
expected: EntryPoint | None, is_file: bool) -> None:
addr = Address("src/python/project")
rule_runner.create_file("src/python/project/app.py")
rule_runner.create_file("src/python/project/f2.py")
ep_field = PexEntryPointField(entry_point, addr)
result = rule_runner.request(ResolvedPexEntryPoint,
[ResolvePexEntryPointRequest(ep_field)])
assert result.val == expected
assert result.file_name_used == is_file
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
union_membership: UnionMembership,
) -> BuiltPackage:
resolved_entry_point, transitive_targets = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
file_tgts = targets_with_sources_types([FileSourceField],
transitive_targets.dependencies,
union_membership)
if file_tgts:
files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
logger.warning(
softwrap(f"""
The `pex_binary` target {field_set.address} transitively depends on the below `files`
targets, but Pants will not include them in the PEX. Filesystem APIs like `open()`
are not able to load files within the binary itself; instead, they read from the
current working directory.
Instead, use `resources` targets or wrap this `pex_binary` in an `archive`.
See {doc_url('resources')}.
Files targets dependencies: {files_addresses}
"""))
output_filename = field_set.output_path.value_or_default(file_ending="pex")
complete_platforms = await Get(CompletePlatforms,
PexCompletePlatformsField,
field_set.complete_platforms)
pex = await Get(
Pex,
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
main=resolved_entry_point.val or field_set.script.value,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
complete_platforms=complete_platforms,
output_filename=output_filename,
layout=PexLayout(field_set.layout.value),
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
include_requirements=field_set.include_requirements.value,
include_local_dists=True,
),
)
return BuiltPackage(pex.digest, (BuiltPackageArtifact(output_filename), ))
async def inject_pex_binary_entry_point_dependency(
request: InjectPexBinaryEntryPointDependency,
python_infer_subsystem: PythonInferSubsystem,
python_setup: PythonSetup,
) -> InjectedDependencies:
if not python_infer_subsystem.entry_points:
return InjectedDependencies()
original_tgt = await Get(
WrappedTarget,
WrappedTargetRequest(
request.dependencies_field.address, description_of_origin="<infallible>"
),
)
entry_point_field = original_tgt.target.get(PexEntryPointField)
if entry_point_field.value is None:
return InjectedDependencies()
explicitly_provided_deps, entry_point = await MultiGet(
Get(ExplicitlyProvidedDependencies, DependenciesRequest(original_tgt.target[Dependencies])),
Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(entry_point_field)),
)
if entry_point.val is None:
return InjectedDependencies()
owners = await Get(
PythonModuleOwners,
PythonModuleOwnersRequest(
entry_point.val.module,
resolve=original_tgt.target[PythonResolveField].normalized_value(python_setup),
),
)
address = original_tgt.target.address
explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
owners.ambiguous,
address,
# If the entry point was specified as a file, like `app.py`, we know the module must
# live in the pex_binary's directory or subdirectory, so the owners must be ancestors.
owners_must_be_ancestors=entry_point.file_name_used,
import_reference="module",
context=softwrap(
f"""
The pex_binary target {address} has the field
`entry_point={repr(entry_point_field.value.spec)}`, which
maps to the Python module `{entry_point.val.module}`
"""
),
)
maybe_disambiguated = explicitly_provided_deps.disambiguated(
owners.ambiguous, owners_must_be_ancestors=entry_point.file_name_used
)
unambiguous_owners = owners.unambiguous or (
(maybe_disambiguated,) if maybe_disambiguated else ()
)
return InjectedDependencies(unambiguous_owners)
def assert_resolved(
*, entry_point: Optional[str], source: Optional[str] = None, expected: Optional[str]
) -> None:
addr = Address("src/python/project")
rule_runner.create_file("src/python/project/app.py")
rule_runner.create_file("src/python/project/f2.py")
ep_field = PexEntryPointField(entry_point, address=addr)
sources = DeprecatedPexBinarySources([source] if source else None, address=addr)
result = rule_runner.request(
ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field, sources)]
)
assert result.val == expected
async def inject_pex_binary_entry_point_dependency(
request: InjectPexBinaryEntryPointDependency, python_infer_subsystem: PythonInferSubsystem
) -> InjectedDependencies:
if not python_infer_subsystem.entry_points:
return InjectedDependencies()
original_tgt = await Get(WrappedTarget, Address, request.dependencies_field.address)
entry_point = await Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(original_tgt.target[PexEntryPointField]),
)
if entry_point.val is None:
return InjectedDependencies()
module, _, _func = entry_point.val.partition(":")
owners = await Get(PythonModuleOwners, PythonModule(module))
return InjectedDependencies(owners)
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
union_membership: UnionMembership,
) -> BuiltPackage:
resolved_entry_point, transitive_targets = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
files_tgts = targets_with_sources_types([FilesSources],
transitive_targets.dependencies,
union_membership)
if files_tgts:
files_addresses = sorted(tgt.address.spec for tgt in files_tgts)
logger.warning(
f"The pex_binary target {field_set.address} transitively depends on the below files "
"targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` "
"are not able to load files within the binary itself; instead, they read from the "
"current working directory."
"\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. See "
f"{bracketed_docs_url('resources')}."
f"\n\nFiles targets dependencies: {files_addresses}")
output_filename = field_set.output_path.value_or_default(field_set.address,
file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
async def inject_pex_binary_entry_point_dependency(
request: InjectPexBinaryEntryPointDependency,
python_infer_subsystem: PythonInferSubsystem) -> InjectedDependencies:
if not python_infer_subsystem.entry_points:
return InjectedDependencies()
original_tgt = await Get(WrappedTarget, Address,
request.dependencies_field.address)
entry_point = await Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(
original_tgt.target[PexEntryPointField],
original_tgt.target[DeprecatedPexBinarySources]),
)
if entry_point.val is None:
return InjectedDependencies()
module, _, _func = entry_point.val.partition(":")
owners = await Get(PythonModuleOwners, PythonModule(module))
# TODO: remove the check for == self once the `sources` field is removed.
return InjectedDependencies(owner for owner in owners
if owner != request.dependencies_field.address)
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage:
resolved_entry_point = await Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point))
output_filename = field_set.output_path.value_or_default(field_set.address,
file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
entry_point=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
python_setup: PythonSetup,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in local_dists_get.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{field_set.address.target_name}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
resolve_and_lockfile=field_set.resolve.resolve_and_lockfile(
python_setup),
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
pex_get = Get(Pex, PexRequest, requirements_pex_request)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
sources=sources,
),
)
merged_digest = await Get(
Digest,
MergeDigests([
pex.digest,
local_dists.pex.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def find_putative_targets(
req: PutativePythonTargetsRequest,
all_owned_sources: AllOwnedSources,
python_setup: PythonSetup,
) -> PutativeTargets:
# Find library/test/test_util targets.
all_py_files_globs: PathGlobs = req.search_paths.path_globs("*.py")
all_py_files = await Get(Paths, PathGlobs, all_py_files_globs)
unowned_py_files = set(all_py_files.files) - set(all_owned_sources)
classified_unowned_py_files = classify_source_files(unowned_py_files)
pts = []
for tgt_type, paths in classified_unowned_py_files.items():
for dirname, filenames in group_by_dir(paths).items():
if issubclass(tgt_type, PythonTestsGeneratorTarget):
name = "tests"
kwargs = {"name": name}
elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget):
name = "test_utils"
kwargs = {"name": name}
else:
name = os.path.basename(dirname)
kwargs = {}
if (
python_setup.tailor_ignore_solitary_init_files
and tgt_type == PythonSourcesGeneratorTarget
and filenames == {"__init__.py"}
):
continue
pts.append(
PutativeTarget.for_target_type(
tgt_type, dirname, name, sorted(filenames), kwargs=kwargs
)
)
if python_setup.tailor_requirements_targets:
# Find requirements files.
all_requirements_files_globs: PathGlobs = req.search_paths.path_globs("*requirements*.txt")
all_requirements_files = await Get(Paths, PathGlobs, all_requirements_files_globs)
unowned_requirements_files = set(all_requirements_files.files) - set(all_owned_sources)
for req_file in unowned_requirements_files:
path, name = os.path.split(req_file)
pts.append(
PutativeTarget(
path=path,
# python_requirements is a macro and doesn't take a name argument, but the
# PutativeTarget still needs a name for display purposes.
name=name,
type_alias="python_requirements",
triggering_sources=[req_file],
owned_sources=[req_file],
addressable=False,
kwargs={} if name == "requirements.txt" else {"requirements_relpath": name},
)
)
if python_setup.tailor_pex_binary_targets:
# Find binary targets.
# Get all files whose content indicates that they are entry points.
digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs)
entry_points = [
file_content.path
for file_content in digest_contents
if is_entry_point(file_content.content)
]
# Get the modules for these entry points.
src_roots = await Get(
SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points)
)
module_to_entry_point = {}
for entry_point in entry_points:
entry_point_path = PurePath(entry_point)
src_root = src_roots.path_to_root[entry_point_path]
stripped_entry_point = entry_point_path.relative_to(src_root.path)
module = PythonModule.create_from_stripped_path(stripped_entry_point)
module_to_entry_point[module.module] = entry_point
# Get existing binary targets for these entry points.
entry_point_dirs = {os.path.dirname(entry_point) for entry_point in entry_points}
possible_existing_binary_targets = await Get(
UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in entry_point_dirs)
)
possible_existing_binary_entry_points = await MultiGet(
Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField]))
for t in possible_existing_binary_targets
if t.has_field(PexEntryPointField)
)
possible_existing_entry_point_modules = {
rep.val.module for rep in possible_existing_binary_entry_points if rep.val
}
unowned_entry_point_modules = (
module_to_entry_point.keys() - possible_existing_entry_point_modules
)
# Generate new targets for entry points that don't already have one.
for entry_point_module in unowned_entry_point_modules:
entry_point = module_to_entry_point[entry_point_module]
path, fname = os.path.split(entry_point)
name = os.path.splitext(fname)[0]
pts.append(
PutativeTarget.for_target_type(
target_type=PexBinary,
path=path,
name=name,
triggering_sources=tuple(),
kwargs={"name": name, "entry_point": fname},
)
)
return PutativeTargets(pts)
async def resolve_python_distribution_entry_points(
request: ResolvePythonDistributionEntryPointsRequest,
) -> ResolvedPythonDistributionEntryPoints:
if request.entry_points_field:
if request.entry_points_field.value is None:
return ResolvedPythonDistributionEntryPoints()
address = request.entry_points_field.address
all_entry_points = cast(_EntryPointsDictType, request.entry_points_field.value)
elif request.provides_field:
address = request.provides_field.address
provides_field_value = cast(
_EntryPointsDictType, request.provides_field.value.kwargs.get("entry_points") or {}
)
if provides_field_value:
all_entry_points = provides_field_value
else:
return ResolvedPythonDistributionEntryPoints()
else:
return ResolvedPythonDistributionEntryPoints()
classified_entry_points = list(_classify_entry_points(all_entry_points))
# Pick out all target addresses up front, so we can use MultiGet later.
#
# This calls for a bit of trickery however (using the "y_by_x" mapping dicts), so we keep track
# of which address belongs to which entry point. I.e. the `address_by_ref` and
# `binary_entry_point_by_address` variables.
target_refs = [
entry_point_str for is_target, _, _, entry_point_str in classified_entry_points if is_target
]
# Intermediate step, as Get(Targets) returns a deduplicated set.. which breaks in case of
# multiple input refs that maps to the same target.
target_addresses = await Get(
Addresses,
UnparsedAddressInputs(
target_refs,
owning_address=address,
description_of_origin="TODO(#14468)",
),
)
address_by_ref = dict(zip(target_refs, target_addresses))
targets = await Get(Targets, Addresses, target_addresses)
# Check that we only have targets with a pex entry_point field.
for target in targets:
if not target.has_field(PexEntryPointField):
raise InvalidEntryPoint(
softwrap(
f"""
All target addresses in the entry_points field must be for pex_binary targets,
but the target {address} includes the value {target.address}, which has the
target type {target.alias}.
Alternatively, you can use a module like "project.app:main".
See {doc_url('python-distributions')}.
"""
)
)
binary_entry_points = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(target[PexEntryPointField]),
)
for target in targets
)
binary_entry_point_by_address = {
target.address: entry_point for target, entry_point in zip(targets, binary_entry_points)
}
entry_points: DefaultDict[str, Dict[str, PythonDistributionEntryPoint]] = defaultdict(dict)
# Parse refs/replace with resolved pex entry point, and validate console entry points have function.
for is_target, category, name, ref in classified_entry_points:
owner: Optional[Address] = None
if is_target:
owner = address_by_ref[ref]
entry_point = binary_entry_point_by_address[owner].val
if entry_point is None:
logger.warning(
softwrap(
f"""
The entry point {name} in {category} references a pex_binary target {ref}
which does not set `entry_point`. Skipping.
"""
)
)
continue
else:
entry_point = EntryPoint.parse(ref, f"{name} for {address} {category}")
if category in ["console_scripts", "gui_scripts"] and not entry_point.function:
url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point"
raise InvalidEntryPoint(
dedent(
f"""\
Every entry point in `{category}` for {address} must end in the format `:my_func`,
but {name} set it to {entry_point.spec!r}. For example, set
`entry_points={{"{category}": {{"{name}": "{entry_point.module}:main}} }}`.
See {url}.
"""
)
)
entry_points[category][name] = PythonDistributionEntryPoint(entry_point, owner)
return ResolvedPythonDistributionEntryPoints(
FrozenDict(
{category: FrozenDict(entry_points) for category, entry_points in entry_points.items()}
)
)
async def find_putative_targets(
req: PutativePythonTargetsRequest,
all_owned_sources: AllOwnedSources,
python_setup: PythonSetup,
) -> PutativeTargets:
pts = []
if python_setup.tailor_source_targets:
# Find library/test/test_util targets.
all_py_files_globs: PathGlobs = req.path_globs("*.py", "*.pyi")
all_py_files = await Get(Paths, PathGlobs, all_py_files_globs)
unowned_py_files = set(all_py_files.files) - set(all_owned_sources)
classified_unowned_py_files = classify_source_files(unowned_py_files)
for tgt_type, paths in classified_unowned_py_files.items():
for dirname, filenames in group_by_dir(paths).items():
name: str | None
if issubclass(tgt_type, PythonTestsGeneratorTarget):
name = "tests"
elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget):
name = "test_utils"
else:
name = None
if (python_setup.tailor_ignore_solitary_init_files
and tgt_type == PythonSourcesGeneratorTarget
and filenames == {"__init__.py"}):
continue
pts.append(
PutativeTarget.for_target_type(
tgt_type,
path=dirname,
name=name,
triggering_sources=sorted(filenames)))
if python_setup.tailor_requirements_targets:
# Find requirements files.
(
all_requirements_files,
all_pipenv_lockfile_files,
all_pyproject_toml_contents,
) = await MultiGet(
Get(DigestContents, PathGlobs,
req.path_globs("*requirements*.txt")),
Get(DigestContents, PathGlobs, req.path_globs("Pipfile.lock")),
Get(DigestContents, PathGlobs, req.path_globs("pyproject.toml")),
)
def add_req_targets(files: Iterable[FileContent], alias: str,
target_name: str) -> None:
contents = {i.path: i.content for i in files}
unowned_files = set(contents) - set(all_owned_sources)
for fp in unowned_files:
path, name = os.path.split(fp)
try:
validate(fp, contents[fp], alias)
except Exception as e:
logger.warning(
f"An error occurred when validating `{fp}`: {e}.\n\n"
"You'll need to create targets for its contents manually.\n"
"To silence this error in future, see "
"https://www.pantsbuild.org/docs/reference-tailor#section-ignore-paths \n"
)
continue
pts.append(
PutativeTarget(
path=path,
name=target_name,
type_alias=alias,
triggering_sources=[fp],
owned_sources=[name],
kwargs=({} if alias != "python_requirements"
or name == "requirements.txt" else {
"source": name
}),
))
def validate(path: str, contents: bytes, alias: str) -> None:
if alias == "python_requirements":
return validate_python_requirements(path, contents)
elif alias == "pipenv_requirements":
return validate_pipenv_requirements(contents)
elif alias == "poetry_requirements":
return validate_poetry_requirements(contents)
def validate_python_requirements(path: str, contents: bytes) -> None:
for _ in parse_requirements_file(contents.decode(), rel_path=path):
pass
def validate_pipenv_requirements(contents: bytes) -> None:
parse_pipenv_requirements(contents)
def validate_poetry_requirements(contents: bytes) -> None:
p = PyProjectToml(PurePath(), PurePath(), contents.decode())
parse_pyproject_toml(p)
add_req_targets(all_requirements_files, "python_requirements", "reqs")
add_req_targets(all_pipenv_lockfile_files, "pipenv_requirements",
"pipenv")
add_req_targets(
{
fc
for fc in all_pyproject_toml_contents
if b"[tool.poetry" in fc.content
},
"poetry_requirements",
"poetry",
)
if python_setup.tailor_pex_binary_targets:
# Find binary targets.
# Get all files whose content indicates that they are entry points or are __main__.py files.
digest_contents = await Get(DigestContents, PathGlobs,
all_py_files_globs)
all_main_py = await Get(Paths, PathGlobs,
req.path_globs("__main__.py"))
entry_points = [
file_content.path for file_content in digest_contents
if is_entry_point(file_content.content)
] + list(all_main_py.files)
# Get the modules for these entry points.
src_roots = await Get(SourceRootsResult, SourceRootsRequest,
SourceRootsRequest.for_files(entry_points))
module_to_entry_point = {}
for entry_point in entry_points:
entry_point_path = PurePath(entry_point)
src_root = src_roots.path_to_root[entry_point_path]
stripped_entry_point = entry_point_path.relative_to(src_root.path)
module = module_from_stripped_path(stripped_entry_point)
module_to_entry_point[module] = entry_point
# Get existing binary targets for these entry points.
entry_point_dirs = {
os.path.dirname(entry_point)
for entry_point in entry_points
}
possible_existing_binary_targets = await Get(
UnexpandedTargets,
RawSpecs(
ancestor_globs=tuple(
AncestorGlobSpec(d) for d in entry_point_dirs),
description_of_origin="the `pex_binary` tailor rule",
),
)
possible_existing_binary_entry_points = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(t[PexEntryPointField]))
for t in possible_existing_binary_targets
if t.has_field(PexEntryPointField))
possible_existing_entry_point_modules = {
rep.val.module
for rep in possible_existing_binary_entry_points if rep.val
}
unowned_entry_point_modules = (module_to_entry_point.keys() -
possible_existing_entry_point_modules)
# Generate new targets for entry points that don't already have one.
for entry_point_module in unowned_entry_point_modules:
entry_point = module_to_entry_point[entry_point_module]
path, fname = os.path.split(entry_point)
name = os.path.splitext(fname)[0]
pts.append(
PutativeTarget.for_target_type(
target_type=PexBinary,
path=path,
name=name,
triggering_sources=tuple(),
kwargs={"entry_point": fname},
))
return PutativeTargets(pts)
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment) -> RunRequest:
run_in_sandbox = field_set.run_in_sandbox.value
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
addresses = [field_set.address]
interpreter_constraints = await Get(
InterpreterConstraints, InterpreterConstraintsRequest(addresses))
pex_filename = (field_set.address.generated_name.replace(".", "_")
if field_set.address.generated_name else
field_set.address.target_name)
pex_get = Get(
Pex,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{pex_filename}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
input_digests = [
pex.digest,
local_dists.pex.digest,
# Note regarding inline mode: You might think that the sources don't need to be copied
# into the chroot when using inline sources. But they do, because some of them might be
# codegenned, and those won't exist in the inline source tree. Rather than incurring the
# complexity of figuring out here which sources were codegenned, we copy everything.
# The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
# sources will take precedence and their copies in the chroot will be ignored.
local_dists.remaining_sources.source_files.snapshot.digest,
]
merged_digest = await Get(Digest, MergeDigests(input_digests))
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
# The order here is important: we want the in-repo sources to take precedence over their
# copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
source_roots = [
*([] if run_in_sandbox else sources.source_roots),
*chrooted_source_roots,
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the PEX itself. It's loaded by setting
# `PEX_EXTRA_SYS_PATH`.
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=entry_point.val,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot:
exported_target = request.exported_target
exported_addr = exported_target.target.address
owned_deps, transitive_targets = await MultiGet(
Get(OwnedDependencies, DependencyOwner(exported_target)),
Get(TransitiveTargets, TransitiveTargetsRequest([exported_target.target.address])),
)
# files() targets aren't owned by a single exported target - they aren't code, so
# we allow them to be in multiple dists. This is helpful for, e.g., embedding
# a standard license file in a dist.
files_targets = (tgt for tgt in transitive_targets.closure if tgt.has_field(FilesSources))
targets = Targets(itertools.chain((od.target for od in owned_deps), files_targets))
sources, requirements = await MultiGet(
Get(SetupPySources, SetupPySourcesRequest(targets, py2=request.py2)),
Get(ExportedTargetRequirements, DependencyOwner(exported_target)),
)
# Generate the kwargs for the setup() call. In addition to using the kwargs that are either
# explicitly provided or generated via a user's plugin, we add additional kwargs based on the
# resolved requirements and sources.
target = exported_target.target
resolved_setup_kwargs = await Get(SetupKwargs, ExportedTarget, exported_target)
setup_kwargs = resolved_setup_kwargs.kwargs.copy()
# NB: We are careful to not overwrite these values, but we also don't expect them to have been
# set. The user must have have gone out of their way to use a `SetupKwargs` plugin, and to have
# specified `SetupKwargs(_allow_banned_keys=True)`.
setup_kwargs.update(
{
"package_dir": {"": CHROOT_SOURCE_ROOT, **setup_kwargs.get("package_dir", {})},
"packages": (*sources.packages, *(setup_kwargs.get("packages", []))),
"namespace_packages": (
*sources.namespace_packages,
*setup_kwargs.get("namespace_packages", []),
),
"package_data": {**dict(sources.package_data), **setup_kwargs.get("package_data", {})},
"install_requires": (*requirements, *setup_kwargs.get("install_requires", [])),
}
)
# Add any `pex_binary` targets from `setup_py().with_binaries()` to the dist's entry points.
key_to_binary_spec = exported_target.provides.binaries
binaries = await Get(
Targets, UnparsedAddressInputs(key_to_binary_spec.values(), owning_address=target.address)
)
entry_point_requests = []
for binary in binaries:
if not binary.has_field(PexEntryPointField):
raise InvalidEntryPoint(
"Expected addresses to `pex_binary` targets in `.with_binaries()` for the "
f"`provides` field for {exported_addr}, but found {binary.address} with target "
f"type {binary.alias}."
)
entry_point = binary[PexEntryPointField].value
url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point"
if not entry_point:
raise InvalidEntryPoint(
"Every `pex_binary` used in `.with_binaries()` for the `provides` field for "
f"{exported_addr} must explicitly set the `entry_point` field, but "
f"{binary.address} left the field off. Set `entry_point` to either "
f"`app.py:func` or the longhand `path.to.app:func`. See {url}."
)
if ":" not in entry_point:
raise InvalidEntryPoint(
"Every `pex_binary` used in `with_binaries()` for the `provides()` field for "
f"{exported_addr} must end in the format `:my_func` for the `entry_point` field, "
f"but {binary.address} set it to {repr(entry_point)}. For example, set "
f"`entry_point='{entry_point}:main'. See {url}."
)
entry_point_requests.append(ResolvePexEntryPointRequest(binary[PexEntryPointField]))
binary_entry_points = await MultiGet(
Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest, request)
for request in entry_point_requests
)
for key, binary_entry_point in zip(key_to_binary_spec.keys(), binary_entry_points):
entry_points = setup_kwargs.setdefault("entry_points", {})
console_scripts = entry_points.setdefault("console_scripts", [])
console_scripts.append(f"{key}={binary_entry_point.val}")
# Generate the setup script.
setup_py_content = SETUP_BOILERPLATE.format(
target_address_spec=target.address.spec,
setup_kwargs_str=distutils_repr(setup_kwargs),
).encode()
files_to_create = [
FileContent("setup.py", setup_py_content),
FileContent("MANIFEST.in", "include *.py".encode()),
]
extra_files_digest, src_digest = await MultiGet(
Get(Digest, CreateDigest(files_to_create)),
# Nest the sources under the src/ prefix.
Get(Digest, AddPrefix(sources.digest, CHROOT_SOURCE_ROOT)),
)
chroot_digest = await Get(Digest, MergeDigests((src_digest, extra_files_digest)))
return SetupPyChroot(chroot_digest, FinalizedSetupKwargs(setup_kwargs, address=target.address))