def test_targets_with_sources_types() -> None:
class Sources1(Sources):
pass
class Sources2(Sources):
pass
class CodegenSources(Sources):
pass
class Tgt1(Target):
alias = "tgt1"
core_fields = (Sources1,)
class Tgt2(Target):
alias = "tgt2"
core_fields = (Sources2,)
class CodegenTgt(Target):
alias = "codegen_tgt"
core_fields = (CodegenSources,)
class GenSources(GenerateSourcesRequest):
input = CodegenSources
output = Sources1
tgt1 = Tgt1({}, address=Address("tgt1"))
tgt2 = Tgt2({}, address=Address("tgt2"))
codegen_tgt = CodegenTgt({}, address=Address("codegen_tgt"))
result = targets_with_sources_types(
[Sources1],
[tgt1, tgt2, codegen_tgt],
union_membership=UnionMembership({GenerateSourcesRequest: [GenSources]}),
)
assert set(result) == {tgt1, codegen_tgt}
async def get_sources(
request: DistBuildChrootRequest, union_membership: UnionMembership
) -> DistBuildSources:
owned_deps, transitive_targets = await MultiGet(
Get(OwnedDependencies, DependencyOwner(request.exported_target)),
Get(
TransitiveTargets,
TransitiveTargetsRequest([request.exported_target.target.address]),
),
)
# files() targets aren't owned by a single exported target - they aren't code, so
# we allow them to be in multiple dists. This is helpful for, e.g., embedding
# a standard license file in a dist.
# TODO: This doesn't actually work, the generated setup.py has no way of referencing
# these, since they aren't in a package, so they won't get included in the built dists.
# There is a separate `license_files()` setup.py kwarg that we should use for this
# special case (see https://setuptools.pypa.io/en/latest/references/keywords.html).
file_targets = targets_with_sources_types(
[FileSourceField], transitive_targets.closure, union_membership
)
targets = Targets(itertools.chain((od.target for od in owned_deps), file_targets))
python_sources_request = PythonSourceFilesRequest(
targets=targets, include_resources=False, include_files=False
)
all_sources_request = PythonSourceFilesRequest(
targets=targets, include_resources=True, include_files=True
)
python_sources, all_sources = await MultiGet(
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, python_sources_request),
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, all_sources_request),
)
python_files = set(python_sources.stripped_source_files.snapshot.files)
all_files = set(all_sources.stripped_source_files.snapshot.files)
resource_files = all_files - python_files
init_py_digest_contents = await Get(
DigestContents,
DigestSubset(
python_sources.stripped_source_files.snapshot.digest, PathGlobs(["**/__init__.py"])
),
)
packages, namespace_packages, package_data = find_packages(
python_files=python_files,
resource_files=resource_files,
init_py_digest_contents=init_py_digest_contents,
# Whether to use py2 or py3 package semantics.
py2=request.interpreter_constraints.includes_python2(),
)
return DistBuildSources(
digest=all_sources.stripped_source_files.snapshot.digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data,
)
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
union_membership: UnionMembership,
) -> BuiltPackage:
resolved_entry_point, transitive_targets = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
file_tgts = targets_with_sources_types([FileSourceField],
transitive_targets.dependencies,
union_membership)
if file_tgts:
files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
logger.warning(
softwrap(f"""
The `pex_binary` target {field_set.address} transitively depends on the below `files`
targets, but Pants will not include them in the PEX. Filesystem APIs like `open()`
are not able to load files within the binary itself; instead, they read from the
current working directory.
Instead, use `resources` targets or wrap this `pex_binary` in an `archive`.
See {doc_url('resources')}.
Files targets dependencies: {files_addresses}
"""))
output_filename = field_set.output_path.value_or_default(file_ending="pex")
complete_platforms = await Get(CompletePlatforms,
PexCompletePlatformsField,
field_set.complete_platforms)
pex = await Get(
Pex,
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
main=resolved_entry_point.val or field_set.script.value,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
complete_platforms=complete_platforms,
output_filename=output_filename,
layout=PexLayout(field_set.layout.value),
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
include_requirements=field_set.include_requirements.value,
include_local_dists=True,
),
)
return BuiltPackage(pex.digest, (BuiltPackageArtifact(output_filename), ))
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
union_membership: UnionMembership,
) -> BuiltPackage:
resolved_entry_point, transitive_targets = await MultiGet(
Get(ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
files_tgts = targets_with_sources_types([FilesSources],
transitive_targets.dependencies,
union_membership)
if files_tgts:
files_addresses = sorted(tgt.address.spec for tgt in files_tgts)
logger.warning(
f"The pex_binary target {field_set.address} transitively depends on the below files "
"targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` "
"are not able to load files within the binary itself; instead, they read from the "
"current working directory."
"\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. See "
f"{bracketed_docs_url('resources')}."
f"\n\nFiles targets dependencies: {files_addresses}")
output_filename = field_set.output_path.value_or_default(field_set.address,
file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
async def package_python_awslambda(
field_set: PythonAwsLambdaFieldSet, lambdex: Lambdex,
union_membership: UnionMembership) -> BuiltPackage:
output_filename = field_set.output_path.value_or_default(
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
file_ending="zip", )
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform += "m"
if (py_major, py_minor) == (2, 7):
platform += "u"
additional_pex_args = (
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
)
pex_request = PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
output_filename=output_filename,
platforms=PexPlatforms([platform]),
additional_args=additional_pex_args,
additional_lockfile_args=additional_pex_args,
)
lambdex_request = PexRequest(
output_filename="lambdex.pex",
internal_only=True,
requirements=lambdex.pex_requirements(),
interpreter_constraints=lambdex.interpreter_constraints,
main=lambdex.main,
)
lambdex_pex, pex_result, handler, transitive_targets = await MultiGet(
Get(VenvPex, PexRequest, lambdex_request),
Get(Pex, PexFromTargetsRequest, pex_request),
Get(ResolvedPythonAwsHandler,
ResolvePythonAwsHandlerRequest(field_set.handler)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
file_tgts = targets_with_sources_types([FileSourceField],
transitive_targets.dependencies,
union_membership)
if file_tgts:
files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
logger.warning(
f"The `python_awslambda` target {field_set.address} transitively depends on the below "
"`files` targets, but Pants will not include them in the built Lambda. Filesystem APIs "
"like `open()` are not able to load files within the binary itself; instead, they "
"read from the current working directory."
f"\n\nInstead, use `resources` targets. See {doc_url('resources')}."
f"\n\nFiles targets dependencies: {files_addresses}")
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
VenvPexProcess(
lambdex_pex,
argv=("build", "-e", handler.val, output_filename),
input_digest=pex_result.digest,
output_files=(output_filename, ),
description=f"Setting up handler in {output_filename}",
),
)
artifact = BuiltPackageArtifact(
output_filename,
extra_log_lines=(
f" Runtime: {field_set.runtime.value}",
# The AWS-facing handler function is always lambdex_handler.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
" Handler: lambdex_handler.handler",
),
)
return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))
async def package_python_awslambda(
field_set: PythonAwsLambdaFieldSet,
lambdex: Lambdex,
platform: Platform,
union_membership: UnionMembership,
) -> BuiltPackage:
if platform.is_macos:
logger.warning(
"AWS Lambdas built on macOS may fail to build. If your lambda uses any third-party"
" dependencies without binary wheels (bdist) for Linux available, it will fail to"
" build. If this happens, you will either need to update your dependencies to only use"
f" dependencies with pre-built wheels, or find a Linux environment to run {bin_name()}"
" package. (See https://realpython.com/python-wheels/ for more about wheels.)\n\n(If"
" the build does not raise an exception, it's safe to use macOS.)")
output_filename = field_set.output_path.value_or_default(
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
file_ending="zip", )
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
pex_platforms = []
interpreter_version = field_set.runtime.to_interpreter_version()
if interpreter_version:
py_major, py_minor = interpreter_version
platform_str = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform_str += "m"
if (py_major, py_minor) == (2, 7):
platform_str += "u"
pex_platforms.append(platform_str)
additional_pex_args = (
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
)
complete_platforms = await Get(CompletePlatforms,
PexCompletePlatformsField,
field_set.complete_platforms)
pex_request = PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
include_requirements=field_set.include_requirements.value,
output_filename=output_filename,
platforms=PexPlatforms(pex_platforms),
complete_platforms=complete_platforms,
additional_args=additional_pex_args,
additional_lockfile_args=additional_pex_args,
)
lambdex_pex, pex_result, handler, transitive_targets = await MultiGet(
Get(VenvPex, PexRequest, lambdex.to_pex_request()),
Get(Pex, PexFromTargetsRequest, pex_request),
Get(ResolvedPythonAwsHandler,
ResolvePythonAwsHandlerRequest(field_set.handler)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
file_tgts = targets_with_sources_types([FileSourceField],
transitive_targets.dependencies,
union_membership)
if file_tgts:
files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
logger.warning(
softwrap(f"""
The `python_awslambda` target {field_set.address} transitively depends on the below
`files` targets, but Pants will not include them in the built Lambda. Filesystem APIs
like `open()` are not able to load files within the binary itself; instead, they
read from the current working directory.
Instead, use `resources` targets. See {doc_url('resources')}.
Files targets dependencies: {files_addresses}
"""))
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
VenvPexProcess(
lambdex_pex,
argv=("build", "-e", handler.val, output_filename),
input_digest=pex_result.digest,
output_files=(output_filename, ),
description=f"Setting up handler in {output_filename}",
),
)
extra_log_data: list[tuple[str, str]] = []
if field_set.runtime.value:
extra_log_data.append(("Runtime", field_set.runtime.value))
extra_log_data.extend(
("Complete platform", path) for path in complete_platforms)
# The AWS-facing handler function is always lambdex_handler.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
extra_log_data.append(("Handler", "lambdex_handler.handler"))
first_column_width = 4 + max(len(header) for header, _ in extra_log_data)
artifact = BuiltPackageArtifact(
output_filename,
extra_log_lines=tuple(
f"{header.rjust(first_column_width, ' ')}: {data}"
for header, data in extra_log_data),
)
return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))