def test_targets_with_sources_types() -> None: class Sources1(Sources): pass class Sources2(Sources): pass class CodegenSources(Sources): pass class Tgt1(Target): alias = "tgt1" core_fields = (Sources1,) class Tgt2(Target): alias = "tgt2" core_fields = (Sources2,) class CodegenTgt(Target): alias = "codegen_tgt" core_fields = (CodegenSources,) class GenSources(GenerateSourcesRequest): input = CodegenSources output = Sources1 tgt1 = Tgt1({}, address=Address("tgt1")) tgt2 = Tgt2({}, address=Address("tgt2")) codegen_tgt = CodegenTgt({}, address=Address("codegen_tgt")) result = targets_with_sources_types( [Sources1], [tgt1, tgt2, codegen_tgt], union_membership=UnionMembership({GenerateSourcesRequest: [GenSources]}), ) assert set(result) == {tgt1, codegen_tgt}
async def get_sources( request: DistBuildChrootRequest, union_membership: UnionMembership ) -> DistBuildSources: owned_deps, transitive_targets = await MultiGet( Get(OwnedDependencies, DependencyOwner(request.exported_target)), Get( TransitiveTargets, TransitiveTargetsRequest([request.exported_target.target.address]), ), ) # files() targets aren't owned by a single exported target - they aren't code, so # we allow them to be in multiple dists. This is helpful for, e.g., embedding # a standard license file in a dist. # TODO: This doesn't actually work, the generated setup.py has no way of referencing # these, since they aren't in a package, so they won't get included in the built dists. # There is a separate `license_files()` setup.py kwarg that we should use for this # special case (see https://setuptools.pypa.io/en/latest/references/keywords.html). file_targets = targets_with_sources_types( [FileSourceField], transitive_targets.closure, union_membership ) targets = Targets(itertools.chain((od.target for od in owned_deps), file_targets)) python_sources_request = PythonSourceFilesRequest( targets=targets, include_resources=False, include_files=False ) all_sources_request = PythonSourceFilesRequest( targets=targets, include_resources=True, include_files=True ) python_sources, all_sources = await MultiGet( Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, python_sources_request), Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, all_sources_request), ) python_files = set(python_sources.stripped_source_files.snapshot.files) all_files = set(all_sources.stripped_source_files.snapshot.files) resource_files = all_files - python_files init_py_digest_contents = await Get( DigestContents, DigestSubset( python_sources.stripped_source_files.snapshot.digest, PathGlobs(["**/__init__.py"]) ), ) packages, namespace_packages, package_data = find_packages( python_files=python_files, resource_files=resource_files, init_py_digest_contents=init_py_digest_contents, # Whether to use py2 or py3 package semantics. py2=request.interpreter_constraints.includes_python2(), ) return DistBuildSources( digest=all_sources.stripped_source_files.snapshot.digest, packages=packages, namespace_packages=namespace_packages, package_data=package_data, )
async def package_pex_binary( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, union_membership: UnionMembership, ) -> BuiltPackage: resolved_entry_point, transitive_targets = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. file_tgts = targets_with_sources_types([FileSourceField], transitive_targets.dependencies, union_membership) if file_tgts: files_addresses = sorted(tgt.address.spec for tgt in file_tgts) logger.warning( softwrap(f""" The `pex_binary` target {field_set.address} transitively depends on the below `files` targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` are not able to load files within the binary itself; instead, they read from the current working directory. Instead, use `resources` targets or wrap this `pex_binary` in an `archive`. See {doc_url('resources')}. Files targets dependencies: {files_addresses} """)) output_filename = field_set.output_path.value_or_default(file_ending="pex") complete_platforms = await Get(CompletePlatforms, PexCompletePlatformsField, field_set.complete_platforms) pex = await Get( Pex, PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, main=resolved_entry_point.val or field_set.script.value, platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), complete_platforms=complete_platforms, output_filename=output_filename, layout=PexLayout(field_set.layout.value), additional_args=field_set.generate_additional_args( pex_binary_defaults), include_requirements=field_set.include_requirements.value, include_local_dists=True, ), ) return BuiltPackage(pex.digest, (BuiltPackageArtifact(output_filename), ))
async def package_pex_binary( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, union_membership: UnionMembership, ) -> BuiltPackage: resolved_entry_point, transitive_targets = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. files_tgts = targets_with_sources_types([FilesSources], transitive_targets.dependencies, union_membership) if files_tgts: files_addresses = sorted(tgt.address.spec for tgt in files_tgts) logger.warning( f"The pex_binary target {field_set.address} transitively depends on the below files " "targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` " "are not able to load files within the binary itself; instead, they read from the " "current working directory." "\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. See " f"{bracketed_docs_url('resources')}." f"\n\nFiles targets dependencies: {files_addresses}") output_filename = field_set.output_path.value_or_default(field_set.address, file_ending="pex") two_step_pex = await Get( TwoStepPex, TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, # TODO(John Sirois): Support ConsoleScript in PexBinary targets: # https://github.com/pantsbuild/pants/issues/11619 main=resolved_entry_point.val, platforms=PexPlatforms.create_from_platforms_field( field_set.platforms), output_filename=output_filename, additional_args=field_set.generate_additional_args( pex_binary_defaults), )), ) return BuiltPackage(two_step_pex.pex.digest, (BuiltPackageArtifact(output_filename), ))
async def package_python_awslambda( field_set: PythonAwsLambdaFieldSet, lambdex: Lambdex, union_membership: UnionMembership) -> BuiltPackage: output_filename = field_set.output_path.value_or_default( # Lambdas typically use the .zip suffix, so we use that instead of .pex. file_ending="zip", ) # We hardcode the platform value to the appropriate one for each AWS Lambda runtime. # (Running the "hello world" lambda in the example code will report the platform, and can be # used to verify correctness of these platform strings.) py_major, py_minor = field_set.runtime.to_interpreter_version() platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}" # set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707 if py_major <= 3 and py_minor < 8: platform += "m" if (py_major, py_minor) == (2, 7): platform += "u" additional_pex_args = ( # Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels. "--manylinux=manylinux2014", # When we're executing Pex on Linux, allow a local interpreter to be resolved if # available and matching the AMI platform. "--resolve-local-platforms", ) pex_request = PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, output_filename=output_filename, platforms=PexPlatforms([platform]), additional_args=additional_pex_args, additional_lockfile_args=additional_pex_args, ) lambdex_request = PexRequest( output_filename="lambdex.pex", internal_only=True, requirements=lambdex.pex_requirements(), interpreter_constraints=lambdex.interpreter_constraints, main=lambdex.main, ) lambdex_pex, pex_result, handler, transitive_targets = await MultiGet( Get(VenvPex, PexRequest, lambdex_request), Get(Pex, PexFromTargetsRequest, pex_request), Get(ResolvedPythonAwsHandler, ResolvePythonAwsHandlerRequest(field_set.handler)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. file_tgts = targets_with_sources_types([FileSourceField], transitive_targets.dependencies, union_membership) if file_tgts: files_addresses = sorted(tgt.address.spec for tgt in file_tgts) logger.warning( f"The `python_awslambda` target {field_set.address} transitively depends on the below " "`files` targets, but Pants will not include them in the built Lambda. Filesystem APIs " "like `open()` are not able to load files within the binary itself; instead, they " "read from the current working directory." f"\n\nInstead, use `resources` targets. See {doc_url('resources')}." f"\n\nFiles targets dependencies: {files_addresses}") # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. result = await Get( ProcessResult, VenvPexProcess( lambdex_pex, argv=("build", "-e", handler.val, output_filename), input_digest=pex_result.digest, output_files=(output_filename, ), description=f"Setting up handler in {output_filename}", ), ) artifact = BuiltPackageArtifact( output_filename, extra_log_lines=( f" Runtime: {field_set.runtime.value}", # The AWS-facing handler function is always lambdex_handler.handler, which is the # wrapper injected by lambdex that manages invocation of the actual handler. " Handler: lambdex_handler.handler", ), ) return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))
async def package_python_awslambda( field_set: PythonAwsLambdaFieldSet, lambdex: Lambdex, platform: Platform, union_membership: UnionMembership, ) -> BuiltPackage: if platform.is_macos: logger.warning( "AWS Lambdas built on macOS may fail to build. If your lambda uses any third-party" " dependencies without binary wheels (bdist) for Linux available, it will fail to" " build. If this happens, you will either need to update your dependencies to only use" f" dependencies with pre-built wheels, or find a Linux environment to run {bin_name()}" " package. (See https://realpython.com/python-wheels/ for more about wheels.)\n\n(If" " the build does not raise an exception, it's safe to use macOS.)") output_filename = field_set.output_path.value_or_default( # Lambdas typically use the .zip suffix, so we use that instead of .pex. file_ending="zip", ) # We hardcode the platform value to the appropriate one for each AWS Lambda runtime. # (Running the "hello world" lambda in the example code will report the platform, and can be # used to verify correctness of these platform strings.) pex_platforms = [] interpreter_version = field_set.runtime.to_interpreter_version() if interpreter_version: py_major, py_minor = interpreter_version platform_str = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}" # set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707 if py_major <= 3 and py_minor < 8: platform_str += "m" if (py_major, py_minor) == (2, 7): platform_str += "u" pex_platforms.append(platform_str) additional_pex_args = ( # Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels. "--manylinux=manylinux2014", # When we're executing Pex on Linux, allow a local interpreter to be resolved if # available and matching the AMI platform. "--resolve-local-platforms", ) complete_platforms = await Get(CompletePlatforms, PexCompletePlatformsField, field_set.complete_platforms) pex_request = PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, include_requirements=field_set.include_requirements.value, output_filename=output_filename, platforms=PexPlatforms(pex_platforms), complete_platforms=complete_platforms, additional_args=additional_pex_args, additional_lockfile_args=additional_pex_args, ) lambdex_pex, pex_result, handler, transitive_targets = await MultiGet( Get(VenvPex, PexRequest, lambdex.to_pex_request()), Get(Pex, PexFromTargetsRequest, pex_request), Get(ResolvedPythonAwsHandler, ResolvePythonAwsHandlerRequest(field_set.handler)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. file_tgts = targets_with_sources_types([FileSourceField], transitive_targets.dependencies, union_membership) if file_tgts: files_addresses = sorted(tgt.address.spec for tgt in file_tgts) logger.warning( softwrap(f""" The `python_awslambda` target {field_set.address} transitively depends on the below `files` targets, but Pants will not include them in the built Lambda. Filesystem APIs like `open()` are not able to load files within the binary itself; instead, they read from the current working directory. Instead, use `resources` targets. See {doc_url('resources')}. Files targets dependencies: {files_addresses} """)) # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. result = await Get( ProcessResult, VenvPexProcess( lambdex_pex, argv=("build", "-e", handler.val, output_filename), input_digest=pex_result.digest, output_files=(output_filename, ), description=f"Setting up handler in {output_filename}", ), ) extra_log_data: list[tuple[str, str]] = [] if field_set.runtime.value: extra_log_data.append(("Runtime", field_set.runtime.value)) extra_log_data.extend( ("Complete platform", path) for path in complete_platforms) # The AWS-facing handler function is always lambdex_handler.handler, which is the # wrapper injected by lambdex that manages invocation of the actual handler. extra_log_data.append(("Handler", "lambdex_handler.handler")) first_column_width = 4 + max(len(header) for header, _ in extra_log_data) artifact = BuiltPackageArtifact( output_filename, extra_log_lines=tuple( f"{header.rjust(first_column_width, ' ')}: {data}" for header, data in extra_log_data), ) return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))