Exemplo n.º 1
0
 def assert_resolved(*, entry_point: Optional[str], expected: Optional[str]) -> None:
     addr = Address("src/python/project")
     rule_runner.create_file("src/python/project/app.py")
     rule_runner.create_file("src/python/project/f2.py")
     ep_field = PexEntryPointField(entry_point, address=addr)
     result = rule_runner.request(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field)])
     assert result.val == expected
Exemplo n.º 2
0
async def inject_pex_binary_entry_point_dependency(
        request: InjectPexBinaryEntryPointDependency,
        python_infer_subsystem: PythonInferSubsystem) -> InjectedDependencies:
    if not python_infer_subsystem.entry_points:
        return InjectedDependencies()
    original_tgt = await Get(WrappedTarget, Address,
                             request.dependencies_field.address)
    explicitly_provided_deps, entry_point = await MultiGet(
        Get(ExplicitlyProvidedDependencies,
            DependenciesRequest(original_tgt.target[Dependencies])),
        Get(
            ResolvedPexEntryPoint,
            ResolvePexEntryPointRequest(
                original_tgt.target[PexEntryPointField]),
        ),
    )
    if entry_point.val is None:
        return InjectedDependencies()
    owners = await Get(PythonModuleOwners,
                       PythonModule(entry_point.val.module))
    address = original_tgt.target.address
    explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
        owners.ambiguous,
        address,
        import_reference="module",
        context=
        (f"The pex_binary target {address} has the field "
         f"`entry_point={repr(original_tgt.target[PexEntryPointField].value.spec)}`, which "
         f"maps to the Python module `{entry_point.val.module}`"),
    )
    maybe_disambiguated = explicitly_provided_deps.disambiguated_via_ignores(
        owners.ambiguous)
    unambiguous_owners = owners.unambiguous or (
        (maybe_disambiguated, ) if maybe_disambiguated else ())
    return InjectedDependencies(unambiguous_owners)
Exemplo n.º 3
0
async def package_pex_binary(
        field_set: PexBinaryFieldSet,
        pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage:
    resolved_entry_point = await Get(
        ResolvedPexEntryPoint,
        ResolvePexEntryPointRequest(field_set.entry_point))
    output_filename = field_set.output_path.value_or_default(field_set.address,
                                                             file_ending="pex")
    two_step_pex = await Get(
        TwoStepPex,
        TwoStepPexFromTargetsRequest(
            PexFromTargetsRequest(
                addresses=[field_set.address],
                internal_only=False,
                # TODO(John Sirois): Support ConsoleScript in PexBinary targets:
                #  https://github.com/pantsbuild/pants/issues/11619
                main=resolved_entry_point.val,
                platforms=PexPlatforms.create_from_platforms_field(
                    field_set.platforms),
                output_filename=output_filename,
                additional_args=field_set.generate_additional_args(
                    pex_binary_defaults),
            )),
    )
    return BuiltPackage(two_step_pex.pex.digest,
                        (BuiltPackageArtifact(output_filename), ))
Exemplo n.º 4
0
 def assert_resolved(*, entry_point: str | None,
                     expected: EntryPoint | None, is_file: bool) -> None:
     addr = Address("src/python/project")
     rule_runner.create_file("src/python/project/app.py")
     rule_runner.create_file("src/python/project/f2.py")
     ep_field = PexEntryPointField(entry_point, addr)
     result = rule_runner.request(ResolvedPexEntryPoint,
                                  [ResolvePexEntryPointRequest(ep_field)])
     assert result.val == expected
     assert result.file_name_used == is_file
Exemplo n.º 5
0
async def package_pex_binary(
    field_set: PexBinaryFieldSet,
    pex_binary_defaults: PexBinaryDefaults,
    union_membership: UnionMembership,
) -> BuiltPackage:
    resolved_entry_point, transitive_targets = await MultiGet(
        Get(ResolvedPexEntryPoint,
            ResolvePexEntryPointRequest(field_set.entry_point)),
        Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
    )

    # Warn if users depend on `files` targets, which won't be included in the PEX and is a common
    # gotcha.
    file_tgts = targets_with_sources_types([FileSourceField],
                                           transitive_targets.dependencies,
                                           union_membership)
    if file_tgts:
        files_addresses = sorted(tgt.address.spec for tgt in file_tgts)
        logger.warning(
            softwrap(f"""
                The `pex_binary` target {field_set.address} transitively depends on the below `files`
                targets, but Pants will not include them in the PEX. Filesystem APIs like `open()`
                are not able to load files within the binary itself; instead, they read from the
                current working directory.

                Instead, use `resources` targets or wrap this `pex_binary` in an `archive`.
                See {doc_url('resources')}.

                Files targets dependencies: {files_addresses}
                """))

    output_filename = field_set.output_path.value_or_default(file_ending="pex")

    complete_platforms = await Get(CompletePlatforms,
                                   PexCompletePlatformsField,
                                   field_set.complete_platforms)

    pex = await Get(
        Pex,
        PexFromTargetsRequest(
            addresses=[field_set.address],
            internal_only=False,
            main=resolved_entry_point.val or field_set.script.value,
            platforms=PexPlatforms.create_from_platforms_field(
                field_set.platforms),
            complete_platforms=complete_platforms,
            output_filename=output_filename,
            layout=PexLayout(field_set.layout.value),
            additional_args=field_set.generate_additional_args(
                pex_binary_defaults),
            include_requirements=field_set.include_requirements.value,
            include_local_dists=True,
        ),
    )
    return BuiltPackage(pex.digest, (BuiltPackageArtifact(output_filename), ))
Exemplo n.º 6
0
async def inject_pex_binary_entry_point_dependency(
    request: InjectPexBinaryEntryPointDependency,
    python_infer_subsystem: PythonInferSubsystem,
    python_setup: PythonSetup,
) -> InjectedDependencies:
    if not python_infer_subsystem.entry_points:
        return InjectedDependencies()
    original_tgt = await Get(
        WrappedTarget,
        WrappedTargetRequest(
            request.dependencies_field.address, description_of_origin="<infallible>"
        ),
    )
    entry_point_field = original_tgt.target.get(PexEntryPointField)
    if entry_point_field.value is None:
        return InjectedDependencies()

    explicitly_provided_deps, entry_point = await MultiGet(
        Get(ExplicitlyProvidedDependencies, DependenciesRequest(original_tgt.target[Dependencies])),
        Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(entry_point_field)),
    )
    if entry_point.val is None:
        return InjectedDependencies()

    owners = await Get(
        PythonModuleOwners,
        PythonModuleOwnersRequest(
            entry_point.val.module,
            resolve=original_tgt.target[PythonResolveField].normalized_value(python_setup),
        ),
    )
    address = original_tgt.target.address
    explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
        owners.ambiguous,
        address,
        # If the entry point was specified as a file, like `app.py`, we know the module must
        # live in the pex_binary's directory or subdirectory, so the owners must be ancestors.
        owners_must_be_ancestors=entry_point.file_name_used,
        import_reference="module",
        context=softwrap(
            f"""
            The pex_binary target {address} has the field
            `entry_point={repr(entry_point_field.value.spec)}`, which
            maps to the Python module `{entry_point.val.module}`
            """
        ),
    )
    maybe_disambiguated = explicitly_provided_deps.disambiguated(
        owners.ambiguous, owners_must_be_ancestors=entry_point.file_name_used
    )
    unambiguous_owners = owners.unambiguous or (
        (maybe_disambiguated,) if maybe_disambiguated else ()
    )
    return InjectedDependencies(unambiguous_owners)
Exemplo n.º 7
0
 def assert_resolved(
     *, entry_point: Optional[str], source: Optional[str] = None, expected: Optional[str]
 ) -> None:
     addr = Address("src/python/project")
     rule_runner.create_file("src/python/project/app.py")
     rule_runner.create_file("src/python/project/f2.py")
     ep_field = PexEntryPointField(entry_point, address=addr)
     sources = DeprecatedPexBinarySources([source] if source else None, address=addr)
     result = rule_runner.request(
         ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field, sources)]
     )
     assert result.val == expected
Exemplo n.º 8
0
async def inject_pex_binary_entry_point_dependency(
    request: InjectPexBinaryEntryPointDependency, python_infer_subsystem: PythonInferSubsystem
) -> InjectedDependencies:
    if not python_infer_subsystem.entry_points:
        return InjectedDependencies()
    original_tgt = await Get(WrappedTarget, Address, request.dependencies_field.address)
    entry_point = await Get(
        ResolvedPexEntryPoint,
        ResolvePexEntryPointRequest(original_tgt.target[PexEntryPointField]),
    )
    if entry_point.val is None:
        return InjectedDependencies()
    module, _, _func = entry_point.val.partition(":")
    owners = await Get(PythonModuleOwners, PythonModule(module))
    return InjectedDependencies(owners)
Exemplo n.º 9
0
async def package_pex_binary(
    field_set: PexBinaryFieldSet,
    pex_binary_defaults: PexBinaryDefaults,
    union_membership: UnionMembership,
) -> BuiltPackage:
    resolved_entry_point, transitive_targets = await MultiGet(
        Get(ResolvedPexEntryPoint,
            ResolvePexEntryPointRequest(field_set.entry_point)),
        Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
    )

    # Warn if users depend on `files` targets, which won't be included in the PEX and is a common
    # gotcha.
    files_tgts = targets_with_sources_types([FilesSources],
                                            transitive_targets.dependencies,
                                            union_membership)
    if files_tgts:
        files_addresses = sorted(tgt.address.spec for tgt in files_tgts)
        logger.warning(
            f"The pex_binary target {field_set.address} transitively depends on the below files "
            "targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` "
            "are not able to load files within the binary itself; instead, they read from the "
            "current working directory."
            "\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. See "
            f"{bracketed_docs_url('resources')}."
            f"\n\nFiles targets dependencies: {files_addresses}")

    output_filename = field_set.output_path.value_or_default(field_set.address,
                                                             file_ending="pex")
    two_step_pex = await Get(
        TwoStepPex,
        TwoStepPexFromTargetsRequest(
            PexFromTargetsRequest(
                addresses=[field_set.address],
                internal_only=False,
                # TODO(John Sirois): Support ConsoleScript in PexBinary targets:
                #  https://github.com/pantsbuild/pants/issues/11619
                main=resolved_entry_point.val,
                platforms=PexPlatforms.create_from_platforms_field(
                    field_set.platforms),
                output_filename=output_filename,
                additional_args=field_set.generate_additional_args(
                    pex_binary_defaults),
            )),
    )
    return BuiltPackage(two_step_pex.pex.digest,
                        (BuiltPackageArtifact(output_filename), ))
Exemplo n.º 10
0
async def inject_pex_binary_entry_point_dependency(
        request: InjectPexBinaryEntryPointDependency,
        python_infer_subsystem: PythonInferSubsystem) -> InjectedDependencies:
    if not python_infer_subsystem.entry_points:
        return InjectedDependencies()
    original_tgt = await Get(WrappedTarget, Address,
                             request.dependencies_field.address)
    entry_point = await Get(
        ResolvedPexEntryPoint,
        ResolvePexEntryPointRequest(
            original_tgt.target[PexEntryPointField],
            original_tgt.target[DeprecatedPexBinarySources]),
    )
    if entry_point.val is None:
        return InjectedDependencies()
    module, _, _func = entry_point.val.partition(":")
    owners = await Get(PythonModuleOwners, PythonModule(module))
    # TODO: remove the check for == self once the `sources` field is removed.
    return InjectedDependencies(owner for owner in owners
                                if owner != request.dependencies_field.address)
Exemplo n.º 11
0
async def package_pex_binary(
        field_set: PexBinaryFieldSet,
        pex_binary_defaults: PexBinaryDefaults) -> BuiltPackage:
    resolved_entry_point = await Get(
        ResolvedPexEntryPoint,
        ResolvePexEntryPointRequest(field_set.entry_point))
    output_filename = field_set.output_path.value_or_default(field_set.address,
                                                             file_ending="pex")
    two_step_pex = await Get(
        TwoStepPex,
        TwoStepPexFromTargetsRequest(
            PexFromTargetsRequest(
                addresses=[field_set.address],
                internal_only=False,
                entry_point=resolved_entry_point.val,
                platforms=PexPlatforms.create_from_platforms_field(
                    field_set.platforms),
                output_filename=output_filename,
                additional_args=field_set.generate_additional_args(
                    pex_binary_defaults),
            )),
    )
    return BuiltPackage(two_step_pex.pex.digest,
                        (BuiltPackageArtifact(output_filename), ))
Exemplo n.º 12
0
async def create_pex_binary_run_request(
    field_set: PexBinaryFieldSet,
    pex_binary_defaults: PexBinaryDefaults,
    pex_env: PexEnvironment,
    python_setup: PythonSetup,
) -> RunRequest:
    entry_point, transitive_targets = await MultiGet(
        Get(
            ResolvedPexEntryPoint,
            ResolvePexEntryPointRequest(field_set.entry_point),
        ),
        Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
    )

    # Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
    # so that we can get the interpreter constraints for use in local_dists_get.
    requirements_pex_request = await Get(
        PexRequest,
        PexFromTargetsRequest(
            [field_set.address],
            output_filename=f"{field_set.address.target_name}.pex",
            internal_only=True,
            include_source_files=False,
            # Note that the file for first-party entry points is not in the PEX itself. In that
            # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
            main=entry_point.val or field_set.script.value,
            resolve_and_lockfile=field_set.resolve.resolve_and_lockfile(
                python_setup),
            additional_args=(
                *field_set.generate_additional_args(pex_binary_defaults),
                # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
                # below, it's important for any app that re-executes itself that these environment
                # variables are not stripped.
                "--no-strip-pex-env",
            ),
        ),
    )
    pex_get = Get(Pex, PexRequest, requirements_pex_request)
    sources_get = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(transitive_targets.closure,
                                 include_files=True))
    pex, sources = await MultiGet(pex_get, sources_get)

    local_dists = await Get(
        LocalDistsPex,
        LocalDistsPexRequest(
            [field_set.address],
            internal_only=True,
            interpreter_constraints=requirements_pex_request.
            interpreter_constraints,
            sources=sources,
        ),
    )

    merged_digest = await Get(
        Digest,
        MergeDigests([
            pex.digest,
            local_dists.pex.digest,
            local_dists.remaining_sources.source_files.snapshot.digest,
        ]),
    )

    def in_chroot(relpath: str) -> str:
        return os.path.join("{chroot}", relpath)

    complete_pex_env = pex_env.in_workspace()
    args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)

    chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
    extra_env = {
        **complete_pex_env.environment_dict(python_configured=pex.python is not None),
        "PEX_PATH":
        in_chroot(local_dists.pex.name),
        "PEX_EXTRA_SYS_PATH":
        os.pathsep.join(chrooted_source_roots),
    }

    return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
Exemplo n.º 13
0
async def find_putative_targets(
    req: PutativePythonTargetsRequest,
    all_owned_sources: AllOwnedSources,
    python_setup: PythonSetup,
) -> PutativeTargets:
    # Find library/test/test_util targets.

    all_py_files_globs: PathGlobs = req.search_paths.path_globs("*.py")
    all_py_files = await Get(Paths, PathGlobs, all_py_files_globs)
    unowned_py_files = set(all_py_files.files) - set(all_owned_sources)
    classified_unowned_py_files = classify_source_files(unowned_py_files)
    pts = []
    for tgt_type, paths in classified_unowned_py_files.items():
        for dirname, filenames in group_by_dir(paths).items():
            if issubclass(tgt_type, PythonTestsGeneratorTarget):
                name = "tests"
                kwargs = {"name": name}
            elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget):
                name = "test_utils"
                kwargs = {"name": name}
            else:
                name = os.path.basename(dirname)
                kwargs = {}
            if (
                python_setup.tailor_ignore_solitary_init_files
                and tgt_type == PythonSourcesGeneratorTarget
                and filenames == {"__init__.py"}
            ):
                continue
            pts.append(
                PutativeTarget.for_target_type(
                    tgt_type, dirname, name, sorted(filenames), kwargs=kwargs
                )
            )

    if python_setup.tailor_requirements_targets:
        # Find requirements files.
        all_requirements_files_globs: PathGlobs = req.search_paths.path_globs("*requirements*.txt")
        all_requirements_files = await Get(Paths, PathGlobs, all_requirements_files_globs)
        unowned_requirements_files = set(all_requirements_files.files) - set(all_owned_sources)
        for req_file in unowned_requirements_files:
            path, name = os.path.split(req_file)
            pts.append(
                PutativeTarget(
                    path=path,
                    # python_requirements is a macro and doesn't take a name argument, but the
                    # PutativeTarget still needs a name for display purposes.
                    name=name,
                    type_alias="python_requirements",
                    triggering_sources=[req_file],
                    owned_sources=[req_file],
                    addressable=False,
                    kwargs={} if name == "requirements.txt" else {"requirements_relpath": name},
                )
            )

    if python_setup.tailor_pex_binary_targets:
        # Find binary targets.

        # Get all files whose content indicates that they are entry points.
        digest_contents = await Get(DigestContents, PathGlobs, all_py_files_globs)
        entry_points = [
            file_content.path
            for file_content in digest_contents
            if is_entry_point(file_content.content)
        ]

        # Get the modules for these entry points.
        src_roots = await Get(
            SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(entry_points)
        )
        module_to_entry_point = {}
        for entry_point in entry_points:
            entry_point_path = PurePath(entry_point)
            src_root = src_roots.path_to_root[entry_point_path]
            stripped_entry_point = entry_point_path.relative_to(src_root.path)
            module = PythonModule.create_from_stripped_path(stripped_entry_point)
            module_to_entry_point[module.module] = entry_point

        # Get existing binary targets for these entry points.
        entry_point_dirs = {os.path.dirname(entry_point) for entry_point in entry_points}
        possible_existing_binary_targets = await Get(
            UnexpandedTargets, AddressSpecs(AscendantAddresses(d) for d in entry_point_dirs)
        )
        possible_existing_binary_entry_points = await MultiGet(
            Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest(t[PexEntryPointField]))
            for t in possible_existing_binary_targets
            if t.has_field(PexEntryPointField)
        )
        possible_existing_entry_point_modules = {
            rep.val.module for rep in possible_existing_binary_entry_points if rep.val
        }
        unowned_entry_point_modules = (
            module_to_entry_point.keys() - possible_existing_entry_point_modules
        )

        # Generate new targets for entry points that don't already have one.
        for entry_point_module in unowned_entry_point_modules:
            entry_point = module_to_entry_point[entry_point_module]
            path, fname = os.path.split(entry_point)
            name = os.path.splitext(fname)[0]
            pts.append(
                PutativeTarget.for_target_type(
                    target_type=PexBinary,
                    path=path,
                    name=name,
                    triggering_sources=tuple(),
                    kwargs={"name": name, "entry_point": fname},
                )
            )

    return PutativeTargets(pts)
Exemplo n.º 14
0
async def resolve_python_distribution_entry_points(
    request: ResolvePythonDistributionEntryPointsRequest,
) -> ResolvedPythonDistributionEntryPoints:
    if request.entry_points_field:
        if request.entry_points_field.value is None:
            return ResolvedPythonDistributionEntryPoints()
        address = request.entry_points_field.address
        all_entry_points = cast(_EntryPointsDictType, request.entry_points_field.value)

    elif request.provides_field:
        address = request.provides_field.address
        provides_field_value = cast(
            _EntryPointsDictType, request.provides_field.value.kwargs.get("entry_points") or {}
        )

        if provides_field_value:
            all_entry_points = provides_field_value
        else:
            return ResolvedPythonDistributionEntryPoints()
    else:
        return ResolvedPythonDistributionEntryPoints()

    classified_entry_points = list(_classify_entry_points(all_entry_points))

    # Pick out all target addresses up front, so we can use MultiGet later.
    #
    # This calls for a bit of trickery however (using the "y_by_x" mapping dicts), so we keep track
    # of which address belongs to which entry point. I.e. the `address_by_ref` and
    # `binary_entry_point_by_address` variables.

    target_refs = [
        entry_point_str for is_target, _, _, entry_point_str in classified_entry_points if is_target
    ]

    # Intermediate step, as Get(Targets) returns a deduplicated set.. which breaks in case of
    # multiple input refs that maps to the same target.
    target_addresses = await Get(
        Addresses,
        UnparsedAddressInputs(
            target_refs,
            owning_address=address,
            description_of_origin="TODO(#14468)",
        ),
    )
    address_by_ref = dict(zip(target_refs, target_addresses))
    targets = await Get(Targets, Addresses, target_addresses)

    # Check that we only have targets with a pex entry_point field.
    for target in targets:
        if not target.has_field(PexEntryPointField):
            raise InvalidEntryPoint(
                softwrap(
                    f"""
                    All target addresses in the entry_points field must be for pex_binary targets,
                    but the target {address} includes the value {target.address}, which has the
                    target type {target.alias}.

                    Alternatively, you can use a module like "project.app:main".
                    See {doc_url('python-distributions')}.
                    """
                )
            )

    binary_entry_points = await MultiGet(
        Get(
            ResolvedPexEntryPoint,
            ResolvePexEntryPointRequest(target[PexEntryPointField]),
        )
        for target in targets
    )
    binary_entry_point_by_address = {
        target.address: entry_point for target, entry_point in zip(targets, binary_entry_points)
    }

    entry_points: DefaultDict[str, Dict[str, PythonDistributionEntryPoint]] = defaultdict(dict)

    # Parse refs/replace with resolved pex entry point, and validate console entry points have function.
    for is_target, category, name, ref in classified_entry_points:
        owner: Optional[Address] = None
        if is_target:
            owner = address_by_ref[ref]
            entry_point = binary_entry_point_by_address[owner].val
            if entry_point is None:
                logger.warning(
                    softwrap(
                        f"""
                        The entry point {name} in {category} references a pex_binary target {ref}
                        which does not set `entry_point`. Skipping.
                        """
                    )
                )
                continue
        else:
            entry_point = EntryPoint.parse(ref, f"{name} for {address} {category}")

        if category in ["console_scripts", "gui_scripts"] and not entry_point.function:
            url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point"
            raise InvalidEntryPoint(
                dedent(
                    f"""\
                Every entry point in `{category}` for {address} must end in the format `:my_func`,
                but {name} set it to {entry_point.spec!r}. For example, set
                `entry_points={{"{category}": {{"{name}": "{entry_point.module}:main}} }}`.
                See {url}.
                """
                )
            )

        entry_points[category][name] = PythonDistributionEntryPoint(entry_point, owner)

    return ResolvedPythonDistributionEntryPoints(
        FrozenDict(
            {category: FrozenDict(entry_points) for category, entry_points in entry_points.items()}
        )
    )
Exemplo n.º 15
0
async def find_putative_targets(
    req: PutativePythonTargetsRequest,
    all_owned_sources: AllOwnedSources,
    python_setup: PythonSetup,
) -> PutativeTargets:
    pts = []

    if python_setup.tailor_source_targets:
        # Find library/test/test_util targets.
        all_py_files_globs: PathGlobs = req.path_globs("*.py", "*.pyi")
        all_py_files = await Get(Paths, PathGlobs, all_py_files_globs)
        unowned_py_files = set(all_py_files.files) - set(all_owned_sources)
        classified_unowned_py_files = classify_source_files(unowned_py_files)
        for tgt_type, paths in classified_unowned_py_files.items():
            for dirname, filenames in group_by_dir(paths).items():
                name: str | None
                if issubclass(tgt_type, PythonTestsGeneratorTarget):
                    name = "tests"
                elif issubclass(tgt_type, PythonTestUtilsGeneratorTarget):
                    name = "test_utils"
                else:
                    name = None
                if (python_setup.tailor_ignore_solitary_init_files
                        and tgt_type == PythonSourcesGeneratorTarget
                        and filenames == {"__init__.py"}):
                    continue
                pts.append(
                    PutativeTarget.for_target_type(
                        tgt_type,
                        path=dirname,
                        name=name,
                        triggering_sources=sorted(filenames)))

    if python_setup.tailor_requirements_targets:
        # Find requirements files.
        (
            all_requirements_files,
            all_pipenv_lockfile_files,
            all_pyproject_toml_contents,
        ) = await MultiGet(
            Get(DigestContents, PathGlobs,
                req.path_globs("*requirements*.txt")),
            Get(DigestContents, PathGlobs, req.path_globs("Pipfile.lock")),
            Get(DigestContents, PathGlobs, req.path_globs("pyproject.toml")),
        )

        def add_req_targets(files: Iterable[FileContent], alias: str,
                            target_name: str) -> None:
            contents = {i.path: i.content for i in files}
            unowned_files = set(contents) - set(all_owned_sources)
            for fp in unowned_files:
                path, name = os.path.split(fp)

                try:
                    validate(fp, contents[fp], alias)
                except Exception as e:
                    logger.warning(
                        f"An error occurred when validating `{fp}`: {e}.\n\n"
                        "You'll need to create targets for its contents manually.\n"
                        "To silence this error in future, see "
                        "https://www.pantsbuild.org/docs/reference-tailor#section-ignore-paths \n"
                    )
                    continue

                pts.append(
                    PutativeTarget(
                        path=path,
                        name=target_name,
                        type_alias=alias,
                        triggering_sources=[fp],
                        owned_sources=[name],
                        kwargs=({} if alias != "python_requirements"
                                or name == "requirements.txt" else {
                                    "source": name
                                }),
                    ))

        def validate(path: str, contents: bytes, alias: str) -> None:
            if alias == "python_requirements":
                return validate_python_requirements(path, contents)
            elif alias == "pipenv_requirements":
                return validate_pipenv_requirements(contents)
            elif alias == "poetry_requirements":
                return validate_poetry_requirements(contents)

        def validate_python_requirements(path: str, contents: bytes) -> None:
            for _ in parse_requirements_file(contents.decode(), rel_path=path):
                pass

        def validate_pipenv_requirements(contents: bytes) -> None:
            parse_pipenv_requirements(contents)

        def validate_poetry_requirements(contents: bytes) -> None:
            p = PyProjectToml(PurePath(), PurePath(), contents.decode())
            parse_pyproject_toml(p)

        add_req_targets(all_requirements_files, "python_requirements", "reqs")
        add_req_targets(all_pipenv_lockfile_files, "pipenv_requirements",
                        "pipenv")
        add_req_targets(
            {
                fc
                for fc in all_pyproject_toml_contents
                if b"[tool.poetry" in fc.content
            },
            "poetry_requirements",
            "poetry",
        )

    if python_setup.tailor_pex_binary_targets:
        # Find binary targets.

        # Get all files whose content indicates that they are entry points or are __main__.py files.
        digest_contents = await Get(DigestContents, PathGlobs,
                                    all_py_files_globs)
        all_main_py = await Get(Paths, PathGlobs,
                                req.path_globs("__main__.py"))
        entry_points = [
            file_content.path for file_content in digest_contents
            if is_entry_point(file_content.content)
        ] + list(all_main_py.files)

        # Get the modules for these entry points.
        src_roots = await Get(SourceRootsResult, SourceRootsRequest,
                              SourceRootsRequest.for_files(entry_points))
        module_to_entry_point = {}
        for entry_point in entry_points:
            entry_point_path = PurePath(entry_point)
            src_root = src_roots.path_to_root[entry_point_path]
            stripped_entry_point = entry_point_path.relative_to(src_root.path)
            module = module_from_stripped_path(stripped_entry_point)
            module_to_entry_point[module] = entry_point

        # Get existing binary targets for these entry points.
        entry_point_dirs = {
            os.path.dirname(entry_point)
            for entry_point in entry_points
        }
        possible_existing_binary_targets = await Get(
            UnexpandedTargets,
            RawSpecs(
                ancestor_globs=tuple(
                    AncestorGlobSpec(d) for d in entry_point_dirs),
                description_of_origin="the `pex_binary` tailor rule",
            ),
        )
        possible_existing_binary_entry_points = await MultiGet(
            Get(ResolvedPexEntryPoint,
                ResolvePexEntryPointRequest(t[PexEntryPointField]))
            for t in possible_existing_binary_targets
            if t.has_field(PexEntryPointField))
        possible_existing_entry_point_modules = {
            rep.val.module
            for rep in possible_existing_binary_entry_points if rep.val
        }
        unowned_entry_point_modules = (module_to_entry_point.keys() -
                                       possible_existing_entry_point_modules)

        # Generate new targets for entry points that don't already have one.
        for entry_point_module in unowned_entry_point_modules:
            entry_point = module_to_entry_point[entry_point_module]
            path, fname = os.path.split(entry_point)
            name = os.path.splitext(fname)[0]
            pts.append(
                PutativeTarget.for_target_type(
                    target_type=PexBinary,
                    path=path,
                    name=name,
                    triggering_sources=tuple(),
                    kwargs={"entry_point": fname},
                ))

    return PutativeTargets(pts)
Exemplo n.º 16
0
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet,
                                        pex_binary_defaults: PexBinaryDefaults,
                                        pex_env: PexEnvironment) -> RunRequest:
    run_in_sandbox = field_set.run_in_sandbox.value
    entry_point, transitive_targets = await MultiGet(
        Get(
            ResolvedPexEntryPoint,
            ResolvePexEntryPointRequest(field_set.entry_point),
        ),
        Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
    )

    addresses = [field_set.address]
    interpreter_constraints = await Get(
        InterpreterConstraints, InterpreterConstraintsRequest(addresses))

    pex_filename = (field_set.address.generated_name.replace(".", "_")
                    if field_set.address.generated_name else
                    field_set.address.target_name)
    pex_get = Get(
        Pex,
        PexFromTargetsRequest(
            [field_set.address],
            output_filename=f"{pex_filename}.pex",
            internal_only=True,
            include_source_files=False,
            # Note that the file for first-party entry points is not in the PEX itself. In that
            # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
            main=entry_point.val or field_set.script.value,
            additional_args=(
                *field_set.generate_additional_args(pex_binary_defaults),
                # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
                # below, it's important for any app that re-executes itself that these environment
                # variables are not stripped.
                "--no-strip-pex-env",
            ),
        ),
    )
    sources_get = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(transitive_targets.closure,
                                 include_files=True))
    pex, sources = await MultiGet(pex_get, sources_get)

    local_dists = await Get(
        LocalDistsPex,
        LocalDistsPexRequest(
            [field_set.address],
            internal_only=True,
            interpreter_constraints=interpreter_constraints,
            sources=sources,
        ),
    )

    input_digests = [
        pex.digest,
        local_dists.pex.digest,
        # Note regarding inline mode: You might think that the sources don't need to be copied
        # into the chroot when using inline sources. But they do, because some of them might be
        # codegenned, and those won't exist in the inline source tree. Rather than incurring the
        # complexity of figuring out here which sources were codegenned, we copy everything.
        # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
        # sources will take precedence and their copies in the chroot will be ignored.
        local_dists.remaining_sources.source_files.snapshot.digest,
    ]
    merged_digest = await Get(Digest, MergeDigests(input_digests))

    def in_chroot(relpath: str) -> str:
        return os.path.join("{chroot}", relpath)

    complete_pex_env = pex_env.in_workspace()
    args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)

    chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
    # The order here is important: we want the in-repo sources to take precedence over their
    # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
    source_roots = [
        *([] if run_in_sandbox else sources.source_roots),
        *chrooted_source_roots,
    ]
    extra_env = {
        **complete_pex_env.environment_dict(python_configured=pex.python is not None),
        "PEX_PATH":
        in_chroot(local_dists.pex.name),
        "PEX_EXTRA_SYS_PATH":
        os.pathsep.join(source_roots),
    }

    return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
Exemplo n.º 17
0
async def create_pex_binary_run_request(
    field_set: PexBinaryFieldSet,
    pex_binary_defaults: PexBinaryDefaults,
    pex_env: PexEnvironment,
) -> RunRequest:
    entry_point, transitive_targets = await MultiGet(
        Get(
            ResolvedPexEntryPoint,
            ResolvePexEntryPointRequest(field_set.entry_point),
        ),
        Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
    )

    # Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
    # so that we can get the interpreter constraints for use in runner_pex_request.
    requirements_pex_request = await Get(
        PexRequest,
        PexFromTargetsRequest,
        PexFromTargetsRequest.for_requirements([field_set.address],
                                               internal_only=True),
    )

    requirements_request = Get(Pex, PexRequest, requirements_pex_request)

    sources_request = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(transitive_targets.closure,
                                 include_files=True))

    output_filename = f"{field_set.address.target_name}.pex"
    runner_pex_request = Get(
        Pex,
        PexRequest(
            output_filename=output_filename,
            interpreter_constraints=requirements_pex_request.
            interpreter_constraints,
            additional_args=field_set.generate_additional_args(
                pex_binary_defaults),
            internal_only=True,
            # Note that the entry point file is not in the PEX itself. It's loaded by setting
            # `PEX_EXTRA_SYS_PATH`.
            # TODO(John Sirois): Support ConsoleScript in PexBinary targets:
            #  https://github.com/pantsbuild/pants/issues/11619
            main=entry_point.val,
        ),
    )

    requirements, sources, runner_pex = await MultiGet(requirements_request,
                                                       sources_request,
                                                       runner_pex_request)

    merged_digest = await Get(
        Digest,
        MergeDigests([
            requirements.digest, sources.source_files.snapshot.digest,
            runner_pex.digest
        ]),
    )

    def in_chroot(relpath: str) -> str:
        return os.path.join("{chroot}", relpath)

    args = pex_env.create_argv(in_chroot(runner_pex.name),
                               python=runner_pex.python)

    chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
    extra_env = {
        **pex_env.environment_dict(python_configured=runner_pex.python is not None),
        "PEX_PATH":
        in_chroot(requirements_pex_request.output_filename),
        "PEX_EXTRA_SYS_PATH":
        ":".join(chrooted_source_roots),
    }

    return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
Exemplo n.º 18
0
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot:
    exported_target = request.exported_target
    exported_addr = exported_target.target.address

    owned_deps, transitive_targets = await MultiGet(
        Get(OwnedDependencies, DependencyOwner(exported_target)),
        Get(TransitiveTargets, TransitiveTargetsRequest([exported_target.target.address])),
    )

    # files() targets aren't owned by a single exported target - they aren't code, so
    # we allow them to be in multiple dists. This is helpful for, e.g., embedding
    # a standard license file in a dist.
    files_targets = (tgt for tgt in transitive_targets.closure if tgt.has_field(FilesSources))
    targets = Targets(itertools.chain((od.target for od in owned_deps), files_targets))

    sources, requirements = await MultiGet(
        Get(SetupPySources, SetupPySourcesRequest(targets, py2=request.py2)),
        Get(ExportedTargetRequirements, DependencyOwner(exported_target)),
    )

    # Generate the kwargs for the setup() call. In addition to using the kwargs that are either
    # explicitly provided or generated via a user's plugin, we add additional kwargs based on the
    # resolved requirements and sources.
    target = exported_target.target
    resolved_setup_kwargs = await Get(SetupKwargs, ExportedTarget, exported_target)
    setup_kwargs = resolved_setup_kwargs.kwargs.copy()
    # NB: We are careful to not overwrite these values, but we also don't expect them to have been
    # set. The user must have have gone out of their way to use a `SetupKwargs` plugin, and to have
    # specified `SetupKwargs(_allow_banned_keys=True)`.
    setup_kwargs.update(
        {
            "package_dir": {"": CHROOT_SOURCE_ROOT, **setup_kwargs.get("package_dir", {})},
            "packages": (*sources.packages, *(setup_kwargs.get("packages", []))),
            "namespace_packages": (
                *sources.namespace_packages,
                *setup_kwargs.get("namespace_packages", []),
            ),
            "package_data": {**dict(sources.package_data), **setup_kwargs.get("package_data", {})},
            "install_requires": (*requirements, *setup_kwargs.get("install_requires", [])),
        }
    )

    # Add any `pex_binary` targets from `setup_py().with_binaries()` to the dist's entry points.
    key_to_binary_spec = exported_target.provides.binaries
    binaries = await Get(
        Targets, UnparsedAddressInputs(key_to_binary_spec.values(), owning_address=target.address)
    )
    entry_point_requests = []
    for binary in binaries:
        if not binary.has_field(PexEntryPointField):
            raise InvalidEntryPoint(
                "Expected addresses to `pex_binary` targets in `.with_binaries()` for the "
                f"`provides` field for {exported_addr}, but found {binary.address} with target "
                f"type {binary.alias}."
            )
        entry_point = binary[PexEntryPointField].value
        url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point"
        if not entry_point:
            raise InvalidEntryPoint(
                "Every `pex_binary` used in `.with_binaries()` for the `provides` field for "
                f"{exported_addr} must explicitly set the `entry_point` field, but "
                f"{binary.address} left the field off. Set `entry_point` to either "
                f"`app.py:func` or the longhand `path.to.app:func`. See {url}."
            )
        if ":" not in entry_point:
            raise InvalidEntryPoint(
                "Every `pex_binary` used in `with_binaries()` for the `provides()` field for "
                f"{exported_addr} must end in the format `:my_func` for the `entry_point` field, "
                f"but {binary.address} set it to {repr(entry_point)}. For example, set "
                f"`entry_point='{entry_point}:main'. See {url}."
            )
        entry_point_requests.append(ResolvePexEntryPointRequest(binary[PexEntryPointField]))
    binary_entry_points = await MultiGet(
        Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest, request)
        for request in entry_point_requests
    )
    for key, binary_entry_point in zip(key_to_binary_spec.keys(), binary_entry_points):
        entry_points = setup_kwargs.setdefault("entry_points", {})
        console_scripts = entry_points.setdefault("console_scripts", [])
        console_scripts.append(f"{key}={binary_entry_point.val}")

    # Generate the setup script.
    setup_py_content = SETUP_BOILERPLATE.format(
        target_address_spec=target.address.spec,
        setup_kwargs_str=distutils_repr(setup_kwargs),
    ).encode()
    files_to_create = [
        FileContent("setup.py", setup_py_content),
        FileContent("MANIFEST.in", "include *.py".encode()),
    ]
    extra_files_digest, src_digest = await MultiGet(
        Get(Digest, CreateDigest(files_to_create)),
        # Nest the sources under the src/ prefix.
        Get(Digest, AddPrefix(sources.digest, CHROOT_SOURCE_ROOT)),
    )

    chroot_digest = await Get(Digest, MergeDigests((src_digest, extra_files_digest)))
    return SetupPyChroot(chroot_digest, FinalizedSetupKwargs(setup_kwargs, address=target.address))