Exemple #1
0
async def get_sources(request: SetupPySourcesRequest,
                      source_root_config: SourceRootConfig) -> SetupPySources:
    targets = request.hydrated_targets
    stripped_srcs_list = await MultiGet(
        Get[SourceRootStrippedSources](HydratedTarget, target)
        for target in targets)

    # Create a chroot with all the sources, and any ancestor __init__.py files that might be needed
    # for imports to work.  Note that if a repo has multiple exported targets under a single ancestor
    # package, then that package must be a namespace package, which in Python 3 means it must not
    # have an __init__.py. We don't validate this here, because it would require inspecting *all*
    # targets, whether or not they are in the target set for this run - basically the entire repo.
    # So it's the repo owners' responsibility to ensure __init__.py hygiene.
    stripped_srcs_digests = [
        stripped_sources.snapshot.directory_digest
        for stripped_sources in stripped_srcs_list
    ]
    ancestor_init_pys = await Get[AncestorInitPyFiles](HydratedTargets,
                                                       targets)
    sources_digest = await Get[Digest](DirectoriesToMerge(directories=tuple(
        [*stripped_srcs_digests, *ancestor_init_pys.digests])))
    init_pys_snapshot = await Get[Snapshot](SnapshotSubset(
        sources_digest, PathGlobs(['**/__init__.py'])))
    init_py_contents = await Get[FilesContent](
        Digest, init_pys_snapshot.directory_digest)

    packages, namespace_packages, package_data = find_packages(
        source_roots=source_root_config.get_source_roots(),
        tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)),
        init_py_contents=init_py_contents,
        py2=request.py2)
    return SetupPySources(digest=sources_digest,
                          packages=packages,
                          namespace_packages=namespace_packages,
                          package_data=package_data)
Exemple #2
0
def all_roots(source_root_config: SourceRootConfig) -> AllSourceRoots:

    source_roots = source_root_config.get_source_roots()

    all_paths: Set[str] = set()
    for path in source_roots.traverse():
        if path.startswith("^/"):
            all_paths.add(f"{path[2:]}/")
        else:
            all_paths.add(f"**/{path}/")

    snapshot = yield Get(Snapshot, PathGlobs(include=tuple(all_paths)))

    all_source_roots: Set[SourceRoot] = set()

    # The globs above can match on subdirectories of the source roots.
    # For instance, `src/*/` might match 'src/rust/' as well as
    # 'src/rust/engine/process_execution/bazel_protos/src/gen'.
    # So we use find_by_path to verify every candidate source root.
    for directory in snapshot.dirs:
        match: SourceRoot = source_roots.find_by_path(directory)
        if match:
            all_source_roots.add(match)

    yield AllSourceRoots(all_source_roots)
Exemple #3
0
async def strip_source_root(
        hydrated_target: HydratedTarget,
        source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
    """Relativize targets to their source root, e.g.
  `src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""

    target_adaptor = hydrated_target.adaptor
    source_roots = source_root_config.get_source_roots()

    # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
    # simplify the hasattr() checks here!
    if not hasattr(target_adaptor, 'sources'):
        return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)

    digest = target_adaptor.sources.snapshot.directory_digest
    source_root = source_roots.find_by_path(target_adaptor.address.spec_path)
    if source_root is None:
        # If we found no source root, use the target's dir.
        # Note that when --source-unmatched is 'create' (the default) we'll never return None,
        # but will return the target's dir. This check allows this code to work even if
        # --source-unmatched is 'fail'.
        source_root_path = target_adaptor.address.spec_path
    else:
        source_root_path = source_root.path

    # Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
    # structure and so we do not remove their source root like we normally do, so that filesystem
    # APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
    if target_adaptor.type_alias == Files.alias():
        source_root_path = ''

    resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
        directory_digest=digest, prefix=source_root_path))
    resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
    return SourceRootStrippedSources(snapshot=resulting_snapshot)
Exemple #4
0
async def strip_source_root(
        hydrated_target: HydratedTarget,
        source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
    """Relativize targets to their source root, e.g.
  `src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""

    target_adaptor = hydrated_target.adaptor
    source_roots = source_root_config.get_source_roots()

    # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
    # simplify the hasattr() checks here!
    if not hasattr(target_adaptor, 'sources'):
        return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)

    digest = target_adaptor.sources.snapshot.directory_digest
    source_root = source_roots.find_by_path(target_adaptor.address.spec_path)

    # Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
    # structure and so we do not remove their source root like we normally do, so that filesystem
    # APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
    if target_adaptor.type_alias == Files.alias():
        source_root = None

    resulting_digest = await Get(
        Digest,
        DirectoryWithPrefixToStrip(
            directory_digest=digest,
            prefix=source_root.path if source_root else ""))
    resulting_snapshot = await Get(Snapshot, Digest, resulting_digest)
    return SourceRootStrippedSources(snapshot=resulting_snapshot)
Exemple #5
0
async def create_coverage_config(
    coverage_config_request: CoverageConfigRequest, source_root_config: SourceRootConfig
) -> CoverageConfig:
    sources = await Get[SourceFiles](
        AllSourceFilesRequest(
            (tgt.get(Sources) for tgt in coverage_config_request.targets), strip_source_roots=False,
        )
    )
    init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(sources.snapshot))
    source_roots = source_root_config.get_source_roots()

    # Generate a map from source root stripped source to its source root. eg:
    #  {'pants/testutil/subsystem/util.py': 'src/python'}. This is so that coverage reports
    #  referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped back to the actual
    #  sources they reference when generating coverage reports.
    def stripped_file_with_source_root(file_name: str) -> Tuple[str, str]:
        source_root_object = source_roots.find_by_path(file_name)
        source_root = source_root_object.path if source_root_object is not None else ""
        stripped_path = file_name[len(source_root) + 1 :]
        return stripped_path, source_root

    stripped_files_to_source_roots = dict(
        stripped_file_with_source_root(filename)
        for filename in sorted(init_injected.snapshot.files)
    )

    default_config = dedent(
        """
        [run]
        branch = True
        timid = False
        relative_files = True
        """
    )

    config_parser = configparser.ConfigParser()
    config_parser.read_file(StringIO(default_config))
    config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
    config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
    config_parser.set(
        COVERAGE_PLUGIN_MODULE_NAME,
        "source_to_target_base",
        json.dumps(stripped_files_to_source_roots),
    )
    config_parser.set(
        COVERAGE_PLUGIN_MODULE_NAME, "test_time", json.dumps(coverage_config_request.is_test_time)
    )

    config_io_stream = StringIO()
    config_parser.write(config_io_stream)
    digest = await Get[Digest](
        InputFilesContent(
            [FileContent(".coveragerc", content=config_io_stream.getvalue().encode())]
        )
    )
    return CoverageConfig(digest)
Exemple #6
0
async def strip_source_roots_from_snapshot(
    request: StripSnapshotRequest,
    source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
    """Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
    `pants/util/strutil.py`."""
    if not request.snapshot.files:
        return SourceRootStrippedSources(request.snapshot)

    source_roots_object = source_root_config.get_source_roots()

    def determine_source_root(path: str) -> str:
        return cast(str, source_roots_object.strict_find_by_path(path).path)

    if request.representative_path is not None:
        source_root = determine_source_root(request.representative_path)
        if source_root == ".":
            return SourceRootStrippedSources(request.snapshot)
        resulting_snapshot = await Get[Snapshot](RemovePrefix(
            request.snapshot.digest, source_root))
        return SourceRootStrippedSources(resulting_snapshot)

    files_grouped_by_source_root = {
        source_root: tuple(files)
        for source_root, files in itertools.groupby(request.snapshot.files,
                                                    key=determine_source_root)
    }

    if len(files_grouped_by_source_root) == 1:
        source_root = next(iter(files_grouped_by_source_root.keys()))
        if source_root == ".":
            return SourceRootStrippedSources(request.snapshot)
        resulting_snapshot = await Get[Snapshot](RemovePrefix(
            request.snapshot.digest, source_root))
        return SourceRootStrippedSources(resulting_snapshot)

    snapshot_subsets = await MultiGet(
        Get[Snapshot](SnapshotSubset(request.snapshot.digest, PathGlobs(
            files))) for files in files_grouped_by_source_root.values())
    resulting_digests = await MultiGet(
        Get[Digest](RemovePrefix(snapshot.digest, source_root))
        for snapshot, source_root in zip(snapshot_subsets,
                                         files_grouped_by_source_root.keys()))

    resulting_snapshot = await Get[Snapshot](MergeDigests(resulting_digests))
    return SourceRootStrippedSources(resulting_snapshot)
Exemple #7
0
async def strip_source_roots_from_snapshot(
    request: StripSnapshotRequest,
    source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
    """Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
    `pants/util/strutil.py`."""
    source_roots_object = source_root_config.get_source_roots()

    def determine_source_root(path: str) -> str:
        source_root = source_roots_object.safe_find_by_path(path)
        if source_root is not None:
            return cast(str, source_root.path)
        if source_root_config.options.unmatched == "fail":
            raise NoSourceRootError(
                f"Could not find a source root for `{path}`.")
        # Otherwise, create a source root by using the parent directory.
        return PurePath(path).parent.as_posix()

    if request.representative_path is not None:
        resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
            directory_digest=request.snapshot.directory_digest,
            prefix=determine_source_root(request.representative_path),
        ))
        resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
        return SourceRootStrippedSources(snapshot=resulting_snapshot)

    files_grouped_by_source_root = {
        source_root: tuple(files)
        for source_root, files in itertools.groupby(request.snapshot.files,
                                                    key=determine_source_root)
    }
    snapshot_subsets = await MultiGet(Get[Snapshot](SnapshotSubset(
        directory_digest=request.snapshot.directory_digest,
        globs=PathGlobs(files),
    )) for files in files_grouped_by_source_root.values())
    resulting_digests = await MultiGet(
        Get[Digest](DirectoryWithPrefixToStrip(
            directory_digest=snapshot.directory_digest, prefix=source_root))
        for snapshot, source_root in zip(snapshot_subsets,
                                         files_grouped_by_source_root.keys()))

    merged_result = await Get[Digest](DirectoriesToMerge(resulting_digests))
    resulting_snapshot = await Get[Snapshot](Digest, merged_result)
    return SourceRootStrippedSources(resulting_snapshot)
Exemple #8
0
async def get_ancestor_init_py(
    targets: Targets, source_root_config: SourceRootConfig
) -> AncestorInitPyFiles:
    """Find any ancestor __init__.py files for the given targets.

    Includes sibling __init__.py files. Returns the files stripped of their source roots.
    """
    source_roots = source_root_config.get_source_roots()
    sources = await Get[SourceFiles](
        AllSourceFilesRequest(tgt[PythonSources] for tgt in targets if tgt.has_field(PythonSources))
    )
    # Find the ancestors of all dirs containing .py files, including those dirs themselves.
    source_dir_ancestors: Set[Tuple[str, str]] = set()  # Items are (src_root, path incl. src_root).
    for fp in sources.snapshot.files:
        source_dir_ancestor = os.path.dirname(fp)
        source_root = source_root_or_raise(source_roots, fp)
        # Do not allow the repository root to leak (i.e., '.' should not be a package in setup.py).
        while source_dir_ancestor != source_root:
            source_dir_ancestors.add((source_root, source_dir_ancestor))
            source_dir_ancestor = os.path.dirname(source_dir_ancestor)

    source_dir_ancestors_list = list(source_dir_ancestors)  # To force a consistent order.

    # Note that we must MultiGet single globs instead of a a single Get for all the globs, because
    # we match each result to its originating glob (see use of zip below).
    ancestor_init_py_snapshots = await MultiGet[Snapshot](
        Get[Snapshot](PathGlobs, PathGlobs([os.path.join(source_dir_ancestor[1], "__init__.py")]))
        for source_dir_ancestor in source_dir_ancestors_list
    )

    source_root_stripped_ancestor_init_pys = await MultiGet[Digest](
        Get[Digest](
            DirectoryWithPrefixToStrip(
                directory_digest=snapshot.directory_digest, prefix=source_dir_ancestor[0]
            )
        )
        for snapshot, source_dir_ancestor in zip(
            ancestor_init_py_snapshots, source_dir_ancestors_list
        )
    )

    return AncestorInitPyFiles(source_root_stripped_ancestor_init_pys)
Exemple #9
0
async def construct_coverage_config(
        source_root_config: SourceRootConfig,
        coverage_config_request: CoveragercRequest) -> Coveragerc:
    sources = await Get[SourceFiles](AllSourceFilesRequest(
        (ht.adaptor for ht in coverage_config_request.hydrated_targets),
        strip_source_roots=False,
    ))
    init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
        sources.snapshot))
    source_roots = source_root_config.get_source_roots()

    # Generate a map from source root stripped source to its source root. eg:
    #  {'pants/testutil/subsystem/util.py': 'src/python'}
    # This is so coverage reports referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped
    # back to the actual sources they reference when generating coverage reports.
    def source_root_stripped_source_and_source_root(
            file_name: str) -> Tuple[str, str]:
        source_root = source_roots.find_by_path(file_name)
        source_root_path = source_root.path if source_root is not None else ""
        source_root_stripped_path = file_name[len(source_root_path) + 1:]
        return (source_root_stripped_path, source_root_path)

    source_to_target_base = dict(
        source_root_stripped_source_and_source_root(filename)
        for filename in sorted(init_injected.snapshot.files))
    config_parser = configparser.ConfigParser()
    config_parser.read_file(StringIO(DEFAULT_COVERAGE_CONFIG))
    ensure_section(config_parser, "run")
    config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
    config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
    config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "source_to_target_base",
                      json.dumps(source_to_target_base))
    config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "test_time",
                      json.dumps(coverage_config_request.test_time))
    config = StringIO()
    config_parser.write(config)
    coveragerc_digest = await Get[Digest](InputFilesContent,
                                          get_coveragerc_input(
                                              config.getvalue()))
    return Coveragerc(coveragerc_digest)
Exemple #10
0
async def list_backends(
    backend_options: BackendsOptions,
    source_roots_config: SourceRootConfig,
    global_options: GlobalOptions,
    console: Console,
) -> Backends:
    source_roots = source_roots_config.get_source_roots()
    discovered_register_pys = await Get[Snapshot](PathGlobs(
        ["**/*/register.py"]))
    register_pys_content = await Get[FilesContent](
        Digest, discovered_register_pys.digest)

    backend_infos = tuple(
        BackendInfo.create(fc, source_roots, global_options)
        for fc in register_pys_content)
    v1_backends = []
    v2_backends = []
    for backend in backend_infos:
        if backend.is_v1:
            v1_backends.append(backend)
        if backend.is_v2:
            v2_backends.append(backend)

    with backend_options.line_oriented(console) as print_stdout:
        if global_options.options.v1:
            print_stdout(
                format_section(v1_backends,
                               console,
                               version_number=1,
                               option_name="backend_packages"))
        if global_options.options.v2:
            print_stdout(
                format_section(v2_backends,
                               console,
                               version_number=2,
                               option_name="backend_packages2"))
    return Backends(exit_code=0)