Exemplo n.º 1
0
async def generate_coverage_reports(
    merged_coverage_data: MergedCoverageData,
    coverage_setup: CoverageSetup,
    coverage_config: CoverageConfig,
    coverage_subsystem: CoverageSubsystem,
    transitive_targets: TransitiveTargets,
) -> CoverageReports:
    """Takes all Python test results and generates a single coverage report."""
    sources = await Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(transitive_targets.closure,
                                 include_resources=False),
    )
    input_digest = await Get(
        Digest,
        MergeDigests((
            merged_coverage_data.coverage_data,
            coverage_config.digest,
            coverage_setup.pex.digest,
            sources.source_files.snapshot.digest,
        )),
    )

    pex_processes = []
    report_types = []
    coverage_reports: List[CoverageReport] = []
    for report_type in coverage_subsystem.reports:
        if report_type == CoverageReportType.RAW:
            coverage_reports.append(
                FilesystemCoverageReport(
                    report_type=CoverageReportType.RAW,
                    result_digest=merged_coverage_data.coverage_data,
                    directory_to_materialize_to=coverage_subsystem.output_dir,
                    report_file=coverage_subsystem.output_dir / ".coverage",
                ))
            continue
        report_types.append(report_type)
        output_file = (f"coverage.{report_type.value}" if report_type in {
            CoverageReportType.XML, CoverageReportType.JSON
        } else None)
        pex_processes.append(
            PexProcess(
                coverage_setup.pex,
                # We pass `--ignore-errors` because Pants dynamically injects missing `__init__.py`
                # files and this will cause Coverage to fail.
                argv=(report_type.report_name, "--ignore-errors"),
                input_digest=input_digest,
                output_directories=("htmlcov", )
                if report_type == CoverageReportType.HTML else None,
                output_files=(output_file, ) if output_file else None,
                description=
                f"Generate Pytest {report_type.report_name} coverage report.",
                level=LogLevel.DEBUG,
            ))
    results = await MultiGet(
        Get(ProcessResult, PexProcess, process) for process in pex_processes)
    coverage_reports.extend(
        _get_coverage_reports(coverage_subsystem.output_dir, report_types,
                              results))
    return CoverageReports(tuple(coverage_reports))
Exemplo n.º 2
0
async def create_ipython_repl_request(
    repl: IPythonRepl, ipython: IPython, pex_env: PexEnvironment
) -> ReplRequest:
    # Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
    # so that we can get the interpreter constraints for use in ipython_request.
    requirements_pex_request = await Get(
        PexRequest,
        PexFromTargetsRequest,
        PexFromTargetsRequest.for_requirements(
            Addresses(tgt.address for tgt in repl.targets), internal_only=True
        ),
    )

    requirements_request = Get(Pex, PexRequest, requirements_pex_request)

    sources_request = Get(
        PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)
    )

    ipython_request = Get(
        Pex,
        PexRequest(
            output_filename="ipython.pex",
            entry_point=ipython.entry_point,
            requirements=PexRequirements(ipython.all_requirements),
            interpreter_constraints=requirements_pex_request.interpreter_constraints,
            internal_only=True,
        ),
    )

    requirements_pex, sources, ipython_pex = await MultiGet(
        requirements_request, sources_request, ipython_request
    )
    merged_digest = await Get(
        Digest,
        MergeDigests(
            (requirements_pex.digest, sources.source_files.snapshot.digest, ipython_pex.digest)
        ),
    )

    args = [repl.in_chroot(ipython_pex.name)]
    if ipython.options.ignore_cwd:
        args.append("--ignore-cwd")

    chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
    extra_env = {
        **pex_env.environment_dict,
        "PEX_PATH": repl.in_chroot(requirements_pex_request.output_filename),
        "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
    }

    return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
Exemplo n.º 3
0
async def create_python_binary_run_request(
        field_set: PythonBinaryFieldSet,
        python_binary_defaults: PythonBinaryDefaults) -> RunRequest:
    entry_point = field_set.entry_point.value
    if entry_point is None:
        # TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
        #  we only need the path relative to the source root.
        binary_sources = await Get(HydratedSources,
                                   HydrateSourcesRequest(field_set.sources))
        stripped_binary_sources = await Get(
            StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ()))
        entry_point = PythonBinarySources.translate_source_file_to_entry_point(
            stripped_binary_sources.snapshot.files)
    if entry_point is None:
        raise InvalidFieldException(
            "You must either specify `sources` or `entry_point` for the target "
            f"{repr(field_set.address)} in order to run it, but both fields were undefined."
        )

    transitive_targets = await Get(TransitiveTargets,
                                   Addresses([field_set.address]))

    output_filename = f"{field_set.address.target_name}.pex"
    pex_request = Get(
        Pex,
        PexFromTargetsRequest(
            addresses=Addresses([field_set.address]),
            platforms=PexPlatforms.create_from_platforms_field(
                field_set.platforms),
            output_filename=output_filename,
            additional_args=field_set.generate_additional_args(
                python_binary_defaults),
            include_source_files=False,
        ),
    )
    sources_request = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(transitive_targets.closure,
                                 include_files=True),
    )
    pex, sources = await MultiGet(pex_request, sources_request)

    merged_digest = await Get(
        Digest,
        MergeDigests([pex.digest, sources.source_files.snapshot.digest]))
    return RunRequest(
        digest=merged_digest,
        binary_name=pex.output_filename,
        prefix_args=("-m", entry_point),
        env={"PEX_EXTRA_SYS_PATH": ":".join(sources.source_roots)},
    )
Exemplo n.º 4
0
async def get_sources(request: SetupPySourcesRequest) -> SetupPySources:
    python_sources = await Get(
        StrippedPythonSourceFiles,
        PythonSourceFilesRequest(targets=request.targets,
                                 include_resources=False,
                                 include_files=False),
    )
    all_sources = await Get(
        StrippedPythonSourceFiles,
        PythonSourceFilesRequest(targets=request.targets,
                                 include_resources=True,
                                 include_files=True),
    )

    python_files = set(python_sources.stripped_source_files.snapshot.files)
    all_files = set(all_sources.stripped_source_files.snapshot.files)
    resource_files = all_files - python_files

    init_py_digest_contents = await Get(
        DigestContents,
        DigestSubset(python_sources.stripped_source_files.snapshot.digest,
                     PathGlobs(["**/__init__.py"])),
    )

    packages, namespace_packages, package_data = find_packages(
        python_files=python_files,
        resource_files=resource_files,
        init_py_digest_contents=init_py_digest_contents,
        py2=request.py2,
    )
    return SetupPySources(
        digest=all_sources.stripped_source_files.snapshot.digest,
        packages=packages,
        namespace_packages=namespace_packages,
        package_data=package_data,
    )
Exemplo n.º 5
0
async def create_python_repl_request(repl: PythonRepl) -> ReplRequest:
    pex_request = Get(
        Pex,
        PexFromTargetsRequest(
            (tgt.address for tgt in repl.targets),
            output_filename="python.pex",
            include_source_files=False,
        ),
    )
    sources_request = Get(PythonSourceFiles, PythonSourceFilesRequest(repl.targets))
    pex, sources = await MultiGet(pex_request, sources_request)
    merged_digest = await Get(
        Digest, MergeDigests((pex.digest, sources.source_files.snapshot.digest))
    )
    return ReplRequest(
        digest=merged_digest,
        binary_name=pex.output_filename,
        env={"PEX_EXTRA_SYS_PATH": ":".join(sources.source_roots)},
    )
Exemplo n.º 6
0
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython) -> ReplRequest:
    pex_request = Get(
        Pex,
        PexFromTargetsRequest(
            (tgt.address for tgt in repl.targets),
            output_filename="ipython.pex",
            entry_point=ipython.entry_point,
            additional_requirements=ipython.all_requirements,
            include_source_files=True,
        ),
    )
    sources_request = Get(PythonSourceFiles, PythonSourceFilesRequest(repl.targets))
    pex, sources = await MultiGet(pex_request, sources_request)
    merged_digest = await Get(
        Digest, MergeDigests((pex.digest, sources.source_files.snapshot.digest))
    )
    return ReplRequest(
        digest=merged_digest,
        binary_name=pex.output_filename,
        env={"PEX_EXTRA_SYS_PATH": ":".join(sources.source_roots)},
    )
Exemplo n.º 7
0
async def create_python_repl_request(repl: PythonRepl, pex_env: PexEnvironment) -> ReplRequest:
    requirements_request = Get(
        Pex,
        PexFromTargetsRequest,
        PexFromTargetsRequest.for_requirements(
            Addresses(tgt.address for tgt in repl.targets), internal_only=True
        ),
    )
    sources_request = Get(
        PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)
    )
    requirements_pex, sources = await MultiGet(requirements_request, sources_request)
    merged_digest = await Get(
        Digest, MergeDigests((requirements_pex.digest, sources.source_files.snapshot.digest))
    )

    chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
    extra_env = {**pex_env.environment_dict, "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots)}

    return ReplRequest(
        digest=merged_digest, args=(repl.in_chroot(requirements_pex.name),), extra_env=extra_env
    )
Exemplo n.º 8
0
 def get_unstripped_sources(
     self,
     targets: Iterable[Target],
     *,
     include_resources: bool = True,
     include_files: bool = False,
     source_roots: Optional[List[str]] = None,
     extra_args: Optional[List[str]] = None,
 ) -> PythonSourceFiles:
     return self.request_product(
         PythonSourceFiles,
         [
             PythonSourceFilesRequest(targets,
                                      include_resources=include_resources,
                                      include_files=include_files),
             create_options_bootstrapper(args=[
                 "--backend-packages=pants.backend.python",
                 f"--source-root-patterns={source_roots or ['src/python']}",
                 *(extra_args or []),
             ]),
         ],
     )
Exemplo n.º 9
0
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest:
    transitive_targets = await Get(TransitiveTargets, Addresses, request.addresses)
    all_targets = transitive_targets.closure

    input_digests = []
    if request.additional_sources:
        input_digests.append(request.additional_sources)
    if request.include_source_files:
        prepared_sources = await Get(
            StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)
        )
        input_digests.append(prepared_sources.stripped_source_files.snapshot.digest)
    merged_input_digest = await Get(Digest, MergeDigests(input_digests))

    interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
        (
            tgt[PythonInterpreterCompatibility]
            for tgt in all_targets
            if tgt.has_field(PythonInterpreterCompatibility)
        ),
        python_setup,
    )

    exact_reqs = PexRequirements.create_from_requirement_fields(
        (
            tgt[PythonRequirementsField]
            for tgt in all_targets
            if tgt.has_field(PythonRequirementsField)
        ),
        additional_requirements=request.additional_requirements,
    )

    requirements = exact_reqs
    description = request.description

    if python_setup.requirement_constraints:
        # In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let
        # packaging canonicalize for us.
        # See: https://www.python.org/dev/peps/pep-0503/#normalized-names

        exact_req_projects = {
            canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs
        }
        constraints_file_contents = await Get(
            DigestContents,
            PathGlobs(
                [python_setup.requirement_constraints],
                glob_match_error_behavior=GlobMatchErrorBehavior.error,
                conjunction=GlobExpansionConjunction.all_match,
                description_of_origin="the option `--python-setup-requirement-constraints`",
            ),
        )
        constraints_file_reqs = set(
            parse_requirements(next(iter(constraints_file_contents)).content.decode())
        )
        constraint_file_projects = {
            canonicalize_project_name(req.project_name) for req in constraints_file_reqs
        }
        unconstrained_projects = exact_req_projects - constraint_file_projects
        if unconstrained_projects:
            logger.warning(
                f"The constraints file {python_setup.requirement_constraints} does not contain "
                f"entries for the following requirements: {', '.join(unconstrained_projects)}"
            )

        if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or (
            python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES
            and request.internal_only
        ):
            if unconstrained_projects:
                logger.warning(
                    "Ignoring resolve_all_constraints setting in [python_setup] scope "
                    "because constraints file does not cover all requirements."
                )
            else:
                requirements = PexRequirements(str(req) for req in constraints_file_reqs)
                description = description or f"Resolving {python_setup.requirement_constraints}"
    elif (
        python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER
        and python_setup.resolve_all_constraints_was_set_explicitly()
    ):
        raise ValueError(
            f"[python-setup].resolve_all_constraints is set to "
            f"{python_setup.resolve_all_constraints.value}, so "
            f"[python-setup].requirement_constraints must also be provided."
        )

    return PexRequest(
        output_filename=request.output_filename,
        internal_only=request.internal_only,
        requirements=requirements,
        interpreter_constraints=interpreter_constraints,
        platforms=request.platforms,
        entry_point=request.entry_point,
        sources=merged_input_digest,
        additional_inputs=request.additional_inputs,
        additional_args=request.additional_args,
        description=description,
    )
Exemplo n.º 10
0
async def mypy_typecheck(request: MyPyRequest, mypy: MyPy) -> TypecheckResults:
    if mypy.skip:
        return TypecheckResults([], typechecker_name="MyPy")

    transitive_targets = await Get(
        TransitiveTargets, Addresses(fs.address for fs in request.field_sets))

    prepared_sources_request = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(transitive_targets.closure),
    )
    pex_request = Get(
        Pex,
        PexRequest(
            output_filename="mypy.pex",
            internal_only=True,
            requirements=PexRequirements(mypy.all_requirements),
            # NB: This only determines what MyPy is run with. The user can specify what version
            # their code is with `--python-version`. See
            # https://mypy.readthedocs.io/en/stable/config_file.html#platform-configuration. We do
            # not auto-configure this for simplicity and to avoid Pants magically setting values for
            # users.
            interpreter_constraints=PexInterpreterConstraints(
                mypy.interpreter_constraints),
            entry_point=mypy.entry_point,
        ),
    )
    config_digest_request = Get(
        Digest,
        PathGlobs(
            globs=[mypy.config] if mypy.config else [],
            glob_match_error_behavior=GlobMatchErrorBehavior.error,
            description_of_origin="the option `--mypy-config`",
        ),
    )
    prepared_sources, pex, config_digest = await MultiGet(
        prepared_sources_request, pex_request, config_digest_request)

    srcs_snapshot = prepared_sources.source_files.snapshot
    file_list_path = "__files.txt"
    python_files = "\n".join(f for f in srcs_snapshot.files
                             if f.endswith(".py"))
    file_list_digest = await Get(
        Digest,
        CreateDigest([FileContent(file_list_path, python_files.encode())]),
    )

    merged_input_files = await Get(
        Digest,
        MergeDigests([
            file_list_digest, srcs_snapshot.digest, pex.digest, config_digest
        ]),
    )

    result = await Get(
        FallibleProcessResult,
        PexProcess(
            pex,
            argv=generate_args(mypy, file_list_path=file_list_path),
            input_digest=merged_input_files,
            extra_env={
                "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots)
            },
            description=
            f"Run MyPy on {pluralize(len(srcs_snapshot.files), 'file')}.",
            level=LogLevel.DEBUG,
        ),
    )
    return TypecheckResults(
        [TypecheckResult.from_fallible_process_result(result)],
        typechecker_name="MyPy")
Exemplo n.º 11
0
async def create_python_binary_run_request(
    field_set: PythonBinaryFieldSet,
    python_binary_defaults: PythonBinaryDefaults,
    pex_env: PexEnvironment,
) -> RunRequest:
    entry_point = field_set.entry_point.value
    if entry_point is None:
        # TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
        #  we only need the path relative to the source root.
        binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources))
        stripped_binary_sources = await Get(
            StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())
        )
        entry_point = PythonBinarySources.translate_source_file_to_entry_point(
            stripped_binary_sources.snapshot.files
        )
    if entry_point is None:
        raise InvalidFieldException(
            "You must either specify `sources` or `entry_point` for the target "
            f"{repr(field_set.address)} in order to run it, but both fields were undefined."
        )

    transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address]))

    # Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
    # so that we can get the interpreter constraints for use in runner_pex_request.
    requirements_pex_request = await Get(
        PexRequest,
        PexFromTargetsRequest,
        PexFromTargetsRequest.for_requirements(Addresses([field_set.address]), internal_only=True),
    )

    requirements_request = Get(Pex, PexRequest, requirements_pex_request)

    sources_request = Get(
        PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)
    )

    output_filename = f"{field_set.address.target_name}.pex"
    runner_pex_request = Get(
        Pex,
        PexRequest(
            output_filename=output_filename,
            interpreter_constraints=requirements_pex_request.interpreter_constraints,
            additional_args=field_set.generate_additional_args(python_binary_defaults),
            internal_only=True,
        ),
    )

    requirements, sources, runner_pex = await MultiGet(
        requirements_request, sources_request, runner_pex_request
    )

    merged_digest = await Get(
        Digest,
        MergeDigests(
            [requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest]
        ),
    )

    def in_chroot(relpath: str) -> str:
        return os.path.join("{chroot}", relpath)

    chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
    extra_env = {
        **pex_env.environment_dict,
        "PEX_PATH": in_chroot(requirements_pex_request.output_filename),
        "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
    }

    return RunRequest(
        digest=merged_digest,
        args=(in_chroot(runner_pex.name), "-m", entry_point),
        extra_env=extra_env,
    )
Exemplo n.º 12
0
async def pex_from_targets(request: PexFromTargetsRequest,
                           python_setup: PythonSetup) -> PexRequest:
    transitive_targets = await Get(TransitiveTargets, Addresses,
                                   request.addresses)
    all_targets = transitive_targets.closure

    input_digests = []
    if request.additional_sources:
        input_digests.append(request.additional_sources)
    if request.include_source_files:
        prepared_sources = await Get(StrippedPythonSourceFiles,
                                     PythonSourceFilesRequest(all_targets))
        input_digests.append(
            prepared_sources.stripped_source_files.snapshot.digest)
    merged_input_digest = await Get(Digest, MergeDigests(input_digests))

    interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
        (tgt[PythonInterpreterCompatibility] for tgt in all_targets
         if tgt.has_field(PythonInterpreterCompatibility)),
        python_setup,
    )

    exact_reqs = PexRequirements.create_from_requirement_fields(
        (tgt[PythonRequirementsField]
         for tgt in all_targets if tgt.has_field(PythonRequirementsField)),
        additional_requirements=request.additional_requirements,
    )

    requirements = exact_reqs

    if python_setup.requirement_constraints:
        exact_req_projects = {
            Requirement.parse(req).project_name
            for req in exact_reqs
        }
        constraints_file_contents = await Get(
            DigestContents,
            PathGlobs(
                [python_setup.requirement_constraints],
                glob_match_error_behavior=GlobMatchErrorBehavior.error,
                conjunction=GlobExpansionConjunction.all_match,
                description_of_origin=
                "the option `--python-setup-requirement-constraints`",
            ),
        )
        constraints_file_reqs = set(
            parse_requirements(
                next(iter(constraints_file_contents)).content.decode()))
        constraint_file_projects = {
            req.project_name
            for req in constraints_file_reqs
        }
        unconstrained_projects = exact_req_projects - constraint_file_projects
        if unconstrained_projects:
            logger.warning(
                f"The constraints file {python_setup.requirement_constraints} does not contain "
                f"entries for the following requirements: {', '.join(unconstrained_projects)}"
            )

        if python_setup.resolve_all_constraints:
            if unconstrained_projects:
                logger.warning(
                    "Ignoring resolve_all_constraints setting in [python_setup] scope"
                    "Because constraints file does not cover all requirements."
                )
            else:
                requirements = PexRequirements(
                    str(req) for req in constraints_file_reqs)
    elif python_setup.resolve_all_constraints:
        raise ValueError(
            "resolve_all_constraints in the [python-setup] scope is set, so "
            "requirement_constraints in [python-setup] must also be provided.")

    return PexRequest(
        output_filename=request.output_filename,
        requirements=requirements,
        interpreter_constraints=interpreter_constraints,
        platforms=request.platforms,
        entry_point=request.entry_point,
        sources=merged_input_digest,
        additional_inputs=request.additional_inputs,
        additional_args=request.additional_args,
        description=request.description,
    )
Exemplo n.º 13
0
async def pylint_lint_partition(partition: PylintPartition,
                                pylint: Pylint) -> LintResult:
    # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies.
    # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`.
    plugin_requirements = PexRequirements.create_from_requirement_fields(
        plugin_tgt[PythonRequirementsField]
        for plugin_tgt in partition.plugin_targets
        if plugin_tgt.has_field(PythonRequirementsField))
    target_requirements = PexRequirements.create_from_requirement_fields(
        tgt[PythonRequirementsField]
        for tgt in partition.targets_with_dependencies
        if tgt.has_field(PythonRequirementsField))
    pylint_pex_request = Get(
        Pex,
        PexRequest(
            output_filename="pylint.pex",
            requirements=PexRequirements(
                [*pylint.all_requirements, *plugin_requirements]),
            interpreter_constraints=partition.interpreter_constraints,
        ),
    )
    requirements_pex_request = Get(
        Pex,
        PexRequest(
            output_filename="requirements.pex",
            requirements=target_requirements,
            interpreter_constraints=partition.interpreter_constraints,
        ),
    )
    # TODO(John Sirois): Support shading python binaries:
    #   https://github.com/pantsbuild/pants/issues/9206
    # Right now any Pylint transitive requirements will shadow corresponding user
    # requirements, which could lead to problems.
    pylint_runner_pex_args = [
        "--pex-path", ":".join(["pylint.pex", "requirements.pex"])
    ]
    pylint_runner_pex_request = Get(
        Pex,
        PexRequest(
            output_filename="pylint_runner.pex",
            entry_point=pylint.entry_point,
            interpreter_constraints=partition.interpreter_constraints,
            additional_args=pylint_runner_pex_args,
        ),
    )

    config_digest_request = Get(
        Digest,
        PathGlobs(
            globs=[pylint.config] if pylint.config else [],
            glob_match_error_behavior=GlobMatchErrorBehavior.error,
            description_of_origin="the option `--pylint-config`",
        ),
    )

    prepare_plugin_sources_request = Get(
        StrippedPythonSourceFiles,
        PythonSourceFilesRequest(partition.plugin_targets),
    )
    prepare_python_sources_request = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(partition.targets_with_dependencies),
    )
    field_set_sources_request = Get(
        SourceFiles,
        SourceFilesRequest(field_set.sources
                           for field_set in partition.field_sets),
    )

    (
        pylint_pex,
        requirements_pex,
        pylint_runner_pex,
        config_digest,
        prepared_plugin_sources,
        prepared_python_sources,
        field_set_sources,
    ) = await MultiGet(
        pylint_pex_request,
        requirements_pex_request,
        pylint_runner_pex_request,
        config_digest_request,
        prepare_plugin_sources_request,
        prepare_python_sources_request,
        field_set_sources_request,
    )

    prefixed_plugin_sources = (await Get(
        Digest,
        AddPrefix(
            prepared_plugin_sources.stripped_source_files.snapshot.digest,
            "__plugins"),
    ) if pylint.source_plugins else EMPTY_DIGEST)

    pythonpath = list(prepared_python_sources.source_roots)
    if pylint.source_plugins:
        # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must
        # point to the plugin's directory, rather than to a parent's directory, because
        # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but
        # not `path.to.plugin`. (This means users must have specified the parent directory as a
        # source root.)
        pythonpath.append("__plugins")

    input_digest = await Get(
        Digest,
        MergeDigests((
            pylint_pex.digest,
            requirements_pex.digest,
            pylint_runner_pex.digest,
            config_digest,
            prefixed_plugin_sources,
            prepared_python_sources.source_files.snapshot.digest,
        )),
    )

    address_references = ", ".join(
        sorted(field_set.address.spec for field_set in partition.field_sets))

    result = await Get(
        FallibleProcessResult,
        PexProcess(
            pylint_runner_pex,
            argv=generate_args(source_files=field_set_sources, pylint=pylint),
            input_digest=input_digest,
            extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
            description=
            (f"Run Pylint on {pluralize(len(partition.field_sets), 'target')}: "
             f"{address_references}."),
        ),
    )
    return LintResult.from_fallible_process_result(result,
                                                   linter_name="Pylint")
Exemplo n.º 14
0
async def setup_pytest_for_target(
    field_set: PythonTestFieldSet,
    pytest: PyTest,
    test_subsystem: TestSubsystem,
    python_setup: PythonSetup,
    coverage_config: CoverageConfig,
    coverage_subsystem: CoverageSubsystem,
) -> TestTargetSetup:
    test_addresses = Addresses((field_set.address, ))

    transitive_targets = await Get(TransitiveTargets, Addresses,
                                   test_addresses)
    all_targets = transitive_targets.closure

    interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
        (tgt[PythonInterpreterCompatibility] for tgt in all_targets
         if tgt.has_field(PythonInterpreterCompatibility)),
        python_setup,
    )

    # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
    # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
    # from zip files. `zipp` has pathologically bad behavior with large zipfiles.
    # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
    # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
    # and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
    additional_args_for_pytest = ("--not-zip-safe", )

    pytest_pex_request = Get(
        Pex,
        PexRequest(
            output_filename="pytest.pex",
            requirements=PexRequirements(pytest.get_requirement_strings()),
            interpreter_constraints=interpreter_constraints,
            additional_args=additional_args_for_pytest,
            internal_only=True,
        ),
    )

    # Defaults to zip_safe=False.
    requirements_pex_request = Get(
        Pex,
        PexFromTargetsRequest,
        PexFromTargetsRequest.for_requirements(test_addresses,
                                               internal_only=True),
    )

    test_runner_pex_request = Get(
        Pex,
        PexRequest(
            interpreter_constraints=interpreter_constraints,
            output_filename="test_runner.pex",
            entry_point="pytest:main",
            additional_args=(
                "--pex-path",
                # TODO(John Sirois): Support shading python binaries:
                #   https://github.com/pantsbuild/pants/issues/9206
                # Right now any pytest transitive requirements will shadow corresponding user
                # requirements which will lead to problems when APIs that are used by either
                # `pytest:main` or the tests themselves break between the two versions.
                ":".join((
                    pytest_pex_request.subject.output_filename,
                    requirements_pex_request.subject.output_filename,
                )),
            ),
            internal_only=True,
        ),
    )

    prepared_sources_request = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(all_targets, include_files=True))

    # Get the file names for the test_target so that we can specify to Pytest precisely which files
    # to test, rather than using auto-discovery.
    field_set_source_files_request = Get(
        SourceFiles, SourceFilesRequest([field_set.sources]))

    (
        pytest_pex,
        requirements_pex,
        test_runner_pex,
        prepared_sources,
        field_set_source_files,
    ) = await MultiGet(
        pytest_pex_request,
        requirements_pex_request,
        test_runner_pex_request,
        prepared_sources_request,
        field_set_source_files_request,
    )

    input_digest = await Get(
        Digest,
        MergeDigests((
            coverage_config.digest,
            prepared_sources.source_files.snapshot.digest,
            requirements_pex.digest,
            pytest_pex.digest,
            test_runner_pex.digest,
        )),
    )

    coverage_args = []
    if test_subsystem.use_coverage:
        cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (
            ".", )
        coverage_args = [
            "--cov-report=",  # Turn off output.
            *itertools.chain.from_iterable(["--cov", cov_path]
                                           for cov_path in cov_paths),
        ]
    return TestTargetSetup(
        test_runner_pex=test_runner_pex,
        args=(*pytest.options.args, *coverage_args,
              *field_set_source_files.files),
        input_digest=input_digest,
        source_roots=prepared_sources.source_roots,
        timeout_seconds=field_set.timeout.calculate_from_global_options(
            pytest),
        xml_dir=pytest.options.junit_xml_dir,
        junit_family=pytest.options.junit_family,
        execution_slot_variable=pytest.options.execution_slot_var,
    )
Exemplo n.º 15
0
async def setup_pytest_for_target(
    request: TestSetupRequest,
    pytest: PyTest,
    test_subsystem: TestSubsystem,
    python_setup: PythonSetup,
    coverage_config: CoverageConfig,
    coverage_subsystem: CoverageSubsystem,
    global_options: GlobalOptions,
) -> TestSetup:
    test_addresses = Addresses((request.field_set.address, ))

    transitive_targets = await Get(TransitiveTargets, Addresses,
                                   test_addresses)
    all_targets = transitive_targets.closure

    interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
        (tgt[PythonInterpreterCompatibility] for tgt in all_targets
         if tgt.has_field(PythonInterpreterCompatibility)),
        python_setup,
    )

    # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
    # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
    # from zip files. `zipp` has pathologically bad behavior with large zipfiles.
    # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
    # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
    # and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
    additional_args_for_pytest = ("--not-zip-safe", )

    pytest_pex_request = Get(
        Pex,
        PexRequest(
            output_filename="pytest.pex",
            requirements=PexRequirements(pytest.get_requirement_strings()),
            interpreter_constraints=interpreter_constraints,
            additional_args=additional_args_for_pytest,
            internal_only=True,
        ),
    )

    # Defaults to zip_safe=False.
    requirements_pex_request = Get(
        Pex,
        PexFromTargetsRequest,
        PexFromTargetsRequest.for_requirements(test_addresses,
                                               internal_only=True),
    )

    test_runner_pex_request = Get(
        Pex,
        PexRequest(
            interpreter_constraints=interpreter_constraints,
            output_filename="test_runner.pex",
            entry_point="pytest:main",
            additional_args=(
                "--pex-path",
                # TODO(John Sirois): Support shading python binaries:
                #   https://github.com/pantsbuild/pants/issues/9206
                # Right now any pytest transitive requirements will shadow corresponding user
                # requirements which will lead to problems when APIs that are used by either
                # `pytest:main` or the tests themselves break between the two versions.
                ":".join((
                    pytest_pex_request.subject.output_filename,
                    requirements_pex_request.subject.output_filename,
                )),
            ),
            internal_only=True,
        ),
    )

    prepared_sources_request = Get(
        PythonSourceFiles,
        PythonSourceFilesRequest(all_targets, include_files=True))

    # Get the file names for the test_target so that we can specify to Pytest precisely which files
    # to test, rather than using auto-discovery.
    field_set_source_files_request = Get(
        SourceFiles, SourceFilesRequest([request.field_set.sources]))

    (
        pytest_pex,
        requirements_pex,
        test_runner_pex,
        prepared_sources,
        field_set_source_files,
    ) = await MultiGet(
        pytest_pex_request,
        requirements_pex_request,
        test_runner_pex_request,
        prepared_sources_request,
        field_set_source_files_request,
    )

    input_digest = await Get(
        Digest,
        MergeDigests((
            coverage_config.digest,
            prepared_sources.source_files.snapshot.digest,
            requirements_pex.digest,
            pytest_pex.digest,
            test_runner_pex.digest,
        )),
    )

    add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
    output_files = []

    results_file_name = None
    if pytest.options.junit_xml_dir and not request.is_debug:
        results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
        add_opts.extend((f"--junitxml={results_file_name}", "-o",
                         f"junit_family={pytest.options.junit_family}"))
        output_files.append(results_file_name)

    coverage_args = []
    if test_subsystem.use_coverage and not request.is_debug:
        output_files.append(".coverage")
        cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (
            ".", )
        coverage_args = [
            "--cov-report=",  # Turn off output.
            *itertools.chain.from_iterable(["--cov", cov_path]
                                           for cov_path in cov_paths),
        ]

    extra_env = {
        "PYTEST_ADDOPTS": " ".join(add_opts),
        "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
    }

    if test_subsystem.force and not request.is_debug:
        # This is a slightly hacky way to force the process to run: since the env var
        #  value is unique, this input combination will never have been seen before,
        #  and therefore never cached. The two downsides are:
        #  1. This leaks into the test's environment, albeit with a funky var name that is
        #     unlikely to cause problems in practice.
        #  2. This run will be cached even though it can never be re-used.
        # TODO: A more principled way of forcing rules to run?
        uuid = await Get(UUID, UUIDRequest())
        extra_env["__PANTS_FORCE_TEST_RUN__"] = str(uuid)

    process = await Get(
        Process,
        PexProcess(
            test_runner_pex,
            argv=(*pytest.options.args, *coverage_args,
                  *field_set_source_files.files),
            extra_env=extra_env,
            input_digest=input_digest,
            output_files=output_files,
            timeout_seconds=request.field_set.timeout.
            calculate_from_global_options(pytest),
            execution_slot_variable=pytest.options.execution_slot_var,
            description=f"Run Pytest for {request.field_set.address}",
            level=LogLevel.DEBUG,
        ),
    )
    return TestSetup(process, results_file_name=results_file_name)