async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField)) pylint_pex_get = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=PexRequirements( [*pylint.all_requirements, *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, ), ) prepare_plugin_sources_get = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets)) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies)) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) ( pylint_pex, requirements_pex, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_plugin_sources_get, prepare_python_sources_get, field_set_sources_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), ), Get(ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)), ) prefixed_plugin_sources = (await Get( Digest, AddPrefix( prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests(( config_files.snapshot.digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, )), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description= f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)))
async def setup_scalapb_shim_classfiles( scalapb: ScalaPBSubsystem, jdk: InternalJdk, ) -> ScalaPBShimCompiledClassfiles: dest_dir = "classfiles" scalapb_shim_content = pkgutil.get_data( "pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala") if not scalapb_shim_content: raise AssertionError("Unable to find ScalaParser.scala resource.") scalapb_shim_source = FileContent("ScalaPBShim.scala", scalapb_shim_content) lockfile_request = await Get(GenerateJvmLockfileFromTool, ScalapbcToolLockfileSentinel()) tool_classpath, shim_classpath, source_digest = await MultiGet( Get( ToolClasspath, ToolClasspathRequest( prefix="__toolcp", artifact_requirements=ArtifactRequirements.from_coordinates([ Coordinate( group="org.scala-lang", artifact="scala-compiler", version=SHIM_SCALA_VERSION, ), Coordinate( group="org.scala-lang", artifact="scala-library", version=SHIM_SCALA_VERSION, ), Coordinate( group="org.scala-lang", artifact="scala-reflect", version=SHIM_SCALA_VERSION, ), ]), ), ), Get(ToolClasspath, ToolClasspathRequest(prefix="__shimcp", lockfile=lockfile_request)), Get(Digest, CreateDigest([scalapb_shim_source, Directory(dest_dir)])), ) merged_digest = await Get( Digest, MergeDigests( (tool_classpath.digest, shim_classpath.digest, source_digest))) process_result = await Get( ProcessResult, JvmProcess( jdk=jdk, classpath_entries=tool_classpath.classpath_entries(), argv=[ "scala.tools.nsc.Main", "-bootclasspath", ":".join(tool_classpath.classpath_entries()), "-classpath", ":".join(shim_classpath.classpath_entries()), "-d", dest_dir, scalapb_shim_source.path, ], input_digest=merged_digest, extra_jvm_options=scalapb.jvm_options, output_directories=(dest_dir, ), description="Compile ScalaPB shim with scalac", level=LogLevel.DEBUG, # NB: We do not use nailgun for this process, since it is launched exactly once. use_nailgun=False, ), ) stripped_classfiles_digest = await Get( Digest, RemovePrefix(process_result.output_digest, dest_dir)) return ScalaPBShimCompiledClassfiles(digest=stripped_classfiles_digest)
async def prepare_shell_command_process(request: ShellCommandProcessRequest, shell_setup: ShellSetup, bash: BashBinary) -> Process: shell_command = request.target interactive = shell_command.has_field(ShellCommandRunWorkdirField) if interactive: working_directory = shell_command[ ShellCommandRunWorkdirField].value or "" else: working_directory = shell_command.address.spec_path command = shell_command[ShellCommandCommandField].value timeout = shell_command.get(ShellCommandTimeoutField).value tools = shell_command.get(ShellCommandToolsField, default_raw_value=()).value outputs = shell_command.get(ShellCommandOutputsField).value or () if not command: raise ValueError( f"Missing `command` line in `{shell_command.alias}` target {shell_command.address}." ) if interactive: command_env = { "CHROOT": "{chroot}", } else: if not tools: raise ValueError( f"Must provide any `tools` used by the `{shell_command.alias}` {shell_command.address}." ) env = await Get(Environment, EnvironmentRequest(["PATH"])) search_path = shell_setup.executable_search_path(env) tool_requests = [ BinaryPathRequest( binary_name=tool, search_path=search_path, ) for tool in {*tools, *["mkdir", "ln"]} if tool not in BASH_BUILTIN_COMMANDS ] tool_paths = await MultiGet( Get(BinaryPaths, BinaryPathRequest, request) for request in tool_requests) command_env = { "TOOLS": " ".join( _shell_tool_safe_env_name(tool.binary_name) for tool in tool_requests), } for binary, tool_request in zip(tool_paths, tool_requests): if binary.first_path: command_env[_shell_tool_safe_env_name( tool_request.binary_name)] = binary.first_path.path else: raise BinaryNotFoundError.from_request( tool_request, rationale= f"execute `{shell_command.alias}` {shell_command.address}", ) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([shell_command.address]), ) sources, pkgs_per_target = await MultiGet( Get( SourceFiles, SourceFilesRequest( sources_fields=[ tgt.get(SourcesField) for tgt in transitive_targets.dependencies ], for_sources_types=(SourcesField, FileSourceField), enable_codegen=True, ), ), Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, transitive_targets.dependencies), ), ) packages = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in pkgs_per_target.field_sets) if interactive or not working_directory or working_directory in sources.snapshot.dirs: work_dir = EMPTY_DIGEST else: work_dir = await Get(Digest, CreateDigest([Directory(working_directory)])) input_digest = await Get( Digest, MergeDigests([ sources.snapshot.digest, work_dir, *(pkg.digest for pkg in packages) ])) output_files = [f for f in outputs if not f.endswith("/")] output_directories = [d for d in outputs if d.endswith("/")] if interactive: relpath = os.path.relpath( working_directory or ".", start="/" if os.path.isabs(working_directory) else ".") boot_script = f"cd {shlex.quote(relpath)}; " if relpath != "." else "" else: # Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH, force # symlinks to avoid issues with repeat runs using the __run.sh script in the sandbox. bin_relpath = ".bin" boot_script = ";".join( dedent(f"""\ $mkdir -p {bin_relpath} for tool in $TOOLS; do $ln -sf ${{!tool}} {bin_relpath}; done export PATH="$PWD/{bin_relpath}" """).split("\n")) return Process( argv=(bash.path, "-c", boot_script + command), description=f"Running {shell_command.alias} {shell_command.address}", env=command_env, input_digest=input_digest, output_directories=output_directories, output_files=output_files, timeout_seconds=timeout, working_directory=working_directory, )
async def pylint_lint( field_sets: PylintFieldSets, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.options.skip: return LintResult.noop() # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. addresses = [] for field_set in field_sets: addresses.append(field_set.address) addresses.extend(field_set.dependencies.value or ()) targets = await Get[Targets](Addresses(addresses)) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (field_set.compatibility for field_set in field_sets), python_setup) requirements_pex_request = Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) config_path: Optional[str] = pylint.options.config config_snapshot_request = Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) prepare_python_sources_request = Get[ImportablePythonSources](Targets, targets) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( ((field_set.sources, field_set.origin) for field_set in field_sets), strip_source_roots=True, )) requirements_pex, config_snapshot, prepared_python_sources, specified_source_files = cast( Tuple[Pex, Snapshot, ImportablePythonSources, SourceFiles], await MultiGet([ requirements_pex_request, config_snapshot_request, prepare_python_sources_request, specified_source_files_request, ]), ) input_digest = await Get[Digest](MergeDigests(( requirements_pex.digest, config_snapshot.digest, prepared_python_sources.snapshot.digest, )), ) address_references = ", ".join( sorted(field_set.address.reference() for field_set in field_sets)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_digest=input_digest, description= f"Run Pylint on {pluralize(len(field_sets), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Pylint")
async def setup_shunit2_for_target( request: TestSetupRequest, shell_setup: ShellSetup, test_subsystem: TestSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: shunit2_download_file = DownloadFile( "https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2", FileDigest( "1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5", 40987), ) shunit2_script, transitive_targets, built_package_dependencies, env = await MultiGet( Get(Digest, DownloadFile, shunit2_download_file), Get(TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])), Get( BuiltPackageDependencies, BuildPackageDependenciesRequest( request.field_set.runtime_package_dependencies), ), Get(Environment, EnvironmentRequest(["PATH"])), ) dependencies_source_files_request = Get( SourceFiles, SourceFilesRequest( (tgt.get(SourcesField) for tgt in transitive_targets.dependencies), for_sources_types=(ShellSourceField, FileSourceField, ResourceSourceField), enable_codegen=True, ), ) dependencies_source_files, field_set_sources = await MultiGet( dependencies_source_files_request, Get(SourceFiles, SourceFilesRequest([request.field_set.sources])), ) field_set_digest_content = await Get(DigestContents, Digest, field_set_sources.snapshot.digest) # `ShellTestSourceField` validates that there's exactly one file. test_file_content = field_set_digest_content[0] updated_test_file_content = add_source_shunit2(test_file_content) updated_test_digest, runner = await MultiGet( Get(Digest, CreateDigest([updated_test_file_content])), Get( Shunit2Runner, Shunit2RunnerRequest(request.field_set.address, test_file_content, request.field_set.shell), ), ) input_digest = await Get( Digest, MergeDigests(( shunit2_script, updated_test_digest, dependencies_source_files.snapshot.digest, *(pkg.digest for pkg in built_package_dependencies), )), ) env_dict = { "PATH": create_path_env_var(shell_setup.executable_search_path(env)), "SHUNIT_COLOR": "always" if global_options.colors else "none", **test_extra_env.env, } argv = ( # Zsh requires extra args. See https://github.com/kward/shunit2/#-zsh. [ runner.binary_path.path, "-o", "shwordsplit", "--", *field_set_sources.snapshot.files ] if runner.shell == Shunit2Shell.zsh else [runner.binary_path.path, *field_set_sources.snapshot.files]) cache_scope = (ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL) process = Process( argv=argv, input_digest=input_digest, description=f"Run shunit2 for {request.field_set.address}.", level=LogLevel.DEBUG, env=env_dict, timeout_seconds=request.field_set.timeout.value, cache_scope=cache_scope, ) return TestSetup(process)
async def bandit_lint_partition(partition: BanditPartition, bandit: Bandit, lint_subsystem: LintSubsystem) -> LintResult: requirements_pex_request = Get( Pex, PexRequest( output_filename="bandit.pex", requirements=PexRequirements(bandit.all_requirements), interpreter_constraints=(partition.interpreter_constraints or PexInterpreterConstraints( bandit.interpreter_constraints)), entry_point=bandit.entry_point, ), ) config_digest_request = Get( Digest, PathGlobs( globs=[bandit.config] if bandit.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--bandit-config`", ), ) source_files_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) requirements_pex, config_digest, source_files = await MultiGet( requirements_pex_request, config_digest_request, source_files_request) input_digest = await Get( Digest, MergeDigests((source_files.snapshot.digest, requirements_pex.digest, config_digest))) address_references = ", ".join( sorted(field_set.address.spec for field_set in partition.field_sets)) report_path = (lint_subsystem.reports_dir / "bandit_report.txt" if lint_subsystem.reports_dir else None) args = generate_args( source_files=source_files, bandit=bandit, output_file=report_path.name if report_path else None, ) result = await Get( FallibleProcessResult, PexProcess( requirements_pex, argv=args, input_digest=input_digest, description= (f"Run Bandit on {pluralize(len(partition.field_sets), 'target')}: " f"{address_references}."), output_files=(report_path.name, ) if report_path else None, ), ) results_file = None if report_path: report_file_snapshot = await Get( Snapshot, DigestSubset(result.output_digest, PathGlobs([report_path.name]))) if len(report_file_snapshot.files) != 1: raise Exception( f"Unexpected report file snapshot: {report_file_snapshot.files}" ) results_file = LintResultFile(output_path=report_path, digest=report_file_snapshot.digest) return LintResult.from_fallible_process_result(result, linter_name="Bandit", results_file=results_file)
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`. plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField) ) target_requirements = PexRequirements.create_from_requirement_fields( tgt[PythonRequirementsField] for tgt in partition.targets_with_dependencies if tgt.has_field(PythonRequirementsField) ) pylint_pex_request = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=PexRequirements([*pylint.all_requirements, *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, ), ) requirements_pex_request = Get( Pex, PexRequest( output_filename="requirements.pex", internal_only=True, requirements=target_requirements, interpreter_constraints=partition.interpreter_constraints, ), ) # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any Pylint transitive requirements will shadow corresponding user # requirements, which could lead to problems. pylint_runner_pex_args = ["--pex-path", ":".join(["pylint.pex", "requirements.pex"])] pylint_runner_pex_request = Get( Pex, PexRequest( output_filename="pylint_runner.pex", internal_only=True, entry_point=pylint.entry_point, interpreter_constraints=partition.interpreter_constraints, additional_args=pylint_runner_pex_args, ), ) config_digest_request = Get( Digest, PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", ), ) prepare_plugin_sources_request = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets), ) prepare_python_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies), ) field_set_sources_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets), ) ( pylint_pex, requirements_pex, pylint_runner_pex, config_digest, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_request, requirements_pex_request, pylint_runner_pex_request, config_digest_request, prepare_plugin_sources_request, prepare_python_sources_request, field_set_sources_request, ) prefixed_plugin_sources = ( await Get( Digest, AddPrefix(prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST ) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests( ( pylint_pex.digest, requirements_pex.digest, pylint_runner_pex.digest, config_digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, ) ), ) result = await Get( FallibleProcessResult, PexProcess( pylint_runner_pex, argv=generate_args(source_files=field_set_sources, pylint=pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(partition.interpreter_constraints)) )
async def run_shellcheck(request: ShellcheckRequest, shellcheck: Shellcheck) -> LintResults: if shellcheck.skip: return LintResults([], linter_name="Shellcheck") # Shellcheck looks at direct dependencies to make sure that every symbol is defined, so we must # include those in the run. all_dependencies = await MultiGet( Get(Targets, DependenciesRequest(field_set.dependencies)) for field_set in request.field_sets) direct_sources_get = Get( SourceFiles, SourceFilesRequest( (field_set.sources for field_set in request.field_sets), for_sources_types=(ShellSources, ), enable_codegen=True, ), ) dependency_sources_get = Get( SourceFiles, SourceFilesRequest( (tgt.get(Sources) for dependencies in all_dependencies for tgt in dependencies), for_sources_types=(ShellSources, ), enable_codegen=True, ), ) config_files_get = Get(ConfigFiles, ConfigFilesRequest, shellcheck.config_request) download_shellcheck_get = Get(DownloadedExternalTool, ExternalToolRequest, shellcheck.get_request(Platform.current)) direct_sources, dependency_sources, downloaded_shellcheck, config_files = await MultiGet( direct_sources_get, dependency_sources_get, download_shellcheck_get, config_files_get) input_digest = await Get( Digest, MergeDigests(( direct_sources.snapshot.digest, dependency_sources.snapshot.digest, downloaded_shellcheck.digest, config_files.snapshot.digest, )), ) process_result = await Get( FallibleProcessResult, Process( argv=[ downloaded_shellcheck.exe, *shellcheck.args, *direct_sources.snapshot.files ], input_digest=input_digest, description= f"Run Shellcheck on {pluralize(len(request.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) result = LintResult.from_fallible_process_result(process_result) return LintResults([result], linter_name="Shellcheck")
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: transitive_targets = await Get(TransitiveTargets, Addresses, request.addresses) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get(StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)) input_digests.append( prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) exact_reqs = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) requirements = exact_reqs description = request.description if python_setup.requirement_constraints: # In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraints_file_contents = await Get( DigestContents, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", ), ) constraints_file_reqs = set( parse_requirements( next(iter(constraints_file_contents)).content.decode())) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {python_setup.requirement_constraints} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only): if unconstrained_projects: logger.warning( "Ignoring resolve_all_constraints setting in [python_setup] scope " "because constraints file does not cover all requirements." ) else: requirements = PexRequirements( str(req) for req in constraints_file_reqs) description = description or f"Resolving {python_setup.requirement_constraints}" elif (python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly()): raise ValueError( f"[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " f"[python-setup].requirement_constraints must also be provided.") return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=description, )
async def fmt( console: Console, targets: Targets, fmt_subsystem: FmtSubsystem, workspace: Workspace, union_membership: UnionMembership, ) -> Fmt: language_target_collection_types = union_membership[LanguageFmtTargets] language_target_collections: Iterable[LanguageFmtTargets] = tuple( language_target_collection_type( Targets(target for target in targets if language_target_collection_type.belongs_to_language( target))) for language_target_collection_type in language_target_collection_types) targets_with_sources: Iterable[TargetsWithSources] = await MultiGet( Get( TargetsWithSources, TargetsWithSourcesRequest(language_target_collection.targets), ) for language_target_collection in language_target_collections) # NB: We must convert back the generic TargetsWithSources objects back into their # corresponding LanguageFmtTargets, e.g. back to PythonFmtTargets, in order for the union # rule to work. valid_language_target_collections: Iterable[LanguageFmtTargets] = tuple( language_target_collection_cls( Targets(target for target in language_target_collection.targets if target in language_targets_with_sources)) for language_target_collection_cls, language_target_collection, language_targets_with_sources in zip( language_target_collection_types, language_target_collections, targets_with_sources) if language_targets_with_sources) if fmt_subsystem.per_file_caching: per_language_results = await MultiGet( Get( LanguageFmtResults, LanguageFmtTargets, language_target_collection.__class__(Targets([target])), ) for language_target_collection in valid_language_target_collections for target in language_target_collection.targets) else: per_language_results = await MultiGet( Get(LanguageFmtResults, LanguageFmtTargets, language_target_collection) for language_target_collection in valid_language_target_collections ) individual_results: List[FmtResult] = list( itertools.chain.from_iterable( language_result.results for language_result in per_language_results)) if not individual_results: return Fmt(exit_code=0) changed_digests = tuple(language_result.output for language_result in per_language_results if language_result.did_change) if changed_digests: # NB: this will fail if there are any conflicting changes, which we want to happen rather # than silently having one result override the other. In practicality, this should never # happen due to us grouping each language's formatters into a single digest. merged_formatted_digest = await Get(Digest, MergeDigests(changed_digests)) workspace.write_digest(merged_formatted_digest) if individual_results: console.print_stderr("") # We group all results for the same formatter so that we can give one final status in the # summary. This is only relevant if there were multiple results because of # `--per-file-caching`. formatter_to_results = defaultdict(set) for result in individual_results: formatter_to_results[result.formatter_name].add(result) for formatter, results in sorted(formatter_to_results.items()): if any(result.did_change for result in results): sigil = console.red("𐄂") status = "made changes" elif all(result.skipped for result in results): sigil = console.yellow("-") status = "skipped" else: sigil = console.green("✓") status = "made no changes" console.print_stderr(f"{sigil} {formatter} {status}.") # Since the rules to produce FmtResult should use ExecuteRequest, rather than # FallibleProcess, we assume that there were no failures. return Fmt(exit_code=0)
async def run_cloc( console: Console, cloc_subsystem: CountLinesOfCodeSubsystem, cloc_binary: ClocBinary, sources_snapshot: SourcesSnapshot, ) -> CountLinesOfCode: """Runs the cloc Perl script.""" if not sources_snapshot.snapshot.files: return CountLinesOfCode(exit_code=0) input_files_filename = "input_files.txt" input_file_digest = await Get( Digest, CreateDigest([ FileContent(input_files_filename, "\n".join(sources_snapshot.snapshot.files).encode()) ]), ) downloaded_cloc_binary = await Get( DownloadedExternalTool, ExternalToolRequest, cloc_binary.get_request(Platform.current)) digest = await Get( Digest, MergeDigests((input_file_digest, downloaded_cloc_binary.digest, sources_snapshot.snapshot.digest)), ) report_filename = "report.txt" ignore_filename = "ignored.txt" cmd = ( "/usr/bin/perl", downloaded_cloc_binary.exe, "--skip-uniqueness", # Skip the file uniqueness check. f"--ignored={ignore_filename}", # Write the names and reasons of ignored files to this file. f"--report-file={report_filename}", # Write the output to this file rather than stdout. f"--list-file={input_files_filename}", # Read an exhaustive list of files to process from this file. ) req = Process( argv=cmd, input_digest=digest, output_files=(report_filename, ignore_filename), description= (f"Count lines of code for {pluralize(len(sources_snapshot.snapshot.files), 'file')}" ), level=LogLevel.DEBUG, ) exec_result = await Get(ProcessResult, Process, req) report_digest_contents = await Get(DigestContents, Digest, exec_result.output_digest) reports = { file_content.path: file_content.content.decode() for file_content in report_digest_contents } for line in reports[report_filename].splitlines(): console.print_stdout(line) if cloc_subsystem.ignored: console.print_stderr("\nIgnored the following files:") for line in reports[ignore_filename].splitlines(): console.print_stderr(line) return CountLinesOfCode(exit_code=0)
async def generate_python_from_protobuf( request: GeneratePythonFromProtobufRequest, protoc: Protoc) -> GeneratedSources: download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)) output_dir = "_generated_files" create_output_dir_request = Get(Digest, CreateDigest([Directory(output_dir)])) # Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't # actually generate those dependencies; it only needs to look at their .proto files to work # with imports. transitive_targets = await Get( TransitiveTargets, Addresses([request.protocol_target.address])) # NB: By stripping the source roots, we avoid having to set the value `--proto_path` # for Protobuf imports to be discoverable. all_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.closure), for_sources_types=(ProtobufSources, ), ), ) target_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSources]])) ( downloaded_protoc_binary, empty_output_dir, all_sources_stripped, target_sources_stripped, ) = await MultiGet( download_protoc_request, create_output_dir_request, all_stripped_sources_request, target_stripped_sources_request, ) input_digest = await Get( Digest, MergeDigests(( all_sources_stripped.snapshot.digest, downloaded_protoc_binary.digest, empty_output_dir, )), ) result = await Get( ProcessResult, Process( ( downloaded_protoc_binary.exe, "--python_out", output_dir, *target_sources_stripped.snapshot.files, ), input_digest=input_digest, description= f"Generating Python sources from {request.protocol_target.address}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), ), ) # We must do some path manipulation on the output digest for it to look like normal sources, # including adding back a source root. py_source_root = request.protocol_target.get(PythonSourceRootField).value if py_source_root: # Verify that the python source root specified by the target is in fact a source root. source_root_request = SourceRootRequest(PurePath(py_source_root)) else: # The target didn't specify a python source root, so use the protobuf_library's source root. source_root_request = SourceRootRequest.for_target( request.protocol_target) normalized_digest, source_root = await MultiGet( Get(Digest, RemovePrefix(result.output_digest, output_dir)), Get(SourceRoot, SourceRootRequest, source_root_request), ) source_root_restored = (await Get( Snapshot, AddPrefix(normalized_digest, source_root.path)) if source_root.path != "." else await Get( Snapshot, Digest, normalized_digest)) return GeneratedSources(source_root_restored)
async def run_tests( console: Console, test_subsystem: TestSubsystem, interactive_runner: InteractiveRunner, workspace: Workspace, union_membership: UnionMembership, ) -> Test: if test_subsystem.debug: targets_to_valid_field_sets = await Get( TargetsToValidFieldSets, TargetsToValidFieldSetsRequest( TestFieldSet, goal_description="`test --debug`", error_if_no_valid_targets=True, expect_single_field_set=True, ), ) field_set = targets_to_valid_field_sets.field_sets[0] request = await Get(TestDebugRequest, TestFieldSet, field_set) debug_result = interactive_runner.run(request.process) return Test(debug_result.exit_code) targets_to_valid_field_sets = await Get( TargetsToValidFieldSets, TargetsToValidFieldSetsRequest( TestFieldSet, goal_description=f"the `{test_subsystem.name}` goal", error_if_no_valid_targets=False, ), ) field_sets_with_sources = await Get( FieldSetsWithSources, FieldSetsWithSourcesRequest(targets_to_valid_field_sets.field_sets)) results = await MultiGet( Get(AddressAndTestResult, WrappedTestFieldSet(field_set)) for field_set in field_sets_with_sources) # Print details. for result in results: if test_subsystem.options.output == ShowOutput.NONE or ( test_subsystem.options.output == ShowOutput.FAILED and result.test_result.status == Status.SUCCESS): continue has_output = result.test_result.stdout or result.test_result.stderr if has_output: status = (console.green("✓") if result.test_result.status == Status.SUCCESS else console.red("𐄂")) console.print_stderr(f"{status} {result.address}") if result.test_result.stdout: console.print_stderr(result.test_result.stdout) if result.test_result.stderr: console.print_stderr(result.test_result.stderr) if has_output and result != results[-1]: console.print_stderr("") # Print summary console.print_stderr("") for result in results: color = console.green if result.test_result.status == Status.SUCCESS else console.red # The right-align logic sees the color control codes as characters, so we have # to account for that. In f-strings the alignment field widths must be literals, # so we have to indirect via a call to .format(). right_align = 19 if console.use_colors else 10 format_str = f"{{addr:80}}.....{{result:>{right_align}}}" console.print_stderr( format_str.format(addr=result.address.spec, result=color(result.test_result.status.value))) merged_xml_results = await Get( Digest, MergeDigests(result.test_result.xml_results for result in results if result.test_result.xml_results), ) workspace.write_digest(merged_xml_results) if test_subsystem.use_coverage: all_coverage_data: Iterable[CoverageData] = [ result.test_result.coverage_data for result in results if result.test_result.coverage_data is not None ] coverage_types_to_collection_types: Dict[ Type[CoverageData], Type[CoverageDataCollection]] = { collection_cls.element_type: collection_cls for collection_cls in union_membership.get( CoverageDataCollection) } coverage_collections: List[CoverageDataCollection] = [] for data_cls, data in itertools.groupby(all_coverage_data, lambda data: type(data)): collection_cls = coverage_types_to_collection_types[data_cls] coverage_collections.append(collection_cls(data)) # We can create multiple reports for each coverage data (console, xml and html) coverage_reports_collections = await MultiGet( Get(CoverageReports, CoverageDataCollection, coverage_collection) for coverage_collection in coverage_collections) coverage_report_files: List[PurePath] = [] for coverage_reports in coverage_reports_collections: report_files = coverage_reports.materialize(console, workspace) coverage_report_files.extend(report_files) if coverage_report_files and test_subsystem.open_coverage: desktop.ui_open(console, interactive_runner, coverage_report_files) exit_code = (PANTS_FAILED_EXIT_CODE if any( res.test_result.status == Status.FAILURE for res in results) else PANTS_SUCCEEDED_EXIT_CODE) return Test(exit_code)
async def create_docker_build_context( request: DockerBuildContextRequest, docker_options: DockerOptions) -> DockerBuildContext: # Get all targets to include in context. transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([request.address])) docker_image = transitive_targets.roots[0] # Get all dependencies for the root target. root_dependencies = await Get( Targets, DependenciesRequest(docker_image.get(Dependencies))) # Get all file sources from the root dependencies. That includes any non-file sources that can # be "codegen"ed into a file source. sources_request = Get( SourceFiles, SourceFilesRequest( sources_fields=[ tgt.get(SourcesField) for tgt in root_dependencies ], for_sources_types=( DockerContextFilesSourcesField, FileSourceField, ), enable_codegen=True, ), ) embedded_pkgs_per_target_request = Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, transitive_targets.dependencies), ) sources, embedded_pkgs_per_target, dockerfile_info = await MultiGet( sources_request, embedded_pkgs_per_target_request, Get(DockerfileInfo, DockerfileInfoRequest(docker_image.address)), ) # Package binary dependencies for build context. embedded_pkgs = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in embedded_pkgs_per_target.field_sets # Exclude docker images, unless build_upstream_images is true. if request.build_upstream_images or not isinstance( getattr(field_set, "source", None), DockerImageSourceField)) if request.build_upstream_images: images_str = ", ".join(a.tags[0] for p in embedded_pkgs for a in p.artifacts if isinstance(a, BuiltDockerImage)) if images_str: logger.debug(f"Built upstream Docker images: {images_str}") else: logger.debug("Did not build any upstream Docker images") packages_str = ", ".join(a.relpath for p in embedded_pkgs for a in p.artifacts if a.relpath) if packages_str: logger.debug(f"Built packages for Docker image: {packages_str}") else: logger.debug("Did not build any packages for Docker image") embedded_pkgs_digest = [ built_package.digest for built_package in embedded_pkgs ] all_digests = (dockerfile_info.digest, sources.snapshot.digest, *embedded_pkgs_digest) # Merge all digests to get the final docker build context digest. context_request = Get(Snapshot, MergeDigests(d for d in all_digests if d)) # Requests for build args and env build_args_request = Get(DockerBuildArgs, DockerBuildArgsRequest(docker_image)) build_env_request = Get(DockerBuildEnvironment, DockerBuildEnvironmentRequest(docker_image)) context, build_args, build_env = await MultiGet(context_request, build_args_request, build_env_request) if request.build_upstream_images: # Update build arg values for FROM image build args. # Get the FROM image build args with defined values in the Dockerfile. dockerfile_build_args = { k: v for k, v in dockerfile_info.from_image_build_args.to_dict().items() if v } # Parse the build args values into Address instances. from_image_addresses = await Get( Addresses, UnparsedAddressInputs( dockerfile_build_args.values(), owning_address=dockerfile_info.address, description_of_origin="TODO(#14468)", ), ) # Map those addresses to the corresponding built image ref (tag). address_to_built_image_tag = { field_set.address: image.tags[0] for field_set, built in zip(embedded_pkgs_per_target.field_sets, embedded_pkgs) for image in built.artifacts if isinstance(image, BuiltDockerImage) } # Create the FROM image build args. from_image_build_args = [ f"{arg_name}={address_to_built_image_tag[addr]}" for arg_name, addr in zip(dockerfile_build_args.keys(), from_image_addresses) ] # Merge all build args. build_args = DockerBuildArgs.from_strings(*build_args, *from_image_build_args) return DockerBuildContext.create( build_args=build_args, snapshot=context, dockerfile_info=dockerfile_info, build_env=build_env, )
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) addresses = [field_set.address] interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest(addresses)) pex_filename = (field_set.address.generated_name.replace(".", "_") if field_set.address.generated_name else field_set.address.target_name) pex_get = Get( Pex, PexFromTargetsRequest( [field_set.address], output_filename=f"{pex_filename}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=interpreter_constraints, sources=sources, ), ) merged_digest = await Get( Digest, MergeDigests([ pex.digest, local_dists.pex.digest, local_dists.remaining_sources.source_files.snapshot.digest, ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pytest_for_target( config: PythonTestConfiguration, pytest: PyTest, test_options: TestOptions, python_setup: PythonSetup, ) -> TestTargetSetup: # TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage # configuration via #7490. test_addresses = Addresses((config.address, )) transitive_targets = await Get[TransitiveTargets](Addresses, test_addresses) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) # Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints # of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for # CreatePex requests. pex_request = functools.partial( PexRequest, interpreter_constraints=interpreter_constraints) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe", ) run_coverage = test_options.values.run_coverage plugin_file_digest: Optional[Digest] = (await Get[Digest]( InputFilesContent, COVERAGE_PLUGIN_INPUT) if run_coverage else None) pytest_pex_request = pex_request( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), additional_args=additional_args_for_pytest, sources=plugin_file_digest, ) requirements_pex_request = PexFromTargetsRequest( addresses=test_addresses, output_filename="requirements.pex", include_source_files=False, additional_args=additional_args_for_pytest, ) test_runner_pex_request = pex_request( output_filename="test_runner.pex", entry_point="pytest:main", interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. ":".join((pytest_pex_request.output_filename, requirements_pex_request.output_filename)), ), ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. specified_source_files_request = SpecifiedSourceFilesRequest( [(config.sources, config.origin)], strip_source_roots=True) # TODO(John Sirois): Support exploiting concurrency better: # https://github.com/pantsbuild/pants/issues/9294 # Some awkward code follows in order to execute 5-6 items concurrently given the current state # of MultiGet typing / API. Improve this since we should encourage full concurrency in general. requests: List[Get[Any]] = [ Get[Pex](PexRequest, pytest_pex_request), Get[Pex](PexFromTargetsRequest, requirements_pex_request), Get[Pex](PexRequest, test_runner_pex_request), Get[ImportablePythonSources](Targets(all_targets)), Get[SourceFiles](SpecifiedSourceFilesRequest, specified_source_files_request), ] if run_coverage: requests.append( Get[CoverageConfig](CoverageConfigRequest( Targets((tgt for tgt in all_targets if tgt.has_field(PythonSources))), is_test_time=True, )), ) ( pytest_pex, requirements_pex, test_runner_pex, prepared_sources, specified_source_files, *rest, ) = cast( Union[Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles], Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles, CoverageConfig], ], await MultiGet(requests), ) digests_to_merge = [ prepared_sources.snapshot.digest, requirements_pex.digest, pytest_pex.digest, test_runner_pex.digest, ] if run_coverage: coverage_config = rest[0] digests_to_merge.append(coverage_config.digest) merged_input_files = await Get[Digest](MergeDigests(digests_to_merge)) coverage_args = [] if run_coverage: coverage_args = [ "--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html ] for package in config.coverage.determine_packages_to_cover( specified_source_files=specified_source_files): coverage_args.extend(["--cov", package]) specified_source_file_names = sorted(specified_source_files.snapshot.files) return TestTargetSetup( test_runner_pex=test_runner_pex, args=(*pytest.options.args, *coverage_args, *specified_source_file_names), input_files_digest=merged_input_files, timeout_seconds=config.timeout.calculate_from_global_options(pytest), xml_dir=pytest.options.junit_xml_dir, junit_family=pytest.options.junit_family, )
async def run_tests( console: Console, test_subsystem: TestSubsystem, interactive_runner: InteractiveRunner, workspace: Workspace, union_membership: UnionMembership, ) -> Test: if test_subsystem.debug: targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( TestFieldSet, goal_description="`test --debug`", no_applicable_targets_behavior=NoApplicableTargetsBehavior. error, ), ) debug_requests = await MultiGet( Get(TestDebugRequest, TestFieldSet, field_set) for field_set in targets_to_valid_field_sets.field_sets) exit_code = 0 for debug_request in debug_requests: if debug_request.process is None: continue debug_result = interactive_runner.run(debug_request.process) if debug_result.exit_code != 0: exit_code = debug_result.exit_code return Test(exit_code) targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( TestFieldSet, goal_description=f"the `{test_subsystem.name}` goal", no_applicable_targets_behavior=NoApplicableTargetsBehavior.warn, ), ) field_sets_with_sources = await Get( FieldSetsWithSources, FieldSetsWithSourcesRequest(targets_to_valid_field_sets.field_sets)) results = await MultiGet( Get(EnrichedTestResult, TestFieldSet, field_set) for field_set in field_sets_with_sources) # Print summary. exit_code = 0 if results: console.print_stderr("") for result in sorted(results): if result.skipped: continue if result.exit_code == 0: sigil = console.green("✓") status = "succeeded" else: sigil = console.red("𐄂") status = "failed" exit_code = cast(int, result.exit_code) console.print_stderr(f"{sigil} {result.address} {status}.") merged_xml_results = await Get( Digest, MergeDigests(result.xml_results.digest for result in results if result.xml_results), ) workspace.write_digest(merged_xml_results) if test_subsystem.use_coverage: # NB: We must pre-sort the data for itertools.groupby() to work properly, using the same # key function for both. However, you can't sort by `types`, so we call `str()` on it. all_coverage_data = sorted( (result.coverage_data for result in results if result.coverage_data is not None), key=lambda cov_data: str(type(cov_data)), ) coverage_types_to_collection_types: Dict[ Type[CoverageData], Type[CoverageDataCollection]] = { collection_cls.element_type: collection_cls for collection_cls in union_membership.get( CoverageDataCollection) } coverage_collections: List[CoverageDataCollection] = [] for data_cls, data in itertools.groupby(all_coverage_data, lambda data: type(data)): collection_cls = coverage_types_to_collection_types[data_cls] coverage_collections.append(collection_cls(data)) # We can create multiple reports for each coverage data (console, xml and html) coverage_reports_collections = await MultiGet( Get(CoverageReports, CoverageDataCollection, coverage_collection) for coverage_collection in coverage_collections) coverage_report_files: List[PurePath] = [] for coverage_reports in coverage_reports_collections: report_files = coverage_reports.materialize(console, workspace) coverage_report_files.extend(report_files) if coverage_report_files and test_subsystem.open_coverage: open_files = await Get( OpenFiles, OpenFilesRequest(coverage_report_files, error_if_open_not_found=False)) for process in open_files.processes: interactive_runner.run(process) return Test(exit_code)
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot: exported_target = request.exported_target exported_addr = exported_target.target.address owned_deps, transitive_targets = await MultiGet( Get(OwnedDependencies, DependencyOwner(exported_target)), Get(TransitiveTargets, TransitiveTargetsRequest([exported_target.target.address])), ) # files() targets aren't owned by a single exported target - they aren't code, so # we allow them to be in multiple dists. This is helpful for, e.g., embedding # a standard license file in a dist. files_targets = (tgt for tgt in transitive_targets.closure if tgt.has_field(FilesSources)) targets = Targets(itertools.chain((od.target for od in owned_deps), files_targets)) sources, requirements = await MultiGet( Get(SetupPySources, SetupPySourcesRequest(targets, py2=request.py2)), Get(ExportedTargetRequirements, DependencyOwner(exported_target)), ) # Generate the kwargs for the setup() call. In addition to using the kwargs that are either # explicitly provided or generated via a user's plugin, we add additional kwargs based on the # resolved requirements and sources. target = exported_target.target resolved_setup_kwargs = await Get(SetupKwargs, ExportedTarget, exported_target) setup_kwargs = resolved_setup_kwargs.kwargs.copy() # NB: We are careful to not overwrite these values, but we also don't expect them to have been # set. The user must have have gone out of their way to use a `SetupKwargs` plugin, and to have # specified `SetupKwargs(_allow_banned_keys=True)`. setup_kwargs.update( { "package_dir": {"": CHROOT_SOURCE_ROOT, **setup_kwargs.get("package_dir", {})}, "packages": (*sources.packages, *(setup_kwargs.get("packages", []))), "namespace_packages": ( *sources.namespace_packages, *setup_kwargs.get("namespace_packages", []), ), "package_data": {**dict(sources.package_data), **setup_kwargs.get("package_data", {})}, "install_requires": (*requirements, *setup_kwargs.get("install_requires", [])), } ) # Add any `pex_binary` targets from `setup_py().with_binaries()` to the dist's entry points. key_to_binary_spec = exported_target.provides.binaries binaries = await Get( Targets, UnparsedAddressInputs(key_to_binary_spec.values(), owning_address=target.address) ) entry_point_requests = [] for binary in binaries: if not binary.has_fields([PexEntryPointField, DeprecatedPexBinarySources]): raise InvalidEntryPoint( "Expected addresses to `pex_binary` targets in `.with_binaries()` for the " f"`provides` field for {exported_addr}, but found {binary.address} with target " f"type {binary.alias}." ) entry_point = binary[PexEntryPointField].value url = "https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point" if not entry_point: raise InvalidEntryPoint( "Every `pex_binary` used in `.with_binaries()` for the `provides` field for " f"{exported_addr} must explicitly set the `entry_point` field, but " f"{binary.address} left the field off. Set `entry_point` to either " f"`app.py:func` or the longhand `path.to.app:func`. See {url}." ) if ":" not in entry_point: raise InvalidEntryPoint( "Every `pex_binary` used in `with_binaries()` for the `provides()` field for " f"{exported_addr} must end in the format `:my_func` for the `entry_point` field, " f"but {binary.address} set it to {repr(entry_point)}. For example, set " f"`entry_point='{entry_point}:main'. See {url}." ) entry_point_requests.append( ResolvePexEntryPointRequest( binary[PexEntryPointField], binary[DeprecatedPexBinarySources] ) ) binary_entry_points = await MultiGet( Get(ResolvedPexEntryPoint, ResolvePexEntryPointRequest, request) for request in entry_point_requests ) for key, binary_entry_point in zip(key_to_binary_spec.keys(), binary_entry_points): entry_points = setup_kwargs["entry_points"] = setup_kwargs.get("entry_points", {}) console_scripts = entry_points["console_scripts"] = entry_points.get("console_scripts", []) console_scripts.append(f"{key}={binary_entry_point.val}") # Generate the setup script. setup_py_content = SETUP_BOILERPLATE.format( target_address_spec=target.address.spec, setup_kwargs_str=distutils_repr(setup_kwargs), ).encode() files_to_create = [ FileContent("setup.py", setup_py_content), FileContent("MANIFEST.in", "include *.py".encode()), ] extra_files_digest, src_digest = await MultiGet( Get(Digest, CreateDigest(files_to_create)), # Nest the sources under the src/ prefix. Get(Digest, AddPrefix(sources.digest, CHROOT_SOURCE_ROOT)), ) chroot_digest = await Get(Digest, MergeDigests((src_digest, extra_files_digest))) return SetupPyChroot(chroot_digest, FinalizedSetupKwargs(setup_kwargs, address=target.address))
async def setup_pytest_for_target( field_set: PythonTestFieldSet, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, ) -> TestTargetSetup: test_addresses = Addresses((field_set.address,)) transitive_targets = await Get(TransitiveTargets, Addresses, test_addresses) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( ( tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility) ), python_setup, ) # Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints # of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for # CreatePex requests. pex_request = functools.partial(PexRequest, interpreter_constraints=interpreter_constraints) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe",) pytest_pex_request = Get( Pex, PexRequest, pex_request( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), additional_args=additional_args_for_pytest, ), ) requirements_pex_request = Get( Pex, PexFromTargetsRequest( addresses=test_addresses, output_filename="requirements.pex", include_source_files=False, additional_args=additional_args_for_pytest, ), ) test_runner_pex_request = Get( Pex, PexRequest, pex_request( output_filename="test_runner.pex", entry_point="pytest:main", interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. ":".join( ( pytest_pex_request.subject.output_filename, requirements_pex_request.subject.output_filename, ) ), ), ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True) ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get(SourceFiles, SourceFilesRequest([field_set.sources])) ( pytest_pex, requirements_pex, test_runner_pex, prepared_sources, field_set_source_files, ) = await MultiGet( pytest_pex_request, requirements_pex_request, test_runner_pex_request, prepared_sources_request, field_set_source_files_request, ) input_digest = await Get( Digest, MergeDigests( ( coverage_config.digest, prepared_sources.source_files.snapshot.digest, requirements_pex.digest, pytest_pex.digest, test_runner_pex.digest, ) ), ) coverage_args = [] if test_subsystem.use_coverage: cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (".",) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] return TestTargetSetup( test_runner_pex=test_runner_pex, args=(*pytest.options.args, *coverage_args, *field_set_source_files.files), input_digest=input_digest, source_roots=prepared_sources.source_roots, timeout_seconds=field_set.timeout.calculate_from_global_options(pytest), xml_dir=pytest.options.junit_xml_dir, junit_family=pytest.options.junit_family, execution_slot_variable=pytest.options.execution_slot_var, )
async def create_pex_from_targets(request: PexFromTargetsRequest) -> PexRequest: interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest, request.to_interpreter_constraints_request(), ) transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) sources_digests = [] if request.additional_sources: sources_digests.append(request.additional_sources) if request.include_source_files: sources = await Get(PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure)) else: sources = PythonSourceFiles.empty() additional_inputs_digests = [] if request.additional_inputs: additional_inputs_digests.append(request.additional_inputs) additional_args = request.additional_args if request.include_local_dists: local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( request.addresses, internal_only=request.internal_only, interpreter_constraints=interpreter_constraints, sources=sources, ), ) remaining_sources = local_dists.remaining_sources additional_inputs_digests.append(local_dists.pex.digest) additional_args += ("--requirements-pex", local_dists.pex.name) else: remaining_sources = sources remaining_sources_stripped = await Get( StrippedPythonSourceFiles, PythonSourceFiles, remaining_sources ) sources_digests.append(remaining_sources_stripped.stripped_source_files.snapshot.digest) merged_sources_digest, additional_inputs = await MultiGet( Get(Digest, MergeDigests(sources_digests)), Get(Digest, MergeDigests(additional_inputs_digests)), ) description = request.description if request.include_requirements: requirements = await Get(PexRequirements, _PexRequirementsRequest(request.addresses)) else: requirements = PexRequirements() if requirements: repository_pex = await Get( OptionalPex, _RepositoryPexRequest( request.addresses, requirements=requirements, hardcoded_interpreter_constraints=request.hardcoded_interpreter_constraints, platforms=request.platforms, complete_platforms=request.complete_platforms, internal_only=request.internal_only, additional_lockfile_args=request.additional_lockfile_args, ), ) requirements = dataclasses.replace(requirements, repository_pex=repository_pex.maybe_pex) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, layout=request.layout, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, complete_platforms=request.complete_platforms, main=request.main, sources=merged_sources_digest, additional_inputs=additional_inputs, additional_args=additional_args, description=description, )
async def generate_python_from_protobuf( request: GeneratePythonFromProtobufRequest, protoc: Protoc, grpc_python_plugin: GrpcPythonPlugin, python_protobuf_subsystem: PythonProtobufSubsystem, ) -> GeneratedSources: download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)) output_dir = "_generated_files" create_output_dir_request = Get(Digest, CreateDigest([Directory(output_dir)])) # Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't # actually generate those dependencies; it only needs to look at their .proto files to work # with imports. transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.protocol_target.address])) # NB: By stripping the source roots, we avoid having to set the value `--proto_path` # for Protobuf imports to be discoverable. all_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.closure), for_sources_types=(ProtobufSources, ), ), ) target_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSources]])) ( downloaded_protoc_binary, empty_output_dir, all_sources_stripped, target_sources_stripped, ) = await MultiGet( download_protoc_request, create_output_dir_request, all_stripped_sources_request, target_stripped_sources_request, ) protoc_gen_mypy_script = "protoc-gen-mypy" mypy_pex = None if python_protobuf_subsystem.mypy_plugin: mypy_pex = await Get( VenvPex, VenvPexRequest( bin_names=[protoc_gen_mypy_script], pex_request=PexRequest( output_filename="mypy_protobuf.pex", internal_only=True, requirements=PexRequirements( [python_protobuf_subsystem.mypy_plugin_version]), # TODO(John Sirois): Fix these interpreter constraints to track the actual # python requirement of the mypy_plugin_version or else plumb an option for # manually setting the constraint to track what mypy_plugin_version needs: # https://github.com/pantsbuild/pants/issues/11565 # Here we guess a constraint that will likely work with any mypy_plugin_version # selected. interpreter_constraints=PexInterpreterConstraints( ["CPython>=3.5"]), ), ), ) downloaded_grpc_plugin = (await Get( DownloadedExternalTool, ExternalToolRequest, grpc_python_plugin.get_request(Platform.current), ) if request.protocol_target.get(ProtobufGrpcToggle).value else None) unmerged_digests = [ all_sources_stripped.snapshot.digest, downloaded_protoc_binary.digest, empty_output_dir, ] if mypy_pex: unmerged_digests.append(mypy_pex.digest) if downloaded_grpc_plugin: unmerged_digests.append(downloaded_grpc_plugin.digest) input_digest = await Get(Digest, MergeDigests(unmerged_digests)) argv = [downloaded_protoc_binary.exe, "--python_out", output_dir] if mypy_pex: argv.extend([ f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}", "--mypy_out", output_dir, ]) if downloaded_grpc_plugin: argv.extend([ f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}", "--grpc_out", output_dir ]) argv.extend(target_sources_stripped.snapshot.files) result = await Get( ProcessResult, Process( argv, input_digest=input_digest, description= f"Generating Python sources from {request.protocol_target.address}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), ), ) # We must do some path manipulation on the output digest for it to look like normal sources, # including adding back a source root. py_source_root = request.protocol_target.get(PythonSourceRootField).value if py_source_root: # Verify that the python source root specified by the target is in fact a source root. source_root_request = SourceRootRequest(PurePath(py_source_root)) else: # The target didn't specify a python source root, so use the protobuf_library's source root. source_root_request = SourceRootRequest.for_target( request.protocol_target) normalized_digest, source_root = await MultiGet( Get(Digest, RemovePrefix(result.output_digest, output_dir)), Get(SourceRoot, SourceRootRequest, source_root_request), ) source_root_restored = (await Get( Snapshot, AddPrefix(normalized_digest, source_root.path)) if source_root.path != "." else await Get( Snapshot, Digest, normalized_digest)) return GeneratedSources(source_root_restored)
async def run_shellcheck(request: ShellcheckRequest, shellcheck: Shellcheck) -> LintResults: if shellcheck.options.skip: return LintResults([], linter_name="Shellcheck") # Shellcheck looks at direct dependencies to make sure that every symbol is defined, so we must # include those in the run. all_dependencies = await MultiGet( Get(Targets, DependenciesRequest(field_set.dependencies)) for field_set in request.field_sets) # Now that we have all dependencies, we flatten the results into a single list of `BashSources` # fields and we filter out all targets without a `BashSources` field registered because those # dependencies are irrelevant to Shellcheck. dependencies_sources_fields = [ tgt[BashSources] for dependencies in all_dependencies for tgt in dependencies if tgt.has_field(BashSources) ] sources_request = Get( SourceFiles, SourceFilesRequest([ *(field_set.sources for field_set in request.field_sets), *dependencies_sources_fields, ]), ) download_shellcheck_request = Get( DownloadedExternalTool, ExternalToolRequest, shellcheck.get_request(Platform.current), ) # If the user specified `--shellcheck-config`, we search for the file they specified with # `PathGlobs` to include it in the `input_digest`. We error if the file cannot be found. # See https://www.pantsbuild.org/v2.0/docs/rules-api-file-system. config_digest_request = Get( Digest, PathGlobs( globs=[shellcheck.options.config] if shellcheck.options.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--shellcheck-config`", ), ) sources, downloaded_shellcheck, config_digest = await MultiGet( sources_request, download_shellcheck_request, config_digest_request) # The Process needs one single `Digest`, so we merge everything together. See # https://www.pantsbuild.org/v2.0/docs/rules-api-file-system. input_digest = await Get( Digest, MergeDigests((sources.snapshot.digest, downloaded_shellcheck.digest, config_digest)), ) # We use `FallibleProcessResult`, rather than `ProcessResult`, because we're okay with the # Process failing. process_result = await Get( FallibleProcessResult, Process( argv=[ downloaded_shellcheck.exe, *shellcheck.options.args, *sources.snapshot.files, ], input_digest=input_digest, description= f"Run Shellcheck on {pluralize(len(request.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) result = LintResult.from_fallible_process_result(process_result) return LintResults([result], linter_name="Shellcheck")
async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> HelmChart: dependencies, source_files, metadata = await MultiGet( Get(Targets, DependenciesRequest(request.field_set.dependencies)), Get( HelmChartSourceFiles, HelmChartSourceFilesRequest, HelmChartSourceFilesRequest.for_field_set( request.field_set, include_metadata=False, include_resources=True, include_files=True, ), ), Get(HelmChartMetadata, HelmChartMetaSourceField, request.field_set.chart), ) third_party_artifacts = await Get( FetchedHelmArtifacts, FetchHelmArfifactsRequest, FetchHelmArfifactsRequest.for_targets( dependencies, description_of_origin=request.field_set.address.spec), ) first_party_subcharts = await MultiGet( Get(HelmChart, HelmChartRequest, HelmChartRequest.from_target(target)) for target in dependencies if HelmChartFieldSet.is_applicable(target)) third_party_charts = await MultiGet( Get(HelmChart, FetchedHelmArtifact, artifact) for artifact in third_party_artifacts) subcharts = [*first_party_subcharts, *third_party_charts] subcharts_digest = EMPTY_DIGEST if subcharts: logger.debug( f"Found {pluralize(len(subcharts), 'subchart')} as direct dependencies on Helm chart at: {request.field_set.address}" ) merged_subcharts = await Get( Digest, MergeDigests([chart.snapshot.digest for chart in subcharts])) subcharts_digest = await Get(Digest, AddPrefix(merged_subcharts, "charts")) # Update subchart dependencies in the metadata and re-render it. remotes = subsystem.remotes() subchart_map: dict[str, HelmChart] = { chart.metadata.name: chart for chart in subcharts } updated_dependencies: OrderedSet[HelmChartDependency] = OrderedSet() for dep in metadata.dependencies: updated_dep = dep if not dep.repository and remotes.default_registry: # If the dependency hasn't specified a repository, then we choose the registry with the 'default' alias. default_remote = remotes.default_registry updated_dep = dataclasses.replace( updated_dep, repository=default_remote.address) elif dep.repository and dep.repository.startswith("@"): remote = next(remotes.get(dep.repository)) updated_dep = dataclasses.replace(updated_dep, repository=remote.address) if dep.name in subchart_map: updated_dep = dataclasses.replace( updated_dep, version=subchart_map[dep.name].metadata.version) updated_dependencies.add(updated_dep) # Include the explicitly provided subchats in the set of dependencies if not already present. updated_dependencies_names = {dep.name for dep in updated_dependencies} remaining_subcharts = [ chart for chart in subcharts if chart.metadata.name not in updated_dependencies_names ] for chart in remaining_subcharts: if chart.artifact: dependency = HelmChartDependency( name=chart.artifact.name, version=chart.artifact.version, repository=chart.artifact.location_url, ) else: dependency = HelmChartDependency( name=chart.metadata.name, version=chart.metadata.version) updated_dependencies.add(dependency) # Update metadata with the information about charts' dependencies. metadata = dataclasses.replace( metadata, dependencies=tuple(updated_dependencies)) # Re-render the Chart.yaml file with the updated dependencies. metadata_digest, sources_without_metadata = await MultiGet( Get(Digest, HelmChartMetadata, metadata), Get( Digest, DigestSubset( source_files.snapshot.digest, PathGlobs([ "**/*", *(f"!**/{filename}" for filename in HELM_CHART_METADATA_FILENAMES) ]), ), ), ) # Merge all digests that conform chart's content. content_digest = await Get( Digest, MergeDigests( [metadata_digest, sources_without_metadata, subcharts_digest])) chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, metadata.name)) return HelmChart(address=request.field_set.address, metadata=metadata, snapshot=chart_snapshot)
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) # Defaults to zip_safe=False. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_request = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, entry_point="pytest:main", internal_only=True, additional_args=( # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import # magic directly from zip files. `zipp` has pathologically bad behavior with large # zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. # Long term, it would be better to fix Zipp (whose fix would then need to be used # by importlib_metadata and then by Pytest). See # https://github.com/jaraco/zipp/pull/26. "--not-zip-safe", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 "--pex-path", requirements_pex_request.input.output_filename, ), ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) # Create any assets that the test depends on through the `runtime_package_dependencies` field. assets: Tuple[BuiltPackage, ...] = () unparsed_runtime_packages = (request.field_set.runtime_package_dependencies .to_unparsed_address_inputs()) if unparsed_runtime_packages.values: runtime_package_targets = await Get(Targets, UnparsedAddressInputs, unparsed_runtime_packages) field_sets_per_target = await Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets), ) assets = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in field_sets_per_target.field_sets) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) pytest_pex, requirements_pex, prepared_sources, field_set_source_files = await MultiGet( pytest_pex_request, requirements_pex_request, prepared_sources_request, field_set_source_files_request, ) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, requirements_pex.digest, pytest_pex.digest, *(binary.digest for binary in assets), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), } extra_env.update(test_extra_env.env) # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, PexProcess( pytest_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def generate_scala_from_protobuf( request: GenerateScalaFromProtobufRequest, protoc: Protoc, scalapb: ScalaPBSubsystem, shim_classfiles: ScalaPBShimCompiledClassfiles, jdk: InternalJdk, ) -> GeneratedSources: output_dir = "_generated_files" toolcp_relpath = "__toolcp" shimcp_relpath = "__shimcp" plugins_relpath = "__plugins" protoc_relpath = "__protoc" lockfile_request = await Get(GenerateJvmLockfileFromTool, ScalapbcToolLockfileSentinel()) ( downloaded_protoc_binary, tool_classpath, empty_output_dir, transitive_targets, inherit_env, ) = await MultiGet( Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)), Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)), Get(Digest, CreateDigest([Directory(output_dir)])), Get(TransitiveTargets, TransitiveTargetsRequest([request.protocol_target.address])), # Need PATH so that ScalaPB can invoke `mkfifo`. Get(Environment, EnvironmentRequest(requested=["PATH"])), ) # NB: By stripping the source roots, we avoid having to set the value `--proto_path` # for Protobuf imports to be discoverable. all_sources_stripped, target_sources_stripped = await MultiGet( Get( StrippedSourceFiles, SourceFilesRequest(tgt[ProtobufSourceField] for tgt in transitive_targets.closure if tgt.has_field(ProtobufSourceField)), ), Get(StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSourceField] ])), ) merged_jvm_plugins_digest = EMPTY_DIGEST maybe_jvm_plugins_setup_args: tuple[str, ...] = () maybe_jvm_plugins_output_args: tuple[str, ...] = () jvm_plugins = scalapb.jvm_plugins if jvm_plugins: materialized_jvm_plugins = await Get( MaterializedJvmPlugins, MaterializeJvmPluginsRequest(jvm_plugins)) merged_jvm_plugins_digest = materialized_jvm_plugins.digest maybe_jvm_plugins_setup_args = materialized_jvm_plugins.setup_args( plugins_relpath) maybe_jvm_plugins_output_args = tuple( f"--{plugin.name}_out={output_dir}" for plugin in materialized_jvm_plugins.plugins) extra_immutable_input_digests = { toolcp_relpath: tool_classpath.digest, shimcp_relpath: shim_classfiles.digest, plugins_relpath: merged_jvm_plugins_digest, protoc_relpath: downloaded_protoc_binary.digest, } input_digest = await Get( Digest, MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir])) result = await Get( ProcessResult, JvmProcess( jdk=jdk, classpath_entries=[ *tool_classpath.classpath_entries(toolcp_relpath), shimcp_relpath ], argv=[ "org.pantsbuild.backend.scala.scalapb.ScalaPBShim", f"--protoc={os.path.join(protoc_relpath, downloaded_protoc_binary.exe)}", *maybe_jvm_plugins_setup_args, f"--scala_out={output_dir}", *maybe_jvm_plugins_output_args, *target_sources_stripped.snapshot.files, ], input_digest=input_digest, extra_immutable_input_digests=extra_immutable_input_digests, extra_nailgun_keys=extra_immutable_input_digests, description= f"Generating Scala sources from {request.protocol_target.address}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), extra_env=inherit_env, ), ) normalized_digest, source_root = await MultiGet( Get(Digest, RemovePrefix(result.output_digest, output_dir)), Get(SourceRoot, SourceRootRequest, SourceRootRequest.for_target(request.protocol_target)), ) source_root_restored = (await Get( Snapshot, AddPrefix(normalized_digest, source_root.path)) if source_root.path != "." else await Get( Snapshot, Digest, normalized_digest)) return GeneratedSources(source_root_restored)
async def setup( request: SetupRequest, isort: Isort, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> Setup: requirements_pex_request = Get[Pex](PexRequest( output_filename="isort.pex", requirements=PexRequirements(isort.get_requirement_specs()), interpreter_constraints=PexInterpreterConstraints( isort.default_interpreter_constraints), entry_point=isort.get_entry_point(), )) config_path: Optional[List[str]] = isort.options.config config_snapshot_request = Get[Snapshot](PathGlobs( globs=config_path or (), glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin="the option `--isort-config`", )) all_source_files_request = Get[SourceFiles](AllSourceFilesRequest( field_set.sources for field_set in request.field_sets)) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest((field_set.sources, field_set.origin) for field_set in request.field_sets)) requests: List[Get] = [ requirements_pex_request, config_snapshot_request, specified_source_files_request, ] if request.field_sets.prior_formatter_result is None: requests.append(all_source_files_request) requirements_pex, config_snapshot, specified_source_files, *rest = cast( Union[Tuple[Pex, Snapshot, SourceFiles], Tuple[Pex, Snapshot, SourceFiles, SourceFiles]], await MultiGet(requests), ) all_source_files_snapshot = (request.field_sets.prior_formatter_result if request.field_sets.prior_formatter_result else rest[0].snapshot) input_digest = await Get[Digest](MergeDigests( (all_source_files_snapshot.digest, requirements_pex.digest, config_snapshot.digest))) address_references = ", ".join( sorted(field_set.address.reference() for field_set in request.field_sets)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./isort.pex", pex_args=generate_args( specified_source_files=specified_source_files, isort=isort, check_only=request.check_only, ), input_digest=input_digest, output_files=all_source_files_snapshot.files, description= (f"Run isort on {pluralize(len(request.field_sets), 'target')}: {address_references}." ), ) return Setup(process, original_digest=all_source_files_snapshot.digest)
async def setup( setup_request: SetupRequest, docformatter: Docformatter, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> Setup: requirements_pex_request = Get[Pex]( PexRequest( output_filename="docformatter.pex", requirements=PexRequirements(docformatter.get_requirement_specs()), interpreter_constraints=PexInterpreterConstraints( docformatter.default_interpreter_constraints ), entry_point=docformatter.get_entry_point(), ) ) all_source_files_request = Get[SourceFiles]( AllSourceFilesRequest(field_set.sources for field_set in setup_request.request.field_sets) ) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( (field_set.sources, field_set.origin) for field_set in setup_request.request.field_sets ) ) requests = requirements_pex_request, specified_source_files_request all_source_files, requirements_pex, specified_source_files = ( await MultiGet(all_source_files_request, *requests) if setup_request.request.prior_formatter_result is None else (SourceFiles(EMPTY_SNAPSHOT), *await MultiGet(*requests)) ) all_source_files_snapshot = ( all_source_files.snapshot if setup_request.request.prior_formatter_result is None else setup_request.request.prior_formatter_result ) input_digest = await Get[Digest]( MergeDigests((all_source_files_snapshot.digest, requirements_pex.digest)) ) address_references = ", ".join( sorted(field_set.address.reference() for field_set in setup_request.request.field_sets) ) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./docformatter.pex", pex_args=generate_args( specified_source_files=specified_source_files, docformatter=docformatter, check_only=setup_request.check_only, ), input_digest=input_digest, output_files=all_source_files_snapshot.files, description=( f"Run Docformatter on {pluralize(len(setup_request.request.field_sets), 'target')}: " f"{address_references}." ), ) return Setup(process, original_digest=all_source_files_snapshot.digest)
async def bandit_lint_partition(partition: BanditPartition, bandit: Bandit, lint_subsystem: LintSubsystem) -> LintResult: bandit_pex_get = Get( VenvPex, PexRequest( output_filename="bandit.pex", internal_only=True, requirements=PexRequirements(bandit.all_requirements), interpreter_constraints=partition.interpreter_constraints, main=bandit.main, ), ) config_files_get = Get(ConfigFiles, ConfigFilesRequest, bandit.config_request) source_files_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) bandit_pex, config_files, source_files = await MultiGet( bandit_pex_get, config_files_get, source_files_get) input_digest = await Get( Digest, MergeDigests( (source_files.snapshot.digest, config_files.snapshot.digest))) report_file_name = "bandit_report.txt" if lint_subsystem.reports_dir else None result = await Get( FallibleProcessResult, VenvPexProcess( bandit_pex, argv=generate_args(source_files=source_files, bandit=bandit, report_file_name=report_file_name), input_digest=input_digest, description= f"Run Bandit on {pluralize(len(partition.field_sets), 'file')}.", output_files=(report_file_name, ) if report_file_name else None, level=LogLevel.DEBUG, ), ) report = None if report_file_name: report_digest = await Get( Digest, DigestSubset( result.output_digest, PathGlobs( [report_file_name], glob_match_error_behavior=GlobMatchErrorBehavior.warn, description_of_origin="Bandit report file", ), ), ) report = LintReport(report_file_name, report_digest) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def build_local_dists( request: LocalDistsPexRequest, ) -> LocalDistsPex: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) applicable_targets = [ tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt) ] python_dist_field_sets = [ PythonDistributionFieldSet.create(target) for target in applicable_targets ] dists = await MultiGet( [Get(BuiltPackage, PackageFieldSet, field_set) for field_set in python_dist_field_sets] ) # The primary use-case of the "local dists" feature is to support consuming native extensions # as wheels without having to publish them first. # It doesn't seem very useful to consume locally-built sdists, and it makes it hard to # reason about possible sys.path collisions between the in-repo sources and whatever the # sdist will place on the sys.path when it's installed. # So for now we simply ignore sdists, with a warning if necessary. provided_files = set() wheels = [] all_contents = await MultiGet(Get(DigestContents, Digest, dist.digest) for dist in dists) for dist, contents, tgt in zip(dists, all_contents, applicable_targets): artifacts = {(a.relpath or "") for a in dist.artifacts} # A given local dist might build a wheel and an sdist (and maybe other artifacts - # we don't know what setup command was run...) # As long as there is a wheel, we can ignore the other artifacts. wheel = next((art for art in artifacts if art.endswith(".whl")), None) if wheel: wheel_content = next(content for content in contents if content.path == wheel) wheels.append(wheel) buf = BytesIO() buf.write(wheel_content.content) buf.seek(0) with zipfile.ZipFile(buf) as zf: provided_files.update(zf.namelist()) else: logger.warning( f"Encountered a dependency on the {tgt.alias} target at {tgt.address.spec}, but " "this target does not produce a Python wheel artifact. Therefore this target's " "code will be used directly from sources, without a distribution being built, " "and therefore any native extensions in it will not be built.\n\n" f"See {doc_url('python-distributions')} for details on how to set up a {tgt.alias} " "target to produce a wheel." ) dists_digest = await Get(Digest, MergeDigests([dist.digest for dist in dists])) wheels_digest = await Get(Digest, DigestSubset(dists_digest, PathGlobs(["**/*.whl"]))) dists_pex = await Get( Pex, PexRequest( output_filename="local_dists.pex", requirements=PexRequirements(wheels), interpreter_constraints=request.interpreter_constraints, additional_inputs=wheels_digest, internal_only=request.internal_only, ), ) # We check source roots in reverse lexicographic order, # so we'll find the innermost root that matches. source_roots = list(reversed(sorted(request.sources.source_roots))) remaining_sources = set(request.sources.source_files.files) unrooted_files_set = set(request.sources.source_files.unrooted_files) for source in request.sources.source_files.files: if source not in unrooted_files_set: for source_root in source_roots: source_relpath = fast_relpath_optional(source, source_root) if source_relpath is not None and source_relpath in provided_files: remaining_sources.remove(source) remaining_sources_snapshot = await Get( Snapshot, DigestSubset( request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources)) ), ) subtracted_sources = PythonSourceFiles( SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files), request.sources.source_roots, ) return LocalDistsPex(dists_pex, subtracted_sources)
async def generate_coverage_reports( merged_coverage_data: MergedCoverageData, coverage_setup: CoverageSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, all_used_addresses: Addresses, ) -> CoverageReports: """Takes all Python test results and generates a single coverage report.""" transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(all_used_addresses)) sources = await Get( PythonSourceFiles, # Coverage sometimes includes non-Python files in its `.coverage` data. We need to # ensure that they're present when generating the report. We include all the files included # by `pytest_runner.py`. PythonSourceFilesRequest(transitive_targets.closure, include_files=True, include_resources=True), ) input_digest = await Get( Digest, MergeDigests(( merged_coverage_data.coverage_data, coverage_config.digest, sources.source_files.snapshot.digest, )), ) pex_processes = [] report_types = [] result_snapshot = await Get(Snapshot, Digest, merged_coverage_data.coverage_data) coverage_reports: List[CoverageReport] = [] for report_type in coverage_subsystem.reports: if report_type == CoverageReportType.RAW: coverage_reports.append( FilesystemCoverageReport( report_type=CoverageReportType.RAW.value, result_snapshot=result_snapshot, directory_to_materialize_to=coverage_subsystem.output_dir, report_file=coverage_subsystem.output_dir / ".coverage", )) continue report_types.append(report_type) output_file = (f"coverage.{report_type.value}" if report_type in { CoverageReportType.XML, CoverageReportType.JSON } else None) pex_processes.append( VenvPexProcess( coverage_setup.pex, argv=(report_type.report_name, f"--rcfile={coverage_config.path}"), input_digest=input_digest, output_directories=("htmlcov", ) if report_type == CoverageReportType.HTML else None, output_files=(output_file, ) if output_file else None, description= f"Generate Pytest {report_type.report_name} coverage report.", level=LogLevel.DEBUG, )) results = await MultiGet( Get(ProcessResult, VenvPexProcess, process) for process in pex_processes) result_stdouts = tuple(res.stdout for res in results) result_snapshots = await MultiGet( Get(Snapshot, Digest, res.output_digest) for res in results) coverage_reports.extend( _get_coverage_report(coverage_subsystem.output_dir, report_type, stdout, snapshot) for (report_type, stdout, snapshot) in zip(report_types, result_stdouts, result_snapshots)) return CoverageReports(tuple(coverage_reports))