def list_target_types( registered_target_types: RegisteredTargetTypes, union_membership: UnionMembership, target_types_subsystem: TargetTypesSubsystem, console: Console, pants_bin: PantsBin, ) -> TargetTypes: with target_types_subsystem.line_oriented(console) as print_stdout: if target_types_subsystem.all: all_target_types = { alias: VerboseTargetInfo.create( target_type, union_membership=union_membership).as_dict() for alias, target_type in registered_target_types.aliases_to_types.items() if not alias.startswith("_") and target_type.deprecated_removal_version is None } print_stdout(json.dumps(all_target_types, sort_keys=True, indent=4)) elif target_types_subsystem.details: alias = target_types_subsystem.details target_type = registered_target_types.aliases_to_types.get(alias) if target_type is None: raise ValueError( f"Unrecognized target type {repr(alias)}. All registered " f"target types: {list(registered_target_types.aliases)}") verbose_target_info = VerboseTargetInfo.create( target_type, union_membership=union_membership) print_stdout("") print_stdout(verbose_target_info.format_for_cli(console)) else: title_text = "Target types" title = console.green(f"{title_text}\n{'-' * len(title_text)}") target_infos = [ AbbreviatedTargetInfo.create(target_type) for target_type in registered_target_types.types if (not target_type.alias.startswith("_") and target_type.deprecated_removal_version is None) ] longest_target_alias = max( len(target_type.alias) for target_type in registered_target_types.types) lines = [ f"\n{title}\n", textwrap.fill( f"Use `{pants_bin.render_command('target-types', '--details=$target_type')}` " "to get detailed information for a particular target type.", 80, ), "\n", *(target_info.format_for_cli( console, longest_target_alias=longest_target_alias) for target_info in target_infos), ] print_stdout("\n".join(lines).rstrip()) return TargetTypes(exit_code=0)
def format_for_cli(self, console: Console) -> str: output = [console.green(f"{self.alias}\n{'-' * len(self.alias)}\n")] if self.description: output.append(f"{self.description}\n") output.extend([ "Valid fields:\n", *sorted(f"{field.format_for_cli(console)}\n" for field in self.fields), ]) return "\n".join(output).rstrip()
def format_for_cli(self, console: Console, *, v1_disabled: bool) -> str: output = [console.green(f"{self.alias}\n{'-' * len(self.alias)}\n")] if self.description: output.append(f"{self.description}\n") output.extend([ "Valid fields:\n", *sorted(f"{field.format_for_cli(console)}\n" for field in self.fields if not field.alias.startswith("_") and (not v1_disabled or not field.v1_only)), ]) return "\n".join(output).rstrip()
def list_target_types( registered_target_types: RegisteredTargetTypes, union_membership: UnionMembership, target_types_options: TargetTypesOptions, global_options: GlobalOptions, console: Console, ) -> TargetTypes: v1_disabled = not global_options.options.v1 with target_types_options.line_oriented(console) as print_stdout: if target_types_options.values.details: alias = target_types_options.values.details target_type = registered_target_types.aliases_to_types.get(alias) if target_type is None: raise ValueError( f"Unrecognized target type {repr(alias)}. All registered " f"target types: {list(registered_target_types.aliases)}" ) verbose_target_info = VerboseTargetInfo.create( target_type, union_membership=union_membership ) print_stdout("") print_stdout(verbose_target_info.format_for_cli(console, v1_disabled=v1_disabled)) else: title_text = "Target types" title = console.green(f"{title_text}\n{'-' * len(title_text)}") target_infos = [ AbbreviatedTargetInfo.create(target_type) for target_type in registered_target_types.types ] longest_target_alias = max( len(target_type.alias) for target_type in registered_target_types.types ) lines = [ f"\n{title}\n", textwrap.fill( "Use `./pants target-types2 --details=$target_type` to get detailed " "information for a particular target type.", 80, ), "\n", *( target_info.format_for_cli(console, longest_target_alias=longest_target_alias) for target_info in target_infos if not target_info.alias.startswith("_") and (not v1_disabled or not target_info.v1_only) ), ] print_stdout("\n".join(lines).rstrip()) return TargetTypes(exit_code=0)
async def typecheck( console: Console, targets: Targets, union_membership: UnionMembership ) -> Typecheck: typecheck_request_types = union_membership[TypecheckRequest] requests: Iterable[StyleRequest] = tuple( lint_request_type( lint_request_type.field_set_type.create(target) for target in targets if lint_request_type.field_set_type.is_applicable(target) ) for lint_request_type in typecheck_request_types ) field_sets_with_sources: Iterable[FieldSetsWithSources] = await MultiGet( Get(FieldSetsWithSources, FieldSetsWithSourcesRequest(request.field_sets)) for request in requests ) valid_requests: Iterable[StyleRequest] = tuple( request_cls(request) for request_cls, request in zip(typecheck_request_types, field_sets_with_sources) if request ) all_results = await MultiGet( Get(TypecheckResults, TypecheckRequest, request) for request in valid_requests ) exit_code = 0 if all_results: console.print_stderr("") for results in sorted(all_results, key=lambda results: results.typechecker_name): if results.skipped: sigil = console.yellow("-") status = "skipped" elif results.exit_code == 0: sigil = console.green("✓") status = "succeeded" else: sigil = console.red("𐄂") status = "failed" exit_code = results.exit_code console.print_stderr(f"{sigil} {results.typechecker_name} {status}.") return Typecheck(exit_code)
def format_section(backends: Sequence[BackendInfo], console: Console, *, version_number: int, option_name: str) -> str: longest_backend = max(len(backend.name) for backend in backends) title = f"V{version_number} backends" formatted_title = console.green(f"{title}\n{'-' * len(title)}") instructions = textwrap.dedent(f"""\ To enable V{version_number} backends, add the backend to `{option_name}.add` in your `pants.toml`, like this: [GLOBAL] {option_name}.add = ["pants.backend.python"] In the below list, all activated backends end with `*`.\n """) lines = [ f"\n{formatted_title}\n", instructions, *(backend.format_for_cli(console, longest_backend=longest_backend, is_v2=version_number == 2) for backend in sorted(backends, key=lambda backend: backend.name)), ] return "\n".join(lines)
async def fmt( console: Console, targets: Targets, fmt_subsystem: FmtSubsystem, workspace: Workspace, union_membership: UnionMembership, ) -> Fmt: language_target_collection_types = union_membership[LanguageFmtTargets] language_target_collections: Iterable[LanguageFmtTargets] = tuple( language_target_collection_type( Targets(target for target in targets if language_target_collection_type.belongs_to_language( target))) for language_target_collection_type in language_target_collection_types) targets_with_sources: Iterable[TargetsWithSources] = await MultiGet( Get( TargetsWithSources, TargetsWithSourcesRequest(language_target_collection.targets), ) for language_target_collection in language_target_collections) # NB: We must convert back the generic TargetsWithSources objects back into their # corresponding LanguageFmtTargets, e.g. back to PythonFmtTargets, in order for the union # rule to work. valid_language_target_collections: Iterable[LanguageFmtTargets] = tuple( language_target_collection_cls( Targets(target for target in language_target_collection.targets if target in language_targets_with_sources)) for language_target_collection_cls, language_target_collection, language_targets_with_sources in zip( language_target_collection_types, language_target_collections, targets_with_sources) if language_targets_with_sources) if fmt_subsystem.per_file_caching: per_language_results = await MultiGet( Get( LanguageFmtResults, LanguageFmtTargets, language_target_collection.__class__(Targets([target])), ) for language_target_collection in valid_language_target_collections for target in language_target_collection.targets) else: per_language_results = await MultiGet( Get(LanguageFmtResults, LanguageFmtTargets, language_target_collection) for language_target_collection in valid_language_target_collections ) individual_results: List[FmtResult] = list( itertools.chain.from_iterable( language_result.results for language_result in per_language_results)) if not individual_results: return Fmt(exit_code=0) changed_digests = tuple(language_result.output for language_result in per_language_results if language_result.did_change) if changed_digests: # NB: this will fail if there are any conflicting changes, which we want to happen rather # than silently having one result override the other. In practice, this should never # happen due to us grouping each language's formatters into a single digest. merged_formatted_digest = await Get(Digest, MergeDigests(changed_digests)) workspace.write_digest(merged_formatted_digest) if individual_results: console.print_stderr("") # We group all results for the same formatter so that we can give one final status in the # summary. This is only relevant if there were multiple results because of # `--per-file-caching`. formatter_to_results = defaultdict(set) for result in individual_results: formatter_to_results[result.formatter_name].add(result) for formatter, results in sorted(formatter_to_results.items()): if any(result.did_change for result in results): sigil = console.red("𐄂") status = "made changes" elif all(result.skipped for result in results): sigil = console.yellow("-") status = "skipped" else: sigil = console.green("✓") status = "made no changes" console.print_stderr(f"{sigil} {formatter} {status}.") # Since the rules to produce FmtResult should use ExecuteRequest, rather than # FallibleProcess, we assume that there were no failures. return Fmt(exit_code=0)
async def run_tests( console: Console, test_subsystem: TestSubsystem, interactive_runner: InteractiveRunner, workspace: Workspace, union_membership: UnionMembership, dist_dir: DistDir, ) -> Test: if test_subsystem.debug: targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( TestFieldSet, goal_description="`test --debug`", no_applicable_targets_behavior=NoApplicableTargetsBehavior.error, ), ) debug_requests = await MultiGet( Get(TestDebugRequest, TestFieldSet, field_set) for field_set in targets_to_valid_field_sets.field_sets ) exit_code = 0 for debug_request in debug_requests: if debug_request.process is None: continue debug_result = interactive_runner.run(debug_request.process) if debug_result.exit_code != 0: exit_code = debug_result.exit_code return Test(exit_code) targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( TestFieldSet, goal_description=f"the `{test_subsystem.name}` goal", no_applicable_targets_behavior=NoApplicableTargetsBehavior.warn, ), ) field_sets_with_sources = await Get( FieldSetsWithSources, FieldSetsWithSourcesRequest(targets_to_valid_field_sets.field_sets) ) results = await MultiGet( Get(EnrichedTestResult, TestFieldSet, field_set) for field_set in field_sets_with_sources ) # Print summary. exit_code = 0 if results: console.print_stderr("") for result in sorted(results): if result.skipped: continue if result.exit_code == 0: sigil = console.green("✓") status = "succeeded" else: sigil = console.red("𐄂") status = "failed" exit_code = cast(int, result.exit_code) console.print_stderr(f"{sigil} {result.address} {status}.") if result.extra_output and result.extra_output.files: workspace.write_digest( result.extra_output.digest, path_prefix=str(dist_dir.relpath / "test" / result.address.path_safe_spec), ) merged_xml_results = await Get( Digest, MergeDigests(result.xml_results.digest for result in results if result.xml_results), ) workspace.write_digest(merged_xml_results) if test_subsystem.use_coverage: # NB: We must pre-sort the data for itertools.groupby() to work properly, using the same # key function for both. However, you can't sort by `types`, so we call `str()` on it. all_coverage_data = sorted( (result.coverage_data for result in results if result.coverage_data is not None), key=lambda cov_data: str(type(cov_data)), ) coverage_types_to_collection_types: Dict[ Type[CoverageData], Type[CoverageDataCollection] ] = { collection_cls.element_type: collection_cls for collection_cls in union_membership.get(CoverageDataCollection) } coverage_collections: List[CoverageDataCollection] = [] for data_cls, data in itertools.groupby(all_coverage_data, lambda data: type(data)): collection_cls = coverage_types_to_collection_types[data_cls] coverage_collections.append(collection_cls(data)) # We can create multiple reports for each coverage data (console, xml and html) coverage_reports_collections = await MultiGet( Get(CoverageReports, CoverageDataCollection, coverage_collection) for coverage_collection in coverage_collections ) coverage_report_files: List[PurePath] = [] for coverage_reports in coverage_reports_collections: report_files = coverage_reports.materialize(console, workspace) coverage_report_files.extend(report_files) if coverage_report_files and test_subsystem.open_coverage: open_files = await Get( OpenFiles, OpenFilesRequest(coverage_report_files, error_if_open_not_found=False) ) for process in open_files.processes: interactive_runner.run(process) return Test(exit_code)
async def lint( console: Console, workspace: Workspace, targets: Targets, lint_subsystem: LintSubsystem, union_membership: UnionMembership, ) -> Lint: request_types = union_membership[LintRequest] requests: Iterable[StyleRequest] = tuple( request_type( request_type.field_set_type.create(target) for target in targets if request_type.field_set_type.is_applicable(target)) for request_type in request_types) field_sets_with_sources: Iterable[FieldSetsWithSources] = await MultiGet( Get(FieldSetsWithSources, FieldSetsWithSourcesRequest(request.field_sets)) for request in requests) valid_requests: Iterable[StyleRequest] = tuple( request_cls(request) for request_cls, request in zip(request_types, field_sets_with_sources) if request) if lint_subsystem.per_file_caching: all_per_file_results = await MultiGet( Get(LintResults, LintRequest, request.__class__([field_set])) for request in valid_requests for field_set in request.field_sets) def key_fn(results: LintResults): return results.linter_name # NB: We must pre-sort the data for itertools.groupby() to work properly. sorted_all_per_files_results = sorted(all_per_file_results, key=key_fn) # We consolidate all results for each linter into a single `LintResults`. all_results = tuple( LintResults( itertools.chain.from_iterable( per_file_results.results for per_file_results in all_linter_results), linter_name=linter_name, ) for linter_name, all_linter_results in itertools.groupby( sorted_all_per_files_results, key=key_fn)) else: all_results = await MultiGet( Get(LintResults, LintRequest, lint_request) for lint_request in valid_requests) all_results = tuple( sorted(all_results, key=lambda results: results.linter_name)) reports = list( itertools.chain.from_iterable(results.reports for results in all_results)) if reports: # TODO(#10532): Tolerate when a linter has multiple reports. linters_with_multiple_reports = [ results.linter_name for results in all_results if len(results.reports) > 1 ] if linters_with_multiple_reports: if lint_subsystem.per_file_caching: suggestion = "Try running without `--lint-per-file-caching` set." else: suggestion = ( "The linters likely partitioned the input targets, such as grouping by Python " "interpreter compatibility. Try running on fewer targets or unset " "`--lint-reports-dir`.") raise InvalidLinterReportsError( "Multiple reports would have been written for these linters: " f"{linters_with_multiple_reports}. The option `--lint-reports-dir` only works if " f"each linter has a single result. {suggestion}") merged_reports = await Get( Digest, MergeDigests(report.digest for report in reports)) workspace.write_digest(merged_reports) logger.info( f"Wrote lint result files to {lint_subsystem.reports_dir}.") exit_code = 0 if all_results: console.print_stderr("") for results in all_results: if results.skipped: sigil = console.yellow("-") status = "skipped" elif results.exit_code == 0: sigil = console.green("✓") status = "succeeded" else: sigil = console.red("𐄂") status = "failed" exit_code = results.exit_code console.print_stderr(f"{sigil} {results.linter_name} {status}.") return Lint(exit_code)
async def run_tests( console: Console, test_subsystem: TestSubsystem, interactive_runner: InteractiveRunner, workspace: Workspace, union_membership: UnionMembership, ) -> Test: if test_subsystem.debug: targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( TestFieldSet, goal_description="`test --debug`", error_if_no_applicable_targets=True ), ) debug_requests = await MultiGet( Get(TestDebugRequest, TestFieldSet, field_set) for field_set in targets_to_valid_field_sets.field_sets ) exit_code = 0 for debug_request in debug_requests: if debug_request.process is None: continue debug_result = interactive_runner.run(debug_request.process) if debug_result.exit_code != 0: exit_code = debug_result.exit_code return Test(exit_code) targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( TestFieldSet, goal_description=f"the `{test_subsystem.name}` goal", error_if_no_applicable_targets=False, ), ) field_sets_with_sources = await Get( FieldSetsWithSources, FieldSetsWithSourcesRequest(targets_to_valid_field_sets.field_sets) ) results = await MultiGet( Get(EnrichedTestResult, TestFieldSet, field_set) for field_set in field_sets_with_sources ) # Print summary. exit_code = 0 if results: console.print_stderr("") for result in sorted(results): if result.skipped: continue if result.exit_code == 0: sigil = console.green("✓") status = "succeeded" else: sigil = console.red("𐄂") status = "failed" exit_code = cast(int, result.exit_code) console.print_stderr(f"{sigil} {result.address} {status}.") merged_xml_results = await Get( Digest, MergeDigests(result.xml_results.digest for result in results if result.xml_results), ) workspace.write_digest(merged_xml_results) if test_subsystem.use_coverage: all_coverage_data: Iterable[CoverageData] = [ result.coverage_data for result in results if result.coverage_data is not None ] coverage_types_to_collection_types: Dict[ Type[CoverageData], Type[CoverageDataCollection] ] = { collection_cls.element_type: collection_cls for collection_cls in union_membership.get(CoverageDataCollection) } coverage_collections: List[CoverageDataCollection] = [] for data_cls, data in itertools.groupby(all_coverage_data, lambda data: type(data)): collection_cls = coverage_types_to_collection_types[data_cls] coverage_collections.append(collection_cls(data)) # We can create multiple reports for each coverage data (console, xml and html) coverage_reports_collections = await MultiGet( Get(CoverageReports, CoverageDataCollection, coverage_collection) for coverage_collection in coverage_collections ) coverage_report_files: List[PurePath] = [] for coverage_reports in coverage_reports_collections: report_files = coverage_reports.materialize(console, workspace) coverage_report_files.extend(report_files) if coverage_report_files and test_subsystem.open_coverage: open_files = await Get( OpenFiles, OpenFilesRequest(coverage_report_files, error_if_open_not_found=False) ) for process in open_files.processes: interactive_runner.run(process) return Test(exit_code)
async def run_tests( console: Console, options: TestOptions, interactive_runner: InteractiveRunner, workspace: Workspace, union_membership: UnionMembership, ) -> Test: if options.values.debug: targets_to_valid_field_sets = await Get[TargetsToValidFieldSets]( TargetsToValidFieldSetsRequest( TestFieldSet, goal_description="`test --debug`", error_if_no_valid_targets=True, expect_single_field_set=True, ) ) field_set = targets_to_valid_field_sets.field_sets[0] request = await Get[TestDebugRequest](TestFieldSet, field_set) debug_result = interactive_runner.run_local_interactive_process(request.ipr) return Test(debug_result.process_exit_code) targets_to_valid_field_sets = await Get[TargetsToValidFieldSets]( TargetsToValidFieldSetsRequest( TestFieldSet, goal_description=f"the `{options.name}` goal", error_if_no_valid_targets=False, ) ) field_sets_with_sources = await Get[FieldSetsWithSources]( FieldSetsWithSourcesRequest(targets_to_valid_field_sets.field_sets) ) results = await MultiGet( Get[AddressAndTestResult](WrappedTestFieldSet(field_set)) for field_set in field_sets_with_sources ) exit_code = PANTS_SUCCEEDED_EXIT_CODE for result in results: if result.test_result.status == Status.FAILURE: exit_code = PANTS_FAILED_EXIT_CODE has_output = result.test_result.stdout or result.test_result.stderr if has_output: status = ( console.green("✓") if result.test_result.status == Status.SUCCESS else console.red("𐄂") ) console.print_stderr(f"{status} {result.address}") if result.test_result.stdout: console.print_stderr(result.test_result.stdout) if result.test_result.stderr: console.print_stderr(result.test_result.stderr) if has_output and result != results[-1]: console.print_stderr("") # Print summary if len(results) > 1: console.print_stderr("") for result in results: console.print_stderr( f"{result.address.reference():80}.....{result.test_result.status.value:>10}" ) for result in results: xml_results = result.test_result.xml_results if not xml_results: continue workspace.materialize_directory(DirectoryToMaterialize(xml_results)) if options.values.use_coverage: all_coverage_data: Iterable[CoverageData] = [ result.test_result.coverage_data for result in results if result.test_result.coverage_data is not None ] coverage_types_to_collection_types: Dict[ Type[CoverageData], Type[CoverageDataCollection] ] = { collection_cls.element_type: collection_cls for collection_cls in union_membership.get(CoverageDataCollection) } coverage_collections: List[CoverageDataCollection] = [] for data_cls, data in itertools.groupby(all_coverage_data, lambda data: type(data)): collection_cls = coverage_types_to_collection_types[data_cls] coverage_collections.append(collection_cls(data)) coverage_reports = await MultiGet( Get[CoverageReport](CoverageDataCollection, coverage_collection) for coverage_collection in coverage_collections ) coverage_report_files = [] for report in coverage_reports: report_file = report.materialize(console, workspace) if report_file is not None: coverage_report_files.append(report_file) if coverage_report_files and options.values.open_coverage: desktop.ui_open(console, interactive_runner, coverage_report_files) return Test(exit_code)
async def run_tests( console: Console, test_subsystem: TestSubsystem, interactive_runner: InteractiveRunner, workspace: Workspace, union_membership: UnionMembership, ) -> Test: if test_subsystem.debug: targets_to_valid_field_sets = await Get( TargetsToValidFieldSets, TargetsToValidFieldSetsRequest( TestFieldSet, goal_description="`test --debug`", error_if_no_valid_targets=True, expect_single_field_set=True, ), ) field_set = targets_to_valid_field_sets.field_sets[0] request = await Get(TestDebugRequest, TestFieldSet, field_set) debug_result = interactive_runner.run(request.process) return Test(debug_result.exit_code) targets_to_valid_field_sets = await Get( TargetsToValidFieldSets, TargetsToValidFieldSetsRequest( TestFieldSet, goal_description=f"the `{test_subsystem.name}` goal", error_if_no_valid_targets=False, ), ) field_sets_with_sources = await Get( FieldSetsWithSources, FieldSetsWithSourcesRequest(targets_to_valid_field_sets.field_sets)) results = await MultiGet( Get(AddressAndTestResult, WrappedTestFieldSet(field_set)) for field_set in field_sets_with_sources) # Print details. for result in results: if test_subsystem.options.output == ShowOutput.NONE or ( test_subsystem.options.output == ShowOutput.FAILED and result.test_result.status == Status.SUCCESS): continue has_output = result.test_result.stdout or result.test_result.stderr if has_output: status = (console.green("✓") if result.test_result.status == Status.SUCCESS else console.red("𐄂")) console.print_stderr(f"{status} {result.address}") if result.test_result.stdout: console.print_stderr(result.test_result.stdout) if result.test_result.stderr: console.print_stderr(result.test_result.stderr) if has_output and result != results[-1]: console.print_stderr("") # Print summary console.print_stderr("") for result in results: color = console.green if result.test_result.status == Status.SUCCESS else console.red # The right-align logic sees the color control codes as characters, so we have # to account for that. In f-strings the alignment field widths must be literals, # so we have to indirect via a call to .format(). right_align = 19 if console.use_colors else 10 format_str = f"{{addr:80}}.....{{result:>{right_align}}}" console.print_stderr( format_str.format(addr=result.address.spec, result=color(result.test_result.status.value))) merged_xml_results = await Get( Digest, MergeDigests(result.test_result.xml_results for result in results if result.test_result.xml_results), ) workspace.write_digest(merged_xml_results) if test_subsystem.use_coverage: all_coverage_data: Iterable[CoverageData] = [ result.test_result.coverage_data for result in results if result.test_result.coverage_data is not None ] coverage_types_to_collection_types: Dict[ Type[CoverageData], Type[CoverageDataCollection]] = { collection_cls.element_type: collection_cls for collection_cls in union_membership.get( CoverageDataCollection) } coverage_collections: List[CoverageDataCollection] = [] for data_cls, data in itertools.groupby(all_coverage_data, lambda data: type(data)): collection_cls = coverage_types_to_collection_types[data_cls] coverage_collections.append(collection_cls(data)) # We can create multiple reports for each coverage data (console, xml and html) coverage_reports_collections = await MultiGet( Get(CoverageReports, CoverageDataCollection, coverage_collection) for coverage_collection in coverage_collections) coverage_report_files: List[PurePath] = [] for coverage_reports in coverage_reports_collections: report_files = coverage_reports.materialize(console, workspace) coverage_report_files.extend(report_files) if coverage_report_files and test_subsystem.open_coverage: desktop.ui_open(console, interactive_runner, coverage_report_files) exit_code = (PANTS_FAILED_EXIT_CODE if any( res.test_result.status == Status.FAILURE for res in results) else PANTS_SUCCEEDED_EXIT_CODE) return Test(exit_code)