def batch(field_sets: Iterable[FieldSet]) -> Iterator[list[FieldSet]]: partitions = partition_sequentially( field_sets, key=lambda fs: fs.address.spec, size_target=lint_subsystem.batch_size, size_max=4 * lint_subsystem.batch_size, ) for partition in partitions: yield partition
async def lint( console: Console, workspace: Workspace, targets: Targets, specs_snapshot: SpecsSnapshot, lint_subsystem: LintSubsystem, union_membership: UnionMembership, dist_dir: DistDir, ) -> Lint: target_request_types = cast("Iterable[type[LintTargetsRequest]]", union_membership[LintTargetsRequest]) file_request_types = union_membership[LintFilesRequest] specified_names = determine_specified_tool_names( "lint", lint_subsystem.only, target_request_types, extra_valid_names={request.name for request in file_request_types}, ) target_requests = tuple( request_type( request_type.field_set_type.create(target) for target in targets if (request_type.name in specified_names and request_type.field_set_type.is_applicable(target))) for request_type in target_request_types) file_requests = (tuple( request_type(specs_snapshot.snapshot.files) for request_type in file_request_types if request_type.name in specified_names) if specs_snapshot.snapshot.files else ()) def address_str(fs: FieldSet) -> str: return fs.address.spec all_requests = [ *(Get(LintResults, LintTargetsRequest, request.__class__(field_set_batch)) for request in target_requests if request.field_sets for field_set_batch in partition_sequentially( request.field_sets, key=address_str, size_target=lint_subsystem.batch_size, size_max=4 * lint_subsystem.batch_size, )), *(Get(LintResults, LintFilesRequest, request) for request in file_requests), ] all_batch_results = cast( "tuple[LintResults, ...]", await MultiGet(all_requests), # type: ignore[arg-type] ) def key_fn(results: LintResults): return results.linter_name # NB: We must pre-sort the data for itertools.groupby() to work properly. sorted_all_batch_results = sorted(all_batch_results, key=key_fn) # We consolidate all results for each linter into a single `LintResults`. all_results = tuple( sorted( (LintResults( itertools.chain.from_iterable( batch_results.results for batch_results in all_linter_results), linter_name=linter_name, ) for linter_name, all_linter_results in itertools.groupby( sorted_all_batch_results, key=key_fn)), key=key_fn, )) def get_name(res: LintResults) -> str: return res.linter_name write_reports( all_results, workspace, dist_dir, goal_name=LintSubsystem.name, get_name=get_name, ) exit_code = 0 if all_results: console.print_stderr("") for results in all_results: if results.skipped: continue elif results.exit_code == 0: sigil = console.sigil_succeeded() status = "succeeded" else: sigil = console.sigil_failed() status = "failed" exit_code = results.exit_code console.print_stderr(f"{sigil} {results.linter_name} {status}.") return Lint(exit_code)
def partitioned_buckets(items: list[str]) -> set[tuple[str, ...]]: return {tuple(p) for p in partition_sequentially(items, key=str, size_target=size_target)}
async def fmt( console: Console, targets: Targets, fmt_subsystem: FmtSubsystem, workspace: Workspace, union_membership: UnionMembership, ) -> Fmt: request_types = union_membership[FmtRequest] specified_names = determine_specified_tool_names("fmt", fmt_subsystem.only, request_types) # Group targets by the sequence of FmtRequests that apply to them. targets_by_fmt_request_order = defaultdict(list) for target in targets: fmt_requests = [] for fmt_request in request_types: valid_name = fmt_request.name in specified_names if valid_name and fmt_request.field_set_type.is_applicable( target): # type: ignore[misc] fmt_requests.append(fmt_request) if fmt_requests: targets_by_fmt_request_order[tuple(fmt_requests)].append(target) # Spawn sequential formatting per unique sequence of FmtRequests. per_language_results = await MultiGet( Get( _LanguageFmtResults, _LanguageFmtRequest(fmt_requests, Targets(target_batch)), ) for fmt_requests, targets in targets_by_fmt_request_order.items() for target_batch in partition_sequentially( targets, key=lambda t: t.address.spec, size_target=fmt_subsystem.batch_size, size_max=4 * fmt_subsystem.batch_size, )) individual_results = list( itertools.chain.from_iterable( language_result.results for language_result in per_language_results)) if not individual_results: return Fmt(exit_code=0) changed_digests = tuple(language_result.output for language_result in per_language_results if language_result.did_change) if changed_digests: # NB: this will fail if there are any conflicting changes, which we want to happen rather # than silently having one result override the other. In practice, this should never # happen due to us grouping each language's formatters into a single digest. merged_formatted_digest = await Get(Digest, MergeDigests(changed_digests)) workspace.write_digest(merged_formatted_digest) if individual_results: console.print_stderr("") # We group all results for the same formatter so that we can give one final status in the # summary. This is only relevant if there were multiple results because of # `--per-file-caching`. formatter_to_results = defaultdict(set) for result in individual_results: formatter_to_results[result.formatter_name].add(result) for formatter, results in sorted(formatter_to_results.items()): if any(result.did_change for result in results): sigil = console.sigil_succeeded_with_edits() status = "made changes" elif all(result.skipped for result in results): continue else: sigil = console.sigil_succeeded() status = "made no changes" console.print_stderr(f"{sigil} {formatter} {status}.") # Since the rules to produce FmtResult should use ExecuteRequest, rather than # FallibleProcess, we assume that there were no failures. return Fmt(exit_code=0)