def test_context_object_on_streaming_workunits(
rule_runner: RuleRunner, run_tracker: RunTracker) -> None:
scheduler = rule_runner.scheduler
def callback(**kwargs) -> None:
context = kwargs["context"]
assert isinstance(context, StreamingWorkunitContext)
completed_workunits = kwargs["completed_workunits"]
for workunit in completed_workunits:
if "artifacts" in workunit and "stdout_digest" in workunit[
"artifacts"]:
digest = workunit["artifacts"]["stdout_digest"]
output = context.single_file_digests_to_bytes([digest])
assert output == (b"stdout output\n", )
handler = StreamingWorkunitHandler(
scheduler,
run_tracker=run_tracker,
callbacks=[callback],
report_interval_seconds=0.01,
max_workunit_verbosity=LogLevel.INFO,
specs=Specs.empty(),
options_bootstrapper=create_options_bootstrapper([]),
)
stdout_process = Process(argv=("/bin/bash", "-c",
"/bin/echo 'stdout output'"),
description="Stdout process")
with handler.session():
rule_runner.request(ProcessResult, [stdout_process])
def test_more_complicated_engine_aware(rule_runner: RuleRunner,
run_tracker: RunTracker) -> None:
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
rule_runner.scheduler,
run_tracker=run_tracker,
callbacks=[tracker.add],
report_interval_seconds=0.01,
max_workunit_verbosity=LogLevel.TRACE,
specs=Specs.empty(),
options_bootstrapper=create_options_bootstrapper([]),
)
with handler.session():
input_1 = CreateDigest((
FileContent(path="a.txt", content=b"alpha"),
FileContent(path="b.txt", content=b"beta"),
))
digest_1 = rule_runner.request(Digest, [input_1])
snapshot_1 = rule_runner.request(Snapshot, [digest_1])
input_2 = CreateDigest((FileContent(path="g.txt", content=b"gamma"), ))
digest_2 = rule_runner.request(Digest, [input_2])
snapshot_2 = rule_runner.request(Snapshot, [digest_2])
input = ComplicatedInput(snapshot_1=snapshot_1, snapshot_2=snapshot_2)
rule_runner.request(Output, [input])
finished = list(
itertools.chain.from_iterable(tracker.finished_workunit_chunks))
workunit = next(
item for item in finished
if item["name"] == "pants.engine.internals.engine_test.a_rule")
streaming_workunit_context = handler._context
artifacts = workunit["artifacts"]
output_snapshot_1 = artifacts["snapshot_1"]
output_snapshot_2 = artifacts["snapshot_2"]
output_contents_list = streaming_workunit_context.snapshots_to_file_contents(
[output_snapshot_1, output_snapshot_2])
assert len(output_contents_list) == 2
assert isinstance(output_contents_list[0], DigestContents)
assert isinstance(output_contents_list[1], DigestContents)
digest_contents_1 = output_contents_list[0]
digest_contents_2 = output_contents_list[1]
assert len(tuple(x for x in digest_contents_1
if x.content == b"alpha")) == 1
assert len(tuple(x for x in digest_contents_1
if x.content == b"beta")) == 1
assert len(tuple(x for x in digest_contents_2
if x.content == b"gamma")) == 1
def _fixture_for_rules(
self, rules, max_workunit_verbosity: LogLevel = LogLevel.INFO
) -> Tuple[SchedulerSession, WorkunitTracker, StreamingWorkunitHandler]:
scheduler = self.mk_scheduler(rules, include_trace_on_error=False)
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler,
run_tracker=new_run_tracker(),
callbacks=[tracker],
report_interval_seconds=0.01,
max_workunit_verbosity=max_workunit_verbosity,
specs=Specs.empty(),
options_bootstrapper=create_options_bootstrapper([]),
pantsd=False,
)
return scheduler, tracker, handler
def test_counters(rule_runner: RuleRunner, run_tracker: RunTracker) -> None:
scheduler = rule_runner.scheduler
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler,
run_tracker=run_tracker,
callbacks=[tracker],
report_interval_seconds=0.01,
max_workunit_verbosity=LogLevel.TRACE,
specs=Specs.empty(),
options_bootstrapper=create_options_bootstrapper([]),
allow_async_completion=False,
)
with handler:
scheduler.record_test_observation(128)
rule_runner.request(
ProcessResult,
[
Process(
["/bin/sh", "-c", "true"],
description="always true",
cache_scope=ProcessCacheScope.PER_SESSION,
)
],
)
histograms_info = scheduler.get_observation_histograms()
finished = list(
itertools.chain.from_iterable(tracker.finished_workunit_chunks))
workunits_with_counters = [item for item in finished if "counters" in item]
assert workunits_with_counters[0]["counters"]["local_cache_requests"] == 1
assert workunits_with_counters[0]["counters"][
"local_cache_requests_uncached"] == 1
assert workunits_with_counters[1]["counters"][
"local_execution_requests"] == 1
assert histograms_info["version"] == 0
assert "histograms" in histograms_info
assert "test_observation" in histograms_info["histograms"]
assert (
histograms_info["histograms"]["test_observation"] ==
b"\x1c\x84\x93\x14\x00\x00\x00\x1fx\x9c\x93i\x99,\xcc\xc0\xc0\xc0\xcc\x00\x010\x9a\x11J3\xd9\x7f\x800\xfe32\x01\x00E\x0c\x03\x81"
)
def test_process_digests_on_streaming_workunits(
rule_runner: RuleRunner, run_tracker: RunTracker) -> None:
scheduler = rule_runner.scheduler
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler,
run_tracker=run_tracker,
callbacks=[tracker],
report_interval_seconds=0.01,
max_workunit_verbosity=LogLevel.DEBUG,
specs=Specs.empty(),
options_bootstrapper=create_options_bootstrapper([]),
allow_async_completion=False,
)
stdout_process = Process(argv=("/bin/bash", "-c",
"/bin/echo 'stdout output'"),
description="Stdout process")
with handler:
result = rule_runner.request(ProcessResult, [stdout_process])
assert tracker.finished
finished = list(
itertools.chain.from_iterable(tracker.finished_workunit_chunks))
process_workunit = next(item for item in finished
if item["name"] == "process")
assert process_workunit is not None
stdout_digest = process_workunit["artifacts"]["stdout_digest"]
stderr_digest = process_workunit["artifacts"]["stderr_digest"]
assert result.stdout == b"stdout output\n"
assert stderr_digest == EMPTY_FILE_DIGEST
assert stdout_digest.serialized_bytes_length == len(result.stdout)
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler,
run_tracker=run_tracker,
callbacks=[tracker],
report_interval_seconds=0.01,
max_workunit_verbosity=LogLevel.DEBUG,
specs=Specs.empty(),
options_bootstrapper=create_options_bootstrapper([]),
allow_async_completion=False,
)
stderr_process = Process(argv=("/bin/bash", "-c",
"1>&2 /bin/echo 'stderr output'"),
description="Stderr process")
with handler:
result = rule_runner.request(ProcessResult, [stderr_process])
assert tracker.finished
finished = list(
itertools.chain.from_iterable(tracker.finished_workunit_chunks))
process_workunit = next(item for item in finished
if item["name"] == "process")
assert process_workunit is not None
stdout_digest = process_workunit["artifacts"]["stdout_digest"]
stderr_digest = process_workunit["artifacts"]["stderr_digest"]
assert result.stderr == b"stderr output\n"
assert stdout_digest == EMPTY_FILE_DIGEST
assert stderr_digest.serialized_bytes_length == len(result.stderr)
assert process_workunit["metadata"]["exit_code"] == 0
try:
scheduler.ensure_remote_has_recursive([stdout_digest, stderr_digest])
except Exception as e:
# This is the exception message we should expect from invoking ensure_remote_has_recursive()
# in rust.
assert str(e) == "Cannot ensure remote has blobs without a remote"
byte_outputs = scheduler.single_file_digests_to_bytes(
[stdout_digest, stderr_digest])
assert byte_outputs[0] == result.stdout
assert byte_outputs[1] == result.stderr
async def lint(
console: Console,
workspace: Workspace,
specs: Specs,
lint_subsystem: LintSubsystem,
union_membership: UnionMembership,
dist_dir: DistDir,
) -> Lint:
lint_target_request_types = cast(
"Iterable[type[LintTargetsRequest]]", union_membership.get(LintTargetsRequest)
)
fmt_target_request_types = cast("Iterable[type[FmtRequest]]", union_membership.get(FmtRequest))
file_request_types = cast(
"Iterable[type[LintFilesRequest]]", union_membership[LintFilesRequest]
)
_check_ambiguous_request_names(
*lint_target_request_types, *fmt_target_request_types, *file_request_types
)
specified_names = determine_specified_tool_names(
"lint",
lint_subsystem.only,
[*lint_target_request_types, *fmt_target_request_types],
extra_valid_names={request.name for request in file_request_types},
)
def is_specified(request_type: type[StyleRequest] | type[LintFilesRequest]):
return request_type.name in specified_names
lint_target_request_types = filter(is_specified, lint_target_request_types)
fmt_target_request_types = filter(is_specified, fmt_target_request_types)
file_request_types = filter(is_specified, file_request_types)
_get_targets = Get(
FilteredTargets,
Specs,
specs if lint_target_request_types or fmt_target_request_types else Specs.empty(),
)
_get_specs_paths = Get(SpecsPaths, Specs, specs if file_request_types else Specs.empty())
targets, specs_paths = await MultiGet(_get_targets, _get_specs_paths)
def batch(field_sets: Iterable[FieldSet]) -> Iterator[list[FieldSet]]:
partitions = partition_sequentially(
field_sets,
key=lambda fs: fs.address.spec,
size_target=lint_subsystem.batch_size,
size_max=4 * lint_subsystem.batch_size,
)
for partition in partitions:
yield partition
def batch_by_type(
request_types: Iterable[type[_SR]],
) -> tuple[tuple[type[_SR], list[FieldSet]], ...]:
return tuple(
(request_type, field_set_batch)
for request_type in request_types
for field_set_batch in batch(
request_type.field_set_type.create(target)
for target in targets
if request_type.field_set_type.is_applicable(target)
)
)
lint_target_requests = (
request_type(batch) for request_type, batch in batch_by_type(lint_target_request_types)
)
fmt_requests: Iterable[FmtRequest] = ()
if not lint_subsystem.skip_formatters:
batched_fmt_request_pairs = batch_by_type(fmt_target_request_types)
all_fmt_source_batches = await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
cast(
SourcesField,
getattr(field_set, "sources", getattr(field_set, "source", None)),
)
for field_set in batch
),
)
for _, batch in batched_fmt_request_pairs
)
fmt_requests = (
request_type(
batch,
snapshot=source_files_snapshot.snapshot,
)
for (request_type, batch), source_files_snapshot in zip(
batched_fmt_request_pairs, all_fmt_source_batches
)
)
file_requests = (
tuple(request_type(specs_paths.files) for request_type in file_request_types)
if specs_paths.files
else ()
)
all_requests = [
*(Get(LintResults, LintTargetsRequest, request) for request in lint_target_requests),
*(Get(FmtResult, FmtRequest, request) for request in fmt_requests),
*(Get(LintResults, LintFilesRequest, request) for request in file_requests),
]
all_batch_results = cast(
"tuple[LintResults | FmtResult, ...]",
await MultiGet(all_requests), # type: ignore[arg-type]
)
def key_fn(results: LintResults | FmtResult):
if isinstance(results, FmtResult):
return results.formatter_name
return results.linter_name
# NB: We must pre-sort the data for itertools.groupby() to work properly.
sorted_all_batch_results = sorted(all_batch_results, key=key_fn)
formatter_failed = False
def coerce_to_lintresult(batch_results: LintResults | FmtResult) -> tuple[LintResult, ...]:
if isinstance(batch_results, FmtResult):
nonlocal formatter_failed
formatter_failed = formatter_failed or batch_results.did_change
return (
LintResult(
1 if batch_results.did_change else 0,
batch_results.stdout,
batch_results.stderr,
),
)
return batch_results.results
# We consolidate all results for each linter into a single `LintResults`.
all_results = tuple(
sorted(
(
LintResults(
itertools.chain.from_iterable(
coerce_to_lintresult(batch_results) for batch_results in results
),
linter_name=linter_name,
)
for linter_name, results in itertools.groupby(sorted_all_batch_results, key=key_fn)
),
key=key_fn,
)
)
def get_name(res: LintResults) -> str:
return res.linter_name
write_reports(
all_results,
workspace,
dist_dir,
goal_name=LintSubsystem.name,
get_name=get_name,
)
_print_results(
console,
all_results,
formatter_failed,
)
return Lint(_get_error_code(all_results))
def test_no_applicable_targets_exception() -> None:
# Check that we correctly render the error message.
class Tgt1(Target):
alias = "tgt1"
core_fields = ()
class Tgt2(Target):
alias = "tgt2"
core_fields = (MultipleSourcesField, )
class Tgt3(Target):
alias = "tgt3"
core_fields = ()
# No targets/files specified. Because none of the relevant targets have a sources field, we do
# not give the filedeps command.
exc = NoApplicableTargetsException(
[],
Specs.empty(),
UnionMembership({}),
applicable_target_types=[Tgt1],
goal_description="the `foo` goal",
)
remedy = (
"Please specify relevant file and/or target arguments. Run `./pants "
"--filter-target-type=tgt1 list ::` to find all applicable targets in your project."
)
assert (dedent(f"""\
No files or targets specified. The `foo` goal works with these target types:
* tgt1
{remedy}""") in str(exc))
invalid_tgt = Tgt3({}, Address("blah"))
exc = NoApplicableTargetsException(
[invalid_tgt],
Specs(
includes=RawSpecs(file_literals=(FileLiteralSpec("foo.ext"), ),
description_of_origin="tests"),
ignores=RawSpecs(description_of_origin="tests"),
),
UnionMembership({}),
applicable_target_types=[Tgt1, Tgt2],
goal_description="the `foo` goal",
)
remedy = (
"Please specify relevant file and/or target arguments. Run `./pants "
"--filter-target-type=tgt1,tgt2 list ::` to find all applicable targets in your project, "
"or run `./pants --filter-target-type=tgt1,tgt2 filedeps ::` to find all "
"applicable files.")
assert (dedent(f"""\
No applicable files or targets matched. The `foo` goal works with these target types:
* tgt1
* tgt2
However, you only specified file arguments with these target types:
* tgt3
{remedy}""") in str(exc))
# Test handling of `Specs`.
exc = NoApplicableTargetsException(
[invalid_tgt],
Specs(
includes=RawSpecs(address_literals=(AddressLiteralSpec(
"foo", "bar"), ),
description_of_origin="tests"),
ignores=RawSpecs(description_of_origin="tests"),
),
UnionMembership({}),
applicable_target_types=[Tgt1],
goal_description="the `foo` goal",
)
assert "However, you only specified target arguments with these target types:" in str(
exc)
exc = NoApplicableTargetsException(
[invalid_tgt],
Specs(
includes=RawSpecs(
address_literals=(AddressLiteralSpec("foo", "bar"), ),
file_literals=(FileLiteralSpec("foo.ext"), ),
description_of_origin="tests",
),
ignores=RawSpecs(description_of_origin="tests"),
),
UnionMembership({}),
applicable_target_types=[Tgt1],
goal_description="the `foo` goal",
)
assert "However, you only specified target and file arguments with these target types:" in str(
exc)
def run_lint_rule(
rule_runner: RuleRunner,
*,
lint_request_types: Sequence[Type[LintTargetsRequest]],
fmt_request_types: Sequence[Type[FmtRequest]] = (),
targets: list[Target],
run_files_linter: bool = False,
batch_size: int = 128,
only: list[str] | None = None,
skip_formatters: bool = False,
) -> Tuple[int, str]:
union_membership = UnionMembership({
LintTargetsRequest:
lint_request_types,
LintFilesRequest: [MockFilesRequest] if run_files_linter else [],
FmtRequest:
fmt_request_types,
})
lint_subsystem = create_goal_subsystem(
LintSubsystem,
batch_size=batch_size,
only=only or [],
skip_formatters=skip_formatters,
)
with mock_console(rule_runner.options_bootstrapper) as (console,
stdio_reader):
result: Lint = run_rule_with_mocks(
lint,
rule_args=[
console,
Workspace(rule_runner.scheduler, _enforce_effects=False),
Specs.empty(),
lint_subsystem,
union_membership,
DistDir(relpath=Path("dist")),
],
mock_gets=[
MockGet(
output_type=SourceFiles,
input_type=SourceFilesRequest,
mock=lambda _: SourceFiles(EMPTY_SNAPSHOT, ()),
),
MockGet(
output_type=LintResults,
input_type=LintTargetsRequest,
mock=lambda mock_request: mock_request.lint_results,
),
MockGet(
output_type=LintResults,
input_type=LintFilesRequest,
mock=lambda mock_request: mock_request.lint_results,
),
MockGet(
output_type=FmtResult,
input_type=FmtRequest,
mock=lambda mock_request: mock_request.fmt_result,
),
MockGet(
output_type=FilteredTargets,
input_type=Specs,
mock=lambda _: FilteredTargets(targets),
),
MockGet(
output_type=SpecsPaths,
input_type=Specs,
mock=lambda _: SpecsPaths(("f.txt", ), ()),
),
],
union_membership=union_membership,
)
assert not stdio_reader.get_stdout()
return result.exit_code, stdio_reader.get_stderr()
