def run(self, start_time: float) -> ExitCode: spec_parser = SpecsParser(get_buildroot()) specs = [str(spec_parser.parse_spec(spec)) for spec in self.options.specs] self.run_tracker.start(run_start_time=start_time, specs=specs) with maybe_profiled(self.profile_path): global_options = self.options.for_global_scope() if self.options.help_request: return self._print_help(self.options.help_request) streaming_reporter = StreamingWorkunitHandler( self.graph_session.scheduler_session, run_tracker=self.run_tracker, callbacks=self._get_workunits_callbacks(), report_interval_seconds=global_options.streaming_workunits_report_interval, ) goals = tuple(self.options.goals) with streaming_reporter.session(): if not goals: return PANTS_SUCCEEDED_EXIT_CODE engine_result = PANTS_FAILED_EXIT_CODE try: engine_result = self._perform_run(goals) except Exception as e: ExceptionSink.log_exception(e) metrics = self.graph_session.scheduler_session.metrics() self.run_tracker.set_pantsd_scheduler_metrics(metrics) self.run_tracker.end_run(engine_result) return engine_result
def run(self, start_time: float) -> ExitCode: self._start_run(start_time) with maybe_profiled(self.profile_path): global_options = self.options.for_global_scope() if self.options.help_request: return self._print_help(self.options.help_request) streaming_reporter = StreamingWorkunitHandler( self.graph_session.scheduler_session, run_tracker=self.run_tracker, callbacks=self._get_workunits_callbacks(), report_interval_seconds=global_options. streaming_workunits_report_interval, ) goals = tuple(self.options.goals) with streaming_reporter.session(): engine_result = PANTS_FAILED_EXIT_CODE try: engine_result = self._run_v2(goals) except Exception as e: ExceptionSink.log_exception(e) self._finish_run(engine_result) return engine_result
def test_context_object_on_streaming_workunits( rule_runner: RuleRunner, run_tracker: RunTracker) -> None: scheduler = rule_runner.scheduler def callback(**kwargs) -> None: context = kwargs["context"] assert isinstance(context, StreamingWorkunitContext) completed_workunits = kwargs["completed_workunits"] for workunit in completed_workunits: if "artifacts" in workunit and "stdout_digest" in workunit[ "artifacts"]: digest = workunit["artifacts"]["stdout_digest"] output = context.single_file_digests_to_bytes([digest]) assert output == (b"stdout output\n", ) handler = StreamingWorkunitHandler( scheduler, run_tracker=run_tracker, callbacks=[callback], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.INFO, ) stdout_process = Process(argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process") with handler.session(): rule_runner.request(ProcessResult, [stdout_process])
def test_more_complicated_engine_aware(rule_runner: RuleRunner, run_tracker: RunTracker) -> None: tracker = WorkunitTracker() handler = StreamingWorkunitHandler( rule_runner.scheduler, run_tracker=run_tracker, callbacks=[tracker.add], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.TRACE, specs=Specs.empty(), options_bootstrapper=create_options_bootstrapper([]), ) with handler.session(): input_1 = CreateDigest(( FileContent(path="a.txt", content=b"alpha"), FileContent(path="b.txt", content=b"beta"), )) digest_1 = rule_runner.request(Digest, [input_1]) snapshot_1 = rule_runner.request(Snapshot, [digest_1]) input_2 = CreateDigest((FileContent(path="g.txt", content=b"gamma"), )) digest_2 = rule_runner.request(Digest, [input_2]) snapshot_2 = rule_runner.request(Snapshot, [digest_2]) input = ComplicatedInput(snapshot_1=snapshot_1, snapshot_2=snapshot_2) rule_runner.request(Output, [input]) finished = list( itertools.chain.from_iterable(tracker.finished_workunit_chunks)) workunit = next( item for item in finished if item["name"] == "pants.engine.internals.engine_test.a_rule") streaming_workunit_context = handler._context artifacts = workunit["artifacts"] output_snapshot_1 = artifacts["snapshot_1"] output_snapshot_2 = artifacts["snapshot_2"] output_contents_list = streaming_workunit_context.snapshots_to_file_contents( [output_snapshot_1, output_snapshot_2]) assert len(output_contents_list) == 2 assert isinstance(output_contents_list[0], DigestContents) assert isinstance(output_contents_list[1], DigestContents) digest_contents_1 = output_contents_list[0] digest_contents_2 = output_contents_list[1] assert len(tuple(x for x in digest_contents_1 if x.content == b"alpha")) == 1 assert len(tuple(x for x in digest_contents_1 if x.content == b"beta")) == 1 assert len(tuple(x for x in digest_contents_2 if x.content == b"gamma")) == 1
def test_streaming_workunits_expanded_specs(run_tracker: RunTracker) -> None: rule_runner = RuleRunner( target_types=[PythonLibrary], rules=[ QueryRule(ProcessResult, (Process, )), ], ) rule_runner.set_options(["--backend-packages=pants.backend.python"]) rule_runner.create_file("src/python/somefiles/BUILD", "python_library()") rule_runner.create_file("src/python/somefiles/a.py", "print('')") rule_runner.create_file("src/python/somefiles/b.py", "print('')") rule_runner.create_file("src/python/others/BUILD", "python_library()") rule_runner.create_file("src/python/others/a.py", "print('')") rule_runner.create_file("src/python/others/b.py", "print('')") specs = SpecsParser(get_buildroot()).parse_specs( ["src/python/somefiles::", "src/python/others/b.py"]) def callback(**kwargs) -> None: context = kwargs["context"] assert isinstance(context, StreamingWorkunitContext) expanded = context.get_expanded_specs() targets = expanded.targets assert len(targets.keys()) == 2 assert targets["src/python/others/b.py"] == [ TargetInfo(filename="src/python/others/b.py") ] assert set(targets["src/python/somefiles"]) == { TargetInfo(filename="src/python/somefiles/a.py"), TargetInfo(filename="src/python/somefiles/b.py"), } handler = StreamingWorkunitHandler( scheduler=rule_runner.scheduler, run_tracker=run_tracker, callbacks=[callback], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.INFO, specs=specs, options_bootstrapper=create_options_bootstrapper( ["--backend-packages=pants.backend.python"]), ) stdout_process = Process(argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process") with handler.session(): rule_runner.request(ProcessResult, [stdout_process])
def run(self, start_time: float) -> ExitCode: with maybe_profiled(self.profile_path): spec_parser = SpecsParser(get_buildroot()) specs = [ str(spec_parser.parse_spec(spec)) for spec in self.options.specs ] self.run_tracker.start(run_start_time=start_time, specs=specs) global_options = self.options.for_global_scope() streaming_reporter = StreamingWorkunitHandler( self.graph_session.scheduler_session, run_tracker=self.run_tracker, specs=self.specs, options_bootstrapper=self.options_bootstrapper, callbacks=self._get_workunits_callbacks(), report_interval_seconds=global_options. streaming_workunits_report_interval, pantsd=global_options.pantsd, ) with streaming_reporter: engine_result = PANTS_FAILED_EXIT_CODE try: engine_result = self._run_inner() finally: metrics = self.graph_session.scheduler_session.metrics() self.run_tracker.set_pantsd_scheduler_metrics(metrics) self.run_tracker.end_run(engine_result) return engine_result
def run(self, start_time: float) -> ExitCode: with maybe_profiled(self.profile_path): spec_parser = SpecsParser() specs = [] for spec_str in self.options.specs: spec, is_ignore = spec_parser.parse_spec(spec_str) specs.append(f"-{spec}" if is_ignore else str(spec)) self.run_tracker.start(run_start_time=start_time, specs=specs) global_options = self.options.for_global_scope() streaming_reporter = StreamingWorkunitHandler( self.graph_session.scheduler_session, run_tracker=self.run_tracker, specs=self.specs, options_bootstrapper=self.options_bootstrapper, callbacks=self._get_workunits_callbacks(), report_interval_seconds=global_options. streaming_workunits_report_interval, allow_async_completion=( global_options.pantsd and global_options.streaming_workunits_complete_async), max_workunit_verbosity=global_options. streaming_workunits_level, ) with streaming_reporter: engine_result = PANTS_FAILED_EXIT_CODE try: engine_result = self._run_inner() finally: metrics = self.graph_session.scheduler_session.metrics() self.run_tracker.set_pantsd_scheduler_metrics(metrics) self.run_tracker.end_run(engine_result) return engine_result
def test_streaming_workunits_expanded_specs(run_tracker: RunTracker) -> None: rule_runner = RuleRunner( target_types=[PythonSourcesGeneratorTarget], rules=[ QueryRule(ProcessResult, (Process, )), ], ) rule_runner.set_options(["--backend-packages=pants.backend.python"]) rule_runner.write_files({ "src/python/somefiles/BUILD": "python_sources()", "src/python/somefiles/a.py": "print('')", "src/python/somefiles/b.py": "print('')", "src/python/others/BUILD": "python_sources()", "src/python/others/a.py": "print('')", "src/python/others/b.py": "print('')", }) specs = SpecsParser().parse_specs( ["src/python/somefiles::", "src/python/others/b.py"], convert_dir_literal_to_address_literal=False, description_of_origin="tests", ) class Callback(WorkunitsCallback): @property def can_finish_async(self) -> bool: return False def __call__(self, **kwargs) -> None: context = kwargs["context"] assert isinstance(context, StreamingWorkunitContext) expanded = context.get_expanded_specs() targets = expanded.targets assert len(targets.keys()) == 2 assert targets["src/python/others/b.py"] == [ TargetInfo(filename="src/python/others/b.py") ] assert set(targets["src/python/somefiles"]) == { TargetInfo(filename="src/python/somefiles/a.py"), TargetInfo(filename="src/python/somefiles/b.py"), } handler = StreamingWorkunitHandler( scheduler=rule_runner.scheduler, run_tracker=run_tracker, callbacks=[Callback()], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.INFO, specs=specs, options_bootstrapper=create_options_bootstrapper( ["--backend-packages=pants.backend.python"]), allow_async_completion=False, ) stdout_process = Process(argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process") with handler: rule_runner.request(ProcessResult, [stdout_process])
def _fixture_for_rules( self, rules, max_workunit_verbosity: LogLevel = LogLevel.INFO ) -> Tuple[SchedulerSession, WorkunitTracker, StreamingWorkunitHandler]: scheduler = self.mk_scheduler(rules, include_trace_on_error=False) tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, run_tracker=new_run_tracker(), callbacks=[tracker], report_interval_seconds=0.01, max_workunit_verbosity=max_workunit_verbosity, specs=Specs.empty(), options_bootstrapper=create_options_bootstrapper([]), pantsd=False, ) return scheduler, tracker, handler
def _fixture_for_rules( self, rules, max_workunit_verbosity: LogLevel = LogLevel.INFO ) -> Tuple[SchedulerSession, WorkunitTracker, StreamingWorkunitHandler]: scheduler = self.mk_scheduler(rules, include_trace_on_error=False) tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, run_tracker=new_run_tracker(), callbacks=[tracker.add], report_interval_seconds=0.01, max_workunit_verbosity=max_workunit_verbosity, ) return (scheduler, tracker, handler)
def test_counters(rule_runner: RuleRunner, run_tracker: RunTracker) -> None: scheduler = rule_runner.scheduler tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, run_tracker=run_tracker, callbacks=[tracker], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.TRACE, specs=Specs.empty(), options_bootstrapper=create_options_bootstrapper([]), allow_async_completion=False, ) with handler: scheduler.record_test_observation(128) rule_runner.request( ProcessResult, [ Process( ["/bin/sh", "-c", "true"], description="always true", cache_scope=ProcessCacheScope.PER_SESSION, ) ], ) histograms_info = scheduler.get_observation_histograms() finished = list( itertools.chain.from_iterable(tracker.finished_workunit_chunks)) workunits_with_counters = [item for item in finished if "counters" in item] assert workunits_with_counters[0]["counters"]["local_cache_requests"] == 1 assert workunits_with_counters[0]["counters"][ "local_cache_requests_uncached"] == 1 assert workunits_with_counters[1]["counters"][ "local_execution_requests"] == 1 assert histograms_info["version"] == 0 assert "histograms" in histograms_info assert "test_observation" in histograms_info["histograms"] assert ( histograms_info["histograms"]["test_observation"] == b"\x1c\x84\x93\x14\x00\x00\x00\x1fx\x9c\x93i\x99,\xcc\xc0\xc0\xc0\xcc\x00\x010\x9a\x11J3\xd9\x7f\x800\xfe32\x01\x00E\x0c\x03\x81" )
def test_context_object_on_streaming_workunits( rule_runner: RuleRunner, run_tracker: RunTracker) -> None: scheduler = rule_runner.scheduler class Callback(WorkunitsCallback): @property def can_finish_async(self) -> bool: return False def __call__(self, **kwargs) -> None: context = kwargs["context"] assert isinstance(context, StreamingWorkunitContext) completed_workunits = kwargs["completed_workunits"] for workunit in completed_workunits: if "artifacts" in workunit and "stdout_digest" in workunit[ "artifacts"]: digest = workunit["artifacts"]["stdout_digest"] output = context.single_file_digests_to_bytes([digest]) assert output == [b"stdout output\n"] handler = StreamingWorkunitHandler( scheduler, run_tracker=run_tracker, callbacks=[Callback()], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.INFO, specs=Specs.empty(), options_bootstrapper=create_options_bootstrapper([]), allow_async_completion=False, ) stdout_process = Process(argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process") with handler: rule_runner.request(ProcessResult, [stdout_process])
def test_process_digests_on_streaming_workunits( rule_runner: RuleRunner, run_tracker: RunTracker) -> None: scheduler = rule_runner.scheduler tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, run_tracker=run_tracker, callbacks=[tracker], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.DEBUG, specs=Specs.empty(), options_bootstrapper=create_options_bootstrapper([]), allow_async_completion=False, ) stdout_process = Process(argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process") with handler: result = rule_runner.request(ProcessResult, [stdout_process]) assert tracker.finished finished = list( itertools.chain.from_iterable(tracker.finished_workunit_chunks)) process_workunit = next(item for item in finished if item["name"] == "process") assert process_workunit is not None stdout_digest = process_workunit["artifacts"]["stdout_digest"] stderr_digest = process_workunit["artifacts"]["stderr_digest"] assert result.stdout == b"stdout output\n" assert stderr_digest == EMPTY_FILE_DIGEST assert stdout_digest.serialized_bytes_length == len(result.stdout) tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, run_tracker=run_tracker, callbacks=[tracker], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.DEBUG, specs=Specs.empty(), options_bootstrapper=create_options_bootstrapper([]), allow_async_completion=False, ) stderr_process = Process(argv=("/bin/bash", "-c", "1>&2 /bin/echo 'stderr output'"), description="Stderr process") with handler: result = rule_runner.request(ProcessResult, [stderr_process]) assert tracker.finished finished = list( itertools.chain.from_iterable(tracker.finished_workunit_chunks)) process_workunit = next(item for item in finished if item["name"] == "process") assert process_workunit is not None stdout_digest = process_workunit["artifacts"]["stdout_digest"] stderr_digest = process_workunit["artifacts"]["stderr_digest"] assert result.stderr == b"stderr output\n" assert stdout_digest == EMPTY_FILE_DIGEST assert stderr_digest.serialized_bytes_length == len(result.stderr) assert process_workunit["metadata"]["exit_code"] == 0 try: scheduler.ensure_remote_has_recursive([stdout_digest, stderr_digest]) except Exception as e: # This is the exception message we should expect from invoking ensure_remote_has_recursive() # in rust. assert str(e) == "Cannot ensure remote has blobs without a remote" byte_outputs = scheduler.single_file_digests_to_bytes( [stdout_digest, stderr_digest]) assert byte_outputs[0] == result.stdout assert byte_outputs[1] == result.stderr