def pre_fork(self): """Pre-fork() callback for ProcessManager.daemonize().""" for service in self._services: service.pre_fork() # Teardown the RunTracker's SubprocPool pre-fork. RunTracker.global_instance().shutdown_worker_pool()
def clean_global_runtime_state(): """Resets the global runtime state of a pants runtime for cleaner forking.""" # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) # Reset Goals and Tasks. Goal.clear()
def clean_global_runtime_state(): """Resets the global runtime state of a pants runtime for cleaner forking.""" # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) # Reset Goals and Tasks. Goal.clear()
def pre_fork(self): """Pre-fork() callback for ProcessManager.daemonize().""" for service in self._services: service.pre_fork() # Teardown the RunTracker's SubprocPool pre-fork. RunTracker.global_instance().shutdown_worker_pool()
def _execute_engine(self): workdir = self._context.options.for_global_scope().pants_workdir if not workdir.endswith('.pants.d'): self._context.log.error( 'Pants working directory should end with \'.pants.d\', currently it is {}\n' .format(workdir)) return 1 unknown_goals = [ goal.name for goal in self._goals if not goal.ordered_task_names() ] if unknown_goals: self._context.log.error('Unknown goal(s): {}\n'.format( ' '.join(unknown_goals))) return 1 engine = RoundEngine() sorted_goal_infos = engine.sort_goals(self._context, self._goals) RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos) result = engine.execute(self._context, self._goals) if self._context.invalidation_report: self._context.invalidation_report.report() return result
def _execute_engine(self): engine = RoundEngine() sorted_goal_infos = engine.sort_goals(self._context, self._goals) RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos) result = engine.execute(self._context, self._goals) if self._context.invalidation_report: self._context.invalidation_report.report() return result
def _clean_runtime_state(self): """Resets the runtime state from running ./pants -> running in the fork()'d daemon context.""" # TODO(kwlzn): Make this logic available to PantsRunner et al for inline state reset before # pants runs to improve testability and avoid potential bitrot. # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) # Reset Goals and Tasks. Goal.clear()
def _execute_engine(self): engine = RoundEngine() sorted_goal_infos = engine.sort_goals(self._context, self._goals) RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos) result = engine.execute(self._context, self._goals) if self._context.invalidation_report: self._context.invalidation_report.report() return result
def _run(self): # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() reporting.initialize(run_tracker, self._run_start_time) try: # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) engine_result = self._maybe_run_v2() goal_runner_result = self._maybe_run_v1(run_tracker, reporting) if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker_result = run_tracker.end() final_exit_code = self._compute_final_exit_code( engine_result, goal_runner_result, run_tracker_result ) self._exiter.exit(final_exit_code)
def prepare_v1_graph_run_v2(self, options, options_bootstrapper): """For v1 (and v2): computing TargetRoots for a later v1 run For v2: running an entire v2 run The exit_code in the return indicates whether any issue was encountered :returns: `(LegacyGraphSession, TargetRoots, exit_code)` """ # If any nodes exist in the product graph, wait for the initial watchman event to avoid # racing watchman startup vs invalidation events. graph_len = self._scheduler.graph_len() if graph_len > 0: self._logger.debug( 'graph len was {}, waiting for initial watchman event'.format( graph_len)) self._watchman_is_running.wait() build_id = RunTracker.global_instance().run_id v2_ui = options.for_global_scope().v2_ui zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2 session = self._graph_helper.new_session(zipkin_trace_v2, build_id, v2_ui) if options.for_global_scope().loop: fn = self._loop else: fn = self._body target_roots, exit_code = fn(session, options, options_bootstrapper) return session, target_roots, exit_code
def run(self, start_time: float) -> ExitCode: run_tracker = RunTracker.global_instance() self._start_run(run_tracker, start_time) with maybe_profiled(self.profile_path): global_options = self.options.for_global_scope() if self.options.help_request: return self._print_help(self.options.help_request) streaming_handlers = global_options.streaming_workunits_handlers callbacks = Subsystem.get_streaming_workunit_callbacks( streaming_handlers) streaming_reporter = StreamingWorkunitHandler( self.graph_session.scheduler_session, callbacks=callbacks, report_interval_seconds=global_options. streaming_workunits_report_interval, ) goals = tuple(self.options.goals) with streaming_reporter.session(): engine_result = PANTS_FAILED_EXIT_CODE try: engine_result = self._run_v2(goals) except Exception as e: ExceptionSink.log_exception(e) self._finish_run(run_tracker, engine_result) return engine_result
def prepare_v1_graph_run_v2( self, options: Options, options_bootstrapper: OptionsBootstrapper, ) -> Tuple[LegacyGraphSession, Specs, int]: """For v1 (and v2): computing Specs for a later v1 run. For v2: running an entire v2 run The exit_code in the return indicates whether any issue was encountered """ # If any nodes exist in the product graph, wait for the initial watchman event to avoid # racing watchman startup vs invalidation events. graph_len = self._scheduler.graph_len() if graph_len > 0: self._logger.debug( "graph len was {}, waiting for initial watchman event".format( graph_len)) self._watchman_is_running.wait() build_id = RunTracker.global_instance().run_id v2_ui = options.for_global_scope().get("v2_ui", False) zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2 session = self._graph_helper.new_session(zipkin_trace_v2, build_id, v2_ui) if options.for_global_scope().get("loop", False): fn = self._loop else: fn = self._body specs, exit_code = fn(session, options, options_bootstrapper) return session, specs, exit_code
def create( cls, env: Mapping[str, str], options_bootstrapper: OptionsBootstrapper, scheduler: Optional[LegacyGraphScheduler] = None, ) -> "LocalPantsRunner": """Creates a new LocalPantsRunner instance by parsing options. By the time this method runs, logging will already have been initialized in either PantsRunner or DaemonPantsRunner. :param env: The environment (e.g. os.environ) for this run. :param options_bootstrapper: The OptionsBootstrapper instance to reuse. :param scheduler: If being called from the daemon, a warmed scheduler to use. """ build_root = get_buildroot() global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) options, build_config = LocalPantsRunner.parse_options( options_bootstrapper) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify configs. if global_bootstrap_options.verify_config: options.verify_configs(options_bootstrapper.config) union_membership = UnionMembership(build_config.union_rules()) # If we're running with the daemon, we'll be handed a warmed Scheduler, which we use # to initialize a session here. graph_session = cls._init_graph_session(options_bootstrapper, build_config, options, scheduler) global_options = options.for_global_scope() specs = SpecsCalculator.create( options=options, build_root=build_root, session=graph_session.scheduler_session, exclude_patterns=tuple(global_options.exclude_target_regexp), tags=tuple(global_options.tag), ) profile_path = env.get("PANTS_PROFILE") return cls( build_root=build_root, options=options, options_bootstrapper=options_bootstrapper, build_config=build_config, specs=specs, graph_session=graph_session, union_membership=union_membership, profile_path=profile_path, _run_tracker=RunTracker.global_instance(), )
def _init_graph_session( cls, options_bootstrapper: OptionsBootstrapper, build_config: BuildConfiguration, options: Options, scheduler: Optional[GraphScheduler] = None, ) -> GraphSession: native = Native() native.set_panic_handler() graph_scheduler_helper = scheduler or EngineInitializer.setup_graph( options_bootstrapper, build_config) try: global_scope = options.for_global_scope() except UnknownFlagsError as err: cls._handle_unknown_flags(err, options_bootstrapper) raise stream_workunits = len( options.for_global_scope().streaming_workunits_handlers) != 0 return graph_scheduler_helper.new_session( RunTracker.global_instance().run_id, dynamic_ui=global_scope.dynamic_ui, use_colors=global_scope.get("colors", True), should_report_workunits=stream_workunits, session_values=SessionValues({ OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment(os.environ), }), )
def _init_graph_session( options_bootstrapper: OptionsBootstrapper, build_config: BuildConfiguration, options: Options, ) -> LegacyGraphSession: native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) v2_ui = options.for_global_scope().get("v2_ui", False) zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2 # TODO(#8658) This should_report_workunits flag must be set to True for # StreamingWorkunitHandler to receive WorkUnits. It should eventually # be merged with the zipkin_trace_v2 flag, since they both involve most # of the same engine functionality, but for now is separate to avoid # breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting. stream_workunits = len( options.for_global_scope().streaming_workunits_handlers) != 0 return graph_scheduler_helper.new_session( zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits=stream_workunits, )
def set_start_time(self, start_time: Optional[float]) -> None: # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() # Propagates parent_build_id to pants runs that may be called from this pants run. os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id self._reporting = Reporting.global_instance() self._reporting.initialize(self._run_tracker, self.options, start_time=start_time) spec_parser = CmdLineSpecParser(get_buildroot()) specs = [ spec_parser.parse_spec(spec).to_spec_string() for spec in self.options.specs ] # Note: This will not include values from `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", specs, stringify=False) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict())
def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper( env=self._env, args=self._args) options, build_config = OptionsInitializer( options_bootstrapper, exiter=self._exiter).setup() global_options = options.for_global_scope() # Apply exiter options. self._exiter.apply_options(options) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify the configs here. if global_options.verify_config: options_bootstrapper.verify_configs_against_options(options) # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() reporting.initialize(run_tracker, self._run_start_time) try: # Determine the build root dir. root_dir = get_buildroot() # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) # Record the preceding product graph size. run_tracker.pantsd_stats.set_preceding_graph_size( self._preceding_graph_size) # Setup and run GoalRunner. goal_runner = GoalRunner.Factory(root_dir, options, build_config, run_tracker, reporting, self._target_roots, self._daemon_build_graph, self._exiter).setup() goal_runner_result = goal_runner.run() if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker_result = run_tracker.end() # Take the exit code with higher abs value in case of negative values. final_exit_code = goal_runner_result if abs(goal_runner_result) > abs( run_tracker_result) else run_tracker_result self._exiter.exit(final_exit_code)
def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env, args=self._args) bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() setup_logging_from_options(bootstrap_options) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) global_options = options.for_global_scope() # Apply exiter options. self._exiter.apply_options(options) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify the configs here. if global_options.verify_config: options_bootstrapper.verify_configs_against_options(options) # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() reporting.initialize(run_tracker, self._run_start_time) try: # Determine the build root dir. root_dir = get_buildroot() # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) # Setup and run GoalRunner. goal_runner = GoalRunner.Factory(root_dir, options, build_config, run_tracker, reporting, self._target_roots, self._daemon_build_graph, self._exiter).setup() goal_runner_result = goal_runner.run() if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker_result = run_tracker.end() # Take the exit code with higher abs value in case of negative values. final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result self._exiter.exit(final_exit_code)
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False): """Resets the global runtime state of a pants runtime for cleaner forking. :param bool reset_runtracker: Whether or not to clean RunTracker global state. :param bool reset_subsystem: Whether or not to clean Subsystem global state. """ if reset_runtracker: # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) if reset_subsystem: # Reset subsystem state. Subsystem.reset() # Reset Goals and Tasks. Goal.clear() # Reset backend/plugins state. OptionsInitializer.reset()
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False): """Resets the global runtime state of a pants runtime for cleaner forking. :param bool reset_runtracker: Whether or not to clean RunTracker global state. :param bool reset_subsystem: Whether or not to clean Subsystem global state. """ if reset_runtracker: # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) if reset_subsystem: # Reset subsystem state. Subsystem.reset() # Reset Goals and Tasks. Goal.clear() # Reset backend/plugins state. OptionsInitializer.reset()
def prepare_graph(self, options: Options) -> LegacyGraphSession: # If any nodes exist in the product graph, wait for the initial watchman event to avoid # racing watchman startup vs invalidation events. graph_len = self._scheduler.graph_len() if graph_len > 0: self._logger.debug(f"graph len was {graph_len}, waiting for initial watchman event") self._watchman_is_running.wait() global_options = options.for_global_scope() build_id = RunTracker.global_instance().run_id v2_ui = global_options.get("v2_ui", False) zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2 return self._graph_helper.new_session(zipkin_trace_v2, build_id, v2_ui)
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False): """Resets the global runtime state of a pants runtime for cleaner forking. :param bool reset_runtracker: Whether or not to clean RunTracker global state. :param bool reset_subsystem: Whether or not to clean Subsystem global state. """ if reset_runtracker: # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) if reset_subsystem: # Reset subsystem state. Subsystem.reset() #TODO: Think of an alternative for IntermediateTargetFactoryBase._targets to avoid this call IntermediateTargetFactoryBase.reset() # Reset Goals and Tasks. Goal.clear() # Reset backend/plugins state. OptionsInitializer.reset()
def _maybe_init_graph_session(graph_session, options_bootstrapper, build_config, options): if not graph_session: native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) v2_ui = options.for_global_scope().v2_ui zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2 graph_session = graph_scheduler_helper.new_session( zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui) return graph_session, graph_session.scheduler_session
def test_raise_no_zipkin_endpoint_set(self): options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}} context = self.context(for_subsystems=[RunTracker, Reporting], options=options) run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() with self.assertRaises(ValueError) as result: reporting.initialize(run_tracker, context.options) self.assertTrue( "The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given." in str(result.exception) )
def set_start_time(self, start_time): # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() self._reporting = Reporting.global_instance() self._run_start_time = start_time self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict())
def _execute_engine(self): workdir = self._context.options.for_global_scope().pants_workdir if not workdir.endswith('.pants.d'): self._context.log.error('Pants working directory should end with \'.pants.d\', currently it is {}\n' .format(workdir)) return 1 unknown_goals = [goal.name for goal in self._goals if not goal.ordered_task_names()] if unknown_goals: self._context.log.error('Unknown goal(s): {}\n'.format(' '.join(unknown_goals))) return 1 engine = RoundEngine() sorted_goal_infos = engine.sort_goals(self._context, self._goals) RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos) result = engine.execute(self._context, self._goals) if self._context.invalidation_report: self._context.invalidation_report.report() return result
def test_raise_no_zipkin_endpoint_set(self): options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}} context = self.context(for_subsystems=[RunTracker, Reporting], options=options) run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() with self.assertRaises(ValueError) as result: reporting.initialize(run_tracker, context.options) self.assertTrue( "The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given." in str(result.exception) )
def test_raise_if_no_trace_id_and_zipkin_endpoint_set(self): options = {'reporting': {'zipkin_parent_id': self.parent_id}} context = self.context(for_subsystems=[RunTracker, Reporting], options=options) run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() with self.assertRaises(ValueError) as result: reporting.initialize(run_tracker, context.options) self.assertTrue( "Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set." in str(result.exception))
def test_raise_if_no_parent_id_and_zipkin_endpoint_set(self): options = {'reporting': {'zipkin_trace_id': self.trace_id}} context = self.context(for_subsystems=[RunTracker, Reporting], options=options) run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() with self.assertRaises(ValueError) as result: reporting.initialize(run_tracker, context.options) self.assertTrue( "Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set." in str(result.exception) )
def set_start_time(self, start_time): # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() self._reporting = Reporting.global_instance() self._run_start_time = start_time self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict()) # The __call__ method of the Exiter allows for the prototype pattern. self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter) ExceptionSink.reset_exiter(self._exiter)
def set_start_time(self, start_time): # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() self._reporting = Reporting.global_instance() self._run_start_time = start_time self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict()) # The __call__ method of the Exiter allows for the prototype pattern. self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter) ExceptionSink.reset_exiter(self._exiter)
def test_raise_if_trace_id_is_of_wrong_ch_format(self): trace_id = 'gggggggggggggggg' options = {'reporting': { 'zipkin_trace_id': trace_id, 'zipkin_parent_id': self.parent_id, 'zipkin_endpoint': self.zipkin_endpoint }} context = self.context(for_subsystems=[RunTracker, Reporting], options=options) run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() with self.assertRaises(ValueError) as result: reporting.initialize(run_tracker, context.options) self.assertTrue( "Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. " + f"Got {trace_id}." in str(result.exception) )
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options): if not graph_session: native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config ) v2_ui = options.for_global_scope().v2_ui zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2 #TODO(gregorys) This should_report_workunits flag must be set to True for # AsyncWorkunitHandler to receive WorkUnits. It should eventually # be merged with the zipkin_trace_v2 flag, since they both involve most # of the same engine functionality, but for now is separate to avoid # breaking functionality associated with zipkin tracing while iterating on async workunit reporting. should_report_workunits = False graph_session = graph_scheduler_helper.new_session(zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits) return graph_session, graph_session.scheduler_session
def set_start_time(self, start_time): # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() self._reporting = Reporting.global_instance() self._run_start_time = start_time self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time) spec_parser = CmdLineSpecParser(get_buildroot()) target_specs = [ spec_parser.parse_spec(spec).to_spec_string() for spec in self._options.positional_args ] # Note: This will not include values from `--owner-of` or `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict())
def test_raise_if_trace_id_is_of_wrong_ch_format(self): trace_id = 'gggggggggggggggg' options = {'reporting': { 'zipkin_trace_id': trace_id, 'zipkin_parent_id': self.parent_id, 'zipkin_endpoint': self.zipkin_endpoint }} context = self.context(for_subsystems=[RunTracker, Reporting], options=options) run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() with self.assertRaises(ValueError) as result: reporting.initialize(run_tracker, context.options) self.assertTrue( "Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. " + "Got {}.".format(trace_id) in str(result.exception) )
def _init_graph_session( options_bootstrapper: OptionsBootstrapper, build_config: BuildConfiguration, options: Options, scheduler: Optional[LegacyGraphScheduler] = None, ) -> LegacyGraphSession: native = Native() native.set_panic_handler() graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph( options_bootstrapper, build_config) global_scope = options.for_global_scope() if global_scope.v2: dynamic_ui = resolve_conflicting_options( old_option="v2_ui", new_option="dynamic_ui", old_scope=GLOBAL_SCOPE, new_scope=GLOBAL_SCOPE, old_container=global_scope, new_container=global_scope, ) else: dynamic_ui = False use_colors = global_scope.get("colors", True) zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2 # TODO(#8658) This should_report_workunits flag must be set to True for # StreamingWorkunitHandler to receive WorkUnits. It should eventually # be merged with the zipkin_trace_v2 flag, since they both involve most # of the same engine functionality, but for now is separate to avoid # breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting. stream_workunits = len( options.for_global_scope().streaming_workunits_handlers) != 0 return graph_scheduler_helper.new_session( zipkin_trace_v2, RunTracker.global_instance().run_id, dynamic_ui=dynamic_ui, use_colors=use_colors, should_report_workunits=stream_workunits, )
def test_raise_if_parent_id_is_of_wrong_ch_format(self): parent_id = "gggggggggggggggg" options = { "reporting": { "zipkin_trace_id": self.trace_id, "zipkin_parent_id": parent_id, "zipkin_endpoint": self.zipkin_endpoint, } } context = self.context(for_subsystems=[RunTracker, Reporting], options=options) run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() with self.assertRaises(ValueError) as result: reporting.initialize(run_tracker, context.options) self.assertTrue( "Value of the flag zipkin-parent-id must be a 16-character hex string. " + f"Got {parent_id}." in str(result.exception))
def _init_graph_session( options_bootstrapper: OptionsBootstrapper, build_config: BuildConfiguration, options: Options, scheduler: Optional[LegacyGraphScheduler] = None, ) -> LegacyGraphSession: native = Native() native.set_panic_handler() graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph( options_bootstrapper, build_config) global_scope = options.for_global_scope() dynamic_ui = global_scope.dynamic_ui if global_scope.v2 else False use_colors = global_scope.get("colors", True) stream_workunits = len( options.for_global_scope().streaming_workunits_handlers) != 0 return graph_scheduler_helper.new_session( RunTracker.global_instance().run_id, dynamic_ui=dynamic_ui, use_colors=use_colors, should_report_workunits=stream_workunits, )
def __init__(self, run_tracker=None, reporting=None): self._run_tracker = run_tracker or RunTracker.global_instance() self._reporting = reporting or Reporting.global_instance()
def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. bootstrap_options = options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scopes = [''] # Add scopes for global subsystem instances. for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types(): known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE)) # Add scopes for all tasks in all goals. for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes) self.register_options() # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()
def __init__(self, run_tracker=None, reporting=None): self._run_tracker = run_tracker or RunTracker.global_instance() self._reporting = reporting or Reporting.global_instance()
def setup(self, options_bootstrapper, working_set): bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually # requested the version on the command line before deciding to print the version and exit. if global_bootstrap_options.is_flagged('pants_version'): print(global_bootstrap_options.pants_version) self._exiter(0) # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (e.g., for loading extra source backends). for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems. subsystems = Subsystem.closure(set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()
def setup(self): options_bootstrapper = OptionsBootstrapper() bootstrap_options = options_bootstrapper.get_bootstrap_options() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = bootstrap_options.for_global_scope().plugins backend_packages = bootstrap_options.for_global_scope().backend_packages build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [ScopeInfo.for_global_scope()] # Add scopes for all needed subsystems. subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM)) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()