def pre_fork(self):
"""Pre-fork() callback for ProcessManager.daemonize()."""
for service in self._services:
service.pre_fork()
# Teardown the RunTracker's SubprocPool pre-fork.
RunTracker.global_instance().shutdown_worker_pool()
def clean_global_runtime_state():
"""Resets the global runtime state of a pants runtime for cleaner forking."""
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
# Reset Goals and Tasks.
Goal.clear()
def clean_global_runtime_state(): """Resets the global runtime state of a pants runtime for cleaner forking.""" # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) # Reset Goals and Tasks. Goal.clear()
def pre_fork(self):
"""Pre-fork() callback for ProcessManager.daemonize()."""
for service in self._services:
service.pre_fork()
# Teardown the RunTracker's SubprocPool pre-fork.
RunTracker.global_instance().shutdown_worker_pool()
def _execute_engine(self):
workdir = self._context.options.for_global_scope().pants_workdir
if not workdir.endswith('.pants.d'):
self._context.log.error(
'Pants working directory should end with \'.pants.d\', currently it is {}\n'
.format(workdir))
return 1
unknown_goals = [
goal.name for goal in self._goals if not goal.ordered_task_names()
]
if unknown_goals:
self._context.log.error('Unknown goal(s): {}\n'.format(
' '.join(unknown_goals)))
return 1
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def _execute_engine(self):
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def _clean_runtime_state(self):
"""Resets the runtime state from running ./pants -> running in the fork()'d daemon context."""
# TODO(kwlzn): Make this logic available to PantsRunner et al for inline state reset before
# pants runs to improve testability and avoid potential bitrot.
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
# Reset Goals and Tasks.
Goal.clear()
def _execute_engine(self):
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def _run(self):
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
engine_result = self._maybe_run_v2()
goal_runner_result = self._maybe_run_v1(run_tracker, reporting)
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
final_exit_code = self._compute_final_exit_code(
engine_result,
goal_runner_result,
run_tracker_result
)
self._exiter.exit(final_exit_code)
def prepare_v1_graph_run_v2(self, options, options_bootstrapper):
"""For v1 (and v2): computing TargetRoots for a later v1 run
For v2: running an entire v2 run
The exit_code in the return indicates whether any issue was encountered
:returns: `(LegacyGraphSession, TargetRoots, exit_code)`
"""
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug(
'graph len was {}, waiting for initial watchman event'.format(
graph_len))
self._watchman_is_running.wait()
build_id = RunTracker.global_instance().run_id
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
session = self._graph_helper.new_session(zipkin_trace_v2, build_id,
v2_ui)
if options.for_global_scope().loop:
fn = self._loop
else:
fn = self._body
target_roots, exit_code = fn(session, options, options_bootstrapper)
return session, target_roots, exit_code
def run(self, start_time: float) -> ExitCode:
run_tracker = RunTracker.global_instance()
self._start_run(run_tracker, start_time)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
if self.options.help_request:
return self._print_help(self.options.help_request)
streaming_handlers = global_options.streaming_workunits_handlers
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=global_options.
streaming_workunits_report_interval,
)
goals = tuple(self.options.goals)
with streaming_reporter.session():
engine_result = PANTS_FAILED_EXIT_CODE
try:
engine_result = self._run_v2(goals)
except Exception as e:
ExceptionSink.log_exception(e)
self._finish_run(run_tracker, engine_result)
return engine_result
def prepare_v1_graph_run_v2(
self,
options: Options,
options_bootstrapper: OptionsBootstrapper,
) -> Tuple[LegacyGraphSession, Specs, int]:
"""For v1 (and v2): computing Specs for a later v1 run.
For v2: running an entire v2 run The exit_code in the return indicates whether any issue was
encountered
"""
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug(
"graph len was {}, waiting for initial watchman event".format(
graph_len))
self._watchman_is_running.wait()
build_id = RunTracker.global_instance().run_id
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
session = self._graph_helper.new_session(zipkin_trace_v2, build_id,
v2_ui)
if options.for_global_scope().get("loop", False):
fn = self._loop
else:
fn = self._body
specs, exit_code = fn(session, options, options_bootstrapper)
return session, specs, exit_code
def create(
cls,
env: Mapping[str, str],
options_bootstrapper: OptionsBootstrapper,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> "LocalPantsRunner":
"""Creates a new LocalPantsRunner instance by parsing options.
By the time this method runs, logging will already have been initialized in either
PantsRunner or DaemonPantsRunner.
:param env: The environment (e.g. os.environ) for this run.
:param options_bootstrapper: The OptionsBootstrapper instance to reuse.
:param scheduler: If being called from the daemon, a warmed scheduler to use.
"""
build_root = get_buildroot()
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
options, build_config = LocalPantsRunner.parse_options(
options_bootstrapper)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify configs.
if global_bootstrap_options.verify_config:
options.verify_configs(options_bootstrapper.config)
union_membership = UnionMembership(build_config.union_rules())
# If we're running with the daemon, we'll be handed a warmed Scheduler, which we use
# to initialize a session here.
graph_session = cls._init_graph_session(options_bootstrapper,
build_config, options,
scheduler)
global_options = options.for_global_scope()
specs = SpecsCalculator.create(
options=options,
build_root=build_root,
session=graph_session.scheduler_session,
exclude_patterns=tuple(global_options.exclude_target_regexp),
tags=tuple(global_options.tag),
)
profile_path = env.get("PANTS_PROFILE")
return cls(
build_root=build_root,
options=options,
options_bootstrapper=options_bootstrapper,
build_config=build_config,
specs=specs,
graph_session=graph_session,
union_membership=union_membership,
profile_path=profile_path,
_run_tracker=RunTracker.global_instance(),
)
def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[GraphScheduler] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
should_report_workunits=stream_workunits,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(os.environ),
}),
)
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
v2_ui,
should_report_workunits=stream_workunits,
)
def set_start_time(self, start_time: Optional[float]) -> None:
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
# Propagates parent_build_id to pants runs that may be called from this pants run.
os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id
self._reporting = Reporting.global_instance()
self._reporting.initialize(self._run_tracker,
self.options,
start_time=start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self.options.specs
]
# Note: This will not include values from `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line",
specs,
stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(
env=self._env, args=self._args)
options, build_config = OptionsInitializer(
options_bootstrapper, exiter=self._exiter).setup()
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Record the preceding product graph size.
run_tracker.pantsd_stats.set_preceding_graph_size(
self._preceding_graph_size)
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir, options, build_config,
run_tracker, reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(
run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
setup_logging_from_options(bootstrap_options)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir,
options,
build_config,
run_tracker,
reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_runtracker: Whether or not to clean RunTracker global state.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_runtracker:
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset backend/plugins state.
OptionsInitializer.reset()
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_runtracker: Whether or not to clean RunTracker global state.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_runtracker:
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset backend/plugins state.
OptionsInitializer.reset()
def prepare_graph(self, options: Options) -> LegacyGraphSession:
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug(f"graph len was {graph_len}, waiting for initial watchman event")
self._watchman_is_running.wait()
global_options = options.for_global_scope()
build_id = RunTracker.global_instance().run_id
v2_ui = global_options.get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
return self._graph_helper.new_session(zipkin_trace_v2, build_id, v2_ui)
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_runtracker: Whether or not to clean RunTracker global state.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_runtracker:
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
#TODO: Think of an alternative for IntermediateTargetFactoryBase._targets to avoid this call
IntermediateTargetFactoryBase.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset backend/plugins state.
OptionsInitializer.reset()
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config, options):
if not graph_session:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
graph_session = graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id, v2_ui)
return graph_session, graph_session.scheduler_session
def test_raise_no_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
in str(result.exception)
)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker,
self._options,
start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def _execute_engine(self):
workdir = self._context.options.for_global_scope().pants_workdir
if not workdir.endswith('.pants.d'):
self._context.log.error('Pants working directory should end with \'.pants.d\', currently it is {}\n'
.format(workdir))
return 1
unknown_goals = [goal.name for goal in self._goals if not goal.ordered_task_names()]
if unknown_goals:
self._context.log.error('Unknown goal(s): {}\n'.format(' '.join(unknown_goals)))
return 1
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def test_raise_no_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
in str(result.exception)
)
def test_raise_if_no_trace_id_and_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_parent_id': self.parent_id}}
context = self.context(for_subsystems=[RunTracker, Reporting],
options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
in str(result.exception))
def test_raise_if_no_parent_id_and_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
in str(result.exception)
)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
# The __call__ method of the Exiter allows for the prototype pattern.
self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
ExceptionSink.reset_exiter(self._exiter)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
# The __call__ method of the Exiter allows for the prototype pattern.
self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
ExceptionSink.reset_exiter(self._exiter)
def test_raise_if_trace_id_is_of_wrong_ch_format(self):
trace_id = 'gggggggggggggggg'
options = {'reporting': {
'zipkin_trace_id': trace_id,
'zipkin_parent_id': self.parent_id,
'zipkin_endpoint': self.zipkin_endpoint
}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
+ f"Got {trace_id}."
in str(result.exception)
)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options):
if not graph_session:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
options_bootstrapper,
build_config
)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
#TODO(gregorys) This should_report_workunits flag must be set to True for
# AsyncWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on async workunit reporting.
should_report_workunits = False
graph_session = graph_scheduler_helper.new_session(zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits)
return graph_session, graph_session.scheduler_session
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
target_specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self._options.positional_args
]
# Note: This will not include values from `--owner-of` or `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def test_raise_if_trace_id_is_of_wrong_ch_format(self):
trace_id = 'gggggggggggggggg'
options = {'reporting': {
'zipkin_trace_id': trace_id,
'zipkin_parent_id': self.parent_id,
'zipkin_endpoint': self.zipkin_endpoint
}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
+ "Got {}.".format(trace_id)
in str(result.exception)
)
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
if global_scope.v2:
dynamic_ui = resolve_conflicting_options(
old_option="v2_ui",
new_option="dynamic_ui",
old_scope=GLOBAL_SCOPE,
new_scope=GLOBAL_SCOPE,
old_container=global_scope,
new_container=global_scope,
)
else:
dynamic_ui = False
use_colors = global_scope.get("colors", True)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
def test_raise_if_parent_id_is_of_wrong_ch_format(self):
parent_id = "gggggggggggggggg"
options = {
"reporting": {
"zipkin_trace_id": self.trace_id,
"zipkin_parent_id": parent_id,
"zipkin_endpoint": self.zipkin_endpoint,
}
}
context = self.context(for_subsystems=[RunTracker, Reporting],
options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Value of the flag zipkin-parent-id must be a 16-character hex string. "
+ f"Got {parent_id}." in str(result.exception))
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
dynamic_ui = global_scope.dynamic_ui if global_scope.v2 else False
use_colors = global_scope.get("colors", True)
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
def __init__(self, run_tracker=None, reporting=None):
self._run_tracker = run_tracker or RunTracker.global_instance()
self._reporting = reporting or Reporting.global_instance()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
# Add scopes for global subsystem instances.
for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types():
known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def __init__(self, run_tracker=None, reporting=None):
self._run_tracker = run_tracker or RunTracker.global_instance()
self._reporting = reporting or Reporting.global_instance()
def setup(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems.
subsystems = Subsystem.closure(set(self.subsystems) |
Goal.subsystems() |
build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
