def _run(self):
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
engine_result = self._maybe_run_v2()
goal_runner_result = self._maybe_run_v1(run_tracker, reporting)
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
final_exit_code = self._compute_final_exit_code(
engine_result,
goal_runner_result,
run_tracker_result
)
self._exiter.exit(final_exit_code)
def set_start_time(self, start_time: Optional[float]) -> None:
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
# Propagates parent_build_id to pants runs that may be called from this pants run.
os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id
self._reporting = Reporting.global_instance()
self._reporting.initialize(self._run_tracker,
self.options,
start_time=start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self.options.specs
]
# Note: This will not include values from `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line",
specs,
stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(
env=self._env, args=self._args)
options, build_config = OptionsInitializer(
options_bootstrapper, exiter=self._exiter).setup()
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Record the preceding product graph size.
run_tracker.pantsd_stats.set_preceding_graph_size(
self._preceding_graph_size)
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir, options, build_config,
run_tracker, reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(
run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
setup_logging_from_options(bootstrap_options)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir,
options,
build_config,
run_tracker,
reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def test_raise_no_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
in str(result.exception)
)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker,
self._options,
start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def test_raise_no_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
in str(result.exception)
)
def test_raise_if_no_trace_id_and_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_parent_id': self.parent_id}}
context = self.context(for_subsystems=[RunTracker, Reporting],
options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
in str(result.exception))
def test_raise_if_no_parent_id_and_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
in str(result.exception)
)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
# The __call__ method of the Exiter allows for the prototype pattern.
self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
ExceptionSink.reset_exiter(self._exiter)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
# The __call__ method of the Exiter allows for the prototype pattern.
self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
ExceptionSink.reset_exiter(self._exiter)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
target_specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self._options.positional_args
]
# Note: This will not include values from `--owner-of` or `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def test_raise_if_trace_id_is_of_wrong_ch_format(self):
trace_id = 'gggggggggggggggg'
options = {'reporting': {
'zipkin_trace_id': trace_id,
'zipkin_parent_id': self.parent_id,
'zipkin_endpoint': self.zipkin_endpoint
}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
+ "Got {}.".format(trace_id)
in str(result.exception)
)
def test_raise_if_trace_id_is_of_wrong_ch_format(self):
trace_id = 'gggggggggggggggg'
options = {'reporting': {
'zipkin_trace_id': trace_id,
'zipkin_parent_id': self.parent_id,
'zipkin_endpoint': self.zipkin_endpoint
}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
+ f"Got {trace_id}."
in str(result.exception)
)
def test_raise_if_parent_id_is_of_wrong_ch_format(self):
parent_id = "gggggggggggggggg"
options = {
"reporting": {
"zipkin_trace_id": self.trace_id,
"zipkin_parent_id": parent_id,
"zipkin_endpoint": self.zipkin_endpoint,
}
}
context = self.context(for_subsystems=[RunTracker, Reporting],
options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Value of the flag zipkin-parent-id must be a 16-character hex string. "
+ f"Got {parent_id}." in str(result.exception))
def __init__(self, run_tracker=None, reporting=None):
self._run_tracker = run_tracker or RunTracker.global_instance()
self._reporting = reporting or Reporting.global_instance()
def setup(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems.
subsystems = Subsystem.closure(set(self.subsystems) |
Goal.subsystems() |
build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def __init__(self, run_tracker=None, reporting=None):
self._run_tracker = run_tracker or RunTracker.global_instance()
self._reporting = reporting or Reporting.global_instance()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()