def pre_fork(self):
"""Pre-fork() callback for ProcessManager.daemonize()."""
for service in self._services:
service.pre_fork()
# Teardown the RunTracker's SubprocPool pre-fork.
RunTracker.global_instance().shutdown_worker_pool()
def pre_fork(self):
"""Pre-fork() callback for ProcessManager.daemonize()."""
for service in self._services:
service.pre_fork()
# Teardown the RunTracker's SubprocPool pre-fork.
RunTracker.global_instance().shutdown_worker_pool()
def clean_global_runtime_state():
"""Resets the global runtime state of a pants runtime for cleaner forking."""
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
# Reset Goals and Tasks.
Goal.clear()
def clean_global_runtime_state(): """Resets the global runtime state of a pants runtime for cleaner forking.""" # Reset RunTracker state. RunTracker.global_instance().reset(reset_options=False) # Reset Goals and Tasks. Goal.clear()
def _execute_engine(self):
workdir = self._context.options.for_global_scope().pants_workdir
if not workdir.endswith('.pants.d'):
self._context.log.error(
'Pants working directory should end with \'.pants.d\', currently it is {}\n'
.format(workdir))
return 1
unknown_goals = [
goal.name for goal in self._goals if not goal.ordered_task_names()
]
if unknown_goals:
self._context.log.error('Unknown goal(s): {}\n'.format(
' '.join(unknown_goals)))
return 1
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def _finish_run(self, run_tracker: RunTracker, code: ExitCode) -> None:
"""Cleans up the run tracker."""
metrics = self.graph_session.scheduler_session.metrics()
run_tracker.set_pantsd_scheduler_metrics(metrics)
outcome = WorkUnit.SUCCESS if code == PANTS_SUCCEEDED_EXIT_CODE else WorkUnit.FAILURE
run_tracker.set_root_outcome(outcome)
def assert_upload_stats(self, *, response_code) -> None:
stats = {'stats': {'foo': 'bar', 'baz': 42}}
class Handler(http.server.BaseHTTPRequestHandler):
def do_POST(handler):
try:
if handler.path.startswith('/redirect'):
code = int(handler.path[-3:])
handler.send_response(code)
handler.send_header('location', mk_url('/upload'))
handler.end_headers()
else:
self.assertEqual('/upload', handler.path)
self.assertEqual('application/x-www-form-urlencoded',
handler.headers['Content-type'])
length = int(handler.headers['Content-Length'])
post_data = parse_qs(
handler.rfile.read(length).decode())
decoded_post_data = {
k: json.loads(v[0])
for k, v in post_data.items()
}
self.assertEqual(stats, decoded_post_data)
self.assertEqual(handler.headers['User-Agent'],
f"pants/v{VERSION}")
self.assertIn(handler.headers['X-Pants-Stats-Version'],
{"1", "2"})
handler.send_response(response_code)
handler.end_headers()
except Exception:
handler.send_response(
400) # Ensure the main thread knows the test failed.
raise
server_address = ('', 0)
server = http.server.HTTPServer(server_address, Handler)
host, port = server.server_address
def mk_url(path):
return f'http://{host}:{port}{path}'
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
self.context(for_subsystems=[Cookies])
self.assertTrue(
RunTracker.post_stats(mk_url('/upload'), stats, stats_version=1))
self.assertTrue(
RunTracker.post_stats(mk_url('/redirect307'),
stats,
stats_version=1))
self.assertFalse(
RunTracker.post_stats(mk_url('/redirect302'),
stats,
stats_version=2))
server.shutdown()
server.server_close()
def test_write_stats_to_json_file(self):
# Set up
stats = {'stats': {'foo': 'bar', 'baz': 42}}
# Execute & verify
with temporary_file_path() as file_name:
RunTracker.write_stats_to_json(file_name, stats)
with open(file_name, 'r') as f:
result = json.load(f)
self.assertEqual(stats, result)
def _clean_runtime_state(self):
"""Resets the runtime state from running ./pants -> running in the fork()'d daemon context."""
# TODO(kwlzn): Make this logic available to PantsRunner et al for inline state reset before
# pants runs to improve testability and avoid potential bitrot.
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
# Reset Goals and Tasks.
Goal.clear()
def _execute_engine(self):
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def test_write_stats_to_json_file(self):
# Set up
stats = {"stats": {"foo": "bar", "baz": 42}}
# Execute & verify
with temporary_file_path() as file_name:
RunTracker.write_stats_to_json(file_name, stats)
with open(file_name, "r") as f:
result = json.load(f)
self.assertEqual(stats, result)
def test_write_stats_to_json_file(self):
# Set up
stats = {'stats': {'foo': 'bar', 'baz': 42}}
# Execute & verify
with temporary_file_path() as file_name:
RunTracker.write_stats_to_json(file_name, stats)
with open(file_name, 'r') as f:
result = json.load(f)
self.assertEqual(stats, result)
def _execute_engine(self):
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def _start_run(self, run_tracker: RunTracker, start_time: float) -> None:
run_tracker.start(self.options, run_start_time=start_time)
spec_parser = SpecsParser(get_buildroot())
specs = [
str(spec_parser.parse_spec(spec)) for spec in self.options.specs
]
# Note: This will not include values from `--changed-*` flags.
run_tracker.run_info.add_info("specs_from_command_line",
specs,
stringify=False)
def test_run_information(exit_code, expected, **kwargs) -> None:
with temporary_dir() as buildroot:
with environment_as(PANTS_BUILDROOT_OVERRIDE=buildroot):
run_tracker = RunTracker(
create_options_bootstrapper([]).bootstrap_options)
specs = ["src/python/pants/goal/run_tracker_test.py"]
run_tracker.start(run_start_time=time.time(), specs=specs)
run_information = run_tracker.run_information()
assert run_information["buildroot"] == get_buildroot()
assert run_information["path"] == get_buildroot()
# freezegun doesn't seem to accurately mock the time zone,
# (i.e. the time zone used depends on that of the machine that
# executes the test), so we can only safely assert that the
# month and year appear in the human-readable string contained
# in the "datetime" key
assert "Jan" in run_information["datetime"]
assert "2020" in run_information["datetime"]
assert run_information["timestamp"] == 1578657601.0
assert run_information["user"] == getpass.getuser()
assert run_information["version"] == VERSION
assert re.match("pants.*run_tracker_test.py",
run_information["cmd_line"])
assert run_information["specs_from_command_line"] == [
"src/python/pants/goal/run_tracker_test.py"
]
frozen_time = kwargs["frozen_time"]
frozen_time.tick(delta=datetime.timedelta(seconds=1))
run_tracker.end_run(exit_code)
run_information_after_ended = run_tracker.run_information()
assert run_information_after_ended["outcome"] == expected
def test_run_information(exit_code: ExitCode, expected: str, tmp_path: Path,
**kwargs) -> None:
frozen_time = kwargs["frozen_time"]
buildroot = tmp_path.as_posix()
with environment_as(PANTS_BUILDROOT_OVERRIDE=buildroot):
spec = "test/example.py"
ob = create_options_bootstrapper(["list", spec])
run_tracker = RunTracker(ob.args, ob.bootstrap_options)
specs = [spec]
run_tracker.start(run_start_time=time.time(), specs=specs)
run_information = run_tracker.run_information()
assert run_information["buildroot"] == get_buildroot()
assert run_information["path"] == get_buildroot()
# freezegun doesn't seem to accurately mock the time zone,
# (i.e. the time zone used depends on that of the machine that
# executes the test), so we can only safely assert that the
# month and year appear in the human-readable string contained
# in the "datetime" key
assert "Jan" in run_information["datetime"]
assert "2020" in run_information["datetime"]
assert run_information["timestamp"] == 1578657601.0
assert run_information["user"] == getpass.getuser()
assert run_information["version"] == VERSION
assert re.match(f"pants.*{spec}", run_information["cmd_line"])
assert run_information["specs_from_command_line"] == [spec]
frozen_time.tick(delta=datetime.timedelta(seconds=1))
run_tracker.end_run(exit_code)
run_information_after_ended = run_tracker.run_information()
assert run_information_after_ended["outcome"] == expected
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
extra_paths = self.config.getlist('backends', 'python-path', [])
if extra_paths:
sys.path.extend(extra_paths)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_config(self.config)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def test_upload_stats(self):
stats = {'stats': {'foo': 'bar', 'baz': 42}}
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(handler):
try:
self.assertEquals('/upload', handler.path)
self.assertEquals('application/x-www-form-urlencoded', handler.headers['Content-type'])
length = int(handler.headers['Content-Length'])
post_data = urlparse.parse_qs(handler.rfile.read(length).decode('utf-8'))
decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
self.assertEquals(stats, decoded_post_data)
handler.send_response(200)
except Exception:
handler.send_response(400) # Ensure the main thread knows the test failed.
raise
server_address = ('', 0)
server = BaseHTTPServer.HTTPServer(server_address, Handler)
host, port = server.server_address
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
self.assertTrue(RunTracker.post_stats('http://{}:{}/upload'.format(host, port), stats))
server.shutdown()
server.server_close()
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
v2_ui,
should_report_workunits=stream_workunits,
)
def _run(self):
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
engine_result = self._maybe_run_v2()
goal_runner_result = self._maybe_run_v1(run_tracker, reporting)
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
final_exit_code = self._compute_final_exit_code(
engine_result,
goal_runner_result,
run_tracker_result
)
self._exiter.exit(final_exit_code)
def test_upload_stats(self):
stats = {'stats': {'foo': 'bar', 'baz': 42}}
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(handler):
try:
self.assertEquals('/upload', handler.path)
self.assertEquals('application/x-www-form-urlencoded', handler.headers['Content-type'])
length = int(handler.headers['Content-Length'])
post_data = urlparse.parse_qs(handler.rfile.read(length).decode('utf-8'))
decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
self.assertEquals(stats, decoded_post_data)
handler.send_response(200)
except Exception:
handler.send_response(400) # Ensure the main thread knows the test failed.
raise
server_address = ('', 0)
server = BaseHTTPServer.HTTPServer(server_address, Handler)
host, port = server.server_address
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
self.assertTrue(RunTracker.post_stats('http://{}:{}/upload'.format(host, port), stats))
server.shutdown()
server.server_close()
def prepare_v1_graph_run_v2(
self,
options: Options,
options_bootstrapper: OptionsBootstrapper,
) -> Tuple[LegacyGraphSession, Specs, int]:
"""For v1 (and v2): computing Specs for a later v1 run.
For v2: running an entire v2 run The exit_code in the return indicates whether any issue was
encountered
"""
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug(
"graph len was {}, waiting for initial watchman event".format(
graph_len))
self._watchman_is_running.wait()
build_id = RunTracker.global_instance().run_id
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
session = self._graph_helper.new_session(zipkin_trace_v2, build_id,
v2_ui)
if options.for_global_scope().get("loop", False):
fn = self._loop
else:
fn = self._body
specs, exit_code = fn(session, options, options_bootstrapper)
return session, specs, exit_code
def prepare_v1_graph_run_v2(self, options, options_bootstrapper):
"""For v1 (and v2): computing TargetRoots for a later v1 run
For v2: running an entire v2 run
The exit_code in the return indicates whether any issue was encountered
:returns: `(LegacyGraphSession, TargetRoots, exit_code)`
"""
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug(
'graph len was {}, waiting for initial watchman event'.format(
graph_len))
self._watchman_is_running.wait()
build_id = RunTracker.global_instance().run_id
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
session = self._graph_helper.new_session(zipkin_trace_v2, build_id,
v2_ui)
if options.for_global_scope().loop:
fn = self._loop
else:
fn = self._body
target_roots, exit_code = fn(session, options, options_bootstrapper)
return session, target_roots, exit_code
def create(
cls,
env: Mapping[str, str],
options_bootstrapper: OptionsBootstrapper,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> "LocalPantsRunner":
"""Creates a new LocalPantsRunner instance by parsing options.
By the time this method runs, logging will already have been initialized in either
PantsRunner or DaemonPantsRunner.
:param env: The environment (e.g. os.environ) for this run.
:param options_bootstrapper: The OptionsBootstrapper instance to reuse.
:param scheduler: If being called from the daemon, a warmed scheduler to use.
"""
build_root = get_buildroot()
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
options, build_config = LocalPantsRunner.parse_options(
options_bootstrapper)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify configs.
if global_bootstrap_options.verify_config:
options.verify_configs(options_bootstrapper.config)
union_membership = UnionMembership(build_config.union_rules())
# If we're running with the daemon, we'll be handed a warmed Scheduler, which we use
# to initialize a session here.
graph_session = cls._init_graph_session(options_bootstrapper,
build_config, options,
scheduler)
global_options = options.for_global_scope()
specs = SpecsCalculator.create(
options=options,
build_root=build_root,
session=graph_session.scheduler_session,
exclude_patterns=tuple(global_options.exclude_target_regexp),
tags=tuple(global_options.tag),
)
profile_path = env.get("PANTS_PROFILE")
return cls(
build_root=build_root,
options=options,
options_bootstrapper=options_bootstrapper,
build_config=build_config,
specs=specs,
graph_session=graph_session,
union_membership=union_membership,
profile_path=profile_path,
_run_tracker=RunTracker.global_instance(),
)
def run(self, start_time: float) -> ExitCode:
run_tracker = RunTracker.global_instance()
self._start_run(run_tracker, start_time)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
if self.options.help_request:
return self._print_help(self.options.help_request)
streaming_handlers = global_options.streaming_workunits_handlers
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=global_options.
streaming_workunits_report_interval,
)
goals = tuple(self.options.goals)
with streaming_reporter.session():
engine_result = PANTS_FAILED_EXIT_CODE
try:
engine_result = self._run_v2(goals)
except Exception as e:
ExceptionSink.log_exception(e)
self._finish_run(run_tracker, engine_result)
return engine_result
def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[GraphScheduler] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
should_report_workunits=stream_workunits,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(os.environ),
}),
)
def set_start_time(self, start_time: Optional[float]) -> None:
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
# Propagates parent_build_id to pants runs that may be called from this pants run.
os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id
self._reporting = Reporting.global_instance()
self._reporting.initialize(self._run_tracker,
self.options,
start_time=start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self.options.specs
]
# Note: This will not include values from `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line",
specs,
stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def test_anonymous_telemetry_with_no_repo_id() -> None:
with temporary_dir() as buildroot:
with environment_as(PANTS_BUILDROOT_OVERRIDE=buildroot):
opts = create_options_bootstrapper([]).bootstrap_options
run_tracker = RunTracker(opts)
run_tracker.start(run_start_time=time.time(), specs=[])
run_tracker.end_run(PANTS_SUCCEEDED_EXIT_CODE)
repo_id = ""
telemetry = run_tracker.get_anonymous_telemetry_data(repo_id)
# Check that these keys have non-trivial values.
for key in (
"run_id",
"timestamp",
"duration",
"outcome",
"platform",
"python_implementation",
"python_version",
"pants_version",
):
assert bool(telemetry.get(key))
for key in ("repo_id", "machine_id", "user_id"):
assert telemetry.get(key) == ""
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
# This is the first case we have of non-task, non-global options.
# The current implementation special-cases RunTracker, and is temporary.
# In the near future it will be replaced with a 'Subsystem' abstraction.
# But for now this is useful for kicking the tires.
def register_run_tracker(*args, **kwargs):
self.options.register('run-tracker', *args, **kwargs)
RunTracker.register_options(register_run_tracker)
for goal in Goal.all():
goal.register_options(self.options)
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
setup_logging_from_options(bootstrap_options)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir,
options,
build_config,
run_tracker,
reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(
env=self._env, args=self._args)
options, build_config = OptionsInitializer(
options_bootstrapper, exiter=self._exiter).setup()
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Record the preceding product graph size.
run_tracker.pantsd_stats.set_preceding_graph_size(
self._preceding_graph_size)
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir, options, build_config,
run_tracker, reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(
run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_runtracker: Whether or not to clean RunTracker global state.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_runtracker:
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset backend/plugins state.
OptionsInitializer.reset()
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_runtracker: Whether or not to clean RunTracker global state.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_runtracker:
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset backend/plugins state.
OptionsInitializer.reset()
def test_upload_stats(self):
stats = {'stats': {'foo': 'bar', 'baz': 42}}
class Handler(http.server.BaseHTTPRequestHandler):
def do_POST(handler):
try:
if handler.path.startswith('/redirect'):
code = int(handler.path[-3:])
handler.send_response(code)
handler.send_header('location', mk_url('/upload'))
handler.end_headers()
else:
self.assertEqual('/upload', handler.path)
self.assertEqual('application/x-www-form-urlencoded', handler.headers['Content-type'])
length = int(handler.headers['Content-Length'])
post_data = parse_qs(handler.rfile.read(length).decode('utf-8'))
decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
self.assertEqual(stats, decoded_post_data)
handler.send_response(200)
handler.end_headers()
except Exception:
handler.send_response(400) # Ensure the main thread knows the test failed.
raise
server_address = ('', 0)
server = http.server.HTTPServer(server_address, Handler)
host, port = server.server_address
def mk_url(path):
return 'http://{}:{}{}'.format(host, port, path)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
self.context(for_subsystems=[Cookies])
self.assertTrue(RunTracker.post_stats(mk_url('/upload'), stats))
self.assertTrue(RunTracker.post_stats(mk_url('/redirect307'), stats))
self.assertFalse(RunTracker.post_stats(mk_url('/redirect302'), stats))
server.shutdown()
server.server_close()
def create_run_tracker(info_dir=None):
"""Creates a ``RunTracker`` and starts it.
:param string info_dir: An optional director for the run tracker to store state; defaults to a
new temp dir that will be be cleaned up on interpreter exit.
"""
# TODO(John Sirois): Rework uses around a context manager for cleanup of the info_dir in a more
# disciplined manner
info_dir = info_dir or safe_mkdtemp()
run_tracker = RunTracker(info_dir)
report = Report()
settings = PlainTextReporter.Settings(outfile=sys.stdout,
log_level=Report.INFO,
color=False,
indent=True,
timing=False,
cache_stats=False)
report.add_reporter('test_debug', PlainTextReporter(run_tracker, settings))
run_tracker.start(report)
return run_tracker
def create_run_tracker(info_dir=None):
"""Creates a ``RunTracker`` and starts it.
:param string info_dir: An optional director for the run tracker to store state; defaults to a
new temp dir that will be be cleaned up on interpreter exit.
"""
# TODO(John Sirois): Rework uses around a context manager for cleanup of the info_dir in a more
# disciplined manner
info_dir = info_dir or safe_mkdtemp()
run_tracker = RunTracker(info_dir)
report = Report()
settings = PlainTextReporter.Settings(outfile=sys.stdout,
log_level=Report.INFO,
color=False,
indent=True,
timing=False,
cache_stats=False)
report.add_reporter('test_debug', PlainTextReporter(run_tracker, settings))
run_tracker.start(report)
return run_tracker
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
# This is the first case we have of non-task, non-global options.
# The current implementation special-cases RunTracker, and is temporary.
# In the near future it will be replaced with a 'Subsystem' abstraction.
# But for now this is useful for kicking the tires.
def register_run_tracker(*args, **kwargs):
self.options.register('run-tracker', *args, **kwargs)
RunTracker.register_options(register_run_tracker)
for goal in Goal.all():
goal.register_options(self.options)
def test_create_dict_with_nested_keys_and_val(self):
keys = []
with self.assertRaises(ValueError):
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something')
keys += ['one']
self.assertEquals(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': 'something'}
)
keys += ['two']
self.assertEquals(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': {'two': 'something'}}
)
keys += ['three']
self.assertEquals(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': {'two': {'three': 'something'}}}
)
keys += ['four']
self.assertEquals(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': {'two': {'three': {'four': 'something'}}}}
)
def test_invalid_stats_version(self):
stats = {'stats': {'foo': 'bar', 'baz': 42}}
url = 'http://example.com/upload/'
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version=0)
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version=None)
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version=9)
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version="not a number")
def test_invalid_stats_version(self):
stats = {"stats": {"foo": "bar", "baz": 42}}
url = "http://example.com/upload/"
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version=0)
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version=None)
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version=9)
with self.assertRaises(ValueError):
RunTracker.post_stats(url, stats, stats_version="not a number")
def test_create_dict_with_nested_keys_and_val(self):
keys = []
with self.assertRaises(ValueError):
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something')
keys += ['one']
self.assertEqual(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': 'something'}
)
keys += ['two']
self.assertEqual(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': {'two': 'something'}}
)
keys += ['three']
self.assertEqual(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': {'two': {'three': 'something'}}}
)
keys += ['four']
self.assertEqual(
RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
{'one': {'two': {'three': {'four': 'something'}}}}
)
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_runtracker: Whether or not to clean RunTracker global state.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_runtracker:
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
#TODO: Think of an alternative for IntermediateTargetFactoryBase._targets to avoid this call
IntermediateTargetFactoryBase.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset backend/plugins state.
OptionsInitializer.reset()
def _execute_engine(self):
workdir = self._context.options.for_global_scope().pants_workdir
if not workdir.endswith('.pants.d'):
self._context.log.error('Pants working directory should end with \'.pants.d\', currently it is {}\n'
.format(workdir))
return 1
unknown_goals = [goal.name for goal in self._goals if not goal.ordered_task_names()]
if unknown_goals:
self._context.log.error('Unknown goal(s): {}\n'.format(' '.join(unknown_goals)))
return 1
engine = RoundEngine()
sorted_goal_infos = engine.sort_goals(self._context, self._goals)
RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
result = engine.execute(self._context, self._goals)
if self._context.invalidation_report:
self._context.invalidation_report.report()
return result
def test_raise_no_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
in str(result.exception)
)
def test_raise_if_no_parent_id_and_zipkin_endpoint_set(self):
options = {'reporting': {'zipkin_trace_id': self.trace_id}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
in str(result.exception)
)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
# The __call__ method of the Exiter allows for the prototype pattern.
self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
ExceptionSink.reset_exiter(self._exiter)
def test_raise_if_trace_id_is_of_wrong_ch_format(self):
trace_id = 'gggggggggggggggg'
options = {'reporting': {
'zipkin_trace_id': trace_id,
'zipkin_parent_id': self.parent_id,
'zipkin_endpoint': self.zipkin_endpoint
}}
context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
with self.assertRaises(ValueError) as result:
reporting.initialize(run_tracker, context.options)
self.assertTrue(
"Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
+ "Got {}.".format(trace_id)
in str(result.exception)
)
def setup(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems.
subsystems = Subsystem.closure(set(self.subsystems) |
Goal.subsystems() |
build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def _run():
# Place the registration of the unhandled exception hook as early as possible in the code.
sys.excepthook = _unhandled_exception_hook
"""
To add additional paths to sys.path, add a block to the config similar to the following:
[main]
roots: ['src/python/pants_internal/test/',]
"""
logging.basicConfig()
version = pants_version()
if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
_do_exit(msg=version, out=sys.stdout)
root_dir = get_buildroot()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)
if len(sys.argv) < 2:
argv = ['goal']
else:
argv = sys.argv[1:]
# Hack to force ./pants -h etc. to redirect to goal.
if argv[0] != 'goal' and set(['-h', '--help', 'help']).intersection(argv):
argv = ['goal'] + argv
parser = optparse.OptionParser(add_help_option=False, version=version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION,
action='store_true',
default=False,
dest='log_exit',
help='Log an exit message on success or failure.')
config = Config.load()
# XXX(wickman) This should be in the command goal, not in pants_exe.py!
run_tracker = RunTracker.from_config(config)
report = initial_reporting(config, run_tracker)
run_tracker.start(report)
url = run_tracker.run_info.get_info('report_url')
if url:
run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
run_tracker.log(Report.INFO, '(To run a reporting server: ./pants goal server)')
backend_packages = config.getlist('backends', 'packages')
build_configuration = load_build_configuration_from_source(additional_backends=backend_packages)
build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=root_dir,
run_tracker=run_tracker)
address_mapper = BuildFileAddressMapper(build_file_parser)
build_graph = BuildGraph(run_tracker=run_tracker, address_mapper=address_mapper)
command_class, command_args = _parse_command(root_dir, argv)
command = command_class(run_tracker,
root_dir,
parser,
command_args,
build_file_parser,
address_mapper,
build_graph)
try:
if command.serialized():
def onwait(pid):
process = psutil.Process(pid)
print('Waiting on pants process %d (%s) to complete' %
(pid, ' '.join(process.cmdline)), file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
if result:
run_tracker.set_root_outcome(WorkUnit.FAILURE)
_do_exit(result)
except KeyboardInterrupt:
command.cleanup()
raise
except Exception:
run_tracker.set_root_outcome(WorkUnit.FAILURE)
raise
finally:
lock.release()
finally:
run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, because there may still
# be pending background work that needs a nailgun.
if (hasattr(command.old_options, 'cleanup_nailguns') and command.old_options.cleanup_nailguns) \
or config.get('nailgun', 'autokill', default=False):
NailgunTask.killall(None)
def test_merge_list_of_keys_into_dict(self):
data = {}
keys = []
with self.assertRaises(ValueError):
RunTracker._merge_list_of_keys_into_dict(data, keys, 'something')
with self.assertRaises(ValueError):
RunTracker._merge_list_of_keys_into_dict(data, keys, 'something', -1)
keys = ['key']
with self.assertRaises(ValueError):
RunTracker._merge_list_of_keys_into_dict(data, keys, 'something', 1)
keys = ['a']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'O-N-E')
self.assertEquals(data, {'a': 'O-N-E'})
keys = ['one', 'two', 'three']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'T-H-R-E-E')
self.assertEquals(data, {'one': {'two': {'three': 'T-H-R-E-E'}}, 'a': 'O-N-E'})
keys = ['one', 'two', 'a']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'L-A')
self.assertEquals(data, {'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E'}}, 'a': 'O-N-E'})
keys = ['c', 'd', 'e', 'f']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'F-O-U-R')
self.assertEquals(data, {
'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E'}}, 'a': 'O-N-E',
'c': {'d': {'e': {'f': 'F-O-U-R'}}}
})
keys = ['one', 'two', 'x', 'y']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'W-H-Y')
self.assertEquals(data, {
'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y'}}}, 'a': 'O-N-E',
'c': {'d': {'e': {'f': 'F-O-U-R'}}}
})
keys = ['c', 'd', 'e', 'g', 'h']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'H-E-L-L-O')
self.assertEquals(data, {
'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y'}}}, 'a': 'O-N-E',
'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O'}}}}
})
keys = ['one', 'two', 'x', 'z']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'Z-E-D')
self.assertEquals(data, {
'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
'a': 'O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O'}}}}
})
keys = ['c', 'd', 'e', 'g', 'i']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'E-Y-E')
self.assertEquals(data, {
'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
'a': 'O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O', 'i': 'E-Y-E'}}}}
})
keys = ['a']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'new O-N-E')
self.assertEquals(data, {
'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
'a': 'new O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O', 'i': 'E-Y-E'}}}}
})
keys = ['one', 'two', 'a']
RunTracker._merge_list_of_keys_into_dict(data, keys, 'L-A-L-A')
self.assertEquals(data, {
'one': {'two': {'a': 'L-A-L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
'a': 'new O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O', 'i': 'E-Y-E'}}}}
})
keys = ['one', 'two', 'a', 'b', 'c']
with self.assertRaises(ValueError):
RunTracker._merge_list_of_keys_into_dict(data, keys, 'new A')
def _run():
"""
To add additional paths to sys.path, add a block to the config similar to the following:
[main]
roots: ['src/python/pants_internal/test/',]
"""
version = get_version()
if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
_do_exit(version)
root_dir = get_buildroot()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)
if len(sys.argv) < 2 or (len(sys.argv) == 2 and sys.argv[1] in _HELP_ALIASES):
_help(version, root_dir)
command_class, command_args = _parse_command(root_dir, sys.argv[1:])
parser = optparse.OptionParser(version=version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION,
action='store_true',
default=False,
dest='log_exit',
help = 'Log an exit message on success or failure.')
config = Config.load()
# TODO: This can be replaced once extensions are enabled with
# https://github.com/pantsbuild/pants/issues/5
roots = config.getlist('parse', 'roots', default=[])
sys.path.extend(map(lambda root: os.path.join(root_dir, root), roots))
# XXX(wickman) This should be in the command goal, not un pants_exe.py!
run_tracker = RunTracker.from_config(config)
report = initial_reporting(config, run_tracker)
run_tracker.start(report)
url = run_tracker.run_info.get_info('report_url')
if url:
run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
command = command_class(run_tracker, root_dir, parser, command_args)
try:
if command.serialized():
def onwait(pid):
print('Waiting on pants process %s to complete' % _process_info(pid), file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
_do_exit(result)
except KeyboardInterrupt:
command.cleanup()
raise
finally:
lock.release()
finally:
run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, because there may still
# be pending background work that needs a nailgun.
if (hasattr(command.options, 'cleanup_nailguns') and command.options.cleanup_nailguns) \
or config.get('nailgun', 'autokill', default=False):
NailgunTask.killall(None)
def __init__(self, run_tracker=None, reporting=None):
self._run_tracker = run_tracker or RunTracker.global_instance()
self._reporting = reporting or Reporting.global_instance()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
# Add scopes for global subsystem instances.
for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types():
known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()