def _init_graph(self, target_roots, graph_helper, exclude_target_regexps, tags):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param TargetRoots target_roots: The existing `TargetRoots` object, if any.
:param LegacyGraphSession graph_helper: A LegacyGraphSession to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, SchedulerSession, TargetRoots).
"""
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(native,
self._global_options,
self._build_config)
graph_helper = graph_scheduler_helper.new_session()
target_roots = target_roots or TargetRootsCalculator.create(
options=self._options,
session=graph_helper.scheduler_session,
build_root=self._root_dir,
symbol_table=graph_helper.symbol_table,
exclude_patterns=tuple(exclude_target_regexps),
tags=tuple(tags)
)
graph, address_mapper = graph_helper.create_build_graph(target_roots,
self._root_dir)
return graph, address_mapper, graph_helper.scheduler_session, target_roots
def _body(self, session, options, options_bootstrapper):
global_options = options.for_global_scope()
target_roots = TargetRootsCalculator.create(
options=options,
session=session.scheduler_session,
exclude_patterns=tuple(global_options.exclude_target_regexp)
if global_options.exclude_target_regexp else tuple(),
tags=tuple(global_options.tag) if global_options.tag else tuple())
exit_code = PANTS_SUCCEEDED_EXIT_CODE
v1_goals, ambiguous_goals, v2_goals = options.goals_by_version
if v2_goals or (ambiguous_goals and global_options.v2):
goals = v2_goals + (ambiguous_goals
if global_options.v2 else tuple())
# N.B. @console_rules run pre-fork in order to cache the products they request during execution.
exit_code = session.run_console_rules(
options_bootstrapper,
options,
goals,
target_roots,
)
return target_roots, exit_code
def _prefork_body(self, session, options, options_bootstrapper):
global_options = options.for_global_scope()
target_roots = TargetRootsCalculator.create(
options=options,
session=session.scheduler_session,
symbol_table=session.symbol_table,
exclude_patterns=tuple(global_options.exclude_target_regexp)
if global_options.exclude_target_regexp else tuple(),
tags=tuple(global_options.tag) if global_options.tag else tuple())
if global_options.v1:
session.warm_product_graph(target_roots)
if global_options.v2:
if not global_options.v1:
session.validate_goals(options.goals_and_possible_v2_goals)
# N.B. @console_rules run pre-fork in order to cache the products they request during execution.
session.run_console_rules(
options_bootstrapper,
options.goals_and_possible_v2_goals,
target_roots,
)
return target_roots
def execute_rule(self, args=tuple(), env=tuple(), exit_code=0):
"""Executes the @console_rule for this test class.
:API: public
Returns the text output of the task.
"""
# Create an OptionsBootstrapper for these args/env, and a captured Console instance.
args = self._implicit_args + (self.goal_cls.name, ) + tuple(args)
env = dict(env)
options_bootstrapper = OptionsBootstrapper.create(args=args, env=env)
BuildConfigInitializer.get(options_bootstrapper)
full_options = options_bootstrapper.get_full_options(
list(self.goal_cls.Options.known_scope_infos()))
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
# Run for the target specs parsed from the args.
specs = TargetRootsCalculator.parse_specs(full_options.target_specs,
self.build_root)
params = Params(specs, console, options_bootstrapper)
actual_exit_code = self.scheduler.run_console_rule(
self.goal_cls, params)
# Flush and capture console output.
console.flush()
stdout = stdout.getvalue()
stderr = stderr.getvalue()
self.assertEqual(
exit_code, actual_exit_code,
"Exited with {} (expected {}):\nstdout:\n{}\nstderr:\n{}".format(
actual_exit_code, exit_code, stdout, stderr))
return stdout
def prefork(self, options, build_config):
"""Runs all pre-fork logic in the process context of the daemon.
:returns: `(LegacyGraphSession, TargetRoots)`
"""
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug('graph len was {}, waiting for initial watchman event'.format(graph_len))
self._watchman_is_running.wait()
session = self._graph_helper.new_session()
with self.fork_lock:
global_options = options.for_global_scope()
target_roots = TargetRootsCalculator.create(
options=options,
session=session.scheduler_session,
symbol_table=session.symbol_table,
exclude_patterns=tuple(global_options.exclude_target_regexp) if global_options.exclude_target_regexp else tuple(),
tags=tuple(global_options.tag) if global_options.tag else tuple()
)
if global_options.v1:
session.warm_product_graph(target_roots)
if global_options.v2:
if not global_options.v1:
session.validate_goals(options.goals)
# N.B. @console_rules run pre-fork in order to cache the products they request during execution.
session.run_console_rules(options.goals, target_roots)
return session, target_roots
def _prefork_body(self, session, options, options_bootstrapper):
global_options = options.for_global_scope()
target_roots = TargetRootsCalculator.create(
options=options,
session=session.scheduler_session,
exclude_patterns=tuple(global_options.exclude_target_regexp) if global_options.exclude_target_regexp else tuple(),
tags=tuple(global_options.tag) if global_options.tag else tuple()
)
exit_code = PANTS_SUCCEEDED_EXIT_CODE
v1_goals, ambiguous_goals, v2_goals = options.goals_by_version
if v1_goals or (ambiguous_goals and global_options.v1):
session.warm_product_graph(target_roots)
if v2_goals or (ambiguous_goals and global_options.v2):
goals = v2_goals + (ambiguous_goals if global_options.v2 else tuple())
# N.B. @console_rules run pre-fork in order to cache the products they request during execution.
exit_code = session.run_console_rules(
options_bootstrapper,
goals,
target_roots,
)
return target_roots, exit_code
def _maybe_init_target_roots(target_roots, graph_session, options, build_root):
if target_roots:
return target_roots
global_options = options.for_global_scope()
return TargetRootsCalculator.create(
options=options,
build_root=build_root,
session=graph_session.scheduler_session,
exclude_patterns=tuple(global_options.exclude_target_regexp),
tags=tuple(global_options.tag)
)
def _maybe_init_target_roots(target_roots, graph_session, options, build_root):
if target_roots:
return target_roots
global_options = options.for_global_scope()
return TargetRootsCalculator.create(
options=options,
build_root=build_root,
session=graph_session.scheduler_session,
exclude_patterns=tuple(global_options.exclude_target_regexp),
tags=tuple(global_options.tag)
)
def _init_graph(self,
pants_ignore_patterns,
build_ignore_patterns,
exclude_target_regexps,
target_specs,
target_roots,
workdir,
graph_helper,
subproject_build_roots):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param str workdir: The pants workdir.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param TargetRoots target_roots: The existing `TargetRoots` object, if any.
:param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, TargetRoots).
"""
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_helper = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
workdir,
self._global_options.build_file_imports,
native=native,
build_file_aliases=self._build_config.registered_aliases(),
rules=self._build_config.rules(),
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_build_roots,
include_trace_on_error=self._options.for_global_scope().print_exception_stacktrace
)
target_roots = target_roots or TargetRootsCalculator.create(
options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator
)
graph, address_mapper = graph_helper.create_build_graph(target_roots, self._root_dir)
return graph, address_mapper, graph_helper.scheduler, target_roots
def execute_rule(
self,
args: Optional[Iterable[str]] = None,
env: Optional[Dict[str, str]] = None,
exit_code: int = 0,
additional_params: Optional[Iterable[Any]] = None,
) -> str:
"""Executes the @goal_rule for this test class.
:API: public
Returns the text output of the task.
"""
# Create an OptionsBootstrapper for these args/env, and a captured Console instance.
options_bootstrapper = create_options_bootstrapper(
args=(self.goal_cls.name, *(args or [])),
env=env,
)
BuildConfigInitializer.get(options_bootstrapper)
full_options = options_bootstrapper.get_full_options(
list(self.goal_cls.subsystem_cls.known_scope_infos()))
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
scheduler = self.scheduler
workspace = Workspace(scheduler)
# Run for the target specs parsed from the args.
address_specs = TargetRootsCalculator.parse_address_specs(
full_options.specs, self.build_root)
params = Params(address_specs, console, options_bootstrapper,
workspace, *(additional_params or []))
actual_exit_code = self.scheduler.run_goal_rule(self.goal_cls, params)
# Flush and capture console output.
console.flush()
stdout_val = stdout.getvalue()
stderr_val = stderr.getvalue()
assert exit_code == actual_exit_code, \
f"Exited with {actual_exit_code} (expected {exit_code}):\nstdout:\n{stdout_val}\nstderr:\n{stderr_val}"
return stdout_val
def execute_rule(self,
args=tuple(),
env=tuple(),
exit_code=0,
additional_params: Iterable[Any] = tuple()):
"""Executes the @console_rule for this test class.
:API: public
Returns the text output of the task.
"""
# Create an OptionsBootstrapper for these args/env, and a captured Console instance.
args = self._implicit_args + (self.goal_cls.name, ) + tuple(args)
env = dict(env)
options_bootstrapper = OptionsBootstrapper.create(args=args, env=env)
BuildConfigInitializer.get(options_bootstrapper)
full_options = options_bootstrapper.get_full_options(
list(self.goal_cls.subsystem_cls.known_scope_infos()))
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
scheduler = self.scheduler
workspace = Workspace(scheduler)
# Run for the target specs parsed from the args.
specs = TargetRootsCalculator.parse_specs(full_options.positional_args,
self.build_root)
params = Params(specs, console, options_bootstrapper, workspace,
*additional_params)
actual_exit_code = self.scheduler.run_console_rule(
self.goal_cls, params)
# Flush and capture console output.
console.flush()
stdout_val = stdout.getvalue()
stderr_val = stderr.getvalue()
assert exit_code == actual_exit_code, \
f"Exited with {actual_exit_code} (expected {exit_code}):\nstdout:\n{stdout_val}\nstderr:\n{stderr_val}"
return stdout_val
def create_target_roots(self, specs, session, symbol_table):
return TargetRootsCalculator.create(self._make_setup_args(specs),
session, symbol_table)
def create_target_roots(self, specs): return TargetRootsCalculator.create(self._make_setup_args(specs))
def create_target_roots(self, specs, session, symbol_table): return TargetRootsCalculator.create(self._make_setup_args(specs), session, symbol_table)
