def run_goal_rule(
self,
goal: Type[Goal],
*,
global_args: Optional[Iterable[str]] = None,
args: Optional[Iterable[str]] = None,
env: Optional[Mapping[str, str]] = None,
) -> GoalRuleResult:
options_bootstrapper = create_options_bootstrapper(
args=(*(global_args or []), goal.name, *(args or [])),
env=env,
)
raw_specs = options_bootstrapper.get_full_options([
*GlobalOptions.known_scope_infos(),
*goal.subsystem_cls.known_scope_infos()
]).specs
specs = SpecsParser(self.build_root).parse_specs(raw_specs)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
exit_code = self.scheduler.run_goal_rule(
goal,
Params(
specs,
console,
options_bootstrapper,
Workspace(self.scheduler),
InteractiveRunner(self.scheduler),
),
)
console.flush()
return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue())
def test_run_rule_goal_rule_generator(self):
res = run_rule(
a_goal_rule_generator,
rule_args=[Console()],
mock_gets=[MockGet(product_type=A, subject_type=str, mock=lambda _: A())],
)
self.assertEqual(res, Example(0))
def execute_rule(self, args=tuple(), env=tuple(), exit_code=0):
"""Executes the @console_rule for this test class.
:API: public
Returns the text output of the task.
"""
# Create an OptionsBootstrapper for these args/env, and a captured Console instance.
args = self._implicit_args + (self.goal_cls.name, ) + tuple(args)
env = dict(env)
options_bootstrapper = OptionsBootstrapper.create(args=args, env=env)
BuildConfigInitializer.get(options_bootstrapper)
full_options = options_bootstrapper.get_full_options(
list(self.goal_cls.Options.known_scope_infos()))
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
# Run for the target specs parsed from the args.
specs = TargetRootsCalculator.parse_specs(full_options.target_specs,
self.build_root)
params = Params(specs, console, options_bootstrapper)
actual_exit_code = self.scheduler.run_console_rule(
self.goal_cls, params)
# Flush and capture console output.
console.flush()
stdout = stdout.getvalue()
stderr = stderr.getvalue()
self.assertEqual(
exit_code, actual_exit_code,
"Exited with {} (expected {}):\nstdout:\n{}\nstderr:\n{}".format(
actual_exit_code, exit_code, stdout, stderr))
return stdout
def test_run_rule_goal_rule_generator(self) -> None:
res = run_rule_with_mocks(
a_goal_rule_generator,
rule_args=[Console()],
mock_gets=[MockGet(output_type=A, input_type=str, mock=lambda _: A())],
)
assert res == Example(0)
def run_goal_rule(
self,
goal: Type[Goal],
*,
global_args: Iterable[str] | None = None,
args: Iterable[str] | None = None,
env: Mapping[str, str] | None = None,
env_inherit: set[str] | None = None,
) -> GoalRuleResult:
merged_args = (*(global_args or []), goal.name, *(args or []))
self.set_options(merged_args, env=env, env_inherit=env_inherit)
raw_specs = self.options_bootstrapper.full_options_for_scopes([
GlobalOptions.get_scope_info(),
goal.subsystem_cls.get_scope_info()
]).specs
specs = SpecsParser(self.build_root).parse_specs(raw_specs)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
exit_code = self.scheduler.run_goal_rule(
goal,
Params(
specs,
console,
Workspace(self.scheduler),
InteractiveRunner(self.scheduler),
),
)
console.flush()
return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue())
def mock_console(
options_bootstrapper: OptionsBootstrapper,
*,
stdin_content: bytes | str | None = None,
) -> Iterator[Tuple[Console, StdioReader]]:
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
@contextmanager
def stdin_context():
if stdin_content is None:
yield open("/dev/null", "r")
else:
with temporary_file(binary_mode=isinstance(stdin_content,
bytes)) as stdin_file:
stdin_file.write(stdin_content)
stdin_file.close()
yield open(stdin_file.name, "r")
with initialize_stdio(global_bootstrap_options), stdin_context(
) as stdin, temporary_file(binary_mode=False) as stdout, temporary_file(
binary_mode=False) as stderr, stdio_destination(
stdin_fileno=stdin.fileno(),
stdout_fileno=stdout.fileno(),
stderr_fileno=stderr.fileno(),
):
# NB: We yield a Console without overriding the destination argument, because we have
# already done a sys.std* level replacement. The replacement is necessary in order for
# InteractiveProcess to have native file handles to interact with.
yield Console(use_colors=global_bootstrap_options.colors), StdioReader(
_stdout=Path(stdout.name), _stderr=Path(stderr.name))
def run_console_rules(self, options_bootstrapper, goals, target_roots):
"""Runs @console_rules sequentially and interactively by requesting their implicit Goal products.
For retryable failures, raises scheduler.ExecutionError.
:param list goals: The list of requested goal names as passed on the commandline.
:param TargetRoots target_roots: The targets root of the request.
:returns: An exit code.
"""
subject = target_roots.specs
console = Console(
use_colors=options_bootstrapper.bootstrap_options.for_global_scope().colors
)
workspace = Workspace(self.scheduler_session)
interactive_runner = InteractiveRunner(self.scheduler_session)
for goal in goals:
goal_product = self.goal_map[goal]
params = Params(subject, options_bootstrapper, console, workspace, interactive_runner)
logger.debug(f'requesting {goal_product} to satisfy execution of `{goal}` goal')
try:
exit_code = self.scheduler_session.run_console_rule(goal_product, params)
finally:
console.flush()
if exit_code != PANTS_SUCCEEDED_EXIT_CODE:
return exit_code
return PANTS_SUCCEEDED_EXIT_CODE
def run_goal_rules(
self,
*,
options_bootstrapper: OptionsBootstrapper,
union_membership: UnionMembership,
options: Options,
goals: Iterable[str],
specs: Specs,
):
"""Runs @goal_rules sequentially and interactively by requesting their implicit Goal
products.
For retryable failures, raises scheduler.ExecutionError.
:returns: An exit code.
"""
global_options = options.for_global_scope()
console = Console(
use_colors=global_options.colors,
session=self.scheduler_session
if global_options.get("v2_ui") else None,
)
workspace = Workspace(self.scheduler_session)
interactive_runner = InteractiveRunner(self.scheduler_session)
for goal in goals:
goal_product = self.goal_map[goal]
# NB: We no-op for goals that have no V2 implementation because no relevant backends are
# registered. This allows us to safely set `--v1 --v2`, even if no V2 backends are registered.
# Once V1 is removed, we might want to reconsider the behavior to instead warn or error when
# trying to run something like `./pants run` without any backends registered.
is_implemented = union_membership.has_members_for_all(
goal_product.subsystem_cls.required_union_implementations)
if not is_implemented:
continue
params = Params(
specs.provided_specs,
options_bootstrapper,
console,
workspace,
interactive_runner,
)
logger.debug(
f"requesting {goal_product} to satisfy execution of `{goal}` goal"
)
try:
exit_code = self.scheduler_session.run_goal_rule(
goal_product, params)
finally:
console.flush()
if exit_code != PANTS_SUCCEEDED_EXIT_CODE:
return exit_code
return PANTS_SUCCEEDED_EXIT_CODE
def new_session(
self,
build_id,
dynamic_ui: bool = False,
use_colors=True,
should_report_workunits=False,
) -> "GraphSession":
session = self.scheduler.new_session(build_id, dynamic_ui,
should_report_workunits)
console = Console(use_colors=use_colors,
session=session if dynamic_ui else None)
return GraphSession(session, console, self.goal_map)
def new_session(
self,
zipkin_trace_v2,
build_id,
dynamic_ui: bool = False,
use_colors=True,
should_report_workunits=False,
) -> "LegacyGraphSession":
session = self.scheduler.new_session(
zipkin_trace_v2, build_id, dynamic_ui, should_report_workunits
)
console = Console(use_colors=use_colors, session=session if dynamic_ui else None,)
return LegacyGraphSession(session, console, self.build_file_aliases, self.goal_map)
def execute_rule(
self,
args: Optional[Iterable[str]] = None,
global_args: Optional[Iterable[str]] = None,
env: Optional[Dict[str, str]] = None,
exit_code: int = 0,
additional_params: Optional[Iterable[Any]] = None,
) -> str:
"""Executes the @goal_rule for this test class.
:API: public
Returns the text output of the task.
"""
# Create an OptionsBootstrapper for these args/env, and a captured Console instance.
options_bootstrapper = create_options_bootstrapper(
args=(*(global_args or []), self.goal_cls.name, *(args or [])), env=env,
)
BuildConfigInitializer.get(options_bootstrapper)
full_options = options_bootstrapper.get_full_options(
[*GlobalOptions.known_scope_infos(), *self.goal_cls.subsystem_cls.known_scope_infos()]
)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
scheduler = self.scheduler
workspace = Workspace(scheduler)
# Run for the target specs parsed from the args.
specs = SpecsCalculator.parse_specs(full_options.specs, self.build_root)
params = Params(
specs.provided_specs,
console,
options_bootstrapper,
workspace,
*(additional_params or []),
)
actual_exit_code = self.scheduler.run_goal_rule(self.goal_cls, params)
# Flush and capture console output.
console.flush()
stdout_val = stdout.getvalue()
stderr_val = stderr.getvalue()
assert (
exit_code == actual_exit_code
), f"Exited with {actual_exit_code} (expected {exit_code}):\nstdout:\n{stdout_val}\nstderr:\n{stderr_val}"
return stdout_val
def new_session(
self,
build_id,
dynamic_ui: bool = False,
use_colors=True,
session_values: Optional[SessionValues] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
session = self.scheduler.new_session(
build_id,
dynamic_ui,
session_values=session_values,
cancellation_latch=cancellation_latch,
)
console = Console(use_colors=use_colors, session=session if dynamic_ui else None)
return GraphSession(session, console, self.goal_map)
def run_goal_rules(
self,
options_bootstrapper: OptionsBootstrapper,
options: Options,
goals: Iterable[str],
specs: Specs,
):
"""Runs @goal_rules sequentially and interactively by requesting their implicit Goal products.
For retryable failures, raises scheduler.ExecutionError.
:returns: An exit code.
"""
global_options = options.for_global_scope()
console = Console(
use_colors=global_options.colors,
session=self.scheduler_session
if global_options.get('v2_ui') else None,
)
workspace = Workspace(self.scheduler_session)
interactive_runner = InteractiveRunner(self.scheduler_session)
for goal in goals:
goal_product = self.goal_map[goal]
params = Params(
specs.provided_specs,
options_bootstrapper,
console,
workspace,
interactive_runner,
)
logger.debug(
f'requesting {goal_product} to satisfy execution of `{goal}` goal'
)
try:
exit_code = self.scheduler_session.run_goal_rule(
goal_product, params)
finally:
console.flush()
if exit_code != PANTS_SUCCEEDED_EXIT_CODE:
return exit_code
return PANTS_SUCCEEDED_EXIT_CODE
def run_goal_rule(
self,
goal: Type[Goal],
*,
global_args: Optional[Iterable[str]] = None,
args: Optional[Iterable[str]] = None,
env: Optional[Mapping[str, str]] = None,
) -> GoalRuleResult:
options_bootstrapper = create_options_bootstrapper(
args=(*(global_args or []), goal.name, *(args or [])),
env=env,
)
raw_specs = options_bootstrapper.get_full_options([
*GlobalOptions.known_scope_infos(),
*goal.subsystem_cls.known_scope_infos()
]).specs
specs = SpecsParser(self.build_root).parse_specs(raw_specs)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
session = self.scheduler.scheduler.new_session(
build_id="buildid_for_test",
should_report_workunits=True,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(env)
}),
)
exit_code = session.run_goal_rule(
goal,
Params(
specs,
console,
Workspace(self.scheduler),
InteractiveRunner(self.scheduler),
),
)
console.flush()
return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue())
def mock_console(
options_bootstrapper: OptionsBootstrapper,
) -> Iterator[Tuple[Console, StdioReader]]:
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
with initialize_stdio(global_bootstrap_options), open(
"/dev/null", "r") as stdin, temporary_file(
binary_mode=False) as stdout, temporary_file(
binary_mode=False) as stderr, stdio_destination(
stdin_fileno=stdin.fileno(),
stdout_fileno=stdout.fileno(),
stderr_fileno=stderr.fileno(),
):
# NB: We yield a Console without overriding the destination argument, because we have
# already done a sys.std* level replacement. The replacement is necessary in order for
# InteractiveProcess to have native file handles to interact with.
yield Console(use_colors=global_bootstrap_options.colors), StdioReader(
_stdout=Path(stdout.name), _stderr=Path(stderr.name))
def new_session(
self,
build_id,
dynamic_ui: bool = False,
ui_use_prodash: bool = False,
use_colors=True,
max_workunit_level: LogLevel = LogLevel.DEBUG,
session_values: SessionValues | None = None,
cancellation_latch: PySessionCancellationLatch | None = None,
) -> GraphSession:
session = self.scheduler.new_session(
build_id,
dynamic_ui,
ui_use_prodash,
max_workunit_level=max_workunit_level,
session_values=session_values,
cancellation_latch=cancellation_latch,
)
console = Console(use_colors=use_colors,
session=session if dynamic_ui else None)
return GraphSession(session, console, self.goal_map)
def _assert_test_summary(
expected: str,
*,
exit_code: int | None,
run_id: int,
result_metadata: ProcessResultMetadata | None,
) -> None:
assert expected == _format_test_summary(
TestResult(
exit_code=exit_code,
stdout="",
stderr="",
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=Address(spec_path="", target_name="dummy_address"),
output_setting=ShowOutput.FAILED,
result_metadata=result_metadata,
),
RunId(run_id),
Console(use_colors=False),
)
def execute_rule(self,
args=tuple(),
env=tuple(),
exit_code=0,
additional_params: Iterable[Any] = tuple()):
"""Executes the @console_rule for this test class.
:API: public
Returns the text output of the task.
"""
# Create an OptionsBootstrapper for these args/env, and a captured Console instance.
args = self._implicit_args + (self.goal_cls.name, ) + tuple(args)
env = dict(env)
options_bootstrapper = OptionsBootstrapper.create(args=args, env=env)
BuildConfigInitializer.get(options_bootstrapper)
full_options = options_bootstrapper.get_full_options(
list(self.goal_cls.subsystem_cls.known_scope_infos()))
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
scheduler = self.scheduler
workspace = Workspace(scheduler)
# Run for the target specs parsed from the args.
specs = TargetRootsCalculator.parse_specs(full_options.positional_args,
self.build_root)
params = Params(specs, console, options_bootstrapper, workspace,
*additional_params)
actual_exit_code = self.scheduler.run_console_rule(
self.goal_cls, params)
# Flush and capture console output.
console.flush()
stdout_val = stdout.getvalue()
stderr_val = stderr.getvalue()
assert exit_code == actual_exit_code, \
f"Exited with {actual_exit_code} (expected {exit_code}):\nstdout:\n{stdout_val}\nstderr:\n{stderr_val}"
return stdout_val
def run_console_rules(self, options_bootstrapper, goals, target_roots):
"""Runs @console_rules sequentially and interactively by requesting their implicit Goal products.
For retryable failures, raises scheduler.ExecutionError.
:param list goals: The list of requested goal names as passed on the commandline.
:param TargetRoots target_roots: The targets root of the request.
"""
# Reduce to only applicable goals - with validation happening by way of `validate_goals()`.
goals = [goal for goal in goals if goal in self.goal_map]
subjects = self._determine_subjects(target_roots)
console = Console()
# Console rule can only have one subject.
assert len(subjects) == 1
for goal in goals:
try:
goal_product = self.goal_map[goal]
params = Params(subjects[0], options_bootstrapper, console)
logger.debug('requesting {} to satisfy execution of `{}` goal'.format(goal_product, goal))
self.scheduler_session.run_console_rule(goal_product, params)
finally:
console.flush()
def mock_console(
options_bootstrapper: OptionsBootstrapper,
*,
stdin_content: bytes | str | None = None,
) -> Iterator[tuple[Console, StdioReader]]:
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
colors = (options_bootstrapper.full_options_for_scopes(
[GlobalOptions.get_scope_info()],
allow_unknown_options=True).for_global_scope().colors)
with initialize_stdio(global_bootstrap_options), stdin_context(
stdin_content) as stdin, temporary_file(
binary_mode=False) as stdout, temporary_file(
binary_mode=False) as stderr, stdio_destination(
stdin_fileno=stdin.fileno(),
stdout_fileno=stdout.fileno(),
stderr_fileno=stderr.fileno(),
):
# NB: We yield a Console without overriding the destination argument, because we have
# already done a sys.std* level replacement. The replacement is necessary in order for
# InteractiveProcess to have native file handles to interact with.
yield Console(use_colors=colors), StdioReader(
_stdout=Path(stdout.name), _stderr=Path(stderr.name))
def setup_legacy_graph_extended(
pants_ignore_patterns,
workdir,
local_store_dir,
build_file_imports_behavior,
options_bootstrapper,
build_configuration,
build_root=None,
native=None,
glob_match_error_behavior=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True,
execution_options=None,
):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements. Valid values: "allow", "warn", "error".
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options.
:type options_bootstrapper: :class:`pants.options.options_bootstrapper.OptionsBootstrapper`
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration`
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:param execution_options: Option values for (remote) process execution.
:type execution_options: :class:`pants.option.global_options.ExecutionOptions`
:returns: A LegacyGraphScheduler.
"""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(
options_bootstrapper)
build_file_aliases = build_configuration.registered_aliases()
rules = build_configuration.rules()
console = Console()
symbol_table = LegacySymbolTable(build_file_aliases)
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases,
build_file_imports_behavior)
address_mapper = AddressMapper(
parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.create()
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
[
SingletonRule.from_instance(console),
SingletonRule.from_instance(
GlobMatchErrorBehavior.create(glob_match_error_behavior)),
SingletonRule.from_instance(build_configuration),
] + create_legacy_graph_tasks(symbol_table) + create_fs_rules() +
create_process_rules() +
create_graph_rules(address_mapper, symbol_table) +
create_options_parsing_rules() + create_core_rules() +
# TODO: This should happen automatically, but most tests (e.g. tests/python/pants_test/auth) fail if it's not here:
[run_python_test] + rules)
goal_map = EngineInitializer._make_goal_map_from_rules(rules)
scheduler = Scheduler(
native,
project_tree,
workdir,
local_store_dir,
rules,
execution_options,
include_trace_on_error=include_trace_on_error,
)
return LegacyGraphScheduler(scheduler, symbol_table, console, goal_map)
def test_run_rule_console_rule_generator(self):
res = run_rule(a_console_rule_generator, Console(), {
(A, str): lambda _: A(),
})
self.assertEquals(res, _GoalProduct.for_name('example')())
def test_run_rule_console_rule_generator(self):
res = run_rule(a_console_rule_generator, Console(), {
(A, str): lambda _: A(),
})
self.assertEquals(res, Example(0))