def get_all_help_info(
cls,
options: Options,
union_membership: UnionMembership,
consumed_scopes_mapper: ConsumedScopesMapper,
registered_target_types: RegisteredTargetTypes,
) -> AllHelpInfo:
scope_to_help_info = {}
name_to_goal_info = {}
for scope_info in sorted(options.known_scope_to_info.values(),
key=lambda x: x.scope):
options.for_scope(scope_info.scope) # Force parsing.
subsystem_cls = scope_info.subsystem_cls
if not scope_info.description:
cls_name = (
f"{subsystem_cls.__module__}.{subsystem_cls.__qualname__}"
if subsystem_cls else "")
raise ValueError(
f"Subsystem {cls_name} with scope `{scope_info.scope}` has no description. "
f"Add a class property `help`.")
is_goal = subsystem_cls is not None and issubclass(
subsystem_cls, GoalSubsystem)
oshi = HelpInfoExtracter(
scope_info.scope).get_option_scope_help_info(
scope_info.description,
options.get_parser(scope_info.scope), is_goal)
scope_to_help_info[oshi.scope] = oshi
if is_goal:
goal_subsystem_cls = cast(Type[GoalSubsystem], subsystem_cls)
is_implemented = union_membership.has_members_for_all(
goal_subsystem_cls.required_union_implementations)
name_to_goal_info[scope_info.scope] = GoalHelpInfo(
goal_subsystem_cls.name,
scope_info.description,
is_implemented,
consumed_scopes_mapper(scope_info.scope),
)
name_to_target_type_info = {
alias: TargetTypeHelpInfo.create(target_type,
union_membership=union_membership)
for alias, target_type in
registered_target_types.aliases_to_types.items()
if (not alias.startswith("_") and target_type.removal_version is
None and alias != target_type.deprecated_alias)
}
return AllHelpInfo(
scope_to_help_info=scope_to_help_info,
name_to_goal_info=name_to_goal_info,
name_to_target_type_info=name_to_target_type_info,
)
def verify_configs_against_options(self, options: Options) -> None:
"""Verify all loaded configs have correct scopes and options.
:param options: Fully bootstrapped valid options.
"""
error_log = []
for config in self.config.configs():
for section in config.sections():
scope = GLOBAL_SCOPE if section == GLOBAL_SCOPE_CONFIG_SECTION else section
try:
valid_options_under_scope = set(
options.for_scope(scope, include_passive_options=True))
# Only catch ConfigValidationError. Other exceptions will be raised directly.
except Config.ConfigValidationError:
error_log.append(
f"Invalid scope [{section}] in {config.config_path}")
else:
# All the options specified under [`section`] in `config` excluding bootstrap defaults.
all_options_under_scope = set(
config.values.options(section)) - set(
config.values.defaults)
for option in sorted(all_options_under_scope):
if option not in valid_options_under_scope:
error_log.append(
f"Invalid option '{option}' under [{section}] in {config.config_path}"
)
if error_log:
for error in error_log:
logger.error(error)
raise Config.ConfigValidationError(
"Invalid config entries detected. See log for details on which entries to update or "
"remove.\n(Specify --no-verify-config to disable this check.)")
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
v2_ui,
should_report_workunits=stream_workunits,
)
def prepare_v1_graph_run_v2(
self,
options: Options,
options_bootstrapper: OptionsBootstrapper,
) -> Tuple[LegacyGraphSession, Specs, int]:
"""For v1 (and v2): computing Specs for a later v1 run.
For v2: running an entire v2 run The exit_code in the return indicates whether any issue was
encountered
"""
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug(
"graph len was {}, waiting for initial watchman event".format(
graph_len))
self._watchman_is_running.wait()
build_id = RunTracker.global_instance().run_id
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
session = self._graph_helper.new_session(zipkin_trace_v2, build_id,
v2_ui)
if options.for_global_scope().get("loop", False):
fn = self._loop
else:
fn = self._body
specs, exit_code = fn(session, options, options_bootstrapper)
return session, specs, exit_code
def create(
cls,
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
) -> Specs:
specs = cls.parse_specs(raw_specs=options.specs, build_root=build_root)
changed_options = ChangedOptions.from_options(
options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs.provided and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
if specs.filesystem_specs and specs.address_specs:
specs_description = "target and file arguments"
elif specs.filesystem_specs:
specs_description = "file arguments"
else:
specs_description = "target arguments"
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. Please "
"use only one.")
if not changed_options.provided:
return specs
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
"The `--changed-*` options are not available without a recognized SCM (usually "
"Git).")
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
dependees=changed_options.dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_specs = []
filesystem_specs = []
for address in cast(ChangedAddresses, changed_addresses):
if not address.is_base_target:
# TODO: Should adjust Specs parsing to support parsing the disambiguated file
# Address, which would bypass-rediscovering owners.
filesystem_specs.append(FilesystemLiteralSpec(
address.filename))
else:
address_specs.append(
SingleAddress(address.spec_path, address.target_name))
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def get_all_help_info(
cls,
options: Options,
union_membership: UnionMembership,
consumed_scopes_mapper: ConsumedScopesMapper,
) -> AllHelpInfo:
scope_to_help_info = {}
name_to_goal_info = {}
for scope_info in sorted(options.known_scope_to_info.values(), key=lambda x: x.scope):
options.for_scope(scope_info.scope) # Force parsing.
optionable_cls = scope_info.optionable_cls
if not scope_info.description:
cls_name = (
f"{optionable_cls.__module__}.{optionable_cls.__qualname__}"
if optionable_cls
else ""
)
raise ValueError(
f"Subsystem {cls_name} with scope `{scope_info.scope}` has no description. "
f"Add a docstring or implement get_description()."
)
is_goal = optionable_cls is not None and issubclass(optionable_cls, GoalSubsystem)
oshi: OptionScopeHelpInfo = HelpInfoExtracter(
scope_info.scope
).get_option_scope_help_info(
scope_info.description, options.get_parser(scope_info.scope), is_goal
)
scope_to_help_info[oshi.scope] = oshi
if is_goal:
goal_subsystem_cls = cast(Type[GoalSubsystem], optionable_cls)
is_implemented = union_membership.has_members_for_all(
goal_subsystem_cls.required_union_implementations
)
name_to_goal_info[scope_info.scope] = GoalHelpInfo(
goal_subsystem_cls.name,
scope_info.description,
is_implemented,
consumed_scopes_mapper(scope_info.scope),
)
return AllHelpInfo(
scope_to_help_info=scope_to_help_info, name_to_goal_info=name_to_goal_info
)
def calculate_specs(
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
*,
build_root: Optional[str] = None,
) -> Specs:
"""Determine the specs for a given Pants run."""
build_root = build_root or get_buildroot()
specs = SpecsParser(build_root).parse_specs(options.specs)
changed_options = ChangedOptions.from_options(options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs.provided and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
if specs.filesystem_specs and specs.address_specs:
specs_description = "target and file arguments"
elif specs.filesystem_specs:
specs_description = "file arguments"
else:
specs_description = "target arguments"
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. Please "
"use only one.")
if not changed_options.provided:
return specs
git = get_git()
if not git:
raise InvalidSpecConstraint(
"The `--changed-*` options are only available if Git is used for the repository."
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(git)),
dependees=changed_options.dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_specs = []
for address in cast(ChangedAddresses, changed_addresses):
address_input = AddressInput.parse(address.spec)
address_specs.append(
AddressLiteralSpec(
path_component=address_input.path_component,
# NB: AddressInput.target_component may be None, but AddressLiteralSpec expects a
# string.
target_component=address_input.target_component
or address.target_name,
))
return Specs(AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs([]))
def create(
cls,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
specs = cls.parse_specs(
raw_specs=options.specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags,
)
changed_options = ChangedOptions.from_options(
options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if changed_options.is_actionable(
) and specs.provided_specs.dependencies:
# We've been provided both a change request and specs.
raise InvalidSpecConstraint(
"Multiple target selection methods provided. Please use only one of "
"`--changed-*`, address specs, or filesystem specs.")
if changed_options.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
"The `--changed-*` options are not available without a recognized SCM (usually Git)."
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
include_dependees=changed_options.include_dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [changed_request])
logger.debug("changed addresses: %s", changed_addresses.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses.addresses)
return Specs(
address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]),
)
return specs
def prepare_graph(self, options: Options) -> LegacyGraphSession:
# If any nodes exist in the product graph, wait for the initial watchman event to avoid
# racing watchman startup vs invalidation events.
graph_len = self._scheduler.graph_len()
if graph_len > 0:
self._logger.debug(f"graph len was {graph_len}, waiting for initial watchman event")
self._watchman_is_running.wait()
global_options = options.for_global_scope()
build_id = RunTracker.global_instance().run_id
v2_ui = global_options.get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
return self._graph_helper.new_session(zipkin_trace_v2, build_id, v2_ui)
def run(
self,
*,
build_config: BuildConfiguration,
graph_session: GraphSession,
options: Options,
specs: Specs,
union_membership: UnionMembership,
) -> ExitCode:
goal_options = options.for_scope(self.name)
if goal_options.server:
return self._run_server(
graph_session=graph_session,
union_membership=union_membership,
)
current_session_values = graph_session.scheduler_session.py_session.session_values
env = current_session_values[CompleteEnvironment]
return self._setup_bsp_connection(
union_membership=union_membership, env=env, options=goal_options
)
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
if global_scope.v2:
dynamic_ui = resolve_conflicting_options(
old_option="v2_ui",
new_option="dynamic_ui",
old_scope=GLOBAL_SCOPE,
new_scope=GLOBAL_SCOPE,
old_container=global_scope,
new_container=global_scope,
)
else:
dynamic_ui = False
use_colors = global_scope.get("colors", True)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
def create(
cls,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> TargetRoots:
# Determine the literal target roots.
address_spec_roots = cls.parse_address_specs(
target_specs=options.specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags)
# Determine `Changed` arguments directly from options to support pre-`Subsystem`
# initialization paths.
changed_options = options.for_scope('changed')
changed_request = ChangedRequest.from_options(changed_options)
# Determine the `--owner-of=` arguments provided from the global options
owned_files = options.for_global_scope().owner_of
logger.debug('address_spec_roots are: %s', address_spec_roots)
logger.debug('changed_request is: %s', changed_request)
logger.debug('owned_files are: %s', owned_files)
targets_specified = sum(1 for item in (changed_request.is_actionable(),
owned_files,
address_spec_roots.dependencies)
if item)
if targets_specified > 1:
# We've been provided more than one of: a change request, an owner request, or spec roots.
raise InvalidSpecConstraint(
'Multiple target selection methods provided. Please use only one of '
'--changed-*, --owner-of, or target specs')
if changed_request.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
'The --changed-* options are not available without a recognized SCM (usually git).'
)
changed_files = cls.changed_files(
scm,
changes_since=changed_request.changes_since,
diffspec=changed_request.diffspec)
# We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute
# alternate target roots.
request = OwnersRequest(
sources=tuple(changed_files),
include_dependees=changed_request.include_dependees,
)
changed_addresses, = session.product_request(
BuildFileAddresses, [request])
logger.debug('changed addresses: %s', changed_addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses)
return TargetRoots(
AddressSpecs(dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags), )
if owned_files:
# We've been provided no spec roots (e.g. `./pants list`) AND a owner request. Compute
# alternate target roots.
request = OwnersRequest(
sources=tuple(owned_files),
include_dependees=IncludeDependeesOption.NONE,
)
owner_addresses, = session.product_request(BuildFileAddresses,
[request])
logger.debug('owner addresses: %s', owner_addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owner_addresses)
return TargetRoots(
AddressSpecs(dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags), )
return TargetRoots(address_spec_roots)
def calculate_specs(
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
) -> Specs:
"""Determine the specs for a given Pants run."""
global_options = options.for_global_scope()
unmatched_cli_globs = global_options.unmatched_cli_globs.to_glob_match_error_behavior(
)
convert_dir_literal_to_address_literal = (
global_options.use_deprecated_directory_cli_args_semantics)
if global_options.is_default(
"use_deprecated_directory_cli_args_semantics"):
warn_or_error(
"2.14.0.dev0",
"`use_deprecated_directory_cli_args_semantics` defaulting to True",
softwrap(f"""
Currently, a directory argument like `{bin_name()} test dir` is shorthand for the
target `dir:dir`, i.e. the target that leaves off `name=`.
In Pants 2.14, by default, a directory argument will instead match all
targets/files in the directory.
To opt into the new and more intuitive semantics early, set
`use_deprecated_directory_cli_args_semantics = false` in the `[GLOBAL]` section in
`pants.toml`. Otherwise, set to `true` to silence this warning.
"""),
)
specs = SpecsParser().parse_specs(
options.specs,
description_of_origin="CLI arguments",
unmatched_glob_behavior=unmatched_cli_globs,
convert_dir_literal_to_address_literal=
convert_dir_literal_to_address_literal,
)
changed_options = ChangedOptions.from_options(options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
specs_description = specs.arguments_provided_description()
assert specs_description is not None
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. You can "
f"only use `{changed_name}` or use normal arguments.")
if not changed_options.provided:
return specs
(git_binary, ) = session.product_request(GitBinary,
[Params(GitBinaryRequest())])
(maybe_git_worktree, ) = session.product_request(
MaybeGitWorktree, [Params(GitWorktreeRequest(), git_binary)])
if not maybe_git_worktree.git_worktree:
raise InvalidSpecConstraint(
"The `--changed-*` options are only available if Git is used for the repository."
)
changed_files = tuple(
changed_options.changed_files(maybe_git_worktree.git_worktree))
file_literal_specs = tuple(FileLiteralSpec(f) for f in changed_files)
changed_request = ChangedRequest(changed_files, changed_options.dependees)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_literal_specs = []
for address in cast(ChangedAddresses, changed_addresses):
address_input = AddressInput.parse(
address.spec, description_of_origin="`--changed-since`")
address_literal_specs.append(
AddressLiteralSpec(
path_component=address_input.path_component,
target_component=address_input.target_component,
generated_component=address_input.generated_component,
parameters=address_input.parameters,
))
return Specs(
includes=RawSpecs(
# We need both address_literals and file_literals to cover all our edge cases, including
# target-aware vs. target-less goals, e.g. `list` vs `count-loc`.
address_literals=tuple(address_literal_specs),
file_literals=file_literal_specs,
unmatched_glob_behavior=unmatched_cli_globs,
filter_by_global_options=True,
from_change_detection=True,
description_of_origin="`--changed-since`",
),
ignores=RawSpecs(description_of_origin="`--changed-since`"),
)
def create(
cls,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
specs = cls.parse_specs(
raw_specs=options.specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags,
)
changed_options = ChangedOptions.from_options(
options.for_scope('changed'))
owned_files = options.for_global_scope().owner_of
logger.debug('specs are: %s', specs)
logger.debug('changed_options are: %s', changed_options)
logger.debug('owned_files are: %s', owned_files)
targets_specified = sum(1
for item in (changed_options.is_actionable(),
owned_files,
specs.provided_specs.dependencies)
if item)
if targets_specified > 1:
# We've been provided more than one of: a change request, an owner request, or specs.
raise InvalidSpecConstraint(
'Multiple target selection methods provided. Please use only one of '
'`--changed-*`, `--owner-of`, address specs, or filesystem specs.'
)
if changed_options.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
'The `--changed-*` options are not available without a recognized SCM (usually git).'
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
include_dependees=changed_options.include_dependees,
)
changed_addresses, = session.product_request(
ChangedAddresses, [changed_request])
logger.debug('changed addresses: %s', changed_addresses.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses.addresses)
return Specs(
address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]),
)
if owned_files:
owner_request = OwnersRequest(sources=tuple(owned_files))
owner_request.validate(
pants_bin_name=options.for_global_scope().pants_bin_name)
owners, = session.product_request(Owners, [owner_request])
logger.debug('owner addresses: %s', owners.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owners.addresses)
return Specs(address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]))
return specs