def validate_instance(cls, opts):
"""Validates an instance of global options for cases that are not prohibited via registration.
For example: mutually exclusive options may be registered by passing a `mutually_exclusive_group`,
but when multiple flags must be specified together, it can be necessary to specify post-parse
checks.
Raises pants.option.errors.OptionsError on validation failure.
"""
if opts.loop and (not opts.v2 or opts.v1):
raise OptionsError(
'The `--loop` option only works with @console_rules, and thus requires '
'`--v2 --no-v1` to function as expected.')
if opts.loop and not opts.enable_pantsd:
raise OptionsError(
'The `--loop` option requires `--enable-pantsd`, in order to watch files.'
)
if opts.v2_ui and not opts.v2:
raise OptionsError(
'The `--v2-ui` option requires `--v2` to be enabled together.')
if opts.remote_execution and not opts.remote_execution_server:
raise OptionsError(
"The `--remote-execution` option requires also setting "
"`--remote-execution-server` to work properly.")
if opts.remote_execution_server and not opts.remote_store_server:
raise OptionsError(
"The `--remote-execution-server` option requires also setting "
"`--remote-store-server`. Often these have the same value.")
def validate_instance(cls, opts):
"""Validates an instance of global options for cases that are not prohibited via
registration.
For example: mutually exclusive options may be registered by passing a `mutually_exclusive_group`,
but when multiple flags must be specified together, it can be necessary to specify post-parse
checks.
Raises pants.option.errors.OptionsError on validation failure.
"""
if opts.remote_execution and not opts.remote_execution_server:
raise OptionsError(
"The `--remote-execution` option requires also setting "
"`--remote-execution-server` to work properly.")
if opts.remote_execution_server and not opts.remote_store_server:
raise OptionsError(
"The `--remote-execution-server` option requires also setting "
"`--remote-store-server`. Often these have the same value.")
if opts.remote_cache_read and not opts.remote_store_server:
raise OptionsError(
"The `--remote-cache-read` option requires also setting "
"`--remote-store-server` to work properly.")
if opts.remote_cache_write and not opts.remote_store_server:
raise OptionsError(
"The `--remote-cache-write` option requires also setting "
"`--remote-store-server` to work properly.")
def validate_scope(cls) -> None:
options_scope = getattr(cls, "options_scope", None)
if options_scope is None:
raise OptionsError(f"{cls.__name__} must set options_scope.")
if not cls.is_valid_scope_name(options_scope):
raise OptionsError(
f'Options scope "{options_scope}" is not valid:\nReplace in code with a new '
"scope name consisting of only lower-case letters, digits, underscores, "
"and non-consecutive dashes.")
def validate_headers(opt_name: str) -> None:
command_line_opt_name = f"--{opt_name.replace('_', '-')}"
for k, v in getattr(opts, opt_name).items():
if not k.isascii():
raise OptionsError(
f"All values in `{command_line_opt_name}` must be ASCII, but the key "
f"in `{k}: {v}` has non-ASCII characters.")
if not v.isascii():
raise OptionsError(
f"All values in `{command_line_opt_name}` must be ASCII, but the value in "
f"`{k}: {v}` has non-ASCII characters.")
def validate_instance(cls, opts):
"""Validates an instance of global options for cases that are not prohibited via registration.
For example: mutually exclusive options may be registered by passing a `mutually_exclusive_group`,
but when multiple flags must be specified together, it can be necessary to specify post-parse
checks.
Raises pants.option.errors.OptionsError on validation failure.
"""
if opts.loop and (not opts.v2 or opts.v1):
raise OptionsError('The --loop option only works with @console_rules, and thus requires '
'`--v2 --no-v1` to function as expected.')
if opts.loop and not opts.enable_pantsd:
raise OptionsError('The --loop option requires `--enable-pantsd`, in order to watch files.')
def verify_configs_against_options(self, options):
"""Verify all loaded configs have correct scopes and options.
:param options: Fully bootstrapped valid options.
:return: None.
"""
has_error = False
for config in self._post_bootstrap_config.configs:
for section in config.sections():
if section == GLOBAL_SCOPE_CONFIG_SECTION:
scope = GLOBAL_SCOPE
else:
scope = section
try:
valid_options_under_scope = set(options.for_scope(scope))
except OptionsError:
logger.error("Invalid scope [{}] in {}".format(
section, config.configpath))
has_error = True
else:
# All the options specified under [`section`] in `config` excluding bootstrap defaults.
all_options_under_scope = set(
config.configparser.options(section)) - set(
config.configparser.defaults())
for option in all_options_under_scope:
if option not in valid_options_under_scope:
logger.error(
"Invalid option '{}' under [{}] in {}".format(
option, section, config.configpath))
has_error = True
if has_error:
raise OptionsError(
"Invalid config entries detected. See log for details.")
def get_scope_info(cls):
"""Returns a ScopeInfo instance representing this Optionable's options scope."""
if cls.options_scope is None or cls.options_scope_category is None:
raise OptionsError(
'{} must set options_scope and options_scope_category.'.format(
cls.__name__))
return ScopeInfo(cls.options_scope, cls.options_scope_category, cls)
def validate_scope_name_component(cls, s):
if not cls.is_valid_scope_name_component(s):
raise OptionsError(
'Options scope "{}" is not valid:\n'
'Replace in code with a new scope name consisting of dash-separated-words, '
'with words consisting only of lower-case letters and digits.'.
format(s))
def main(self) -> MainSpecification:
is_default_console_script = self.options.is_default("console_script")
is_default_entry_point = self.options.is_default("entry_point")
if not is_default_console_script and not is_default_entry_point:
raise OptionsError(
f"Both [{self.scope}].console-script={self.options.console_script} and "
f"[{self.scope}].entry-point={self.options.entry_point} are configured but these "
f"options are mutually exclusive. Please pick one.")
if not is_default_console_script:
return ConsoleScript(cast(str, self.options.console_script))
if not is_default_entry_point:
return EntryPoint.parse(cast(str, self.options.entry_point))
return self.default_main
def validate_instance(cls, opts):
"""Validates an instance of global options for cases that are not prohibited via
registration.
For example: mutually exclusive options may be registered by passing a `mutually_exclusive_group`,
but when multiple flags must be specified together, it can be necessary to specify post-parse
checks.
Raises pants.option.errors.OptionsError on validation failure.
"""
if opts.remote_execution and not opts.remote_execution_server:
raise OptionsError(
"The `--remote-execution` option requires also setting "
"`--remote-execution-server` to work properly.")
if opts.remote_execution_server and not opts.remote_store_server:
raise OptionsError(
"The `--remote-execution-server` option requires also setting "
"`--remote-store-server`. Often these have the same value.")
if opts.remote_cache_read and not opts.remote_store_server:
raise OptionsError(
"The `--remote-cache-read` option requires also setting "
"`--remote-store-server` to work properly.")
if opts.remote_cache_write and not opts.remote_store_server:
raise OptionsError(
"The `--remote-cache-write` option requires also setting "
"`--remote-store-server` to work properly.")
if opts.remote_execution and (opts.remote_cache_read
or opts.remote_cache_write):
raise OptionsError(
"`--remote-execution` cannot be set at the same time as either "
"`--remote-cache-read` or `--remote-cache-write`.\n\nIf remote execution is "
"enabled, it will already use remote caching.")
# Ensure that timeout values are non-zero.
if opts.remote_store_initial_timeout <= 0:
raise OptionsError(
"The --remote-store-initial-timeout option requires a positive number of "
"milliseconds.")
if opts.remote_store_timeout_multiplier <= 0.0:
raise OptionsError(
"The --remote-store-timeout-multiplier option requires a positive number for the "
"multiplier.")
if opts.remote_store_maximum_timeout <= 0:
raise OptionsError(
"The --remote-store-initial-timeout option requires a positive number of "
"milliseconds.")
def main(self) -> MainSpecification:
is_default_console_script = self.options.is_default("console_script")
is_default_entry_point = self.options.is_default("entry_point")
if not is_default_console_script and not is_default_entry_point:
raise OptionsError(
softwrap(f"""
Both [{self.options_scope}].console-script={self.console_script} and
[{self.options_scope}].entry-point={self.entry_point} are configured
but these options are mutually exclusive. Please pick one.
"""))
if not is_default_console_script:
assert self.console_script is not None
return ConsoleScript(self.console_script)
if not is_default_entry_point:
assert self.entry_point is not None
return EntryPoint.parse(self.entry_point)
return self.default_main
def get_scope_info(cls):
"""Returns a ScopeInfo instance representing this Optionable's options scope."""
if cls.options_scope is None:
raise OptionsError(f"{cls.__name__} must set options_scope.")
return ScopeInfo(cls.options_scope, cls)
def validate_scope_name_component(cls, s: str) -> None:
if not cls.is_valid_scope_name_component(s):
raise OptionsError(
f'Options scope "{s}" is not valid:\nReplace in code with a new scope name consisting of '
f"dash-separated-words, with words consisting only of lower-case letters and digits."
)
def get_parser_by_scope(self, scope):
try:
return self._parser_by_scope[scope]
except KeyError:
raise OptionsError('No such options scope: {}'.format(scope))
def from_options(cls, options: Options) -> ExecutionOptions:
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
# Possibly insert some headers and disable remote execution/caching.
remote_execution_headers = cast(
Dict[str, str], bootstrap_options.remote_execution_headers)
remote_store_headers = cast(Dict[str, str],
bootstrap_options.remote_store_headers)
remote_execution = cast(bool, bootstrap_options.remote_execution)
remote_cache_read = cast(bool, bootstrap_options.remote_cache_read)
remote_cache_write = cast(bool, bootstrap_options.remote_cache_write)
if bootstrap_options.remote_oauth_bearer_token_path:
oauth_token = (Path(
bootstrap_options.remote_oauth_bearer_token_path).resolve().
read_text().strip())
if set(oauth_token).intersection({"\n", "\r"}):
raise OptionsError(
f"OAuth bearer token path {bootstrap_options.remote_oauth_bearer_token_path} "
"must not contain multiple lines.")
token_header = {"authorization": f"Bearer {oauth_token}"}
remote_execution_headers.update(token_header)
remote_store_headers.update(token_header)
if bootstrap_options.remote_auth_plugin and (remote_execution
or remote_cache_read
or remote_cache_write):
if ":" not in bootstrap_options.remote_auth_plugin:
raise OptionsError(
"Invalid value for `--remote-auth-plugin`: "
f"{bootstrap_options.remote_auth_plugin}. Please use the format "
f"`path.to.module:my_func`.")
auth_plugin_path, auth_plugin_func = bootstrap_options.remote_auth_plugin.split(
":")
auth_plugin_module = importlib.import_module(auth_plugin_path)
auth_plugin_func = getattr(auth_plugin_module, auth_plugin_func)
auth_plugin_result = cast(
AuthPluginResult,
auth_plugin_func(
initial_execution_headers=remote_execution_headers,
initial_store_headers=remote_store_headers,
options=options,
),
)
if not auth_plugin_result.is_available:
# NB: This is debug because we expect plugins to log more informative messages.
logger.debug(
"Disabling remote caching and remote execution because authentication was not "
"available via the plugin from `--remote-auth-plugin`.")
remote_execution = False
remote_cache_read = False
remote_cache_write = False
else:
remote_execution_headers = auth_plugin_result.execution_headers
remote_store_headers = auth_plugin_result.store_headers
return cls(
# Remote execution strategy.
remote_execution=remote_execution,
remote_cache_read=remote_cache_read,
remote_cache_write=remote_cache_write,
# General remote setup.
remote_instance_name=bootstrap_options.remote_instance_name,
remote_ca_certs_path=bootstrap_options.remote_ca_certs_path,
# Process execution setup.
process_execution_local_parallelism=bootstrap_options.
process_execution_local_parallelism,
process_execution_remote_parallelism=bootstrap_options.
process_execution_remote_parallelism,
process_execution_cleanup_local_dirs=bootstrap_options.
process_execution_cleanup_local_dirs,
process_execution_use_local_cache=bootstrap_options.
process_execution_use_local_cache,
process_execution_cache_namespace=bootstrap_options.
process_execution_cache_namespace,
process_execution_local_enable_nailgun=bootstrap_options.
process_execution_local_enable_nailgun,
# Remote store setup.
remote_store_server=bootstrap_options.remote_store_server,
remote_store_headers=remote_store_headers,
remote_store_thread_count=bootstrap_options.
remote_store_thread_count,
remote_store_chunk_bytes=bootstrap_options.
remote_store_chunk_bytes,
remote_store_chunk_upload_timeout_seconds=bootstrap_options.
remote_store_chunk_upload_timeout_seconds,
remote_store_rpc_retries=bootstrap_options.
remote_store_rpc_retries,
remote_store_connection_limit=bootstrap_options.
remote_store_connection_limit,
remote_store_initial_timeout=bootstrap_options.
remote_store_initial_timeout,
remote_store_timeout_multiplier=bootstrap_options.
remote_store_timeout_multiplier,
remote_store_maximum_timeout=bootstrap_options.
remote_store_maximum_timeout,
# Remote cache setup.
remote_cache_eager_fetch=bootstrap_options.
remote_cache_eager_fetch,
# Remote execution setup.
remote_execution_server=bootstrap_options.remote_execution_server,
remote_execution_extra_platform_properties=bootstrap_options.
remote_execution_extra_platform_properties,
remote_execution_headers=remote_execution_headers,
remote_execution_overall_deadline_secs=bootstrap_options.
remote_execution_overall_deadline_secs,
)
def validate_instance(cls, opts):
"""Validates an instance of global options for cases that are not prohibited via
registration.
For example: mutually exclusive options may be registered by passing a `mutually_exclusive_group`,
but when multiple flags must be specified together, it can be necessary to specify post-parse
checks.
Raises pants.option.errors.OptionsError on validation failure.
"""
if opts.remote_execution and (opts.remote_cache_read
or opts.remote_cache_write):
raise OptionsError(
"`--remote-execution` cannot be set at the same time as either "
"`--remote-cache-read` or `--remote-cache-write`.\n\nIf remote execution is "
"enabled, it will already use remote caching.")
if opts.remote_execution and not opts.remote_execution_server:
raise OptionsError(
"The `--remote-execution` option requires also setting "
"`--remote-execution-server` to work properly.")
if opts.remote_execution_server and not opts.remote_store_server:
raise OptionsError(
"The `--remote-execution-server` option requires also setting "
"`--remote-store-server`. Often these have the same value.")
if opts.remote_cache_read and not opts.remote_store_server:
raise OptionsError(
"The `--remote-cache-read` option requires also setting "
"`--remote-store-server` to work properly.")
if opts.remote_cache_write and not opts.remote_store_server:
raise OptionsError(
"The `--remote-cache-write` option requires also setting "
"`--remote-store-server` to work properly.")
# Ensure that timeout values are non-zero.
if opts.remote_store_initial_timeout <= 0:
raise OptionsError(
"The --remote-store-initial-timeout option requires a positive number of "
"milliseconds.")
if opts.remote_store_timeout_multiplier <= 0.0:
raise OptionsError(
"The --remote-store-timeout-multiplier option requires a positive number for the "
"multiplier.")
if opts.remote_store_maximum_timeout <= 0:
raise OptionsError(
"The --remote-store-initial-timeout option requires a positive number of "
"milliseconds.")
# Ensure that remote headers are ASCII.
def validate_headers(opt_name: str) -> None:
command_line_opt_name = f"--{opt_name.replace('_', '-')}"
for k, v in getattr(opts, opt_name).items():
if not k.isascii():
raise OptionsError(
f"All values in `{command_line_opt_name}` must be ASCII, but the key "
f"in `{k}: {v}` has non-ASCII characters.")
if not v.isascii():
raise OptionsError(
f"All values in `{command_line_opt_name}` must be ASCII, but the value in "
f"`{k}: {v}` has non-ASCII characters.")
validate_headers("remote_execution_headers")
validate_headers("remote_store_headers")