def register_options(cls, register):
super(CoursierSubsystem, cls).register_options(register)
register('--cache-dir', type=str, fingerprint=True,
default=os.path.join(get_pants_cachedir(), 'coursier'),
help='Version paired with --bootstrap-jar-url, in order to invalidate and fetch the new version.')
register('--repos', type=list, fingerprint=True,
help='Maven style repos', default=['https://repo1.maven.org/maven2'])
register('--fetch-options', type=list, fingerprint=True,
default=[
# Quiet mode, so coursier does not show resolve progress,
# but still prints results if --report is specified.
'-q',
# Do not use default public maven repo.
'--no-default',
# Concurrent workers
'-n', '8',
],
help='Additional options to pass to coursier fetch. See `coursier fetch --help`')
register('--artifact-types', type=list, fingerprint=True,
default=['jar', 'bundle', 'test-jar', 'maven-plugin', 'src', 'doc'],
help='Specify the type of artifacts to fetch. See `packaging` at https://maven.apache.org/pom.html#Maven_Coordinates, '
'except `src` and `doc` being coursier specific terms for sources and javadoc.')
# TODO(yic): Use a published version of Coursier. https://github.com/pantsbuild/pants/issues/6852
register('--bootstrap-jar-url', fingerprint=True,
default='https://github.com/coursier/coursier/releases/download/pants_release_1.5.x/coursier-cli-1.1.0.cf365ea27a710d5f09db1f0a6feee129aa1fc417.jar',
help='Location to download a bootstrap version of Coursier.')
register('--version', type=str, fingerprint=True,
default='1.1.0.cf365ea27a710d5f09db1f0a6feee129aa1fc417',
help='Version paired with --bootstrap-jar-url, in order to invalidate and fetch the new version.')
register('--bootstrap-fetch-timeout-secs', type=int, advanced=True, default=10,
help='Timeout the fetch if the connection is idle for longer than this value.')
def _create_parser(cls, seed_values=None):
"""Creates a config parser that supports %([key-name])s value substitution.
A handful of seed values will be set to act as if specified in the loaded config file's DEFAULT
section, and be available for use in substitutions. The caller may override some of these
seed values.
:param seed_values: A dict with optional override seed values for buildroot, pants_workdir,
pants_supportdir and pants_distdir.
"""
seed_values = seed_values or {}
buildroot = seed_values.get('buildroot', get_buildroot())
all_seed_values = {
'buildroot': buildroot,
'homedir': os.path.expanduser('~'),
'user': getpass.getuser(),
'pants_bootstrapdir': get_pants_cachedir(),
'pants_configdir': get_pants_configdir(),
}
def update_dir_from_seed_values(key, default):
all_seed_values[key] = seed_values.get(key, os.path.join(buildroot, default))
update_dir_from_seed_values('pants_workdir', '.pants.d')
update_dir_from_seed_values('pants_supportdir', 'build-support')
update_dir_from_seed_values('pants_distdir', 'dist')
return configparser.SafeConfigParser(all_seed_values)
def register_options(cls, register):
super(CoursierSubsystem, cls).register_options(register)
register('--cache-dir', type=str, fingerprint=True,
default=os.path.join(get_pants_cachedir(), 'coursier'),
help='Version paired with --bootstrap-jar-url, in order to invalidate and fetch the new version.')
register('--repos', type=list, fingerprint=True,
help='Maven style repos', default=['https://repo1.maven.org/maven2'])
register('--fetch-options', type=list, fingerprint=True,
default=[
# Quiet mode, so coursier does not show resolve progress,
# but still prints results if --report is specified.
'-q',
# Do not use default public maven repo.
'--no-default',
# Concurrent workers
'-n', '8',
],
help='Additional options to pass to coursier fetch. See `coursier fetch --help`')
register('--artifact-types', type=list, fingerprint=True,
default=['jar', 'bundle', 'test-jar', 'maven-plugin', 'src', 'doc', 'aar'],
help='Specify the type of artifacts to fetch. See `packaging` at https://maven.apache.org/pom.html#Maven_Coordinates, '
'except `src` and `doc` being coursier specific terms for sources and javadoc.')
register('--bootstrap-jar-url', fingerprint=True,
default='https://dl.dropboxusercontent.com/s/zwh074l9kxhqlwp/coursier-cli-1.1.0.cf365ea27a710d5f09db1f0a6feee129aa1fc417.jar?dl=0',
help='Location to download a bootstrap version of Coursier.')
# TODO(wisechengyi): currently using a custom url for fast iteration.
# Once the coursier builds are stable, move the logic to binary_util. https://github.com/pantsbuild/pants/issues/5381
# Ths sha in the version corresponds to the sha in the PR https://github.com/coursier/coursier/pull/774
# The jar is built by following https://github.com/coursier/coursier/blob/master/DEVELOPMENT.md#build-with-pants
register('--version', type=str, fingerprint=True,
default='1.1.0.cf365ea27a710d5f09db1f0a6feee129aa1fc417',
help='Version paired with --bootstrap-jar-url, in order to invalidate and fetch the new version.')
register('--bootstrap-fetch-timeout-secs', type=int, advanced=True, default=10,
help='Timeout the fetch if the connection is idle for longer than this value.')
def setUp(self):
super(JvmToolTaskTestBase, self).setUp()
# Use a synthetic subclass for proper isolation when bootstrapping within the test.
bootstrap_scope = 'bootstrap_scope'
self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
JvmToolMixin.reset_registered_tools()
# Set some options:
# 1. Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using
# -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
# tool, constructing a fat jar and then shading that fat jar.
#
# 2. Allow tests to read/write tool jars from the real artifact cache, so they don't
# each have to resolve and shade them every single time, which is a huge slowdown.
# Note that local artifact cache writes are atomic, so it's fine for multiple concurrent
# tests to write to it.
#
# Note that we don't have access to the invoking pants instance's options, so we assume that
# its artifact cache is in the standard location. If it isn't, worst case the tests will
# populate a second cache at the standard location, which is no big deal.
# TODO: We really need a straightforward way for pants's own tests to get to the enclosing
# pants instance's options values.
artifact_caches = [os.path.join(get_pants_cachedir(), 'artifact_cache')]
self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])
self.set_options_for_scope('cache.{}'.format(bootstrap_scope),
read_from=artifact_caches,
write_to=artifact_caches)
# Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
# into our test workspace.
shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)
Bootstrapper.reset_instance()
def store_stats(self):
"""Store stats about this run in local and optionally remote stats dbs."""
stats = {
'run_info': self.run_info.get_as_dict(),
'cumulative_timings': self.cumulative_timings.get_all(),
'self_timings': self.self_timings.get_all(),
'artifact_cache_stats': self.artifact_cache_stats.get_all()
}
# Dump individual stat file.
# TODO(benjy): Do we really need these, once the statsdb is mature?
stats_file = os.path.join(get_pants_cachedir(), 'stats',
'{}.json'.format(self.run_info.get_info('id')))
safe_file_dump(stats_file, json.dumps(stats))
# Add to local stats db.
StatsDBFactory.global_instance().get_db().insert_stats(stats)
# Upload to remote stats db.
stats_url = self.get_options().stats_upload_url
if stats_url:
self.post_stats(stats_url, stats, timeout=self.get_options().stats_upload_timeout)
# Write stats to local json file.
stats_json_file_name = self.get_options().stats_local_json_file
if stats_json_file_name:
self.write_stats_to_json(stats_json_file_name, stats)
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
register('--plugins', advanced=True, type=Options.list, help='Load these plugins.')
register('--backend-packages', advanced=True, type=Options.list,
help='Load backends from these packages that are already on the path.')
register('--pants-bootstrapdir', advanced=True, metavar='<dir>', default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir', advanced=True, metavar='<dir>', default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir', metavar='<dir>', default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir', metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir', metavar='<dir>', default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register('--config-override', help='A second config file, to override pants.ini.')
register('--pantsrc', action='store_true', default=True,
help='Use pantsrc files.')
register('--pantsrc-files', action='append', metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register('--pythonpath', action='append',
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file', action='append', dest='target_spec_files',
help='Read additional specs from this file, one per line')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d', '--logdir', metavar='<dir>',
help='Write logs to files under this directory.')
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l', '--level', choices=['debug', 'info', 'warn'], default='info', recursive=True,
help='Set the logging level.')
register('-q', '--quiet', action='store_true',
help='Squelches all console output apart from errors.')
def _generate_fingerprinted_pex_path(self, tool_subsystem, interpreter):
# `tool_subsystem.get_requirement_specs()` is a list, but order shouldn't actually matter. This
# should probably be sorted, but it's possible a user could intentionally tweak order to work
# around a particular requirement resolution resolve-order issue. In practice the lists are
# expected to be mostly static, so we accept the risk of too-fine-grained caching creating lots
# of pexes in the cache dir.
specs_fingerprint = stable_json_sha1(tool_subsystem.get_requirement_specs())
return os.path.join(
get_pants_cachedir(),
'python',
str(interpreter.identity),
self.fingerprint,
'{}-{}.pex'.format(tool_subsystem.options_scope, specs_fingerprint),
)
def bootstrap_conan(self):
pex_info = PexInfo.default()
pex_info.entry_point = 'conans.conan'
conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
interpreter = PythonInterpreter.get()
if not os.path.exists(conan_pex_path):
with safe_concurrent_creation(conan_pex_path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
dump_requirements(builder, interpreter, reqs, logger)
builder.freeze()
conan_binary = PEX(conan_pex_path, interpreter)
return self.ConanBinary(pex=conan_binary)
def upload_stats(self):
"""Write stats to local cache, and upload to server, if needed."""
stats = {
'run_info': self.run_info.get_as_dict(),
'cumulative_timings': self.cumulative_timings.get_all(),
'self_timings': self.self_timings.get_all(),
'artifact_cache_stats': self.artifact_cache_stats.get_all()
}
stats_file = os.path.join(get_pants_cachedir(), 'stats',
'{}.json'.format(self.run_info.get_info('id')))
safe_file_dump(stats_file, json.dumps(stats))
stats_url = self.get_options().stats_upload_url
if stats_url:
self.post_stats(stats_url, stats, timeout=self.get_options().stats_upload_timeout)
def setUp(self):
super(TaskTestBase, self).setUp()
self.options_scope = 'test_scope'
self._testing_task_type = self.synthesize_task_subtype(self.task_type(), self.options_scope)
# We locate the workdir below the pants_workdir, which BaseTest locates within the BuildRoot.
# BaseTest cleans this up, so we don't need to. We give it a stable name, so that we can
# use artifact caching to speed up tests.
self._test_workdir = os.path.join(self.pants_workdir, self.task_type().stable_name())
os.mkdir(self._test_workdir)
# TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific
# action in this non-JVM-specific, high-level base class.
Bootstrapper.reset_instance()
self.artifact_cache = os.path.join(get_pants_cachedir(), 'artifact_cache')
self.set_options_for_scope('cache',
read_from=[self.artifact_cache],
write_to=[self.artifact_cache])
def setup_pexrc_with_pex_python_path(interpreter_paths):
"""A helper function for writing interpreter paths to a PEX_PYTHON_PATH variable in a .pexrc file.
NB: Mutates HOME and XDG_CACHE_HOME to ensure a `~/.pexrc` that won't trample any existing file
and will also be found.
:param list interpreter_paths: a list of paths to interpreter binaries to include on
PEX_PYTHON_PATH.
"""
cache_dir = get_pants_cachedir()
with temporary_dir() as home:
xdg_cache_home = os.path.join(home, '.cache')
with environment_as(HOME=home, XDG_CACHE_HOME=xdg_cache_home):
target = os.path.join(xdg_cache_home, os.path.basename(cache_dir))
safe_mkdir_for(target)
os.symlink(cache_dir, target)
with open(os.path.join(home, '.pexrc'), 'w') as pexrc:
pexrc.write('PEX_PYTHON_PATH={}'.format(':'.join(interpreter_paths)))
yield
def store_stats(self):
"""Store stats about this run in local and optionally remote stats dbs."""
run_information = self.run_info.get_as_dict()
target_data = run_information.get('target_data', None)
if target_data:
run_information['target_data'] = ast.literal_eval(target_data)
stats = {
'run_info': run_information,
'cumulative_timings': self.cumulative_timings.get_all(),
'self_timings': self.self_timings.get_all(),
'artifact_cache_stats': self.artifact_cache_stats.get_all(),
'outcomes': self.outcomes
}
# Dump individual stat file.
# TODO(benjy): Do we really need these, once the statsdb is mature?
stats_file = os.path.join(get_pants_cachedir(), 'stats',
'{}.json'.format(self.run_info.get_info('id')))
safe_file_dump(stats_file, json.dumps(stats))
# Add to local stats db.
StatsDBFactory.global_instance().get_db().insert_stats(stats)
# Upload to remote stats db.
stats_url = self.get_options().stats_upload_url
if stats_url:
pid = os.fork()
if pid == 0:
try:
self.post_stats(stats_url, stats, timeout=self.get_options().stats_upload_timeout)
finally:
os._exit(0)
# Write stats to local json file.
stats_json_file_name = self.get_options().stats_local_json_file
if stats_json_file_name:
self.write_stats_to_json(stats_json_file_name, stats)
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are the set of options necessary to create a Scheduler. If an option is
not consumed during creation of a Scheduler, it should be in `register_options` instead.
Bootstrap option values can be interpolated into the config file, and can be referenced
programmatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = "dist"
default_rel_distdir = f"/{default_distdir_name}/"
register(
"--backend-packages",
advanced=True,
type=list,
default=[],
help=
("Register functionality from these backends.\n\nThe backend packages must be "
"present on the PYTHONPATH, typically because they are in the Pants core dist, in a "
"plugin dist, or available as sources in the repo."),
)
register(
"--plugins",
advanced=True,
type=list,
help=
("Allow backends to be loaded from these plugins (usually released through PyPI). "
"The default backends for each plugin will be loaded automatically. Other backends "
"in a plugin can be loaded by listing them in `backend_packages` in the "
"`[GLOBAL]` scope."),
)
register(
"--plugins-force-resolve",
advanced=True,
type=bool,
default=False,
help="Re-resolve plugins, even if previously resolved.",
)
register(
"--plugin-cache-dir",
advanced=True,
default=os.path.join(get_pants_cachedir(), "plugins"),
help="Cache resolved plugin requirements here.",
)
register("-l",
"--level",
type=LogLevel,
default=LogLevel.INFO,
help="Set the logging level.")
register(
"--show-log-target",
type=bool,
default=False,
advanced=True,
help=
"Display the target where a log message originates in that log message's output. "
"This can be helpful when paired with --log-levels-by-target.",
)
register(
"--log-levels-by-target",
type=dict,
default={},
advanced=True,
help=
"Set a more specific logging level for one or more logging targets. The names of "
"logging targets are specified in log strings when the --show-log-target option is set. "
"The logging levels are one of: "
'"error", "warn", "info", "debug", "trace". '
"All logging targets not specified here use the global log level set with --level. For example, "
'you can set `--log-levels-by-target=\'{"workunit_store": "info", "pants.engine.rules": "warn"}\'`.',
)
register(
"--log-show-rust-3rdparty",
type=bool,
default=False,
advanced=True,
help=
"Whether to show/hide logging done by 3rdparty Rust crates used by the Pants "
"engine.",
)
register(
"--colors",
type=bool,
default=sys.stdout.isatty(),
help=
("Whether Pants should use colors in output or not. This may also impact whether "
"some tools Pants run use color."),
)
# TODO(#7203): make a regexp option type!
register(
"--ignore-pants-warnings",
type=list,
member_type=str,
default=[],
advanced=True,
help="Regexps matching warning strings to ignore, e.g. "
'["DEPRECATED: the option `--my-opt` will be removed"]. The regex patterns will be '
"matched from the start of the warning string, and are case-insensitive.",
)
register(
"--pants-version",
advanced=True,
default=pants_version(),
daemon=True,
help=
"Use this Pants version. Note that Pants only uses this to verify that you are "
"using the requested version, as Pants cannot dynamically change the version it "
"is using once the program is already running.\n\nIf you use the `./pants` script from "
f"{docs_url('installation')}, however, changing the value in your "
"`pants.toml` will cause the new version to be installed and run automatically.\n\n"
"Run `./pants --version` to check what is being used.",
)
register(
"--pants-bin-name",
advanced=True,
default="./pants",
help="The name of the script or binary used to invoke Pants. "
"Useful when printing help messages.",
)
register(
"--pants-bootstrapdir",
advanced=True,
metavar="<dir>",
default=None,
removal_version="2.3.0.dev1",
removal_hint="Unused.",
help="Unused.",
)
register(
"--pants-configdir",
advanced=True,
metavar="<dir>",
default=None,
removal_version="2.3.0.dev1",
removal_hint="Unused.",
help="Unused.",
)
register(
"--pants-workdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, ".pants.d"),
daemon=True,
help="Write intermediate logs and output files to this dir.",
)
register(
"--pants-physical-workdir-base",
advanced=True,
metavar="<dir>",
default=None,
daemon=True,
help=
"When set, a base directory in which to store `--pants-workdir` contents. "
"If this option is a set, the workdir will be created as symlink into a "
"per-workspace subdirectory.",
)
register(
"--pants-supportdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "build-support"),
help="Unused. Will be deprecated in 2.2.0.",
)
register(
"--pants-distdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "dist"),
help=
"Write end products, such as the results of `./pants package`, to this dir.",
)
register(
"--pants-subprocessdir",
advanced=True,
default=os.path.join(buildroot, ".pids"),
daemon=True,
help=
"The directory to use for tracking subprocess metadata. This should "
"live outside of the dir used by `pants_workdir` to allow for tracking "
"subprocesses that outlive the workdir data.",
)
register(
"--pants-config-files",
advanced=True,
type=list,
# NB: We don't fingerprint the list of config files, because the content of the config
# files independently affects fingerprints.
fingerprint=False,
default=[get_default_pants_config_file()],
help=
("Paths to Pants config files. This may only be set through the environment variable "
"`PANTS_CONFIG_FILES` and the command line argument `--pants-config-files`; it will "
"be ignored if in a config file like `pants.toml`."),
)
register(
"--pantsrc",
advanced=True,
type=bool,
default=True,
# NB: See `--pants-config-files`.
fingerprint=False,
help=
("Use pantsrc files located at the paths specified in the global option "
"`pantsrc_files`."),
)
register(
"--pantsrc-files",
advanced=True,
type=list,
metavar="<path>",
# NB: See `--pants-config-files`.
fingerprint=False,
default=["/etc/pantsrc", "~/.pants.rc"],
help=
("Override config with values from these files, using syntax matching that of "
"`--pants-config-files`."),
)
register(
"--pythonpath",
advanced=True,
type=list,
help=
("Add these directories to PYTHONPATH to search for plugins. This does not impact "
"the PYTHONPATH used by Pants when running your Python code."),
)
register(
"--spec-files",
type=list,
# NB: We don't fingerprint spec files because the content of the files independently
# affects fingerprints.
fingerprint=False,
help=
("Read additional specs (target addresses, files, and/or globs), one per line,"
"from these files."),
)
register(
"--verify-config",
type=bool,
default=True,
advanced=True,
help=
"Verify that all config file values correspond to known options.",
)
register(
"--stats-record-option-scopes",
advanced=True,
type=list,
default=["*"],
help=
("Option scopes to record in stats on run completion. "
"Options may be selected by joining the scope and the option with a ^ character, "
"i.e. to get option `pantsd` in the GLOBAL scope, you'd pass `GLOBAL^pantsd`. "
"Add a '*' to the list to capture all known scopes."),
)
register(
"--pants-ignore",
advanced=True,
type=list,
member_type=str,
default=[".*/", default_rel_distdir],
help=
"Paths to ignore for all filesystem operations performed by pants "
"(e.g. BUILD file scanning, glob matching, etc). "
"Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). "
"The `pants_distdir` and `pants_workdir` locations are automatically ignored. "
"`pants_ignore` can be used in tandem with `pants_ignore_use_gitignore`; any rules "
"specified here are applied after rules specified in a .gitignore file.",
)
register(
"--pants-ignore-use-gitignore",
advanced=True,
type=bool,
default=True,
help=
"Make use of a root .gitignore file when determining whether to ignore filesystem "
"operations performed by Pants. If used together with `--pants-ignore`, any exclude/include "
"patterns specified there apply after .gitignore rules.",
)
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register(
"-d",
"--logdir",
advanced=True,
metavar="<dir>",
daemon=True,
help="Write logs to files under this directory.",
)
register(
"--pantsd",
advanced=True,
type=bool,
default=True,
daemon=True,
help=
("Enables use of the Pants daemon (pantsd). pantsd can significantly improve "
"runtime performance by lowering per-run startup cost, and by memoizing filesystem "
"operations and rule execution."),
)
# Whether or not to make necessary arrangements to have concurrent runs in pants.
# In practice, this means that if this is set, a run will not even try to use pantsd.
# NB: Eventually, we would like to deprecate this flag in favor of making pantsd runs parallelizable.
register(
"--concurrent",
advanced=True,
type=bool,
default=False,
help=
"Enable concurrent runs of Pants. Without this enabled, Pants will "
"start up all concurrent invocations (e.g. in other terminals) without pantsd. "
"Enabling this option requires parallel Pants invocations to block on the first",
)
# NB: We really don't want this option to invalidate the daemon, because different clients might have
# different needs. For instance, an IDE might have a very long timeout because it only wants to refresh
# a project in the background, while a user might want a shorter timeout for interactivity.
register(
"--pantsd-timeout-when-multiple-invocations",
advanced=True,
type=float,
default=60.0,
help=
"The maximum amount of time to wait for the invocation to start until "
"raising a timeout exception. "
"Because pantsd currently does not support parallel runs, "
"any prior running Pants command must be finished for the current one to start. "
"To never timeout, use the value -1.",
)
register(
"--pantsd-max-memory-usage",
advanced=True,
type=int,
default=2**30,
help=
("The maximum memory usage of a pantsd process (in bytes). There is at most one "
"pantsd process per workspace."),
)
# These facilitate configuring the native engine.
register(
"--print-stacktrace",
advanced=True,
type=bool,
default=False,
help="Print the full exception stack trace for any errors.",
)
register(
"--native-engine-visualize-to",
advanced=True,
default=None,
type=dir_option,
help=
"A directory to write execution and rule graphs to as `dot` files. The contents "
"of the directory will be overwritten if any filenames collide.",
)
# Pants Daemon options.
register(
"--pantsd-pailgun-port",
advanced=True,
type=int,
default=0,
daemon=True,
help=
"The port to bind the Pants nailgun server to. Defaults to a random port.",
)
register(
"--pantsd-invalidation-globs",
advanced=True,
type=list,
default=[],
daemon=True,
help=
"Filesystem events matching any of these globs will trigger a daemon restart. "
"Pants's own code, plugins, and `--pants-config-files` are inherently invalidated.",
)
process_execution_local_parallelism = "--process-execution-local-parallelism"
rule_threads_core = "--rule-threads-core"
rule_threads_max = "--rule-threads-max"
register(
rule_threads_core,
type=int,
default=max(2, _CPU_COUNT // 2),
advanced=True,
help=
("The number of threads to keep active and ready to execute `@rule` logic (see "
f"also: `{rule_threads_max}`). Values less than 2 are not currently supported. "
"This value is independent of the number of processes that may be spawned in "
f"parallel locally (controlled by `{process_execution_local_parallelism}`)."
),
)
register(
rule_threads_max,
type=int,
default=None,
advanced=True,
help=
("The maximum number of threads to use to execute `@rule` logic. Defaults to "
f"a small multiple of `{rule_threads_core}`."),
)
cache_instructions = (
"The path may be absolute or relative. If the directory is within the build root, be "
"sure to include it in `--pants-ignore`.")
register(
"--local-store-dir",
advanced=True,
help=
(f"Directory to use for the local file store, which stores the results of "
f"subprocesses run by Pants. {cache_instructions}"),
# This default is also hard-coded into the engine's rust code in
# fs::Store::default_path so that tools using a Store outside of pants
# are likely to be able to use the same storage location.
default=os.path.join(get_pants_cachedir(), "lmdb_store"),
)
register(
"--named-caches-dir",
advanced=True,
help=
("Directory to use for named global caches for tools and processes with trusted, "
f"concurrency-safe caches. {cache_instructions}"),
default=os.path.join(get_pants_cachedir(), "named_caches"),
)
register(
"--local-execution-root-dir",
advanced=True,
help=
f"Directory to use for local process execution sandboxing. {cache_instructions}",
default=tempfile.gettempdir(),
)
register(
"--process-execution-use-local-cache",
type=bool,
default=True,
advanced=True,
help=
"Whether to keep process executions in a local cache persisted to disk.",
)
register(
"--process-execution-cleanup-local-dirs",
type=bool,
default=True,
advanced=True,
help=
"Whether or not to cleanup directories used for local process execution "
"(primarily useful for e.g. debugging).",
)
register(
"--ca-certs-path",
advanced=True,
type=str,
default=None,
help=
"Path to a file containing PEM-format CA certificates used for verifying secure "
"connections when downloading files required by a build.",
)
register(
process_execution_local_parallelism,
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_parallelism,
advanced=True,
help="Number of concurrent processes that may be executed locally.",
)
register(
"--process-execution-remote-parallelism",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_remote_parallelism,
advanced=True,
help=
"Number of concurrent processes that may be executed remotely.",
)
register(
"--process-execution-cache-namespace",
advanced=True,
type=str,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_cache_namespace,
help=
("The cache namespace for process execution. "
"Change this value to invalidate every artifact's execution, or to prevent "
"process cache entries from being (re)used for different usecases or users."
),
)
register(
"--process-execution-speculation-delay",
type=float,
default=1,
advanced=True,
help=
"Number of seconds to wait before speculating a second request for a slow process. "
" see `--process-execution-speculation-strategy`",
removal_version="2.4.0.dev0",
removal_hint=
("This option now no-ops, as speculation has been removed. It will be "
"re-implemented in the future."),
)
register(
"--process-execution-speculation-strategy",
choices=["remote_first", "local_first", "none"],
default="none",
help=
"Speculate a second request for an underlying process if the first one does not complete within "
"`--process-execution-speculation-delay` seconds.\n"
"`local_first` (default): Try to run the process locally first, "
"and fall back to remote execution if available.\n"
"`remote_first`: Run the process on the remote execution backend if available, "
"and fall back to the local host if remote calls take longer than the speculation timeout.\n"
"`none`: Do not speculate about long running processes.",
advanced=True,
removal_version="2.4.0.dev0",
removal_hint=
("This option now no-ops, as speculation has been removed. It will be "
"re-implemented in the future."),
)
register(
"--process-execution-local-enable-nailgun",
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_enable_nailgun,
help=
"Whether or not to use nailgun to run the requests that are marked as nailgunnable.",
advanced=True,
)
register(
"--remote-execution",
advanced=True,
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.remote_execution,
help=
("Enables remote workers for increased parallelism. (Alpha)\n\nAlternatively, you "
"can use `--remote-cache-read` and `--remote-cache-write` to still run everything "
"locally, but to use a remote cache."),
)
register(
"--remote-cache-read",
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.remote_cache_read,
advanced=True,
help=
("Whether to enable reading from a remote cache.\n\nThis cannot be used at the same "
"time as `--remote-execution`."),
)
register(
"--remote-cache-write",
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.remote_cache_write,
advanced=True,
help=
("Whether to enable writing results to a remote cache.\n\nThis cannot be used at "
"the same time as `--remote-execution`."),
)
register(
"--remote-cache-eager-fetch",
type=bool,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_cache_eager_fetch,
help=
("Eagerly fetch relevant content from the remote store instead of lazily fetching."
"\n\nThis may result in worse performance, but reduce the frequency of errors "
"encountered by reducing the surface area of when remote caching is used."
),
)
register(
"--remote-store-server",
advanced=True,
type=list,
default=[],
help=
"host:port of grpc server to use as remote execution file store.",
)
# TODO: Infer this from remote-store-connection-limit.
register(
"--remote-store-thread-count",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_thread_count,
help=
"Thread count to use for the pool that interacts with the remote file store.",
)
register(
"--remote-execution-server",
advanced=True,
help=
"host:port of grpc server to use as remote execution scheduler.",
)
register(
"--remote-store-chunk-bytes",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_bytes,
help=
"Size in bytes of chunks transferred to/from the remote file store.",
)
register(
"--remote-store-chunk-upload-timeout-seconds",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.
remote_store_chunk_upload_timeout_seconds,
help=
"Timeout (in seconds) for uploads of individual chunks to the remote file store.",
)
register(
"--remote-store-rpc-retries",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_rpc_retries,
help=
"Number of times to retry any RPC to the remote store before giving up.",
)
register(
"--remote-store-connection-limit",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_connection_limit,
help=
"Number of remote stores to concurrently allow connections to.",
)
register(
"--remote-store-initial-timeout",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_initial_timeout,
help=
"Initial timeout (in milliseconds) when there is a failure in accessing a remote store.",
)
register(
"--remote-store-timeout-multiplier",
type=float,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_timeout_multiplier,
help=
"Multiplier used to increase the timeout (starting with value of --remote-store-initial-timeout) between retry attempts in accessing a remote store.",
)
register(
"--remote-store-maximum-timeout",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_maximum_timeout,
help=
"Maximum timeout (in millseconds) to allow between retry attempts in accessing a remote store.",
)
register(
"--remote-instance-name",
advanced=True,
help=
"Name of the remote execution instance to use. Used for routing within "
"--remote-execution-server and --remote-store-server.",
)
register(
"--remote-ca-certs-path",
advanced=True,
help=
"Path to a PEM file containing CA certificates used for verifying secure "
"connections to --remote-execution-server and --remote-store-server. "
"If not specified, TLS will not be used.",
)
register(
"--remote-oauth-bearer-token-path",
advanced=True,
help=
"Path to a file containing an oauth token to use for grpc connections to "
"--remote-execution-server and --remote-store-server. If not specified, no "
"authorization will be performed.",
)
register(
"--remote-execution-extra-platform-properties",
advanced=True,
help="Platform properties to set on remote execution requests. "
"Format: property=value. Multiple values should be specified as multiple "
"occurrences of this flag. Pants itself may add additional platform properties.",
type=list,
default=[],
)
register(
"--remote-execution-headers",
advanced=True,
help="Headers to set on remote execution requests. "
"Format: header=value. Pants itself may add additional headers.",
type=dict,
default={},
)
register(
"--remote-execution-overall-deadline-secs",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
remote_execution_overall_deadline_secs,
advanced=True,
help=
"Overall timeout in seconds for each remote execution request from time of submission",
)
def test_get_cachedir(self) -> None:
with environment_as(XDG_CACHE_HOME=""):
self.assertEqual(os.path.expanduser("~/.cache/pants"),
get_pants_cachedir())
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l',
'--level',
choices=['debug', 'info', 'warn'],
default='info',
recursive=True,
help='Set the logging level.')
register('-q',
'--quiet',
type=bool,
recursive=True,
help='Squelches most console output.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors',
type=bool,
default=True,
recursive=True,
help='Set whether log messages are displayed in color.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version',
advanced=True,
default=pants_version(),
help='Use this pants version.')
register('--plugins',
advanced=True,
type=list,
help='Load these plugins.')
register('--plugin-cache-dir',
advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register(
'--backend-packages',
advanced=True,
type=list,
default=[
'pants.backend.graph_info', 'pants.backend.python',
'pants.backend.jvm', 'pants.backend.codegen',
'pants.backend.project_info'
],
help=
'Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir',
advanced=True,
metavar='<dir>',
default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir',
advanced=True,
metavar='<dir>',
default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register(
'--pants-subprocessdir',
advanced=True,
default=os.path.join(buildroot, '.pids'),
help=
'The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).'
)
register('--pants-config-files',
advanced=True,
type=list,
default=[get_default_pants_config_file()],
help='Paths to Pants config files.')
# TODO: Deprecate --config-override in favor of --pants-config-files.
# But only once we're able to both append and override list-valued options, as there are
# use-cases for both here.
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--config-override',
advanced=True,
type=list,
metavar='<path>',
help='A second config file, to override pants.ini.')
register('--pantsrc',
advanced=True,
type=bool,
default=True,
help='Use pantsrc files.')
register('--pantsrc-files',
advanced=True,
type=list,
metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register(
'--pythonpath',
advanced=True,
type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file',
type=list,
dest='target_spec_files',
help='Read additional specs from this file, one per line')
register(
'--verify-config',
type=bool,
default=True,
help=
'Verify that all config file values correspond to known options.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d',
'--logdir',
advanced=True,
metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
'--enable-pantsd',
advanced=True,
type=bool,
default=False,
help=
'Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)'
)
# This facilitates use of the v2 engine for BuildGraph construction, sans daemon.
register('--enable-v2-engine',
advanced=True,
type=bool,
default=False,
help='Enables use of the v2 engine. (Beta)')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = "dist"
default_rel_distdir = f"/{default_distdir_name}/"
register(
"-l",
"--level",
choices=["trace", "debug", "info", "warn", "error"],
default="info",
recursive=True,
help="Set the logging level.",
)
register(
"--log-show-rust-3rdparty",
type=bool,
default=False,
advanced=True,
help=
"Whether to show/hide logging done by 3rdparty rust crates used by the pants "
"engine.",
)
# Toggles v1/v2 `Task` vs `@rule` pipelines on/off.
# Having these in bootstrap options allows them to affect registration of non-bootstrap options.
register("--v1",
advanced=True,
type=bool,
default=True,
help="Enables execution of v1 Tasks.")
register(
"--v2",
advanced=True,
type=bool,
default=True,
help="Enables execution of v2 @goal_rules.",
)
# TODO(#7203): make a regexp option type!
register(
"--ignore-pants-warnings",
type=list,
member_type=str,
default=[],
advanced=True,
help="Regexps matching warning strings to ignore, e.g. "
'["DEPRECATED: scope some_scope will be removed"]. The regexps will be matched '
"from the start of the warning string, and will always be case-insensitive. "
"See the `warnings` module documentation for more background on these are used.",
)
register(
"--option-name-check-distance",
advanced=True,
type=int,
default=2,
help=
"The maximum Levenshtein distance to use when offering suggestions for invalid "
"option names.",
)
register(
"--pants-version",
advanced=True,
default=pants_version(),
help=
"Use this pants version. Note Pants code only uses this to verify that you are "
"using the requested version, as Pants cannot dynamically change the version it "
"is using once the program is already running. This option is useful to set in "
"your pants.toml, however, and then you can grep the value to select which "
"version to use for setup scripts (e.g. `./pants`), runner scripts, IDE plugins, "
"etc. For example, the setup script we distribute at https://www.pantsbuild.org/install.html#recommended-installation "
"uses this value to determine which Python version to run with. You may find the "
"version of the pants instance you are running using -v, -V, or --version.",
)
register(
"--pants-bin-name",
advanced=True,
default="./pants",
help="The name of the script or binary used to invoke pants. "
"Useful when printing help messages.",
)
register(
"--plugins",
advanced=True,
type=list,
help=
"Allow v1 backends to be loaded from these plugins. The default backends for "
"each plugin will be loaded automatically. Other backends in a plugin can be "
"loaded by listing them in --backend-packages.",
)
register(
"--plugins2",
advanced=True,
type=list,
help=
"Allow v2 backends to be loaded from these plugins. The default backends for "
"each plugin will be loaded automatically. Other backends in a plugin can be "
"loaded by listing them in --backend-packages.",
)
register(
"--plugins-force-resolve",
advanced=True,
type=bool,
default=False,
help="Re-resolve plugins even if previously resolved.",
)
register(
"--plugin-cache-dir",
advanced=True,
default=os.path.join(get_pants_cachedir(), "plugins"),
help="Cache resolved plugin requirements here.",
)
register(
"--backend-packages",
advanced=True,
type=list,
default=[
"pants.backend.graph_info",
"pants.backend.python",
"pants.backend.python.lint.isort",
"pants.backend.jvm",
"pants.backend.native",
"pants.backend.codegen.antlr.java",
"pants.backend.codegen.antlr.python",
"pants.backend.codegen.jaxb",
"pants.backend.codegen.protobuf.java",
"pants.backend.codegen.ragel.java",
"pants.backend.codegen.thrift.java",
"pants.backend.codegen.thrift.python",
"pants.backend.codegen.grpcio.python",
"pants.backend.codegen.wire.java",
"pants.backend.project_info",
"pants.cache",
],
help=
("Register v1 tasks from these backends. The backend packages must be present on "
"the PYTHONPATH, typically because they are in the Pants core dist, in a "
"plugin dist, or available as sources in the repo."),
)
register(
"--backend-packages2",
advanced=True,
type=list,
default=[],
help=
("Register v2 rules from these backends. The backend packages must be present on "
"the PYTHONPATH, typically because they are in the Pants core dist, in a "
"plugin dist, or available as sources in the repo."),
)
register(
"--pants-bootstrapdir",
advanced=True,
metavar="<dir>",
default=get_pants_cachedir(),
help="Use this dir for global cache.",
)
register(
"--pants-configdir",
advanced=True,
metavar="<dir>",
default=get_pants_configdir(),
help="Use this dir for global config files.",
)
register(
"--pants-workdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, ".pants.d"),
help="Write intermediate output files to this dir.",
)
register(
"--pants-physical-workdir-base",
advanced=True,
metavar="<dir>",
default=None,
help=
"When set, a base directory in which to store `--pants-workdir` contents. "
"If this option is a set, the workdir will be created as symlink into a "
"per-workspace subdirectory.",
)
register(
"--pants-supportdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "build-support"),
help="Use support files from this dir.",
)
register(
"--pants-distdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "dist"),
help="Write end-product artifacts to this dir.",
)
register(
"--pants-subprocessdir",
advanced=True,
default=os.path.join(buildroot, ".pids"),
help=
"The directory to use for tracking subprocess metadata, if any. This should "
"live outside of the dir used by `--pants-workdir` to allow for tracking "
"subprocesses that outlive the workdir data (e.g. `./pants server`).",
)
register(
"--pants-config-files",
advanced=True,
type=list,
daemon=False,
default=[get_default_pants_config_file()],
help="Paths to Pants config files.",
)
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register("--pantsrc",
advanced=True,
type=bool,
default=True,
help="Use pantsrc files.")
register(
"--pantsrc-files",
advanced=True,
type=list,
metavar="<path>",
daemon=False,
default=["/etc/pantsrc", "~/.pants.rc"],
help="Override config with values from these files. "
"Later files override earlier ones.",
)
register(
"--pythonpath",
advanced=True,
type=list,
help="Add these directories to PYTHONPATH to search for plugins.",
)
register(
"--spec-file",
type=list,
dest="spec_files",
daemon=False,
help=
"Read additional specs from this file (e.g. target addresses or file names). "
"Each spec should be one per line.",
)
register(
"--verify-config",
type=bool,
default=True,
daemon=False,
advanced=True,
help=
"Verify that all config file values correspond to known options.",
)
register(
"--build-ignore",
advanced=True,
type=list,
default=[
".*/", "bower_components/", "node_modules/", "*.egg-info/"
],
help="Paths to ignore when identifying BUILD files. "
"This does not affect any other filesystem operations. "
"Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).",
)
register(
"--pants-ignore",
advanced=True,
type=list,
member_type=str,
default=[".*/", default_rel_distdir],
help=
"Paths to ignore for all filesystem operations performed by pants "
"(e.g. BUILD file scanning, glob matching, etc). "
"Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). "
"The `--pants-distdir` and `--pants-workdir` locations are inherently ignored.",
)
register(
"--owners-not-found-behavior",
advanced=True,
type=OwnersNotFoundBehavior,
default=OwnersNotFoundBehavior.error,
help=
"What to do when file arguments do not have any owning target. This happens when there "
"are no targets whose `sources` fields include the file argument.",
)
register(
"--files-not-found-behavior",
advanced=True,
type=FileNotFoundBehavior,
default=FileNotFoundBehavior.warn,
help=
"What to do when files and globs specified in BUILD files, such as in the "
"`sources` field, cannot be found. This happens when the files do not exist on "
"your machine or when they are ignored by the `--pants-ignore` option.",
)
# TODO(#7203): make a regexp option type!
register(
"--exclude-target-regexp",
advanced=True,
type=list,
default=[],
daemon=False,
metavar="<regexp>",
help="Exclude target roots that match these regexes.",
)
register(
"--subproject-roots",
type=list,
advanced=True,
default=[],
help=
"Paths that correspond with build roots for any subproject that this "
"project depends on.",
)
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register(
"-d",
"--logdir",
advanced=True,
metavar="<dir>",
help="Write logs to files under this directory.",
)
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
"--enable-pantsd",
advanced=True,
type=bool,
default=False,
help=
"Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)",
)
# Whether or not to make necessary arrangements to have concurrent runs in pants.
# In practice, this means that if this is set, a run will not even try to use pantsd.
# NB: Eventually, we would like to deprecate this flag in favor of making pantsd runs parallelizable.
register(
"--concurrent",
advanced=True,
type=bool,
default=False,
daemon=False,
help=
"Enable concurrent runs of pants. Without this enabled, pants will "
"start up all concurrent invocations (e.g. in other terminals) without pantsd. "
"Enabling this option requires parallel pants invocations to block on the first",
)
# Calling pants command (inner run) from other pants command is unusual behaviour,
# and most users should never set this flag.
# It is automatically set by pants when an inner run is detected.
# Currently, pants commands with this option set don't use pantsd,
# but this effect should not be relied upon.
# This option allows us to know who was the parent of pants inner runs for informational purposes.
register(
"--parent-build-id",
advanced=True,
default=None,
help=
"The build ID of the other pants run which spawned this one, if any.",
)
# Shutdown pantsd after the current run.
# This needs to be accessed at the same time as enable_pantsd,
# so we register it at bootstrap time.
register(
"--shutdown-pantsd-after-run",
advanced=True,
type=bool,
default=False,
help=
"Create a new pantsd server, and use it, and shut it down immediately after. "
"If pantsd is already running, it will shut it down and spawn a new instance (Beta)",
)
# NB: We really don't want this option to invalidate the daemon, because different clients might have
# different needs. For instance, an IDE might have a very long timeout because it only wants to refresh
# a project in the background, while a user might want a shorter timeout for interactivity.
register(
"--pantsd-timeout-when-multiple-invocations",
advanced=True,
type=float,
default=60.0,
daemon=False,
help=
"The maximum amount of time to wait for the invocation to start until "
"raising a timeout exception. "
"Because pantsd currently does not support parallel runs, "
"any prior running Pants command must be finished for the current one to start. "
"To never timeout, use the value -1.",
)
# These facilitate configuring the native engine.
register(
"--native-engine-visualize-to",
advanced=True,
default=None,
type=dir_option,
daemon=False,
help=
"A directory to write execution and rule graphs to as `dot` files. The contents "
"of the directory will be overwritten if any filenames collide.",
)
register(
"--print-exception-stacktrace",
advanced=True,
type=bool,
help=
"Print to console the full exception stack trace if encountered.",
)
# BinaryUtil options.
register(
"--binaries-baseurls",
type=list,
advanced=True,
default=["https://binaries.pantsbuild.org"],
help="List of URLs from which binary tools are downloaded. URLs are "
"searched in order until the requested path is found.",
)
register(
"--binaries-fetch-timeout-secs",
type=int,
default=30,
advanced=True,
daemon=False,
help=
"Timeout in seconds for URL reads when fetching binary tools from the "
"repos specified by --baseurls.",
)
register(
"--binaries-path-by-id",
type=dict,
advanced=True,
help=
("Maps output of uname for a machine to a binary search path: "
"(sysname, id) -> (os, arch), e.g. {('darwin', '15'): ('mac', '10.11'), "
"('linux', 'arm32'): ('linux', 'arm32')}."),
)
register(
"--allow-external-binary-tool-downloads",
type=bool,
default=True,
advanced=True,
help=
"If False, require BinaryTool subclasses to download their contents from urls "
"generated from --binaries-baseurls, even if the tool has an external url "
"generator. This can be necessary if using Pants in an environment which cannot "
"contact the wider Internet.",
)
# Pants Daemon options.
register(
"--pantsd-pailgun-host",
advanced=True,
default="127.0.0.1",
help="The host to bind the pants nailgun server to.",
)
register(
"--pantsd-pailgun-port",
advanced=True,
type=int,
default=0,
help=
"The port to bind the pants nailgun server to. Defaults to a random port.",
)
# TODO(#7514): Make this default to 1.0 seconds if stdin is a tty!
register(
"--pantsd-pailgun-quit-timeout",
advanced=True,
type=float,
default=5.0,
help=
"The length of time (in seconds) to wait for further output after sending a "
"signal to the remote pantsd process before killing it.",
)
register(
"--pantsd-log-dir",
advanced=True,
default=None,
help="The directory to log pantsd output to.",
)
register(
"--pantsd-invalidation-globs",
advanced=True,
type=list,
default=[],
help=
"Filesystem events matching any of these globs will trigger a daemon restart. "
"The `--pythonpath` and `--pants-config-files` are inherently invalidated.",
)
# Watchman options.
register("--watchman-version",
advanced=True,
default="4.9.0-pants1",
help="Watchman version.")
register(
"--watchman-supportdir",
advanced=True,
default="bin/watchman",
help=
"Find watchman binaries under this dir. Used as part of the path to lookup "
"the binary with --binaries-baseurls and --pants-bootstrapdir.",
)
register(
"--watchman-startup-timeout",
type=float,
advanced=True,
default=60.0,
help=
"The watchman socket timeout (in seconds) for the initial `watch-project` command. "
"This may need to be set higher for larger repos due to watchman startup cost.",
)
register(
"--watchman-socket-timeout",
type=float,
advanced=True,
default=0.1,
help=
"The watchman client socket timeout in seconds. Setting this to too high a "
"value can negatively impact the latency of runs forked by pantsd.",
)
register(
"--watchman-socket-path",
type=str,
advanced=True,
default=None,
help=
"The path to the watchman UNIX socket. This can be overridden if the default "
"absolute path length exceeds the maximum allowed by the OS.",
)
# This option changes the parser behavior in a fundamental way (which currently invalidates
# all caches), and needs to be parsed out early, so we make it a bootstrap option.
register(
"--build-file-imports",
type=BuildFileImportsBehavior,
default=BuildFileImportsBehavior.error,
advanced=True,
removal_version="1.29.0.dev0",
removal_hint=
("Import statements should be avoided in BUILD files because they can easily break "
"Pants caching and lead to stale results. If you still need to keep the "
"functionality you have from import statements, consider rewriting your code into "
"a Pants plugin: https://www.pantsbuild.org/howto_plugin.html."),
help="Whether to allow import statements in BUILD files",
)
register(
"--local-store-dir",
advanced=True,
help="Directory to use for engine's local file store.",
# This default is also hard-coded into the engine's rust code in
# fs::Store::default_path
default=os.path.expanduser("~/.cache/pants/lmdb_store"),
)
register(
"--remote-execution",
advanced=True,
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.remote_execution,
help="Enables remote workers for increased parallelism. (Alpha)",
)
register(
"--remote-store-server",
advanced=True,
type=list,
default=[],
help=
"host:port of grpc server to use as remote execution file store.",
)
# TODO: Infer this from remote-store-connection-limit.
register(
"--remote-store-thread-count",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_thread_count,
help=
"Thread count to use for the pool that interacts with the remote file store.",
)
register(
"--remote-execution-server",
advanced=True,
help=
"host:port of grpc server to use as remote execution scheduler.",
)
register(
"--remote-store-chunk-bytes",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_bytes,
help=
"Size in bytes of chunks transferred to/from the remote file store.",
)
register(
"--remote-store-chunk-upload-timeout-seconds",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.
remote_store_chunk_upload_timeout_seconds,
help=
"Timeout (in seconds) for uploads of individual chunks to the remote file store.",
)
register(
"--remote-store-rpc-retries",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_rpc_retries,
help=
"Number of times to retry any RPC to the remote store before giving up.",
)
register(
"--remote-store-connection-limit",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_connection_limit,
help=
"Number of remote stores to concurrently allow connections to.",
)
register(
"--remote-execution-process-cache-namespace",
advanced=True,
help="The cache namespace for remote process execution. "
"Bump this to invalidate every artifact's remote execution. "
"This is the remote execution equivalent of the legacy cache-key-gen-version "
"flag.",
)
register(
"--remote-instance-name",
advanced=True,
help=
"Name of the remote execution instance to use. Used for routing within "
"--remote-execution-server and --remote-store-server.",
)
register(
"--remote-ca-certs-path",
advanced=True,
help=
"Path to a PEM file containing CA certificates used for verifying secure "
"connections to --remote-execution-server and --remote-store-server. "
"If not specified, TLS will not be used.",
)
register(
"--remote-oauth-bearer-token-path",
advanced=True,
help=
"Path to a file containing an oauth token to use for grpc connections to "
"--remote-execution-server and --remote-store-server. If not specified, no "
"authorization will be performed.",
)
register(
"--remote-execution-extra-platform-properties",
advanced=True,
help="Platform properties to set on remote execution requests. "
"Format: property=value. Multiple values should be specified as multiple "
"occurrences of this flag. Pants itself may add additional platform properties.",
type=list,
default=[],
)
register(
"--remote-execution-headers",
advanced=True,
help="Headers to set on remote execution requests. "
"Format: header=value. Pants itself may add additional headers.",
type=dict,
default={},
)
register(
"--process-execution-local-parallelism",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_parallelism,
advanced=True,
help="Number of concurrent processes that may be executed locally.",
)
register(
"--process-execution-remote-parallelism",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_remote_parallelism,
advanced=True,
help=
"Number of concurrent processes that may be executed remotely.",
)
register(
"--process-execution-cleanup-local-dirs",
type=bool,
default=True,
advanced=True,
help=
"Whether or not to cleanup directories used for local process execution "
"(primarily useful for e.g. debugging).",
)
register(
"--process-execution-speculation-delay",
type=float,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_speculation_delay,
advanced=True,
help=
"Number of seconds to wait before speculating a second request for a slow process. "
" see `--process-execution-speculation-strategy`",
)
register(
"--process-execution-speculation-strategy",
choices=["remote_first", "local_first", "none"],
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_speculation_strategy,
help=
"Speculate a second request for an underlying process if the first one does not complete within "
"`--process-execution-speculation-delay` seconds.\n"
"`local_first` (default): Try to run the process locally first, "
"and fall back to remote execution if available.\n"
"`remote_first`: Run the process on the remote execution backend if available, "
"and fall back to the local host if remote calls take longer than the speculation timeout.\n"
"`none`: Do not speculate about long running processes.",
advanced=True,
)
register(
"--process-execution-use-local-cache",
type=bool,
default=True,
advanced=True,
help=
"Whether to keep process executions in a local cache persisted to disk.",
)
register(
"--process-execution-local-enable-nailgun",
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_enable_nailgun,
help=
"Whether or not to use nailgun to run the requests that are marked as nailgunnable.",
advanced=True,
)
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, "WARN")
register(
"-l",
"--level",
choices=["debug", "info", "warn"],
default="info",
recursive=True,
help="Set the logging level.",
)
register("-q", "--quiet", action="store_true", recursive=True, help="Squelches most console output.")
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register(
"--colors",
action="store_true",
default=True,
recursive=True,
help="Set whether log messages are displayed in color.",
)
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register("--pants-version", advanced=True, default=pants_version(), help="Use this pants version.")
register("--plugins", advanced=True, type=list_option, help="Load these plugins.")
register(
"--plugin-cache-dir",
advanced=True,
default=os.path.join(get_pants_cachedir(), "plugins"),
help="Cache resolved plugin requirements here.",
)
register(
"--backend-packages",
advanced=True,
type=list_option,
help="Load backends from these packages that are already on the path.",
)
register(
"--pants-bootstrapdir",
advanced=True,
metavar="<dir>",
default=get_pants_cachedir(),
help="Use this dir for global cache.",
)
register(
"--pants-configdir",
advanced=True,
metavar="<dir>",
default=get_pants_configdir(),
help="Use this dir for global config files.",
)
register(
"--pants-workdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, ".pants.d"),
help="Write intermediate output files to this dir.",
)
register(
"--pants-supportdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "build-support"),
help="Use support files from this dir.",
)
register(
"--pants-distdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "dist"),
help="Write end-product artifacts to this dir.",
)
register(
"--config-override",
advanced=True,
action="append",
metavar="<path>",
help="A second config file, to override pants.ini.",
)
register("--pantsrc", advanced=True, action="store_true", default=True, help="Use pantsrc files.")
register(
"--pantsrc-files",
advanced=True,
action="append",
metavar="<path>",
default=["/etc/pantsrc", "~/.pants.rc"],
help="Override config with values from these files. " "Later files override earlier ones.",
)
register(
"--pythonpath",
advanced=True,
action="append",
help="Add these directories to PYTHONPATH to search for plugins.",
)
register(
"--target-spec-file",
action="append",
dest="target_spec_files",
help="Read additional specs from this file, one per line",
)
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register("-d", "--logdir", advanced=True, metavar="<dir>", help="Write logs to files under this directory.")
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
"--enable-pantsd",
advanced=True,
action="store_true",
default=False,
help="Enables use of the pants daemon. (Beta)",
)
def test_get_cachedir(self):
with environment_as(XDG_CACHE_HOME=''):
self.assertEqual(os.path.expanduser('~/.cache/pants'),
get_pants_cachedir())
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l',
'--level',
choices=['debug', 'info', 'warn'],
default='info',
recursive=True,
help='Set the logging level.')
register('-q',
'--quiet',
action='store_true',
help='Squelches all console output apart from errors.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors',
action='store_true',
default=True,
recursive=True,
help='Set whether log messages are displayed in color.')
# NB: Right now this option is a placeholder that is unused within pants itself except when
# specified on the command line to print the OSS pants version. Both the IntelliJ Pants plugin
# and the pantsbuild/setup bootstrap script grep for pants_version though so this option
# registration serves in part as documentation of the dependency.
# TODO(John Sirois): Move pantsbuild.pants bootstrapping into pants itself and have it use this
# version option directly.
register(
'-V',
'--pants-version',
'--version', # NB: pants_version is the 1st long option
# since that's the one read from pants.ini;
# the version form only works from the CLI.
nargs=
'?', # Allows using the flag with no args on the CLI to print version as well
# as setting the version in pants.ini
default=pants_version(
), # Displays the current version correctly in `./pants -h`.
const=pants_version(
), # Displays the current version via `./pants -V`.
help="Prints pants' version number and exits.")
register('--plugins',
advanced=True,
type=list_option,
help='Load these plugins.')
register(
'--backend-packages',
advanced=True,
type=list_option,
help=
'Load backends from these packages that are already on the path.')
register('--pants-bootstrapdir',
advanced=True,
metavar='<dir>',
default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir',
advanced=True,
metavar='<dir>',
default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register('--config-override',
advanced=True,
action='append',
metavar='<path>',
help='A second config file, to override pants.ini.')
register('--pantsrc',
advanced=True,
action='store_true',
default=True,
help='Use pantsrc files.')
register('--pantsrc-files',
advanced=True,
action='append',
metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register(
'--pythonpath',
advanced=True,
action='append',
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file',
action='append',
dest='target_spec_files',
help='Read additional specs from this file, one per line')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d',
'--logdir',
advanced=True,
metavar='<dir>',
help='Write logs to files under this directory.')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l', '--level', choices=['debug', 'info', 'warn'], default='info', recursive=True,
help='Set the logging level.')
register('-q', '--quiet', type=bool, recursive=True,
help='Squelches most console output.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors', type=bool, default=sys.stdout.isatty(), recursive=True,
help='Set whether log messages are displayed in color.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version', advanced=True, default=pants_version(),
help='Use this pants version.')
register('--plugins', advanced=True, type=list, help='Load these plugins.')
register('--plugin-cache-dir', advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register('--backend-packages', advanced=True, type=list,
default=['pants.backend.graph_info',
'pants.backend.python',
'pants.backend.jvm',
'pants.backend.codegen.antlr.java',
'pants.backend.codegen.antlr.python',
'pants.backend.codegen.jaxb',
'pants.backend.codegen.protobuf.java',
'pants.backend.codegen.ragel.java',
'pants.backend.codegen.thrift.java',
'pants.backend.codegen.thrift.python',
'pants.backend.codegen.wire.java',
'pants.backend.project_info'],
help='Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir', advanced=True, metavar='<dir>', default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir', advanced=True, metavar='<dir>', default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register('--pants-subprocessdir', advanced=True, default=os.path.join(buildroot, '.pids'),
help='The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).')
register('--pants-config-files', advanced=True, type=list,
default=[get_default_pants_config_file()], help='Paths to Pants config files.')
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--config-override', advanced=True, type=list, metavar='<path>',
removal_version='1.6.0.dev0',
removal_hint='Use --pants-config-files=<second config file path> instead.',
help='A second config file, to override pants.ini.')
register('--pantsrc', advanced=True, type=bool, default=True,
help='Use pantsrc files.')
register('--pantsrc-files', advanced=True, type=list, metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register('--pythonpath', advanced=True, type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file', type=list, dest='target_spec_files',
help='Read additional specs from this file, one per line')
register('--verify-config', type=bool, default=True,
help='Verify that all config file values correspond to known options.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d', '--logdir', advanced=True, metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register('--enable-pantsd', advanced=True, type=bool, default=False,
help='Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)')
# This facilitates use of the v2 engine, sans daemon.
# TODO: Add removal_version='1.5.0.dev0' before 1.4 lands.
register('--enable-v2-engine', advanced=True, type=bool, default=True,
help='Enables use of the v2 engine.')
# These facilitate configuring the native engine.
register('--native-engine-version', advanced=True,
default=pkg_resources.resource_string('pants.engine', 'native_engine_version').strip(),
help='Native engine version.')
register('--native-engine-supportdir', advanced=True, default='bin/native-engine',
help='Find native engine binaries under this dir. Used as part of the path to '
'lookup the binary with --binary-util-baseurls and --pants-bootstrapdir.')
register('--native-engine-visualize-to', advanced=True, default=None, type=dir_option,
help='A directory to write execution and rule graphs to as `dot` files. The contents '
'of the directory will be overwritten if any filenames collide.')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, "WARN")
register(
"-l",
"--level",
choices=["debug", "info", "warn"],
default="info",
recursive=True,
help="Set the logging level.",
)
register("-q", "--quiet", type=bool, recursive=True, help="Squelches most console output.")
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register(
"--colors", type=bool, default=True, recursive=True, help="Set whether log messages are displayed in color."
)
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register("--pants-version", advanced=True, default=pants_version(), help="Use this pants version.")
register("--plugins", advanced=True, type=list, help="Load these plugins.")
register(
"--plugin-cache-dir",
advanced=True,
default=os.path.join(get_pants_cachedir(), "plugins"),
help="Cache resolved plugin requirements here.",
)
register(
"--backend-packages",
advanced=True,
type=list,
default=[
"pants.backend.graph_info",
"pants.backend.python",
"pants.backend.jvm",
"pants.backend.codegen",
"pants.backend.project_info",
],
help="Load backends from these packages that are already on the path. "
"Add contrib and custom backends to this list.",
)
register(
"--default-backend-packages",
advanced=True,
type=list,
removal_version="1.3.0",
removal_hint="All backends must be specified using the backend_packages option. "
"That option has the same defaults as this one, and you can append"
"and filter those using +[...] and -[...] syntax, as described here: "
"http://www.pantsbuild.org/options.html#list-options.",
default=[
"pants.backend.graph_info",
"pants.backend.python",
"pants.backend.jvm",
"pants.backend.codegen",
"pants.backend.project_info",
],
help="Load these backends by default. These backends come distributed with Pants. "
"Remove unused backends from this list to speed up execution. "
"Use --backend-packages to configure additional backends with Pants.",
)
register(
"--pants-bootstrapdir",
advanced=True,
metavar="<dir>",
default=get_pants_cachedir(),
help="Use this dir for global cache.",
)
register(
"--pants-configdir",
advanced=True,
metavar="<dir>",
default=get_pants_configdir(),
help="Use this dir for global config files.",
)
register(
"--pants-workdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, ".pants.d"),
help="Write intermediate output files to this dir.",
)
register(
"--pants-supportdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "build-support"),
help="Use support files from this dir.",
)
register(
"--pants-distdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "dist"),
help="Write end-product artifacts to this dir.",
)
register(
"--pants-subprocessdir",
advanced=True,
default=os.path.join(buildroot, ".pids"),
help="The directory to use for tracking subprocess metadata, if any. This should "
"live outside of the dir used by `--pants-workdir` to allow for tracking "
"subprocesses that outlive the workdir data (e.g. `./pants server`).",
)
register(
"--pants-config-files",
advanced=True,
type=list,
default=[get_default_pants_config_file()],
help="Paths to Pants config files.",
)
# TODO: Deprecate --config-override in favor of --pants-config-files.
# But only once we're able to both append and override list-valued options, as there are
# use-cases for both here.
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register(
"--config-override",
advanced=True,
type=list,
metavar="<path>",
help="A second config file, to override pants.ini.",
)
register("--pantsrc", advanced=True, type=bool, default=True, help="Use pantsrc files.")
register(
"--pantsrc-files",
advanced=True,
type=list,
metavar="<path>",
default=["/etc/pantsrc", "~/.pants.rc"],
help="Override config with values from these files. " "Later files override earlier ones.",
)
register(
"--pythonpath", advanced=True, type=list, help="Add these directories to PYTHONPATH to search for plugins."
)
register(
"--target-spec-file",
type=list,
dest="target_spec_files",
help="Read additional specs from this file, one per line",
)
register(
"--verify-config",
type=bool,
default=True,
help="Verify that all config file values correspond to known options.",
)
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register("-d", "--logdir", advanced=True, metavar="<dir>", help="Write logs to files under this directory.")
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
"--enable-pantsd",
advanced=True,
type=bool,
default=False,
help="Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)",
)
# This facilitates use of the v2 engine for BuildGraph construction, sans daemon.
register(
"--enable-v2-engine", advanced=True, type=bool, default=False, help="Enables use of the v2 engine. (Beta)"
)
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are the set of options necessary to create a Scheduler. If an option is
not consumed during creation of a Scheduler, it should be in `register_options` instead.
Bootstrap option values can be interpolated into the config file, and can be referenced
programmatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = "dist"
default_rel_distdir = f"/{default_distdir_name}/"
register(
"-l",
"--level",
type=LogLevel,
default=LogLevel.INFO,
help="Set the logging level. The logging levels are one of: "
'"error", "warn", "info", "debug", "trace".',
)
register(
"--show-log-target",
type=bool,
default=False,
advanced=True,
help="Display the target where a log message originates in that log message's output. "
"This can be helpful when paired with --log-levels-by-target.",
)
register(
"--log-levels-by-target",
type=dict,
default={},
advanced=True,
help="Set a more specific logging level for one or more logging targets. The names of "
"logging targets are specified in log strings when the --show-log-target option is set. "
"The logging levels are one of: "
'"error", "warn", "info", "debug", "trace". '
"All logging targets not specified here use the global log level set with --level. For example, "
'you can set `--log-levels-by-target=\'{"workunit_store": "info", "pants.engine.rules": "warn"}\'` ',
)
register(
"--log-show-rust-3rdparty",
type=bool,
default=False,
advanced=True,
help="Whether to show/hide logging done by 3rdparty Rust crates used by the Pants "
"engine.",
)
register(
"--colors",
type=bool,
default=sys.stdout.isatty(),
help=(
"Whether Pants should use colors in output or not. This may also impact whether "
"some tools Pants run use color."
),
)
# TODO(#7203): make a regexp option type!
register(
"--ignore-pants-warnings",
type=list,
member_type=str,
default=[],
advanced=True,
help="Regexps matching warning strings to ignore, e.g. "
'["DEPRECATED: scope some_scope will be removed"]. The regexps will be matched '
"from the start of the warning string, and will always be case-insensitive. "
"See the `warnings` Python module documentation for more background "
"on how these are used.",
)
register(
"--pants-version",
advanced=True,
default=pants_version(),
daemon=True,
help="Use this pants version. Note Pants code only uses this to verify that you are "
"using the requested version, as Pants cannot dynamically change the version it "
"is using once the program is already running. This option is useful to set in "
"your pants.toml, however, and then you can grep the value to select which "
"version to use for setup scripts (e.g. `./pants`), runner scripts, IDE plugins, "
"etc. For example, the setup script we distribute at "
"https://www.pantsbuild.org/docs/installation uses this value to determine which Python"
"version to run with. You may find the version of the Pants instance you are running "
"by using -v, -V, or --version.",
)
register(
"--pants-bin-name",
advanced=True,
default="./pants",
help="The name of the script or binary used to invoke Pants. "
"Useful when printing help messages.",
)
register(
"--plugins",
advanced=True,
type=list,
help="Allow backends to be loaded from these plugins. The default backends for "
"each plugin will be loaded automatically. Other backends in a plugin can be "
"loaded by listing them in `backend_packages` in the `[GLOBAL]` scope.",
)
register(
"--plugins-force-resolve",
advanced=True,
type=bool,
default=False,
help="Re-resolve plugins even if previously resolved.",
)
register(
"--plugin-cache-dir",
advanced=True,
default=os.path.join(get_pants_cachedir(), "plugins"),
help="Cache resolved plugin requirements here.",
)
register(
"--backend-packages",
advanced=True,
type=list,
default=[],
help=(
"Register rules from these backends. The backend packages must be present on "
"the PYTHONPATH, typically because they are in the Pants core dist, in a "
"plugin dist, or available as sources in the repo."
),
)
register(
"--pants-bootstrapdir",
advanced=True,
metavar="<dir>",
default=get_pants_cachedir(),
help="Use this dir for global cache.",
)
register(
"--pants-configdir",
advanced=True,
metavar="<dir>",
default=get_pants_configdir(),
help="Use this dir for global config files.",
)
register(
"--pants-workdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, ".pants.d"),
daemon=True,
help="Write intermediate output files to this dir.",
)
register(
"--pants-physical-workdir-base",
advanced=True,
metavar="<dir>",
default=None,
daemon=True,
help="When set, a base directory in which to store `--pants-workdir` contents. "
"If this option is a set, the workdir will be created as symlink into a "
"per-workspace subdirectory.",
)
register(
"--pants-supportdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "build-support"),
help="Use support files from this dir.",
)
register(
"--pants-distdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "dist"),
help="Write end-product artifacts to this dir.",
)
# TODO: Change the default to false in 2.1, deprecate the option in 2.2 and remove in 2.3.
register(
"--pants-distdir-legacy-paths",
type=bool,
advanced=True,
default=True,
help="Whether to write binaries to the pre-2.0 paths under distdir. These legacy "
"paths are not qualified by target address, so may be ambiguous. This option "
"is a temporary mechanism for easing transition to 2.0. We recommemd switching "
"to the new, unambiguous paths ASAP, by setting this option to true.",
)
register(
"--pants-subprocessdir",
advanced=True,
default=os.path.join(buildroot, ".pids"),
daemon=True,
help="The directory to use for tracking subprocess metadata, if any. This should "
"live outside of the dir used by `--pants-workdir` to allow for tracking "
"subprocesses that outlive the workdir data.",
)
register(
"--pants-config-files",
advanced=True,
type=list,
# NB: We don't fingerprint the list of config files, because the content of the config
# files independently affects fingerprints.
fingerprint=False,
default=[get_default_pants_config_file()],
help="Paths to Pants config files.",
)
register(
"--pantsrc",
advanced=True,
type=bool,
default=True,
# NB: See `--pants-config-files`.
fingerprint=False,
help="Use pantsrc files.",
)
register(
"--pantsrc-files",
advanced=True,
type=list,
metavar="<path>",
# NB: See `--pants-config-files`.
fingerprint=False,
default=["/etc/pantsrc", "~/.pants.rc"],
help=(
"Override config with values from these files, using syntax matching that of "
"`--pants-config-files`."
),
)
register(
"--pythonpath",
advanced=True,
type=list,
help="Add these directories to PYTHONPATH to search for plugins.",
)
register(
"--spec-files",
type=list,
# NB: See `--pants-config-files`.
fingerprint=False,
help="Read additional specs (e.g., target addresses or file names), one per line,"
"from these files.",
)
register(
"--spec-file",
type=list,
removal_version="2.1.0.dev0",
removal_hint="Use --spec-files",
fingerprint=False,
help="Read additional specs from this file (e.g. target addresses or file names). "
"Each spec should be one per line.",
)
register(
"--verify-config",
type=bool,
default=True,
advanced=True,
help="Verify that all config file values correspond to known options.",
)
register(
"--pants-ignore",
advanced=True,
type=list,
member_type=str,
default=[".*/", default_rel_distdir],
help="Paths to ignore for all filesystem operations performed by pants "
"(e.g. BUILD file scanning, glob matching, etc). "
"Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). "
"The `--pants-distdir` and `--pants-workdir` locations are inherently ignored. "
"--pants-ignore can be used in tandem with --pants-ignore-use-gitignore, and any rules "
"specified here apply after rules specified in a .gitignore file.",
)
register(
"--pants-ignore-use-gitignore",
advanced=True,
type=bool,
default=True,
help="Make use of a root .gitignore file when determining whether to ignore filesystem "
"operations performed by Pants. If used together with `--pants-ignore`, any exclude/include "
"patterns specified there apply after .gitignore rules.",
)
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register(
"-d",
"--logdir",
advanced=True,
metavar="<dir>",
daemon=True,
help="Write logs to files under this directory.",
)
register(
"--pantsd",
advanced=True,
type=bool,
default=True,
daemon=True,
help=(
"Enables use of the Pants daemon (pantsd). pantsd can significantly improve "
"runtime performance by lowering per-run startup cost, and by caching filesystem "
"operations and rule execution."
),
)
# Whether or not to make necessary arrangements to have concurrent runs in pants.
# In practice, this means that if this is set, a run will not even try to use pantsd.
# NB: Eventually, we would like to deprecate this flag in favor of making pantsd runs parallelizable.
register(
"--concurrent",
advanced=True,
type=bool,
default=False,
help="Enable concurrent runs of Pants. Without this enabled, Pants will "
"start up all concurrent invocations (e.g. in other terminals) without pantsd. "
"Enabling this option requires parallel Pants invocations to block on the first",
)
# NB: We really don't want this option to invalidate the daemon, because different clients might have
# different needs. For instance, an IDE might have a very long timeout because it only wants to refresh
# a project in the background, while a user might want a shorter timeout for interactivity.
register(
"--pantsd-timeout-when-multiple-invocations",
advanced=True,
type=float,
default=60.0,
help="The maximum amount of time to wait for the invocation to start until "
"raising a timeout exception. "
"Because pantsd currently does not support parallel runs, "
"any prior running Pants command must be finished for the current one to start. "
"To never timeout, use the value -1.",
)
register(
"--pantsd-max-memory-usage",
advanced=True,
type=int,
default=2 ** 30,
help=(
"The maximum memory usage of a pantsd process (in bytes). There is at most one "
"pantsd process per workspace."
),
)
# These facilitate configuring the native engine.
register(
"--native-engine-visualize-to",
advanced=True,
default=None,
type=dir_option,
help="A directory to write execution and rule graphs to as `dot` files. The contents "
"of the directory will be overwritten if any filenames collide.",
)
register(
"--print-stacktrace",
advanced=True,
type=bool,
default=False,
help="Print the full exception stack trace for any errors.",
)
register(
"--print-exception-stacktrace",
advanced=True,
type=bool,
default=False,
help="Print to console the full exception stack trace if encountered.",
removal_version="2.1.0.dev0",
removal_hint="Use `--print-stacktrace` instead of `--print-exception-stacktrace`.",
)
# Pants Daemon options.
register(
"--pantsd-pailgun-port",
advanced=True,
type=int,
default=0,
daemon=True,
help="The port to bind the Pants nailgun server to. Defaults to a random port.",
)
# TODO(#7514): Make this default to 1.0 seconds if stdin is a tty!
register(
"--pantsd-pailgun-quit-timeout",
advanced=True,
type=float,
default=5.0,
help="The length of time (in seconds) to wait for further output after sending a "
"signal to the remote pantsd process before killing it.",
)
register(
"--pantsd-log-dir",
advanced=True,
default=None,
daemon=True,
help="The directory to log pantsd output to.",
)
register(
"--pantsd-invalidation-globs",
advanced=True,
type=list,
default=[],
daemon=True,
help="Filesystem events matching any of these globs will trigger a daemon restart. "
"Pants's own code, plugins, and `--pants-config-files` are inherently invalidated.",
)
cache_instructions = (
"The path may be absolute or relative. If the directory is within the build root, be "
"sure to include it in `--pants-ignore`."
)
register(
"--local-store-dir",
advanced=True,
help=(
f"Directory to use for the local file store, which stores the results of "
f"subprocesses run by Pants. {cache_instructions}"
),
# This default is also hard-coded into the engine's rust code in
# fs::Store::default_path so that tools using a Store outside of pants
# are likely to be able to use the same storage location.
default=os.path.join(get_pants_cachedir(), "lmdb_store"),
)
register(
"--local-execution-root-dir",
advanced=True,
help=f"Directory to use for local process execution sandboxing. {cache_instructions}",
default=tempfile.gettempdir(),
)
register(
"--named-caches-dir",
advanced=True,
help=(
"Directory to use for named global caches for tools and processes with trusted, "
f"concurrency-safe caches. {cache_instructions}"
),
default=os.path.join(get_pants_cachedir(), "named_caches"),
)
register(
"--ca-certs-path",
advanced=True,
type=str,
default=None,
help="Path to a file containing PEM-format CA certificates used for verifying secure "
"connections when downloading files required by a build.",
)
register(
"--remote-execution",
advanced=True,
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.remote_execution,
help="Enables remote workers for increased parallelism. (Alpha)",
)
register(
"--remote-store-server",
advanced=True,
type=list,
default=[],
help="host:port of grpc server to use as remote execution file store.",
)
# TODO: Infer this from remote-store-connection-limit.
register(
"--remote-store-thread-count",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_thread_count,
help="Thread count to use for the pool that interacts with the remote file store.",
)
register(
"--remote-execution-server",
advanced=True,
help="host:port of grpc server to use as remote execution scheduler.",
)
register(
"--remote-store-chunk-bytes",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_bytes,
help="Size in bytes of chunks transferred to/from the remote file store.",
)
register(
"--remote-store-chunk-upload-timeout-seconds",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_upload_timeout_seconds,
help="Timeout (in seconds) for uploads of individual chunks to the remote file store.",
)
register(
"--remote-store-rpc-retries",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_rpc_retries,
help="Number of times to retry any RPC to the remote store before giving up.",
)
register(
"--remote-store-connection-limit",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_connection_limit,
help="Number of remote stores to concurrently allow connections to.",
)
register(
"--remote-execution-process-cache-namespace",
advanced=True,
help="The cache namespace for remote process execution. "
"Bump this to invalidate every artifact's remote execution. "
"This is the remote execution equivalent of the legacy cache-key-gen-version "
"flag.",
)
register(
"--remote-instance-name",
advanced=True,
help="Name of the remote execution instance to use. Used for routing within "
"--remote-execution-server and --remote-store-server.",
)
register(
"--remote-ca-certs-path",
advanced=True,
help="Path to a PEM file containing CA certificates used for verifying secure "
"connections to --remote-execution-server and --remote-store-server. "
"If not specified, TLS will not be used.",
)
register(
"--remote-oauth-bearer-token-path",
advanced=True,
help="Path to a file containing an oauth token to use for grpc connections to "
"--remote-execution-server and --remote-store-server. If not specified, no "
"authorization will be performed.",
)
register(
"--remote-execution-extra-platform-properties",
advanced=True,
help="Platform properties to set on remote execution requests. "
"Format: property=value. Multiple values should be specified as multiple "
"occurrences of this flag. Pants itself may add additional platform properties.",
type=list,
default=[],
)
register(
"--remote-execution-headers",
advanced=True,
help="Headers to set on remote execution requests. "
"Format: header=value. Pants itself may add additional headers.",
type=dict,
default={},
)
register(
"--remote-execution-overall-deadline-secs",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.remote_execution_overall_deadline_secs,
advanced=True,
help="Overall timeout in seconds for each remote execution request from time of submission",
)
register(
"--process-execution-local-parallelism",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.process_execution_local_parallelism,
advanced=True,
help="Number of concurrent processes that may be executed locally.",
)
register(
"--process-execution-remote-parallelism",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.process_execution_remote_parallelism,
advanced=True,
help="Number of concurrent processes that may be executed remotely.",
)
register(
"--process-execution-cleanup-local-dirs",
type=bool,
default=True,
advanced=True,
help="Whether or not to cleanup directories used for local process execution "
"(primarily useful for e.g. debugging).",
)
register(
"--process-execution-speculation-delay",
type=float,
default=DEFAULT_EXECUTION_OPTIONS.process_execution_speculation_delay,
advanced=True,
help="Number of seconds to wait before speculating a second request for a slow process. "
" see `--process-execution-speculation-strategy`",
)
register(
"--process-execution-speculation-strategy",
choices=["remote_first", "local_first", "none"],
default=DEFAULT_EXECUTION_OPTIONS.process_execution_speculation_strategy,
help="Speculate a second request for an underlying process if the first one does not complete within "
"`--process-execution-speculation-delay` seconds.\n"
"`local_first` (default): Try to run the process locally first, "
"and fall back to remote execution if available.\n"
"`remote_first`: Run the process on the remote execution backend if available, "
"and fall back to the local host if remote calls take longer than the speculation timeout.\n"
"`none`: Do not speculate about long running processes.",
advanced=True,
)
register(
"--process-execution-use-local-cache",
type=bool,
default=True,
advanced=True,
help="Whether to keep process executions in a local cache persisted to disk.",
)
register(
"--process-execution-local-enable-nailgun",
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.process_execution_local_enable_nailgun,
help="Whether or not to use nailgun to run the requests that are marked as nailgunnable.",
advanced=True,
)
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l', '--level', choices=['debug', 'info', 'warn'], default='info', recursive=True,
help='Set the logging level.')
register('-q', '--quiet', action='store_true', recursive=True,
help='Squelches most console output.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors', '--color', action='store_true', default=True, recursive=True,
help='Set whether log messages are displayed in color. NB: The singular `color` form '
'is deprecated until 0.0.66: prefer `colors`.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version', advanced=True, default=pants_version(),
help='Use this pants version.')
register('--plugins', advanced=True, type=list_option, help='Load these plugins.')
register('--plugin-cache-dir', advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register('--backend-packages', advanced=True, type=list_option,
help='Load backends from these packages that are already on the path.')
register('--pants-bootstrapdir', advanced=True, metavar='<dir>', default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir', advanced=True, metavar='<dir>', default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register('--config-override', advanced=True, action='append', metavar='<path>',
help='A second config file, to override pants.ini.')
register('--pantsrc', advanced=True, action='store_true', default=True,
help='Use pantsrc files.')
register('--pantsrc-files', advanced=True, action='append', metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register('--pythonpath', advanced=True, action='append',
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file', action='append', dest='target_spec_files',
help='Read additional specs from this file, one per line')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d', '--logdir', advanced=True, metavar='<dir>',
help='Write logs to files under this directory.')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = 'dist'
default_rel_distdir = f'/{default_distdir_name}/'
register('-l',
'--level',
choices=['trace', 'debug', 'info', 'warn'],
default='info',
recursive=True,
help='Set the logging level.')
register(
'-q',
'--quiet',
type=bool,
recursive=True,
daemon=False,
help=
'Squelches most console output. NOTE: Some tasks default to behaving quietly: '
'inverting this option supports making them noisier than they would be otherwise.'
)
register(
'--log-show-rust-3rdparty',
type=bool,
default=False,
advanced=True,
help=
'Whether to show/hide logging done by 3rdparty rust crates used by the pants '
'engine.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors',
type=bool,
default=sys.stdout.isatty(),
recursive=True,
daemon=False,
help='Set whether log messages are displayed in color.')
# TODO(#7203): make a regexp option type!
register(
'--ignore-pants-warnings',
type=list,
member_type=str,
default=[],
help='Regexps matching warning strings to ignore, e.g. '
'["DEPRECATED: scope some_scope will be removed"]. The regexps will be matched '
'from the start of the warning string, and will always be case-insensitive. '
'See the `warnings` module documentation for more background on these are used.'
)
register(
'--option-name-check-distance',
advanced=True,
type=int,
default=2,
help=
'The maximum Levenshtein distance to use when offering suggestions for invalid '
'option names.')
register(
'--pants-version',
advanced=True,
default=pants_version(),
help=
'Use this pants version. Note Pants code only uses this to verify that you are '
'using the requested version, as Pants cannot dynamically change the version it '
'is using once the program is already running. This option is useful to set in '
'your pants.ini, however, and then you can grep the value to select which '
'version to use for setup scripts (e.g. `./pants`), runner scripts, IDE plugins, '
'etc. For example, the setup script we distribute at https://www.pantsbuild.org/install.html#recommended-installation '
'uses this value to determine which Python version to run with. You may find the '
'version of the pants instance you are running using -v, -V, or --version.'
)
register('--pants-bin-name',
advanced=True,
default='./pants',
help='The name of the script or binary used to invoke pants. '
'Useful when printing help messages.')
register('--plugins',
advanced=True,
type=list,
help='Load these plugins.')
register('--plugin-cache-dir',
advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register(
'--backend-packages',
advanced=True,
type=list,
default=[
'pants.backend.graph_info', 'pants.backend.python',
'pants.backend.jvm', 'pants.backend.native',
'pants.backend.codegen.antlr.java',
'pants.backend.codegen.antlr.python',
'pants.backend.codegen.jaxb',
'pants.backend.codegen.protobuf.java',
'pants.backend.codegen.ragel.java',
'pants.backend.codegen.thrift.java',
'pants.backend.codegen.thrift.python',
'pants.backend.codegen.grpcio.python',
'pants.backend.codegen.wire.java', 'pants.backend.project_info'
],
help=
'Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir',
advanced=True,
metavar='<dir>',
default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir',
advanced=True,
metavar='<dir>',
default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register(
'--pants-physical-workdir-base',
advanced=True,
metavar='<dir>',
default=None,
help=
'When set, a base directory in which to store `--pants-workdir` contents. '
'If this option is a set, the workdir will be created as symlink into a '
'per-workspace subdirectory.')
register('--pants-supportdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register(
'--pants-subprocessdir',
advanced=True,
default=os.path.join(buildroot, '.pids'),
help=
'The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).'
)
register('--pants-config-files',
advanced=True,
type=list,
daemon=False,
default=[get_default_pants_config_file()],
help='Paths to Pants config files.')
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--pantsrc',
advanced=True,
type=bool,
default=True,
help='Use pantsrc files.')
register('--pantsrc-files',
advanced=True,
type=list,
metavar='<path>',
daemon=False,
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register(
'--pythonpath',
advanced=True,
type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file',
type=list,
dest='target_spec_files',
daemon=False,
help='Read additional specs from this file, one per line')
register(
'--verify-config',
type=bool,
default=True,
daemon=False,
advanced=True,
help=
'Verify that all config file values correspond to known options.')
register(
'--build-ignore',
advanced=True,
type=list,
default=[
'.*/', 'bower_components/', 'node_modules/', '*.egg-info/'
],
help='Paths to ignore when identifying BUILD files. '
'This does not affect any other filesystem operations. '
'Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).'
)
register(
'--pants-ignore',
advanced=True,
type=list,
default=['.*/', default_rel_distdir],
help=
'Paths to ignore for all filesystem operations performed by pants '
'(e.g. BUILD file scanning, glob matching, etc). '
'Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). '
'The `--pants-distdir` and `--pants-workdir` locations are inherently ignored.'
)
register(
'--glob-expansion-failure',
advanced=True,
default=GlobMatchErrorBehavior.warn,
type=GlobMatchErrorBehavior,
help="Raise an exception if any targets declaring source files "
"fail to match any glob provided in the 'sources' argument.")
# TODO(#7203): make a regexp option type!
register('--exclude-target-regexp',
advanced=True,
type=list,
default=[],
daemon=False,
metavar='<regexp>',
help='Exclude target roots that match these regexes.')
register(
'--subproject-roots',
type=list,
advanced=True,
default=[],
help=
'Paths that correspond with build roots for any subproject that this '
'project depends on.')
register(
'--owner-of',
type=list,
member_type=file_option,
default=[],
daemon=False,
metavar='<path>',
help='Select the targets that own these files. '
'This is the third target calculation strategy along with the --changed-* '
'options and specifying the targets directly. These three types of target '
'selection are mutually exclusive.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d',
'--logdir',
advanced=True,
metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
'--enable-pantsd',
advanced=True,
type=bool,
default=False,
help=
'Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)'
)
# Whether or not to make necessary arrangements to have concurrent runs in pants.
# In practice, this means that if this is set, a run will not even try to use pantsd.
# NB: Eventually, we would like to deprecate this flag in favor of making pantsd runs parallelizable.
register(
'--concurrent',
advanced=True,
type=bool,
default=False,
daemon=False,
help=
'Enable concurrent runs of pants. Without this enabled, pants will '
'start up all concurrent invocations (e.g. in other terminals) without pantsd. '
'Enabling this option requires parallel pants invocations to block on the first'
)
# Shutdown pantsd after the current run.
# This needs to be accessed at the same time as enable_pantsd,
# so we register it at bootstrap time.
register(
'--shutdown-pantsd-after-run',
advanced=True,
type=bool,
default=False,
help=
'Create a new pantsd server, and use it, and shut it down immediately after. '
'If pantsd is already running, it will shut it down and spawn a new instance (Beta)'
)
# NB: We really don't want this option to invalidate the daemon, because different clients might have
# different needs. For instance, an IDE might have a very long timeout because it only wants to refresh
# a project in the background, while a user might want a shorter timeout for interactivity.
register(
'--pantsd-timeout-when-multiple-invocations',
advanced=True,
type=float,
default=60.0,
daemon=False,
help=
'The maximum amount of time to wait for the invocation to start until '
'raising a timeout exception. '
'Because pantsd currently does not support parallel runs, '
'any prior running Pants command must be finished for the current one to start. '
'To never timeout, use the value -1.')
# These facilitate configuring the native engine.
register(
'--native-engine-visualize-to',
advanced=True,
default=None,
type=dir_option,
daemon=False,
help=
'A directory to write execution and rule graphs to as `dot` files. The contents '
'of the directory will be overwritten if any filenames collide.')
register(
'--print-exception-stacktrace',
advanced=True,
type=bool,
help=
'Print to console the full exception stack trace if encountered.')
# BinaryUtil options.
register(
'--binaries-baseurls',
type=list,
advanced=True,
default=['https://binaries.pantsbuild.org'],
help='List of URLs from which binary tools are downloaded. URLs are '
'searched in order until the requested path is found.')
register(
'--binaries-fetch-timeout-secs',
type=int,
default=30,
advanced=True,
daemon=False,
help=
'Timeout in seconds for URL reads when fetching binary tools from the '
'repos specified by --baseurls.')
register(
'--binaries-path-by-id',
type=dict,
advanced=True,
help=
("Maps output of uname for a machine to a binary search path: "
"(sysname, id) -> (os, arch), e.g. {('darwin', '15'): ('mac', '10.11'), "
"('linux', 'arm32'): ('linux', 'arm32')}."))
register(
'--allow-external-binary-tool-downloads',
type=bool,
default=True,
advanced=True,
help=
"If False, require BinaryTool subclasses to download their contents from urls "
"generated from --binaries-baseurls, even if the tool has an external url "
"generator. This can be necessary if using Pants in an environment which cannot "
"contact the wider Internet.")
# Pants Daemon options.
register('--pantsd-pailgun-host',
advanced=True,
default='127.0.0.1',
help='The host to bind the pants nailgun server to.')
register(
'--pantsd-pailgun-port',
advanced=True,
type=int,
default=0,
help=
'The port to bind the pants nailgun server to. Defaults to a random port.'
)
# TODO(#7514): Make this default to 1.0 seconds if stdin is a tty!
register(
'--pantsd-pailgun-quit-timeout',
advanced=True,
type=float,
default=5.0,
help=
'The length of time (in seconds) to wait for further output after sending a '
'signal to the remote pantsd process before killing it.')
register('--pantsd-log-dir',
advanced=True,
default=None,
help='The directory to log pantsd output to.')
register(
'--pantsd-invalidation-globs',
advanced=True,
type=list,
default=[],
help=
'Filesystem events matching any of these globs will trigger a daemon restart. '
'The `--pythonpath` and `--pants-config-files` are inherently invalidated.'
)
# Watchman options.
register('--watchman-version',
advanced=True,
default='4.9.0-pants1',
help='Watchman version.')
register(
'--watchman-supportdir',
advanced=True,
default='bin/watchman',
help=
'Find watchman binaries under this dir. Used as part of the path to lookup '
'the binary with --binaries-baseurls and --pants-bootstrapdir.')
register(
'--watchman-startup-timeout',
type=float,
advanced=True,
default=60.0,
help=
'The watchman socket timeout (in seconds) for the initial `watch-project` command. '
'This may need to be set higher for larger repos due to watchman startup cost.'
)
register(
'--watchman-socket-timeout',
type=float,
advanced=True,
default=0.1,
help=
'The watchman client socket timeout in seconds. Setting this to too high a '
'value can negatively impact the latency of runs forked by pantsd.'
)
register(
'--watchman-socket-path',
type=str,
advanced=True,
default=None,
help=
'The path to the watchman UNIX socket. This can be overridden if the default '
'absolute path length exceeds the maximum allowed by the OS.')
# This option changes the parser behavior in a fundamental way (which currently invalidates
# all caches), and needs to be parsed out early, so we make it a bootstrap option.
register('--build-file-imports',
choices=['allow', 'warn', 'error'],
default='warn',
advanced=True,
help='Whether to allow import statements in BUILD files')
register(
'--local-store-dir',
advanced=True,
help="Directory to use for engine's local file store.",
# This default is also hard-coded into the engine's rust code in
# fs::Store::default_path
default=os.path.expanduser('~/.cache/pants/lmdb_store'))
register(
'--remote-execution',
advanced=True,
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.remote_execution,
help="Enables remote workers for increased parallelism. (Alpha)")
register(
'--remote-store-server',
advanced=True,
type=list,
default=[],
help=
'host:port of grpc server to use as remote execution file store.')
# TODO: Infer this from remote-store-connection-limit.
register(
'--remote-store-thread-count',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_thread_count,
help=
'Thread count to use for the pool that interacts with the remote file store.'
)
register(
'--remote-execution-server',
advanced=True,
help=
'host:port of grpc server to use as remote execution scheduler.')
register(
'--remote-store-chunk-bytes',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_bytes,
help=
'Size in bytes of chunks transferred to/from the remote file store.'
)
register(
'--remote-store-chunk-upload-timeout-seconds',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.
remote_store_chunk_upload_timeout_seconds,
help=
'Timeout (in seconds) for uploads of individual chunks to the remote file store.'
)
register(
'--remote-store-rpc-retries',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_rpc_retries,
help=
'Number of times to retry any RPC to the remote store before giving up.'
)
register(
'--remote-store-connection-limit',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_connection_limit,
help='Number of remote stores to concurrently allow connections to.'
)
register(
'--remote-execution-process-cache-namespace',
advanced=True,
help="The cache namespace for remote process execution. "
"Bump this to invalidate every artifact's remote execution. "
"This is the remote execution equivalent of the legacy cache-key-gen-version "
"flag.")
register(
'--remote-instance-name',
advanced=True,
help=
'Name of the remote execution instance to use. Used for routing within '
'--remote-execution-server and --remote-store-server.')
register(
'--remote-ca-certs-path',
advanced=True,
help=
'Path to a PEM file containing CA certificates used for verifying secure '
'connections to --remote-execution-server and --remote-store-server. '
'If not specified, TLS will not be used.')
register(
'--remote-oauth-bearer-token-path',
advanced=True,
help=
'Path to a file containing an oauth token to use for grpc connections to '
'--remote-execution-server and --remote-store-server. If not specified, no '
'authorization will be performed.')
register(
'--remote-execution-extra-platform-properties',
advanced=True,
help='Platform properties to set on remote execution requests. '
'Format: property=value. Multiple values should be specified as multiple '
'occurrences of this flag. Pants itself may add additional platform properties.',
type=list,
default=[])
register(
'--remote-execution-headers',
advanced=True,
help='Headers to set on remote execution requests. '
'Format: header=value. Pants itself may add additional headers.',
type=dict,
default={})
register(
'--process-execution-local-parallelism',
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_parallelism,
advanced=True,
help='Number of concurrent processes that may be executed locally.'
)
register(
'--process-execution-remote-parallelism',
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_remote_parallelism,
advanced=True,
help='Number of concurrent processes that may be executed remotely.'
)
register(
'--process-execution-cleanup-local-dirs',
type=bool,
default=True,
advanced=True,
help=
'Whether or not to cleanup directories used for local process execution '
'(primarily useful for e.g. debugging).')
register(
'--process-execution-speculation-delay',
type=float,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_speculation_delay,
advanced=True,
help=
'Number of seconds to wait before speculating a second request for a slow process. '
' see `--process-execution-speculation-strategy`')
register(
'--process-execution-speculation-strategy',
choices=['remote_first', 'local_first', 'none'],
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_speculation_strategy,
help=
"Speculate a second request for an underlying process if the first one does not complete within "
'`--process-execution-speculation-delay` seconds.\n'
'`local_first` (default): Try to run the process locally first, '
'and fall back to remote execution if available.\n'
'`remote_first`: Run the process on the remote execution backend if available, '
'and fall back to the local host if remote calls take longer than the speculation timeout.\n'
'`none`: Do not speculate about long running processes.',
advanced=True)
register(
'--process-execution-use-local-cache',
type=bool,
default=True,
advanced=True,
help=
'Whether to keep process executions in a local cache persisted to disk.'
)
register(
'--process-execution-local-enable-nailgun',
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_enable_nailgun,
help=
'Whether or not to use nailgun to run the requests that are marked as nailgunnable.',
advanced=True)
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = 'dist'
default_distdir = os.path.join(buildroot, default_distdir_name)
default_rel_distdir = '/{}/'.format(default_distdir_name)
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l',
'--level',
choices=['debug', 'info', 'warn'],
default='info',
recursive=True,
help='Set the logging level.')
register(
'-q',
'--quiet',
type=bool,
recursive=True,
daemon=False,
help=
'Squelches most console output. NOTE: Some tasks default to behaving quietly: '
'inverting this option supports making them noisier than they would be otherwise.'
)
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors',
type=bool,
default=sys.stdout.isatty(),
recursive=True,
daemon=False,
help='Set whether log messages are displayed in color.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version',
advanced=True,
default=pants_version(),
help='Use this pants version.')
register('--plugins',
advanced=True,
type=list,
help='Load these plugins.')
register('--plugin-cache-dir',
advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register(
'--backend-packages',
advanced=True,
type=list,
default=[
'pants.backend.graph_info', 'pants.backend.python',
'pants.backend.jvm', 'pants.backend.codegen.antlr.java',
'pants.backend.codegen.antlr.python',
'pants.backend.codegen.jaxb',
'pants.backend.codegen.protobuf.java',
'pants.backend.codegen.ragel.java',
'pants.backend.codegen.thrift.java',
'pants.backend.codegen.thrift.python',
'pants.backend.codegen.wire.java', 'pants.backend.project_info'
],
help=
'Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir',
advanced=True,
metavar='<dir>',
default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir',
advanced=True,
metavar='<dir>',
default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register(
'--pants-distdir',
advanced=True,
metavar='<dir>',
default=default_distdir,
help=
'Write end-product artifacts to this dir. If you modify this path, you '
'should also update --build-ignore and --pants-ignore to include the '
'custom dist dir path as well.')
register(
'--pants-subprocessdir',
advanced=True,
default=os.path.join(buildroot, '.pids'),
help=
'The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).'
)
register('--pants-config-files',
advanced=True,
type=list,
default=[get_default_pants_config_file()],
help='Paths to Pants config files.')
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--config-override',
advanced=True,
type=list,
metavar='<path>',
removal_version='1.6.0.dev0',
removal_hint=
'Use --pants-config-files=<second config file path> instead.',
help='A second config file, to override pants.ini.')
register('--pantsrc',
advanced=True,
type=bool,
default=True,
help='Use pantsrc files.')
register('--pantsrc-files',
advanced=True,
type=list,
metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register(
'--pythonpath',
advanced=True,
type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file',
type=list,
dest='target_spec_files',
daemon=False,
help='Read additional specs from this file, one per line')
register(
'--verify-config',
type=bool,
default=True,
daemon=False,
help=
'Verify that all config file values correspond to known options.')
register(
'--build-ignore',
advanced=True,
type=list,
fromfile=True,
default=[
'.*/', default_rel_distdir, 'bower_components/',
'node_modules/', '*.egg-info/'
],
help='Paths to ignore when identifying BUILD files. '
'This does not affect any other filesystem operations. '
'Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).'
)
register(
'--pants-ignore',
advanced=True,
type=list,
fromfile=True,
default=['.*/', default_rel_distdir],
help=
'Paths to ignore for all filesystem operations performed by pants '
'(e.g. BUILD file scanning, glob matching, etc). '
'Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). '
'This currently only affects the v2 engine. '
'To experiment with v2 engine, try --enable-v2-engine option.')
register('--exclude-target-regexp',
advanced=True,
type=list,
default=[],
daemon=False,
metavar='<regexp>',
help='Exclude target roots that match these regexes.')
register(
'--subproject-roots',
type=list,
advanced=True,
fromfile=True,
default=[],
help=
'Paths that correspond with build roots for any subproject that this '
'project depends on.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d',
'--logdir',
advanced=True,
metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
'--enable-pantsd',
advanced=True,
type=bool,
default=False,
help=
'Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)'
)
# This facilitates use of the v2 engine, sans daemon.
# TODO: Add removal_version='1.5.0.dev0' before 1.4 lands.
register('--enable-v2-engine',
advanced=True,
type=bool,
default=True,
help='Enables use of the v2 engine.')
# These facilitate configuring the native engine.
register(
'--native-engine-version',
advanced=True,
default='DEPRECATED',
removal_version='1.6.0.dev0',
removal_hint=
'Unused, the native engine is now embedded in the pantsbuild.pants wheel',
help='Native engine version.')
register(
'--native-engine-supportdir',
advanced=True,
default='DEPRECATED',
removal_version='1.6.0.dev0',
removal_hint=
'Unused, the native engine is now embedded in the pantsbuild.pants wheel',
help=
'Find native engine binaries under this dir. Used as part of the path to '
'lookup the binary with --binary-util-baseurls and --pants-bootstrapdir.'
)
register(
'--native-engine-visualize-to',
advanced=True,
default=None,
type=dir_option,
daemon=False,
help=
'A directory to write execution and rule graphs to as `dot` files. The contents '
'of the directory will be overwritten if any filenames collide.')
# BinaryUtil options.
register(
'--binaries-baseurls',
type=list,
advanced=True,
default=['https://binaries.pantsbuild.org'],
help='List of URLs from which binary tools are downloaded. URLs are '
'searched in order until the requested path is found.')
register(
'--binaries-fetch-timeout-secs',
type=int,
default=30,
advanced=True,
daemon=False,
help=
'Timeout in seconds for URL reads when fetching binary tools from the '
'repos specified by --baseurls.')
register(
'--binaries-path-by-id',
type=dict,
advanced=True,
help=
('Maps output of uname for a machine to a binary search path. e.g. '
'{("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux"'
', "arm32"]}'))
# Pants Daemon options.
register('--pantsd-pailgun-host',
advanced=True,
default='127.0.0.1',
help='The host to bind the pants nailgun server to.')
register(
'--pantsd-pailgun-port',
advanced=True,
type=int,
default=0,
help=
'The port to bind the pants nailgun server to. Defaults to a random port.'
)
register('--pantsd-log-dir',
advanced=True,
default=None,
help='The directory to log pantsd output to.')
register(
'--pantsd-fs-event-detection',
advanced=True,
type=bool,
removal_version='1.5.0.dev0',
removal_hint='This option is now implied by `--enable-pantsd`.',
help='Whether or not to use filesystem event detection.')
register(
'--pantsd-fs-event-workers',
advanced=True,
type=int,
default=4,
help=
'The number of workers to use for the filesystem event service executor pool.'
)
# Watchman options.
register('--watchman-version',
advanced=True,
default='4.5.0',
help='Watchman version.')
register(
'--watchman-supportdir',
advanced=True,
default='bin/watchman',
help=
'Find watchman binaries under this dir. Used as part of the path to lookup '
'the binary with --binary-util-baseurls and --pants-bootstrapdir.')
register(
'--watchman-startup-timeout',
type=float,
advanced=True,
default=30.0,
help=
'The watchman socket timeout (in seconds) for the initial `watch-project` command. '
'This may need to be set higher for larger repos due to watchman startup cost.'
)
register('--watchman-socket-timeout',
type=float,
advanced=True,
default=5.0,
help='The watchman client socket timeout in seconds.')
register(
'--watchman-socket-path',
type=str,
advanced=True,
default=None,
help=
'The path to the watchman UNIX socket. This can be overridden if the default '
'absolute path length exceeds the maximum allowed by the OS.')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = 'dist'
default_distdir = os.path.join(buildroot, default_distdir_name)
default_rel_distdir = '/{}/'.format(default_distdir_name)
register('-l', '--level', choices=['trace', 'debug', 'info', 'warn'], default='info',
recursive=True, help='Set the logging level.')
register('-q', '--quiet', type=bool, recursive=True, daemon=False,
help='Squelches most console output. NOTE: Some tasks default to behaving quietly: '
'inverting this option supports making them noisier than they would be otherwise.')
register('--log-show-rust-3rdparty', type=bool, default=False, advanced=True,
help='Whether to show/hide logging done by 3rdparty rust crates used by the pants '
'engine.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors', type=bool, default=sys.stdout.isatty(), recursive=True, daemon=False,
help='Set whether log messages are displayed in color.')
# TODO(#7203): make a regexp option type!
register('--ignore-pants-warnings', type=list, member_type=str, default=[],
help='Regexps matching warning strings to ignore, e.g. '
'["DEPRECATED: scope some_scope will be removed"]. The regexps will be matched '
'from the start of the warning string, and will always be case-insensitive. '
'See the `warnings` module documentation for more background on these are used.')
register('--option-name-check-distance', advanced=True, type=int, default=2,
help='The maximum Levenshtein distance to use when offering suggestions for invalid '
'option names.')
register('--pants-version', advanced=True, default=pants_version(),
help='Use this pants version. Note Pants code only uses this to verify that you are '
'using the requested version, as Pants cannot dynamically change the version it '
'is using once the program is already running. This option is useful to set in '
'your pants.ini, however, and then you can grep the value to select which '
'version to use for setup scripts (e.g. `./pants`), runner scripts, IDE plugins, '
'etc. For example, the setup script we distribute at https://www.pantsbuild.org/install.html#recommended-installation '
'uses this value to determine which Python version to run with. You may find the '
'version of the pants instance you are running using -v, -V, or --version.')
register('--pants-runtime-python-version', advanced=True,
removal_version='1.19.0.dev0',
deprecation_start_version='1.17.0.dev0',
removal_hint=dedent("""
This option was only used to help with Pants' migration to run on Python 3. \
Pants will now correctly default to whichever Python versions are supported for \
the current `pants_version` you are using. Please make sure you are using the \
most up-to-date version of the `./pants` script with:
curl -L -O https://pantsbuild.github.io/setup/pants
and then unset this option."""),
help='Use this Python version to run Pants. The option expects the major and minor '
'version, e.g. 2.7 or 3.6. Note Pants code only uses this to verify that you are '
'using the requested interpreter, as Pants cannot dynamically change the '
'interpreter it is using once the program is already running. This option is '
'useful to set in your pants.ini, however, and then you can grep the value to '
'select which interpreter to use for setup scripts (e.g. `./pants`), runner '
'scripts, IDE plugins, etc. For example, the setup script we distribute at '
'https://www.pantsbuild.org/install.html#recommended-installation uses this '
'value to determine which Python version to run with. Also note this does not mean '
'your own code must use this Python version. See '
'https://www.pantsbuild.org/python_readme.html#configure-the-python-version '
'for how to configure your code\'s compatibility.')
register('--plugins', advanced=True, type=list, help='Load these plugins.')
register('--plugin-cache-dir', advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register('--backend-packages', advanced=True, type=list,
default=['pants.backend.graph_info',
'pants.backend.python',
'pants.backend.jvm',
'pants.backend.native',
# TODO: Move into the graph_info backend.
'pants.rules.core',
'pants.backend.codegen.antlr.java',
'pants.backend.codegen.antlr.python',
'pants.backend.codegen.jaxb',
'pants.backend.codegen.protobuf.java',
'pants.backend.codegen.ragel.java',
'pants.backend.codegen.thrift.java',
'pants.backend.codegen.thrift.python',
'pants.backend.codegen.grpcio.python',
'pants.backend.codegen.wire.java',
'pants.backend.project_info'],
help='Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir', advanced=True, metavar='<dir>', default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir', advanced=True, metavar='<dir>', default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir', advanced=True, metavar='<dir>',
default=default_distdir,
help='Write end-product artifacts to this dir. If you modify this path, you '
'should also update --build-ignore and --pants-ignore to include the '
'custom dist dir path as well.')
register('--pants-subprocessdir', advanced=True, default=os.path.join(buildroot, '.pids'),
help='The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).')
register('--pants-config-files', advanced=True, type=list, daemon=False,
default=[get_default_pants_config_file()], help='Paths to Pants config files.')
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--pantsrc', advanced=True, type=bool, default=True,
help='Use pantsrc files.')
register('--pantsrc-files', advanced=True, type=list, metavar='<path>', daemon=False,
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register('--pythonpath', advanced=True, type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file', type=list, dest='target_spec_files', daemon=False,
help='Read additional specs from this file, one per line')
register('--verify-config', type=bool, default=True, daemon=False,
advanced=True,
help='Verify that all config file values correspond to known options.')
register('--build-ignore', advanced=True, type=list,
default=['.*/', default_rel_distdir, 'bower_components/',
'node_modules/', '*.egg-info/'],
help='Paths to ignore when identifying BUILD files. '
'This does not affect any other filesystem operations. '
'Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).')
register('--pants-ignore', advanced=True, type=list,
default=['.*/', default_rel_distdir],
help='Paths to ignore for all filesystem operations performed by pants '
'(e.g. BUILD file scanning, glob matching, etc). '
'Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore).')
register('--glob-expansion-failure', advanced=True,
default=GlobMatchErrorBehavior.warn, type=GlobMatchErrorBehavior,
help="Raise an exception if any targets declaring source files "
"fail to match any glob provided in the 'sources' argument.")
# TODO(#7203): make a regexp option type!
register('--exclude-target-regexp', advanced=True, type=list, default=[], daemon=False,
metavar='<regexp>', help='Exclude target roots that match these regexes.')
register('--subproject-roots', type=list, advanced=True, default=[],
help='Paths that correspond with build roots for any subproject that this '
'project depends on.')
register('--owner-of', type=list, member_type=file_option, default=[], daemon=False, metavar='<path>',
help='Select the targets that own these files. '
'This is the third target calculation strategy along with the --changed-* '
'options and specifying the targets directly. These three types of target '
'selection are mutually exclusive.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d', '--logdir', advanced=True, metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register('--enable-pantsd', advanced=True, type=bool, default=False,
help='Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)')
# Shutdown pantsd after the current run.
# This needs to be accessed at the same time as enable_pantsd,
# so we register it at bootstrap time.
register('--shutdown-pantsd-after-run', advanced=True, type=bool, default=False,
help='Create a new pantsd server, and use it, and shut it down immediately after. '
'If pantsd is already running, it will shut it down and spawn a new instance (Beta)')
# These facilitate configuring the native engine.
register('--native-engine-visualize-to', advanced=True, default=None, type=dir_option, daemon=False,
help='A directory to write execution and rule graphs to as `dot` files. The contents '
'of the directory will be overwritten if any filenames collide.')
register('--print-exception-stacktrace', advanced=True, type=bool,
help='Print to console the full exception stack trace if encountered.')
# BinaryUtil options.
register('--binaries-baseurls', type=list, advanced=True,
default=['https://binaries.pantsbuild.org'],
help='List of URLs from which binary tools are downloaded. URLs are '
'searched in order until the requested path is found.')
register('--binaries-fetch-timeout-secs', type=int, default=30, advanced=True, daemon=False,
help='Timeout in seconds for URL reads when fetching binary tools from the '
'repos specified by --baseurls.')
register('--binaries-path-by-id', type=dict, advanced=True,
help=("Maps output of uname for a machine to a binary search path: "
"(sysname, id) -> (os, arch), e.g. {('darwin', '15'): ('mac', '10.11'), "
"('linux', 'arm32'): ('linux', 'arm32')}."))
register('--allow-external-binary-tool-downloads', type=bool, default=True, advanced=True,
help="If False, require BinaryTool subclasses to download their contents from urls "
"generated from --binaries-baseurls, even if the tool has an external url "
"generator. This can be necessary if using Pants in an environment which cannot "
"contact the wider Internet.")
# Pants Daemon options.
register('--pantsd-pailgun-host', advanced=True, default='127.0.0.1',
help='The host to bind the pants nailgun server to.')
register('--pantsd-pailgun-port', advanced=True, type=int, default=0,
help='The port to bind the pants nailgun server to. Defaults to a random port.')
# TODO(#7514): Make this default to 1.0 seconds if stdin is a tty!
register('--pantsd-pailgun-quit-timeout', advanced=True, type=float, default=5.0,
help='The length of time (in seconds) to wait for further output after sending a '
'signal to the remote pantsd process before killing it.')
register('--pantsd-log-dir', advanced=True, default=None,
help='The directory to log pantsd output to.')
register('--pantsd-invalidation-globs', advanced=True, type=list, default=[],
help='Filesystem events matching any of these globs will trigger a daemon restart.')
# Watchman options.
register('--watchman-version', advanced=True, default='4.9.0-pants1', help='Watchman version.')
register('--watchman-supportdir', advanced=True, default='bin/watchman',
help='Find watchman binaries under this dir. Used as part of the path to lookup '
'the binary with --binaries-baseurls and --pants-bootstrapdir.')
register('--watchman-startup-timeout', type=float, advanced=True, default=30.0,
help='The watchman socket timeout (in seconds) for the initial `watch-project` command. '
'This may need to be set higher for larger repos due to watchman startup cost.')
register('--watchman-socket-timeout', type=float, advanced=True, default=0.1,
help='The watchman client socket timeout in seconds. Setting this to too high a '
'value can negatively impact the latency of runs forked by pantsd.')
register('--watchman-socket-path', type=str, advanced=True, default=None,
help='The path to the watchman UNIX socket. This can be overridden if the default '
'absolute path length exceeds the maximum allowed by the OS.')
# This option changes the parser behavior in a fundamental way (which currently invalidates
# all caches), and needs to be parsed out early, so we make it a bootstrap option.
register('--build-file-imports', choices=['allow', 'warn', 'error'], default='warn',
advanced=True,
help='Whether to allow import statements in BUILD files')
register('--local-store-dir', advanced=True,
help="Directory to use for engine's local file store.",
# This default is also hard-coded into the engine's rust code in
# fs::Store::default_path
default=os.path.expanduser('~/.cache/pants/lmdb_store'))
register('--remote-store-server', advanced=True, type=list, default=[],
help='host:port of grpc server to use as remote execution file store.')
register('--remote-store-thread-count', type=int, advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_thread_count,
help='Thread count to use for the pool that interacts with the remote file store.')
register('--remote-execution-server', advanced=True,
help='host:port of grpc server to use as remote execution scheduler.')
register('--remote-store-chunk-bytes', type=int, advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_bytes,
help='Size in bytes of chunks transferred to/from the remote file store.')
register('--remote-store-chunk-upload-timeout-seconds', type=int, advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_upload_timeout_seconds,
help='Timeout (in seconds) for uploads of individual chunks to the remote file store.')
register('--remote-store-rpc-retries', type=int, advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_rpc_retries,
help='Number of times to retry any RPC to the remote store before giving up.')
register('--remote-execution-process-cache-namespace', advanced=True,
help="The cache namespace for remote process execution. "
"Bump this to invalidate every artifact's remote execution. "
"This is the remote execution equivalent of the legacy cache-key-gen-version "
"flag.")
register('--remote-instance-name', advanced=True,
help='Name of the remote execution instance to use. Used for routing within '
'--remote-execution-server and --remote-store-server.')
register('--remote-ca-certs-path', advanced=True,
help='Path to a PEM file containing CA certificates used for verifying secure '
'connections to --remote-execution-server and --remote-store-server. '
'If not specified, TLS will not be used.')
register('--remote-oauth-bearer-token-path', advanced=True,
help='Path to a file containing an oauth token to use for grpc connections to '
'--remote-execution-server and --remote-store-server. If not specified, no '
'authorization will be performed.')
register('--remote-execution-extra-platform-properties', advanced=True,
help='Platform properties to set on remote execution requests. '
'Format: property=value. Multiple values should be specified as multiple '
'occurrences of this flag. Pants itself may add additional platform properties.',
type=list, default=[])
# This should eventually deprecate the RunTracker worker count, which is used for legacy cache
# lookups via CacheSetup in TaskBase.
register('--process-execution-parallelism', type=int, default=multiprocessing.cpu_count(),
advanced=True,
help='Number of concurrent processes that may be executed either locally and remotely.')
register('--process-execution-cleanup-local-dirs', type=bool, default=True, advanced=True,
help='Whether or not to cleanup directories used for local process execution '
'(primarily useful for e.g. debugging).')
def test_set_cachedir(self):
with temporary_file() as temp:
with environment_as(XDG_CACHE_HOME=temp.name):
self.assertEqual(os.path.join(temp.name, 'pants'),
get_pants_cachedir())
def test_get_cachedir(self):
with environment_as(XDG_CACHE_HOME=''):
self.assertEqual(os.path.expanduser('~/.cache/pants'), get_pants_cachedir())
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = 'dist'
default_distdir = os.path.join(buildroot, default_distdir_name)
default_rel_distdir = '/{}/'.format(default_distdir_name)
register('-l',
'--level',
choices=['trace', 'debug', 'info', 'warn'],
default='info',
recursive=True,
help='Set the logging level.')
register(
'-q',
'--quiet',
type=bool,
recursive=True,
daemon=False,
help=
'Squelches most console output. NOTE: Some tasks default to behaving quietly: '
'inverting this option supports making them noisier than they would be otherwise.'
)
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors',
type=bool,
default=sys.stdout.isatty(),
recursive=True,
daemon=False,
help='Set whether log messages are displayed in color.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version',
advanced=True,
default=pants_version(),
help='Use this pants version.')
register('--plugins',
advanced=True,
type=list,
help='Load these plugins.')
register('--plugin-cache-dir',
advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register(
'--backend-packages',
advanced=True,
type=list,
default=[
'pants.backend.graph_info', 'pants.backend.python',
'pants.backend.jvm', 'pants.backend.native',
'pants.backend.codegen.antlr.java',
'pants.backend.codegen.antlr.python',
'pants.backend.codegen.jaxb',
'pants.backend.codegen.protobuf.java',
'pants.backend.codegen.ragel.java',
'pants.backend.codegen.thrift.java',
'pants.backend.codegen.thrift.python',
'pants.backend.codegen.wire.java', 'pants.backend.project_info'
],
help=
'Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir',
advanced=True,
metavar='<dir>',
default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir',
advanced=True,
metavar='<dir>',
default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register(
'--pants-distdir',
advanced=True,
metavar='<dir>',
default=default_distdir,
help=
'Write end-product artifacts to this dir. If you modify this path, you '
'should also update --build-ignore and --pants-ignore to include the '
'custom dist dir path as well.')
register(
'--pants-subprocessdir',
advanced=True,
default=os.path.join(buildroot, '.pids'),
help=
'The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).'
)
register('--pants-config-files',
advanced=True,
type=list,
daemon=False,
default=[get_default_pants_config_file()],
help='Paths to Pants config files.')
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--pantsrc',
advanced=True,
type=bool,
default=True,
help='Use pantsrc files.')
register('--pantsrc-files',
advanced=True,
type=list,
metavar='<path>',
daemon=False,
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register(
'--pythonpath',
advanced=True,
type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file',
type=list,
dest='target_spec_files',
daemon=False,
help='Read additional specs from this file, one per line')
register(
'--verify-config',
type=bool,
default=True,
daemon=False,
advanced=True,
help=
'Verify that all config file values correspond to known options.')
register(
'--build-ignore',
advanced=True,
type=list,
fromfile=True,
default=[
'.*/', default_rel_distdir, 'bower_components/',
'node_modules/', '*.egg-info/'
],
help='Paths to ignore when identifying BUILD files. '
'This does not affect any other filesystem operations. '
'Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).'
)
register(
'--pants-ignore',
advanced=True,
type=list,
fromfile=True,
default=['.*/', default_rel_distdir],
help=
'Paths to ignore for all filesystem operations performed by pants '
'(e.g. BUILD file scanning, glob matching, etc). '
'Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore).'
)
register(
'--glob-expansion-failure',
type=str,
choices=GlobMatchErrorBehavior.allowed_values,
default=GlobMatchErrorBehavior.default_option_value,
advanced=True,
help="Raise an exception if any targets declaring source files "
"fail to match any glob provided in the 'sources' argument.")
register('--exclude-target-regexp',
advanced=True,
type=list,
default=[],
daemon=False,
metavar='<regexp>',
help='Exclude target roots that match these regexes.')
register(
'--subproject-roots',
type=list,
advanced=True,
fromfile=True,
default=[],
help=
'Paths that correspond with build roots for any subproject that this '
'project depends on.')
register(
'--owner-of',
type=list,
default=[],
daemon=False,
fromfile=True,
metavar='<path>',
help='Select the targets that own these files. '
'This is the third target calculation strategy along with the --changed-* '
'options and specifying the targets directly. These three types of target '
'selection are mutually exclusive.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d',
'--logdir',
advanced=True,
metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
'--enable-pantsd',
advanced=True,
type=bool,
default=False,
help=
'Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)'
)
# These facilitate configuring the native engine.
register(
'--native-engine-visualize-to',
advanced=True,
default=None,
type=dir_option,
daemon=False,
help=
'A directory to write execution and rule graphs to as `dot` files. The contents '
'of the directory will be overwritten if any filenames collide.')
register(
'--print-exception-stacktrace',
advanced=True,
type=bool,
help=
'Print to console the full exception stack trace if encountered.')
# BinaryUtil options.
register(
'--binaries-baseurls',
type=list,
advanced=True,
default=['https://binaries.pantsbuild.org'],
help='List of URLs from which binary tools are downloaded. URLs are '
'searched in order until the requested path is found.')
register(
'--binaries-fetch-timeout-secs',
type=int,
default=30,
advanced=True,
daemon=False,
help=
'Timeout in seconds for URL reads when fetching binary tools from the '
'repos specified by --baseurls.')
register(
'--binaries-path-by-id',
type=dict,
advanced=True,
help=
("Maps output of uname for a machine to a binary search path: "
"(sysname, id) -> (os, arch), e.g. {('darwin', '15'): ('mac', '10.11'), "
"('linux', 'arm32'): ('linux', 'arm32')}."))
register(
'--allow-external-binary-tool-downloads',
type=bool,
default=True,
advanced=True,
help=
"If False, require BinaryTool subclasses to download their contents from urls "
"generated from --binaries-baseurls, even if the tool has an external url "
"generator. This can be necessary if using Pants in an environment which cannot "
"contact the wider Internet.")
# Pants Daemon options.
register('--pantsd-pailgun-host',
advanced=True,
default='127.0.0.1',
help='The host to bind the pants nailgun server to.')
register(
'--pantsd-pailgun-port',
advanced=True,
type=int,
default=0,
help=
'The port to bind the pants nailgun server to. Defaults to a random port.'
)
register('--pantsd-log-dir',
advanced=True,
default=None,
help='The directory to log pantsd output to.')
register(
'--pantsd-fs-event-workers',
advanced=True,
type=int,
default=4,
removal_version='1.14.0.dev2',
removal_hint=
'Filesystem events are now handled by a single dedicated thread.',
help=
'The number of workers to use for the filesystem event service executor pool.'
)
register(
'--pantsd-invalidation-globs',
advanced=True,
type=list,
fromfile=True,
default=[],
help=
'Filesystem events matching any of these globs will trigger a daemon restart.'
)
# Watchman options.
register('--watchman-version',
advanced=True,
default='4.9.0-pants1',
help='Watchman version.')
register(
'--watchman-supportdir',
advanced=True,
default='bin/watchman',
help=
'Find watchman binaries under this dir. Used as part of the path to lookup '
'the binary with --binaries-baseurls and --pants-bootstrapdir.')
register(
'--watchman-startup-timeout',
type=float,
advanced=True,
default=30.0,
help=
'The watchman socket timeout (in seconds) for the initial `watch-project` command. '
'This may need to be set higher for larger repos due to watchman startup cost.'
)
register('--watchman-socket-timeout',
type=float,
advanced=True,
default=5.0,
help='The watchman client socket timeout in seconds.')
register(
'--watchman-socket-path',
type=str,
advanced=True,
default=None,
help=
'The path to the watchman UNIX socket. This can be overridden if the default '
'absolute path length exceeds the maximum allowed by the OS.')
# This option changes the parser behavior in a fundamental way (which currently invalidates
# all caches), and needs to be parsed out early, so we make it a bootstrap option.
register('--build-file-imports',
choices=['allow', 'warn', 'error'],
default='warn',
advanced=True,
help='Whether to allow import statements in BUILD files')
register(
'--remote-store-server',
advanced=True,
help=
'host:port of grpc server to use as remote execution file store.')
register(
'--remote-store-thread-count',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_thread_count,
help=
'Thread count to use for the pool that interacts with the remote file store.'
)
register(
'--remote-execution-server',
advanced=True,
help=
'host:port of grpc server to use as remote execution scheduler.')
register(
'--remote-store-chunk-bytes',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_bytes,
help=
'Size in bytes of chunks transferred to/from the remote file store.'
)
register(
'--remote-store-chunk-upload-timeout-seconds',
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.
remote_store_chunk_upload_timeout_seconds,
help=
'Timeout (in seconds) for uploads of individual chunks to the remote file store.'
)
register(
'--remote-instance-name',
advanced=True,
help=
'Name of the remote execution instance to use. Used for routing within '
'--remote-execution-server and --remote-store-server.')
register(
'--remote-ca-certs-path',
advanced=True,
help=
'Path to a PEM file containing CA certificates used for verifying secure '
'connections to --remote-execution-server and --remote-store-server. '
'If not specified, TLS will not be used.')
register(
'--remote-oauth-bearer-token-path',
advanced=True,
help=
'Path to a file containing an oauth token to use for grpc connections to '
'--remote-execution-server and --remote-store-server. If not specified, no '
'authorization will be performed.')
# This should eventually deprecate the RunTracker worker count, which is used for legacy cache
# lookups via CacheSetup in TaskBase.
register(
'--process-execution-parallelism',
type=int,
default=multiprocessing.cpu_count(),
advanced=True,
help=
'Number of concurrent processes that may be executed either locally and remotely.'
)
register(
'--process-execution-cleanup-local-dirs',
type=bool,
default=True,
advanced=True,
help=
'Whether or not to cleanup directories used for local process execution '
'(primarily useful for e.g. debugging).')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are the set of options necessary to create a Scheduler. If an option is
not consumed during creation of a Scheduler, it should be in `register_options` instead.
Bootstrap option values can be interpolated into the config file, and can be referenced
programmatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = "dist"
default_rel_distdir = f"/{default_distdir_name}/"
register(
"-l",
"--level",
type=LogLevel,
default=LogLevel.INFO,
help="Set the logging level.",
)
register(
"--log-show-rust-3rdparty",
type=bool,
default=False,
advanced=True,
help=
"Whether to show/hide logging done by 3rdparty Rust crates used by the Pants "
"engine.",
)
register(
"--colors",
type=bool,
default=sys.stdout.isatty(),
help=
("Whether Pants should use colors in output or not. This may also impact whether "
"some tools Pants run use color."),
)
register(
"--v1",
advanced=True,
type=bool,
default=False,
help="Enables execution of v1 Tasks.",
removal_version="2.1.0.dev0",
removal_hint=
"The v1 engine no longer exists. This option does nothing.",
)
register(
"--v2",
advanced=True,
type=bool,
default=True,
help="Enables execution of v2 @goal_rules.",
removal_version="2.1.0.dev0",
removal_hint=
"The v2 engine is the only one available. This option does nothing.",
)
# TODO(#7203): make a regexp option type!
register(
"--ignore-pants-warnings",
type=list,
member_type=str,
default=[],
advanced=True,
help="Regexps matching warning strings to ignore, e.g. "
'["DEPRECATED: scope some_scope will be removed"]. The regexps will be matched '
"from the start of the warning string, and will always be case-insensitive. "
"See the `warnings` module documentation for more background on these are used.",
)
register(
"--option-name-check-distance",
advanced=True,
type=int,
default=2,
help=
"The maximum Levenshtein distance to use when offering suggestions for invalid "
"option names.",
)
register(
"--pants-version",
advanced=True,
default=pants_version(),
daemon=True,
help=
"Use this pants version. Note Pants code only uses this to verify that you are "
"using the requested version, as Pants cannot dynamically change the version it "
"is using once the program is already running. This option is useful to set in "
"your pants.toml, however, and then you can grep the value to select which "
"version to use for setup scripts (e.g. `./pants`), runner scripts, IDE plugins, "
"etc. For example, the setup script we distribute at "
"https://www.pantsbuild.org/docs/installation uses this value to determine which Python"
"version to run with. You may find the version of the Pants instance you are running "
"by using -v, -V, or --version.",
)
register(
"--pants-bin-name",
advanced=True,
default="./pants",
help="The name of the script or binary used to invoke Pants. "
"Useful when printing help messages.",
)
register(
"--plugins",
advanced=True,
type=list,
help=
"Allow backends to be loaded from these plugins. The default backends for "
"each plugin will be loaded automatically. Other backends in a plugin can be "
"loaded by listing them in --backend-packages.",
)
register(
"--plugins2",
advanced=True,
type=list,
removal_version="2.1.0.dev0",
removal_hint="Use --plugins instead.",
help=
"Allow backends to be loaded from these plugins. The default backends for "
"each plugin will be loaded automatically. Other backends in a plugin can be "
"loaded by listing them in --backend-packages.",
)
register(
"--plugins-force-resolve",
advanced=True,
type=bool,
default=False,
help="Re-resolve plugins even if previously resolved.",
)
register(
"--plugin-cache-dir",
advanced=True,
default=os.path.join(get_pants_cachedir(), "plugins"),
help="Cache resolved plugin requirements here.",
)
register(
"--backend-packages",
advanced=True,
type=list,
default=[],
help=
("Register rules from these backends. The backend packages must be present on "
"the PYTHONPATH, typically because they are in the Pants core dist, in a "
"plugin dist, or available as sources in the repo."),
)
register(
"--backend-packages2",
advanced=True,
type=list,
default=[],
removal_version="2.1.0.dev0",
removal_hint="Use --backend-packages instead.",
help=
("Register rules from these backends. The backend packages must be present on "
"the PYTHONPATH, typically because they are in the Pants core dist, in a "
"plugin dist, or available as sources in the repo."),
)
register(
"--pants-bootstrapdir",
advanced=True,
metavar="<dir>",
default=get_pants_cachedir(),
help="Use this dir for global cache.",
)
register(
"--pants-configdir",
advanced=True,
metavar="<dir>",
default=get_pants_configdir(),
help="Use this dir for global config files.",
)
register(
"--pants-workdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, ".pants.d"),
daemon=True,
help="Write intermediate output files to this dir.",
)
register(
"--pants-physical-workdir-base",
advanced=True,
metavar="<dir>",
default=None,
daemon=True,
help=
"When set, a base directory in which to store `--pants-workdir` contents. "
"If this option is a set, the workdir will be created as symlink into a "
"per-workspace subdirectory.",
)
register(
"--pants-supportdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "build-support"),
help="Use support files from this dir.",
)
register(
"--pants-distdir",
advanced=True,
metavar="<dir>",
default=os.path.join(buildroot, "dist"),
help="Write end-product artifacts to this dir.",
)
register(
"--pants-subprocessdir",
advanced=True,
default=os.path.join(buildroot, ".pids"),
daemon=True,
help=
"The directory to use for tracking subprocess metadata, if any. This should "
"live outside of the dir used by `--pants-workdir` to allow for tracking "
"subprocesses that outlive the workdir data.",
)
register(
"--pants-config-files",
advanced=True,
type=list,
# NB: We don't fingerprint the list of config files, because the content of the config
# files independently affects fingerprints.
fingerprint=False,
default=[get_default_pants_config_file()],
help="Paths to Pants config files.",
)
register(
"--pantsrc",
advanced=True,
type=bool,
default=True,
# NB: See `--pants-config-files`.
fingerprint=False,
help="Use pantsrc files.",
)
register(
"--pantsrc-files",
advanced=True,
type=list,
metavar="<path>",
# NB: See `--pants-config-files`.
fingerprint=False,
default=["/etc/pantsrc", "~/.pants.rc"],
help=
("Override config with values from these files, using syntax matching that of "
"`--pants-config-files`."),
)
register(
"--pythonpath",
advanced=True,
type=list,
help="Add these directories to PYTHONPATH to search for plugins.",
)
register(
"--spec-files",
type=list,
# NB: See `--pants-config-files`.
fingerprint=False,
help=
"Read additional specs (e.g., target addresses or file names), one per line,"
"from these files.",
)
register(
"--spec-file",
type=list,
removal_version="2.1.0.dev0",
removal_hint="Use --spec-files",
fingerprint=False,
help=
"Read additional specs from this file (e.g. target addresses or file names). "
"Each spec should be one per line.",
)
register(
"--verify-config",
type=bool,
default=True,
advanced=True,
help=
"Verify that all config file values correspond to known options.",
)
register(
"--build-patterns",
advanced=True,
type=list,
default=["BUILD", "BUILD.*"],
help=
("The naming scheme for BUILD files, i.e. where you define targets. This only sets "
"the naming scheme, not the directory paths to look for. To add ignore"
"patterns, use the option `--build-ignore`."),
)
register(
"--build-ignore",
advanced=True,
type=list,
default=[
".*/", "bower_components/", "node_modules/", "*.egg-info/"
],
help=
("Paths to ignore when identifying BUILD files. This does not affect any other "
"filesystem operations; use `--pants-ignore` for that instead. Patterns use the "
"gitignore pattern syntax (https://git-scm.com/docs/gitignore)."),
)
register(
"--pants-ignore",
advanced=True,
type=list,
member_type=str,
default=[".*/", default_rel_distdir],
help=
"Paths to ignore for all filesystem operations performed by pants "
"(e.g. BUILD file scanning, glob matching, etc). "
"Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). "
"The `--pants-distdir` and `--pants-workdir` locations are inherently ignored. "
"--pants-ignore can be used in tandem with --pants-ignore-use-gitignore, and any rules "
"specified here apply after rules specified in a .gitignore file.",
)
register(
"--pants-ignore-use-gitignore",
advanced=True,
type=bool,
default=True,
help=
"Make use of a root .gitignore file when determining whether to ignore filesystem "
"operations performed by Pants. If used together with `--pants-ignore`, any exclude/include "
"patterns specified there apply after .gitignore rules.",
)
register(
"--owners-not-found-behavior",
advanced=True,
type=OwnersNotFoundBehavior,
default=OwnersNotFoundBehavior.error,
help=
"What to do when file arguments do not have any owning target. This happens when there "
"are no targets whose `sources` fields include the file argument.",
)
register(
"--files-not-found-behavior",
advanced=True,
type=FileNotFoundBehavior,
default=FileNotFoundBehavior.warn,
help=
"What to do when files and globs specified in BUILD files, such as in the "
"`sources` field, cannot be found. This happens when the files do not exist on "
"your machine or when they are ignored by the `--pants-ignore` option.",
)
register(
"--tag",
type=list,
metavar="[+-]tag1,tag2,...",
help=
("Include only targets with these tags (optional '+' prefix) or without these "
"tags ('-' prefix). See https://www.pantsbuild.org/docs/advanced-target-selection."
),
)
register(
"--exclude-target-regexp",
advanced=True,
type=list,
default=[],
metavar="<regexp>",
help="Exclude target roots that match these regexes.",
)
register(
"--subproject-roots",
type=list,
advanced=True,
default=[],
help=
"Paths that correspond with build roots for any subproject that this "
"project depends on.",
)
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register(
"-d",
"--logdir",
advanced=True,
metavar="<dir>",
daemon=True,
help="Write logs to files under this directory.",
)
register(
"--pantsd",
advanced=True,
type=bool,
default=True,
daemon=True,
help=
("Enables use of the Pants daemon (pantsd). pantsd can significantly improve "
"runtime performance by lowering per-run startup cost, and by caching filesystem "
"operations and rule execution."),
)
# Whether or not to make necessary arrangements to have concurrent runs in pants.
# In practice, this means that if this is set, a run will not even try to use pantsd.
# NB: Eventually, we would like to deprecate this flag in favor of making pantsd runs parallelizable.
register(
"--concurrent",
advanced=True,
type=bool,
default=False,
help=
"Enable concurrent runs of Pants. Without this enabled, Pants will "
"start up all concurrent invocations (e.g. in other terminals) without pantsd. "
"Enabling this option requires parallel Pants invocations to block on the first",
)
# Calling pants command (inner run) from other pants command is unusual behaviour,
# and most users should never set this flag.
# It is automatically set by pants when an inner run is detected.
# Currently, pants commands with this option set don't use pantsd,
# but this effect should not be relied upon.
# This option allows us to know who was the parent of pants inner runs for informational purposes.
register(
"--parent-build-id",
advanced=True,
default=None,
help=
"The build ID of the other Pants run which spawned this one, if any.",
)
# NB: We really don't want this option to invalidate the daemon, because different clients might have
# different needs. For instance, an IDE might have a very long timeout because it only wants to refresh
# a project in the background, while a user might want a shorter timeout for interactivity.
register(
"--pantsd-timeout-when-multiple-invocations",
advanced=True,
type=float,
default=60.0,
help=
"The maximum amount of time to wait for the invocation to start until "
"raising a timeout exception. "
"Because pantsd currently does not support parallel runs, "
"any prior running Pants command must be finished for the current one to start. "
"To never timeout, use the value -1.",
)
register(
"--pantsd-max-memory-usage",
advanced=True,
type=int,
default=2**30,
help=
("The maximum memory usage of a pantsd process (in bytes). There is at most one "
"pantsd process per workspace."),
)
# These facilitate configuring the native engine.
register(
"--native-engine-visualize-to",
advanced=True,
default=None,
type=dir_option,
help=
"A directory to write execution and rule graphs to as `dot` files. The contents "
"of the directory will be overwritten if any filenames collide.",
)
register(
"--print-exception-stacktrace",
advanced=True,
type=bool,
default=False,
# TODO: We must restart the daemon because of global mutable state in `init/logging.py`
# from `ExceptionSink.should_print_exception_stacktrace`. Fix this and only use
# `fingerprint=True` instead.
daemon=True,
help=
"Print to console the full exception stack trace if encountered.",
)
# BinaryUtil options.
# TODO: Nuke these once we get rid of src/python/pants/binaries/binary_util.py
# (see there for what that will take).
register(
"--binaries-baseurls",
type=list,
advanced=True,
default=["https://binaries.pantsbuild.org"],
help="List of URLs from which binary tools are downloaded. URLs are "
"searched in order until the requested path is found.",
)
register(
"--binaries-fetch-timeout-secs",
type=int,
default=30,
advanced=True,
help=
"Timeout in seconds for URL reads when fetching binary tools from the "
"repos specified by --baseurls.",
)
register(
"--binaries-path-by-id",
type=dict,
advanced=True,
help=
("Maps output of uname for a machine to a binary search path: "
"(sysname, id) -> (os, arch), e.g. {('darwin', '15'): ('mac', '10.11'), "
"('linux', 'arm32'): ('linux', 'arm32')}."),
)
register(
"--allow-external-binary-tool-downloads",
type=bool,
default=True,
advanced=True,
help=
"If False, require BinaryTool subclasses to download their contents from urls "
"generated from --binaries-baseurls, even if the tool has an external url "
"generator. This can be necessary if using Pants in an environment which cannot "
"contact the wider Internet.",
)
# Pants Daemon options.
register(
"--pantsd-pailgun-port",
advanced=True,
type=int,
default=0,
daemon=True,
help=
"The port to bind the Pants nailgun server to. Defaults to a random port.",
)
# TODO(#7514): Make this default to 1.0 seconds if stdin is a tty!
register(
"--pantsd-pailgun-quit-timeout",
advanced=True,
type=float,
default=5.0,
help=
"The length of time (in seconds) to wait for further output after sending a "
"signal to the remote pantsd process before killing it.",
)
register(
"--pantsd-log-dir",
advanced=True,
default=None,
daemon=True,
help="The directory to log pantsd output to.",
)
register(
"--pantsd-invalidation-globs",
advanced=True,
type=list,
default=[],
daemon=True,
help=
"Filesystem events matching any of these globs will trigger a daemon restart. "
"Pants's own code, plugins, and `--pants-config-files` are inherently invalidated.",
)
register(
"--build-file-prelude-globs",
advanced=True,
type=list,
default=[],
help=
"Python files to evaluate and whose symbols should be exposed to all BUILD files ."
"This allows for writing functions which create multiple rules, or set default "
"arguments for rules. The order these files will be evaluated is undefined - they should not rely on each "
"other, or override symbols from each other.",
)
cache_instructions = (
"The path may be absolute or relative. If the directory is within the build root, be "
"sure to include it in `--pants-ignore`.")
register(
"--local-store-dir",
advanced=True,
help=
(f"Directory to use for the local file store, which stores the results of "
f"subprocesses run by Pants. {cache_instructions}"),
# This default is also hard-coded into the engine's rust code in
# fs::Store::default_path so that tools using a Store outside of pants
# are likely to be able to use the same storage location.
default=os.path.join(get_pants_cachedir(), "lmdb_store"),
)
register(
"--local-execution-root-dir",
advanced=True,
help=
f"Directory to use for local process execution sandboxing. {cache_instructions}",
default=tempfile.gettempdir(),
)
register(
"--named-caches-dir",
advanced=True,
help=
("Directory to use for named global caches for tools and processes with trusted, "
f"concurrency-safe caches. {cache_instructions}"),
default=os.path.join(get_pants_cachedir(), "named_caches"),
)
register(
"--remote-execution",
advanced=True,
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.remote_execution,
help="Enables remote workers for increased parallelism. (Alpha)",
)
register(
"--remote-store-server",
advanced=True,
type=list,
default=[],
help=
"host:port of grpc server to use as remote execution file store.",
)
# TODO: Infer this from remote-store-connection-limit.
register(
"--remote-store-thread-count",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_thread_count,
help=
"Thread count to use for the pool that interacts with the remote file store.",
)
register(
"--remote-execution-server",
advanced=True,
help=
"host:port of grpc server to use as remote execution scheduler.",
)
register(
"--remote-store-chunk-bytes",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_chunk_bytes,
help=
"Size in bytes of chunks transferred to/from the remote file store.",
)
register(
"--remote-store-chunk-upload-timeout-seconds",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.
remote_store_chunk_upload_timeout_seconds,
help=
"Timeout (in seconds) for uploads of individual chunks to the remote file store.",
)
register(
"--remote-store-rpc-retries",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_rpc_retries,
help=
"Number of times to retry any RPC to the remote store before giving up.",
)
register(
"--remote-store-connection-limit",
type=int,
advanced=True,
default=DEFAULT_EXECUTION_OPTIONS.remote_store_connection_limit,
help=
"Number of remote stores to concurrently allow connections to.",
)
register(
"--remote-execution-process-cache-namespace",
advanced=True,
help="The cache namespace for remote process execution. "
"Bump this to invalidate every artifact's remote execution. "
"This is the remote execution equivalent of the legacy cache-key-gen-version "
"flag.",
)
register(
"--remote-instance-name",
advanced=True,
help=
"Name of the remote execution instance to use. Used for routing within "
"--remote-execution-server and --remote-store-server.",
)
register(
"--remote-ca-certs-path",
advanced=True,
help=
"Path to a PEM file containing CA certificates used for verifying secure "
"connections to --remote-execution-server and --remote-store-server. "
"If not specified, TLS will not be used.",
)
register(
"--remote-oauth-bearer-token-path",
advanced=True,
help=
"Path to a file containing an oauth token to use for grpc connections to "
"--remote-execution-server and --remote-store-server. If not specified, no "
"authorization will be performed.",
)
register(
"--remote-execution-extra-platform-properties",
advanced=True,
help="Platform properties to set on remote execution requests. "
"Format: property=value. Multiple values should be specified as multiple "
"occurrences of this flag. Pants itself may add additional platform properties.",
type=list,
default=[],
)
register(
"--remote-execution-headers",
advanced=True,
help="Headers to set on remote execution requests. "
"Format: header=value. Pants itself may add additional headers.",
type=dict,
default={},
)
register(
"--remote-execution-enable-streaming",
type=bool,
default=True,
advanced=True,
help=
"This option no longer does anything. (It used to enable the streaming remote execution client "
"which is now the only remote execution client.)",
removal_version="2.1.0.dev0",
removal_hint="This option is no longer applicable.",
)
register(
"--remote-execution-overall-deadline-secs",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
remote_execution_overall_deadline_secs,
advanced=True,
help=
"Overall timeout in seconds for each remote execution request from time of submission",
)
register(
"--process-execution-local-parallelism",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_parallelism,
advanced=True,
help="Number of concurrent processes that may be executed locally.",
)
register(
"--process-execution-remote-parallelism",
type=int,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_remote_parallelism,
advanced=True,
help=
"Number of concurrent processes that may be executed remotely.",
)
register(
"--process-execution-cleanup-local-dirs",
type=bool,
default=True,
advanced=True,
help=
"Whether or not to cleanup directories used for local process execution "
"(primarily useful for e.g. debugging).",
)
register(
"--process-execution-speculation-delay",
type=float,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_speculation_delay,
advanced=True,
help=
"Number of seconds to wait before speculating a second request for a slow process. "
" see `--process-execution-speculation-strategy`",
)
register(
"--process-execution-speculation-strategy",
choices=["remote_first", "local_first", "none"],
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_speculation_strategy,
help=
"Speculate a second request for an underlying process if the first one does not complete within "
"`--process-execution-speculation-delay` seconds.\n"
"`local_first` (default): Try to run the process locally first, "
"and fall back to remote execution if available.\n"
"`remote_first`: Run the process on the remote execution backend if available, "
"and fall back to the local host if remote calls take longer than the speculation timeout.\n"
"`none`: Do not speculate about long running processes.",
advanced=True,
)
register(
"--process-execution-use-local-cache",
type=bool,
default=True,
advanced=True,
help=
"Whether to keep process executions in a local cache persisted to disk.",
)
register(
"--process-execution-local-enable-nailgun",
type=bool,
default=DEFAULT_EXECUTION_OPTIONS.
process_execution_local_enable_nailgun,
help=
"Whether or not to use nailgun to run the requests that are marked as nailgunnable.",
advanced=True,
)
def register_options(cls, register):
super(CoursierSubsystem, cls).register_options(register)
register(
'--cache-dir',
type=str,
fingerprint=True,
default=os.path.join(get_pants_cachedir(), 'coursier'),
help=
'Version paired with --bootstrap-jar-url, in order to invalidate and fetch the new version.'
)
register('--repos',
type=list,
fingerprint=True,
help='Maven style repos',
default=['https://repo1.maven.org/maven2'])
register(
'--fetch-options',
type=list,
fingerprint=True,
default=[
# Quiet mode, so coursier does not show resolve progress,
# but still prints results if --report is specified.
'-q',
# Do not use default public maven repo.
'--no-default',
# Concurrent workers
'-n',
'8',
],
help=
'Additional options to pass to coursier fetch. See `coursier fetch --help`'
)
register(
'--artifact-types',
type=list,
fingerprint=True,
default=[
'jar', 'bundle', 'test-jar', 'maven-plugin', 'src', 'doc',
'aar'
],
help=
'Specify the type of artifacts to fetch. See `packaging` at https://maven.apache.org/pom.html#Maven_Coordinates, '
'except `src` and `doc` being coursier specific terms for sources and javadoc.'
)
register(
'--bootstrap-jar-url',
fingerprint=True,
default=
'https://dl.dropboxusercontent.com/s/zwh074l9kxhqlwp/coursier-cli-1.1.0.cf365ea27a710d5f09db1f0a6feee129aa1fc417.jar?dl=0',
help='Location to download a bootstrap version of Coursier.')
# TODO(wisechengyi): currently using a custom url for fast iteration.
# Once the coursier builds are stable, move the logic to binary_util. https://github.com/pantsbuild/pants/issues/5381
# Ths sha in the version corresponds to the sha in the PR https://github.com/coursier/coursier/pull/774
# The jar is built by following https://github.com/coursier/coursier/blob/master/DEVELOPMENT.md#build-with-pants
register(
'--version',
type=str,
fingerprint=True,
default='1.1.0.cf365ea27a710d5f09db1f0a6feee129aa1fc417',
help=
'Version paired with --bootstrap-jar-url, in order to invalidate and fetch the new version.'
)
register(
'--bootstrap-fetch-timeout-secs',
type=int,
advanced=True,
default=10,
help=
'Timeout the fetch if the connection is idle for longer than this value.'
)
class Config(object):
"""Encapsulates ini-style config file loading and access.
Supports recursive variable substitution using standard python format strings. E.g.,
%(var_name)s will be replaced with the value of var_name.
"""
DEFAULT_SECTION = ConfigParser.DEFAULTSECT
_defaults = {
'homedir': os.path.expanduser('~'),
'user': getpass.getuser(),
'pants_bootstrapdir': get_pants_cachedir(),
'pants_configdir': get_pants_configdir()
}
reset_default_bootstrap_option_values(_defaults)
class ConfigError(Exception):
pass
@classmethod
def reset_default_bootstrap_option_values(cls,
values=None,
buildroot=None):
reset_default_bootstrap_option_values(cls._defaults, values, buildroot)
_cached_config = None
@classmethod
def _munge_configpaths_arg(cls, configpaths):
"""Converts a string or iterable-of-strings argument into a tuple of strings.
Result is hashable, so may be used as a cache key.
"""
if is_text_or_binary(configpaths):
return (configpaths, )
return tuple(configpaths) if configpaths else (os.path.join(
get_buildroot(), 'pants.ini'), )
@classmethod
def from_cache(cls):
if not cls._cached_config:
raise cls.ConfigError('No config cached.')
return cls._cached_config
@classmethod
def cache(cls, config):
cls._cached_config = config
@classmethod
def load(cls, configpaths=None):
"""Loads config from the given paths.
By default this is the path to the pants.ini file in the current build root directory.
Callers may specify a single path, or a list of the paths of configs to be chained, with
later instances taking precedence over eariler ones.
Any defaults supplied will act as if specified in the loaded config file's DEFAULT section.
The 'buildroot', invoking 'user' and invoking user's 'homedir' are automatically defaulted.
"""
configpaths = cls._munge_configpaths_arg(configpaths)
single_file_configs = []
for configpath in configpaths:
parser = cls.create_parser()
with open(configpath, 'r') as ini:
parser.readfp(ini)
single_file_configs.append(SingleFileConfig(configpath, parser))
return ChainedConfig(single_file_configs)
@classmethod
def create_parser(cls):
"""Creates a config parser that supports %([key-name])s value substitution.
Any defaults supplied will act as if specified in the loaded config file's DEFAULT section and
be available for substitutions, along with all the standard defaults defined above.
"""
return ConfigParser.SafeConfigParser(cls._defaults)
def getbool(self, section, option, default=None):
"""Equivalent to calling get with expected type bool."""
return self.get(section, option, type=bool, default=default)
def getint(self, section, option, default=None):
"""Equivalent to calling get with expected type int."""
return self.get(section, option, type=int, default=default)
def getfloat(self, section, option, default=None):
"""Equivalent to calling get with expected type float."""
return self.get(section, option, type=float, default=default)
def getlist(self, section, option, default=None):
"""Equivalent to calling get with expected type list."""
return self.get(section, option, type=list, default=default)
def getdict(self, section, option, default=None):
"""Equivalent to calling get with expected type dict."""
return self.get(section, option, type=dict, default=default)
def getdefault(self, option, type=str, default=None):
"""
Retrieves option from the DEFAULT section if it exists and attempts to parse it as type.
If there is no definition found, the default value supplied is returned.
"""
return self.get(Config.DEFAULT_SECTION, option, type, default=default)
def get(self, section, option, type=str, default=None):
"""
Retrieves option from the specified section if it exists and attempts to parse it as type.
If the specified section is missing a definition for the option, the value is looked up in the
DEFAULT section. If there is still no definition found, the default value supplied is
returned.
"""
return self._getinstance(section, option, type, default=default)
def get_required(self, section, option, type=str):
"""Retrieves option from the specified section and attempts to parse it as type.
If the specified section is missing a definition for the option, the value is
looked up in the DEFAULT section. If there is still no definition found,
a `ConfigError` is raised.
:param string section: Section to lookup the option in, before looking in DEFAULT.
:param string option: Option to retrieve.
:param type: Type to retrieve the option as.
:returns: The option as the specified type.
:raises: :class:`pants.base.config.Config.ConfigError` if option is not found.
"""
val = self.get(section, option, type=type)
# Empty str catches blank options. If blank entries are ok, use get(..., default='') instead.
if val is None or val == '':
raise Config.ConfigError('Required option %s.%s is not defined.' %
(section, option))
return val
@staticmethod
def format_raw_value(raw_value):
lines = raw_value.splitlines()
for line_number in range(0, len(lines)):
lines[line_number] = "{line_number:{width}}: {line}".format(
line_number=line_number + 1,
line=lines[line_number],
width=len(str(len(lines))))
return '\n'.join(lines)
def _getinstance(self, section, option, type, default=None):
if not self.has_option(section, option):
return default
raw_value = self.get_value(section, option)
if issubclass(type, str):
return raw_value
try:
parsed_value = eval(raw_value, {}, {})
except SyntaxError as e:
raise Config.ConfigError(
'No valid {type_name} for {section}.{option}:\n{value}\n{error}'
.format(type_name=type.__name__,
section=section,
option=option,
value=Config.format_raw_value(raw_value),
error=e))
if not isinstance(parsed_value, type):
raise Config.ConfigError(
'No valid {type_name} for {section}.{option}:\n{value}'.format(
type_name=type.__name__,
section=section,
option=option,
value=Config.format_raw_value(raw_value)))
return parsed_value
# Subclasses must implement.
def sources(self):
"""Return the sources of this config as a list of filenames."""
raise NotImplementedError()
def has_section(self, section):
"""Return whether this config has the section."""
raise NotImplementedError()
def has_option(self, section, option):
"""Return whether this config specified a value the option."""
raise NotImplementedError()
def get_value(self, section, option):
"""Return the value of the option in this config, as a string, or None if no value specified."""
raise NotImplementedError()
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
register('--plugins',
advanced=True,
type=Options.list,
help='Load these plugins.')
register(
'--backend-packages',
advanced=True,
type=Options.list,
help=
'Load backends from these packages that are already on the path.')
register('--pants-bootstrapdir',
advanced=True,
metavar='<dir>',
default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir',
advanced=True,
metavar='<dir>',
default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir',
metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir',
metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir',
metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register('--config-override',
help='A second config file, to override pants.ini.')
register('--pantsrc',
action='store_true',
default=True,
help='Use pantsrc files.')
register('--pantsrc-files',
action='append',
metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register(
'--pythonpath',
action='append',
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file',
action='append',
dest='target_spec_files',
help='Read additional specs from this file, one per line')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d',
'--logdir',
metavar='<dir>',
help='Write logs to files under this directory.')
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l',
'--level',
choices=['debug', 'info', 'warn'],
default='info',
recursive=True,
help='Set the logging level.')
register('-q',
'--quiet',
action='store_true',
help='Squelches all console output apart from errors.')
def _conan_pex_path(self):
return os.path.join(get_pants_cachedir(), self.conan_pex_subdir,
self.conan_pex_filename)
def test_get_pants_cachedir() -> None:
with environment_as(XDG_CACHE_HOME=""):
assert os.path.expanduser("~/.cache/pants") == get_pants_cachedir()
with temporary_file() as temp, environment_as(XDG_CACHE_HOME=temp.name):
assert os.path.join(temp.name, "pants") == get_pants_cachedir()
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
default_distdir_name = 'dist'
default_distdir = os.path.join(buildroot, default_distdir_name)
default_rel_distdir = '/{}/'.format(default_distdir_name)
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l', '--level', choices=['debug', 'info', 'warn'], default='info', recursive=True,
help='Set the logging level.')
register('-q', '--quiet', type=bool, recursive=True, daemon=False,
help='Squelches most console output. NOTE: Some tasks default to behaving quietly: '
'inverting this option supports making them noisier than they would be otherwise.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors', type=bool, default=sys.stdout.isatty(), recursive=True, daemon=False,
help='Set whether log messages are displayed in color.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version', advanced=True, default=pants_version(),
help='Use this pants version.')
register('--plugins', advanced=True, type=list, help='Load these plugins.')
register('--plugin-cache-dir', advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register('--backend-packages', advanced=True, type=list,
default=['pants.backend.graph_info',
'pants.backend.python',
'pants.backend.jvm',
'pants.backend.native',
'pants.backend.codegen.antlr.java',
'pants.backend.codegen.antlr.python',
'pants.backend.codegen.jaxb',
'pants.backend.codegen.protobuf.java',
'pants.backend.codegen.ragel.java',
'pants.backend.codegen.thrift.java',
'pants.backend.codegen.thrift.python',
'pants.backend.codegen.wire.java',
'pants.backend.project_info'],
help='Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir', advanced=True, metavar='<dir>', default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir', advanced=True, metavar='<dir>', default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir', advanced=True, metavar='<dir>',
default=default_distdir,
help='Write end-product artifacts to this dir. If you modify this path, you '
'should also update --build-ignore and --pants-ignore to include the '
'custom dist dir path as well.')
register('--pants-subprocessdir', advanced=True, default=os.path.join(buildroot, '.pids'),
help='The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).')
register('--pants-config-files', advanced=True, type=list, daemon=False,
default=[get_default_pants_config_file()], help='Paths to Pants config files.')
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--pantsrc', advanced=True, type=bool, default=True,
help='Use pantsrc files.')
register('--pantsrc-files', advanced=True, type=list, metavar='<path>', daemon=False,
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register('--pythonpath', advanced=True, type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file', type=list, dest='target_spec_files', daemon=False,
help='Read additional specs from this file, one per line')
register('--verify-config', type=bool, default=True, daemon=False,
help='Verify that all config file values correspond to known options.')
register('--build-ignore', advanced=True, type=list, fromfile=True,
default=['.*/', default_rel_distdir, 'bower_components/',
'node_modules/', '*.egg-info/'],
help='Paths to ignore when identifying BUILD files. '
'This does not affect any other filesystem operations. '
'Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).')
register('--pants-ignore', advanced=True, type=list, fromfile=True,
default=['.*/', default_rel_distdir],
help='Paths to ignore for all filesystem operations performed by pants '
'(e.g. BUILD file scanning, glob matching, etc). '
'Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore).')
register('--glob-expansion-failure', type=str,
choices=GlobMatchErrorBehavior.allowed_values,
default=GlobMatchErrorBehavior.default_option_value,
help="Raise an exception if any targets declaring source files "
"fail to match any glob provided in the 'sources' argument.")
register('--exclude-target-regexp', advanced=True, type=list, default=[], daemon=False,
metavar='<regexp>', help='Exclude target roots that match these regexes.')
register('--subproject-roots', type=list, advanced=True, fromfile=True, default=[],
help='Paths that correspond with build roots for any subproject that this '
'project depends on.')
register('--owner-of', type=list, default=[], metavar='<path>',
help='Select the targets that own these files. '
'This is the third target calculation strategy along with the --changed '
'options and specifying the targets directly. These three types of target '
'selection are mutually exclusive.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d', '--logdir', advanced=True, metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register('--enable-pantsd', advanced=True, type=bool, default=False,
help='Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)')
# These facilitate configuring the native engine.
register('--native-engine-visualize-to', advanced=True, default=None, type=dir_option, daemon=False,
help='A directory to write execution and rule graphs to as `dot` files. The contents '
'of the directory will be overwritten if any filenames collide.')
register('--print-exception-stacktrace', advanced=True, type=bool,
help='Print to console the full exception stack trace if encountered.')
# BinaryUtil options.
register('--binaries-baseurls', type=list, advanced=True,
default=['https://binaries.pantsbuild.org'],
help='List of URLs from which binary tools are downloaded. URLs are '
'searched in order until the requested path is found.')
register('--binaries-fetch-timeout-secs', type=int, default=30, advanced=True, daemon=False,
help='Timeout in seconds for URL reads when fetching binary tools from the '
'repos specified by --baseurls.')
register('--binaries-path-by-id', type=dict, advanced=True,
help=("Maps output of uname for a machine to a binary search path: "
"(sysname, id) -> (os, arch), e.g. {('darwin', '15'): ('mac', '10.11'), "
"('linux', 'arm32'): ('linux', 'arm32')}."))
register('--allow-external-binary-tool-downloads', type=bool, default=True, advanced=True,
help="If False, require BinaryTool subclasses to download their contents from urls "
"generated from --binaries-baseurls, even if the tool has an external url "
"generator. This can be necessary if using Pants in an environment which cannot "
"contact the wider Internet.")
# Pants Daemon options.
register('--pantsd-pailgun-host', advanced=True, default='127.0.0.1',
help='The host to bind the pants nailgun server to.')
register('--pantsd-pailgun-port', advanced=True, type=int, default=0,
help='The port to bind the pants nailgun server to. Defaults to a random port.')
register('--pantsd-log-dir', advanced=True, default=None,
help='The directory to log pantsd output to.')
register('--pantsd-fs-event-workers', advanced=True, type=int, default=4,
help='The number of workers to use for the filesystem event service executor pool.')
register('--pantsd-invalidation-globs', advanced=True, type=list, fromfile=True, default=[],
help='Filesystem events matching any of these globs will trigger a daemon restart.')
# Watchman options.
register('--watchman-version', advanced=True, default='4.9.0-pants1', help='Watchman version.')
register('--watchman-supportdir', advanced=True, default='bin/watchman',
help='Find watchman binaries under this dir. Used as part of the path to lookup '
'the binary with --binaries-baseurls and --pants-bootstrapdir.')
register('--watchman-startup-timeout', type=float, advanced=True, default=30.0,
help='The watchman socket timeout (in seconds) for the initial `watch-project` command. '
'This may need to be set higher for larger repos due to watchman startup cost.')
register('--watchman-socket-timeout', type=float, advanced=True, default=5.0,
help='The watchman client socket timeout in seconds.')
register('--watchman-socket-path', type=str, advanced=True, default=None,
help='The path to the watchman UNIX socket. This can be overridden if the default '
'absolute path length exceeds the maximum allowed by the OS.')
# This option changes the parser behavior in a fundamental way (which currently invalidates
# all caches), and needs to be parsed out early, so we make it a bootstrap option.
register('--build-file-imports', choices=['allow', 'warn', 'error'], default='warn',
help='Whether to allow import statements in BUILD files')
register('--remote-store-server',
help='host:port of grpc server to use as remote execution file store')
register('--remote-execution-server',
help='host:port of grpc server to use as remote execution scheduler')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l', '--level', choices=['debug', 'info', 'warn'], default='info', recursive=True,
help='Set the logging level.')
register('-q', '--quiet', type=bool, recursive=True,
help='Squelches most console output.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors', type=bool, default=True, recursive=True,
help='Set whether log messages are displayed in color.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version', advanced=True, default=pants_version(),
help='Use this pants version.')
register('--plugins', advanced=True, type=list, help='Load these plugins.')
register('--plugin-cache-dir', advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register('--backend-packages', advanced=True, type=list,
help='Load backends from these packages that are already on the path.')
register('--pants-bootstrapdir', advanced=True, metavar='<dir>', default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir', advanced=True, metavar='<dir>', default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register('--pants-config-files', advanced=True, type=list,
default=[get_default_pants_config_file()], help='Paths to Pants config files.')
# TODO: Deprecate --config-override in favor of --pants-config-files.
# But only once we're able to both append and override list-valued options, as there are
# use-cases for both here.
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--config-override', advanced=True, type=list, metavar='<path>',
help='A second config file, to override pants.ini.')
register('--pantsrc', advanced=True, type=bool, default=True,
help='Use pantsrc files.')
register('--pantsrc-files', advanced=True, type=list, metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register('--pythonpath', advanced=True, type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file', type=list, dest='target_spec_files',
help='Read additional specs from this file, one per line')
register('--verify-config', type=bool, default=True,
help='Verify that all config file values correspond to known options.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d', '--logdir', advanced=True, metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register('--enable-pantsd', advanced=True, type=bool, default=False,
help='Enables use of the pants daemon. (Beta)')
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l',
'--level',
choices=['debug', 'info', 'warn'],
default='info',
recursive=True,
help='Set the logging level.')
register('-q',
'--quiet',
type=bool,
recursive=True,
help='Squelches most console output.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors',
type=bool,
default=sys.stdout.isatty(),
recursive=True,
help='Set whether log messages are displayed in color.')
# Pants code uses this only to verify that we are of the requested version. However
# setup scripts, runner scripts, IDE plugins, etc., may grep this out of pants.ini
# and use it to select the right version.
# Note that to print the version of the pants instance you're running, use -v, -V or --version.
register('--pants-version',
advanced=True,
default=pants_version(),
help='Use this pants version.')
register('--plugins',
advanced=True,
type=list,
help='Load these plugins.')
register('--plugin-cache-dir',
advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register(
'--backend-packages',
advanced=True,
type=list,
default=[
'pants.backend.graph_info', 'pants.backend.python',
'pants.backend.jvm', 'pants.backend.codegen.antlr.java',
'pants.backend.codegen.antlr.python',
'pants.backend.codegen.jaxb',
'pants.backend.codegen.protobuf.java',
'pants.backend.codegen.ragel.java',
'pants.backend.codegen.thrift.java',
'pants.backend.codegen.thrift.python',
'pants.backend.codegen.wire.java', 'pants.backend.project_info'
],
help=
'Load backends from these packages that are already on the path. '
'Add contrib and custom backends to this list.')
register('--pants-bootstrapdir',
advanced=True,
metavar='<dir>',
default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir',
advanced=True,
metavar='<dir>',
default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir',
advanced=True,
metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register(
'--pants-subprocessdir',
advanced=True,
default=os.path.join(buildroot, '.pids'),
help=
'The directory to use for tracking subprocess metadata, if any. This should '
'live outside of the dir used by `--pants-workdir` to allow for tracking '
'subprocesses that outlive the workdir data (e.g. `./pants server`).'
)
register('--pants-config-files',
advanced=True,
type=list,
default=[get_default_pants_config_file()],
help='Paths to Pants config files.')
# TODO: Deprecate the --pantsrc/--pantsrc-files options? This would require being able
# to set extra config file locations in an initial bootstrap config file.
register('--config-override',
advanced=True,
type=list,
metavar='<path>',
removal_version='1.6.0.dev0',
removal_hint=
'Use --pants-config-files=<second config file path> instead.',
help='A second config file, to override pants.ini.')
register('--pantsrc',
advanced=True,
type=bool,
default=True,
help='Use pantsrc files.')
register('--pantsrc-files',
advanced=True,
type=list,
metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register(
'--pythonpath',
advanced=True,
type=list,
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file',
type=list,
dest='target_spec_files',
help='Read additional specs from this file, one per line')
register(
'--verify-config',
type=bool,
default=True,
help=
'Verify that all config file values correspond to known options.')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d',
'--logdir',
advanced=True,
metavar='<dir>',
help='Write logs to files under this directory.')
# This facilitates bootstrap-time configuration of pantsd usage such that we can
# determine whether or not to use the Pailgun client to invoke a given pants run
# without resorting to heavier options parsing.
register(
'--enable-pantsd',
advanced=True,
type=bool,
default=False,
help=
'Enables use of the pants daemon (and implicitly, the v2 engine). (Beta)'
)
# This facilitates use of the v2 engine, sans daemon.
# TODO: Add removal_version='1.5.0.dev0' before 1.4 lands.
register('--enable-v2-engine',
advanced=True,
type=bool,
default=True,
help='Enables use of the v2 engine.')
# These facilitate configuring the native engine.
register('--native-engine-version',
advanced=True,
default=pkg_resources.resource_string(
'pants.engine', 'native_engine_version').strip(),
help='Native engine version.')
register(
'--native-engine-supportdir',
advanced=True,
default='bin/native-engine',
help=
'Find native engine binaries under this dir. Used as part of the path to '
'lookup the binary with --binary-util-baseurls and --pants-bootstrapdir.'
)
register(
'--native-engine-visualize-to',
advanced=True,
default=None,
type=dir_option,
help=
'A directory to write execution and rule graphs to as `dot` files. The contents '
'of the directory will be overwritten if any filenames collide.')
# BinaryUtil options.
register(
'--binaries-baseurls',
type=list,
advanced=True,
default=['https://binaries.pantsbuild.org'],
help=
'List of URLs from which binary tools are downloaded. URLs are searched in '
'order until the requested path is found.')
register(
'--binaries-fetch-timeout-secs',
type=int,
default=30,
advanced=True,
help=
'Timeout in seconds for URL reads when fetching binary tools from the '
'repos specified by --baseurls.')
register(
'--binaries-path-by-id',
type=dict,
advanced=True,
help=
('Maps output of uname for a machine to a binary search path. e.g. '
'{("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux", "arm32"]}'
))
def test_set_cachedir(self):
with temporary_file() as temp:
with environment_as(XDG_CACHE_HOME=temp.name):
self.assertEqual(os.path.join(temp.name, 'pants'), get_pants_cachedir())
def test_get_pants_cachedir() -> None:
assert Native().default_cache_path() == get_pants_cachedir()
def register_bootstrap_options(cls, register):
"""Register bootstrap options.
"Bootstrap options" are a small set of options whose values are useful when registering other
options. Therefore we must bootstrap them early, before other options are registered, let
alone parsed.
Bootstrap option values can be interpolated into the config file, and can be referenced
programatically in registration code, e.g., as register.bootstrap.pants_workdir.
Note that regular code can also access these options as normal global-scope options. Their
status as "bootstrap options" is only pertinent during option registration.
"""
buildroot = get_buildroot()
# Although logging supports the WARN level, its not documented and could conceivably be yanked.
# Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly
# setup a 'WARN' logging level name that maps to 'WARNING'.
logging.addLevelName(logging.WARNING, 'WARN')
register('-l', '--level', choices=['debug', 'info', 'warn'], default='info', recursive=True,
help='Set the logging level.')
register('-q', '--quiet', action='store_true',
help='Squelches all console output apart from errors.')
# Not really needed in bootstrap options, but putting it here means it displays right
# after -l and -q in help output, which is conveniently contextual.
register('--colors', action='store_true', default=True, recursive=True,
help='Set whether log messages are displayed in color.')
# NB: Right now this option is a placeholder that is unused within pants itself except when
# specified on the command line to print the OSS pants version. Both the IntelliJ Pants plugin
# and the pantsbuild/setup bootstrap script grep for pants_version though so this option
# registration serves in part as documentation of the dependency.
# TODO(John Sirois): Move pantsbuild.pants bootstrapping into pants itself and have it use this
# version option directly.
register('-V', '--pants-version', '--version', # NB: pants_version is the 1st long option
# since that's the one read from pants.ini;
# the version form only works from the CLI.
nargs='?', # Allows using the flag with no args on the CLI to print version as well
# as setting the version in pants.ini
default=pants_version(), # Displays the current version correctly in `./pants -h`.
const=pants_version(), # Displays the current version via `./pants -V`.
help="Prints pants' version number and exits.")
register('--plugins', advanced=True, type=list_option, help='Load these plugins.')
register('--plugin-cache-dir', advanced=True,
default=os.path.join(get_pants_cachedir(), 'plugins'),
help='Cache resolved plugin requirements here.')
register('--backend-packages', advanced=True, type=list_option,
help='Load backends from these packages that are already on the path.')
register('--pants-bootstrapdir', advanced=True, metavar='<dir>', default=get_pants_cachedir(),
help='Use this dir for global cache.')
register('--pants-configdir', advanced=True, metavar='<dir>', default=get_pants_configdir(),
help='Use this dir for global config files.')
register('--pants-workdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, '.pants.d'),
help='Write intermediate output files to this dir.')
register('--pants-supportdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'build-support'),
help='Use support files from this dir.')
register('--pants-distdir', advanced=True, metavar='<dir>',
default=os.path.join(buildroot, 'dist'),
help='Write end-product artifacts to this dir.')
register('--config-override', advanced=True, action='append', metavar='<path>',
help='A second config file, to override pants.ini.')
register('--pantsrc', advanced=True, action='store_true', default=True,
help='Use pantsrc files.')
register('--pantsrc-files', advanced=True, action='append', metavar='<path>',
default=['/etc/pantsrc', '~/.pants.rc'],
help='Override config with values from these files. '
'Later files override earlier ones.')
register('--pythonpath', advanced=True, action='append',
help='Add these directories to PYTHONPATH to search for plugins.')
register('--target-spec-file', action='append', dest='target_spec_files',
help='Read additional specs from this file, one per line')
# These logging options are registered in the bootstrap phase so that plugins can log during
# registration and not so that their values can be interpolated in configs.
register('-d', '--logdir', advanced=True, metavar='<dir>',
help='Write logs to files under this directory.')
def _make_ensime_cache_dir(self):
bootstrap_dir = get_pants_cachedir()
cache_dir = os.path.join(bootstrap_dir, 'ensime')
safe_mkdir(cache_dir)
return cache_dir