def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
extra_paths = self.config.getlist('backends', 'python-path', [])
if extra_paths:
sys.path.extend(extra_paths)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_config(self.config)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def run(self):
# Register our exiter at the beginning of the run() method so that any code in this process from
# this point onwards will use that exiter in the case of a fatal error.
ExceptionSink.reset_exiter(self._exiter)
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
if global_bootstrap_options.enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.warn(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def _initialize_build_configuration(
plugin_resolver: PluginResolver,
options_bootstrapper: OptionsBootstrapper,
env: CompleteEnvironment,
) -> BuildConfiguration:
"""Initialize a BuildConfiguration for the given OptionsBootstrapper.
NB: This method:
1. has the side-effect of (idempotently) adding PYTHONPATH entries for this process
2. is expensive to call, because it might resolve plugins from the network
"""
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
working_set = plugin_resolver.resolve(options_bootstrapper, env)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in bootstrap_options.pythonpath:
if path not in sys.path:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
return load_backends_and_plugins(
bootstrap_options.plugins,
working_set,
bootstrap_options.backend_packages,
)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
ExceptionSink.set_destination(
bootstrap_options.for_global_scope().pants_workdir)
if bootstrap_options.for_global_scope().enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.warn(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
return self._run(global_bootstrap_options.enable_pantsd,
exiter=self._exiter,
args=self._args,
env=self._env,
options_bootstrapper=options_bootstrapper)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
return self._run(is_remote=global_bootstrap_options.enable_pantsd,
exiter=self._exiter,
args=self._args,
env=self._env,
process_metadata_dir=global_bootstrap_options.pants_subprocessdir,
options_bootstrapper=options_bootstrapper)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
global_bootstrap_options = options_bootstrapper.get_bootstrap_options(
).for_global_scope()
return self._run(global_bootstrap_options.enable_pantsd,
exiter=self._exiter,
args=self._args,
env=self._env,
options_bootstrapper=options_bootstrapper)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
global_bootstrap_options = options_bootstrapper.get_bootstrap_options(
).for_global_scope()
return self._run(
is_remote=global_bootstrap_options.enable_pantsd,
exiter=self._exiter,
args=self._args,
env=self._env,
process_metadata_dir=global_bootstrap_options.pants_subprocessdir,
options_bootstrapper=options_bootstrapper)
def test_setting_pants_config_in_config(self):
# Test that setting pants_config in the config file has no effect.
with temporary_dir() as tmpdir:
config1 = os.path.join(tmpdir, 'config1')
config2 = os.path.join(tmpdir, 'config2')
with open(config1, 'w') as out1:
out1.write(b"[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2))
with open(config2, 'w') as out2:
out2.write(b'[DEFAULT]\nlogdir: logdir2\n')
ob = OptionsBootstrapper(env={}, args=["--pants-config-files=['{}']".format(config1)])
logdir = ob.get_bootstrap_options().for_global_scope().logdir
self.assertEqual('logdir1', logdir)
def test_setting_pants_config_in_config(self):
# Test that setting pants_config in the config file has no effect.
with temporary_dir() as tmpdir:
config1 = os.path.join(tmpdir, "config1")
config2 = os.path.join(tmpdir, "config2")
with open(config1, "w") as out1:
out1.write(b"[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2))
with open(config2, "w") as out2:
out2.write(b"[DEFAULT]\nlogdir: logdir2\n")
ob = OptionsBootstrapper(env={}, args=["--pants-config-files=['{}']".format(config1)])
logdir = ob.get_bootstrap_options().for_global_scope().logdir
self.assertEqual("logdir1", logdir)
def _test_bootstrap_options(self, config, env, args, **expected_entries):
with temporary_file() as fp:
fp.write('[DEFAULT]\n')
if config:
for k, v in config.items():
fp.write('{0}: {1}\n'.format(k, v))
fp.close()
bootstrapper = OptionsBootstrapper(env=env, configpath=fp.name, args=args)
vals = bootstrapper.get_bootstrap_options().for_global_scope()
vals_dict = {k: getattr(vals, k) for k in expected_entries}
self.assertEquals(expected_entries, vals_dict)
def _test_bootstrap_options(self, config, env, args, **expected_entries):
with temporary_file() as fp:
fp.write('[DEFAULT]\n')
if config:
for k, v in config.items():
fp.write('{0}: {1}\n'.format(k, v))
fp.close()
args = args + self._config_path(fp.name)
bootstrapper = OptionsBootstrapper(env=env, args=args)
vals = bootstrapper.get_bootstrap_options().for_global_scope()
vals_dict = {k: getattr(vals, k) for k in expected_entries}
self.assertEquals(expected_entries, vals_dict)
def test_file_spec_args(self):
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(dedent(
'''
foo
bar
'''
))
tmp.flush()
cmdline = './pants --target-spec-file={filename} compile morx fleem'.format(filename=tmp.name)
bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline))
bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope()
options = self._parse(cmdline, bootstrap_option_values=bootstrap_options)
sorted_specs = sorted(options.target_specs)
self.assertEqual(['bar', 'fleem', 'foo', 'morx'], sorted_specs)
def test_file_spec_args(self):
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(dedent(
'''
foo
bar
'''
))
tmp.flush()
cmdline = './pants --target-spec-file={filename} compile morx fleem'.format(filename=tmp.name)
bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline))
bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope()
options = self._parse(cmdline, bootstrap_option_values=bootstrap_options)
sorted_specs = sorted(options.target_specs)
self.assertEqual(['bar', 'fleem', 'foo', 'morx'], sorted_specs)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
if bootstrap_options.for_global_scope().enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.debug('caught client exception: {!r}, falling back to non-daemon mode'.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
return LocalPantsRunner(self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper).run()
def test_file_spec_args(self):
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(dedent(
"""
foo
bar
"""
))
tmp.flush()
# Note that we prevent loading a real pants.ini during get_bootstrap_options().
cmdline = './pants --target-spec-file={filename} --pants-config-files="[]" ' \
'compile morx:tgt fleem:tgt'.format(
filename=tmp.name)
bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline))
bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope()
options = self._parse(cmdline, bootstrap_option_values=bootstrap_options)
sorted_specs = sorted(options.target_specs)
self.assertEqual(['bar', 'fleem:tgt', 'foo', 'morx:tgt'], sorted_specs)
def test_file_spec_args(self):
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(dedent(
"""
foo
bar
"""
))
tmp.flush()
# Note that we prevent loading a real pants.ini during get_bootstrap_options().
cmdline = './pants --target-spec-file={filename} --pants-config-files="[]" ' \
'compile morx:tgt fleem:tgt'.format(
filename=tmp.name)
bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline))
bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope()
options = self._parse(cmdline, bootstrap_option_values=bootstrap_options)
sorted_specs = sorted(options.target_specs)
self.assertEqual(['bar', 'fleem:tgt', 'foo', 'morx:tgt'], sorted_specs)
def create(
cls,
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
) -> Options:
global_bootstrap_options = options_bootstrapper.get_bootstrap_options(
).for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
f"Version mismatch: Requested version was {global_bootstrap_options.pants_version}, "
f"our version is {pants_version()}.")
# Parse and register options.
known_scope_infos = [
si for optionable in build_configuration.all_optionables
for si in optionable.known_scope_infos()
]
options = options_bootstrapper.get_full_options(known_scope_infos)
GlobalOptions.validate_instance(options.for_global_scope())
return options
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
if bootstrap_options.for_global_scope().enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.debug(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
return LocalPantsRunner(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper).run()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def __init__(self, options_bootstrapper: OptionsBootstrapper) -> None:
self._options_bootstrapper = options_bootstrapper
self._bootstrap_options = options_bootstrapper.get_bootstrap_options(
).for_global_scope()
self._working_set = PluginResolver(
self._options_bootstrapper).resolve()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
# Add scopes for global subsystem instances.
for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types():
known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def _parse_bootstrap_options():
options_bootstrapper = OptionsBootstrapper()
return options_bootstrapper.get_bootstrap_options(
).for_global_scope()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
