def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Add any extra paths to python path (eg for loading extra source backends) extra_paths = self.config.getlist('backends', 'python-path', []) if extra_paths: sys.path.extend(extra_paths) # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scopes = [''] for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes) self.register_options() self.run_tracker = RunTracker.from_config(self.config) report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: %s' % url) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() self._expand_goals_and_specs()
def test_create_bootstrapped_options(self): # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate # correctly into regular config. with temporary_file() as fp: fp.write(dedent(""" [foo] bar: %(pants_workdir)s/baz [fruit] apple: %(pants_supportdir)s/banana """)) fp.close() bootstrapper = OptionsBootstrapper(env={ 'PANTS_SUPPORTDIR': '/pear' }, configpath=fp.name, args=['--pants-workdir=/qux']) opts = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('fruit', ScopeInfo.TASK) ]) opts.register('', '--pants-workdir') # So we don't choke on it on the cmd line. opts.register('foo', '--bar') opts.register('fruit', '--apple') self.assertEquals('/qux/baz', opts.for_scope('foo').bar) self.assertEquals('/pear/banana', opts.for_scope('fruit').apple)
def test_create_bootstrapped_options(self): # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate # correctly into regular config. with temporary_file() as fp: fp.write(dedent(""" [foo] bar: %(pants_workdir)s/baz [fruit] apple: %(pants_supportdir)s/banana """)) fp.close() args = ['--pants-workdir=/qux'] + self._config_path(fp.name) bootstrapper = OptionsBootstrapper(env={ 'PANTS_SUPPORTDIR': '/pear' }, args=args) opts = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('fruit', ScopeInfo.TASK) ]) # So we don't choke on these on the cmd line. opts.register('', '--pants-workdir') opts.register('', '--pants-config-files') opts.register('foo', '--bar') opts.register('fruit', '--apple') self.assertEquals('/qux/baz', opts.for_scope('foo').bar) self.assertEquals('/pear/banana', opts.for_scope('fruit').apple)
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) bootstrap_options = options_bootstrapper.get_bootstrap_options() ExceptionSink.set_destination( bootstrap_options.for_global_scope().pants_workdir) if bootstrap_options.for_global_scope().enable_pantsd: try: return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run() except RemotePantsRunner.Fallback as e: logger.warn( 'caught client exception: {!r}, falling back to non-daemon mode' .format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner runner = LocalPantsRunner.create( self._exiter, self._args, self._env, options_bootstrapper=options_bootstrapper) runner.set_start_time(self._start_time) return runner.run()
def test_create_bootstrapped_options(self): # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate # correctly into regular config. with temporary_file() as fp: fp.write( dedent( """ [foo] bar: %(pants_workdir)s/baz [fruit] apple: %(pants_supportdir)s/banana """ ) ) fp.close() args = ["--pants-workdir=/qux"] + self._config_path(fp.name) bootstrapper = OptionsBootstrapper(env={"PANTS_SUPPORTDIR": "/pear"}, args=args) opts = bootstrapper.get_full_options( known_scope_infos=[ ScopeInfo("", ScopeInfo.GLOBAL), ScopeInfo("foo", ScopeInfo.TASK), ScopeInfo("fruit", ScopeInfo.TASK), ] ) # So we don't choke on these on the cmd line. opts.register("", "--pants-workdir") opts.register("", "--pants-config-files") opts.register("foo", "--bar") opts.register("fruit", "--apple") self.assertEquals("/qux/baz", opts.for_scope("foo").bar) self.assertEquals("/pear/banana", opts.for_scope("fruit").apple)
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() return self._run(global_bootstrap_options.enable_pantsd, exiter=self._exiter, args=self._args, env=self._env, options_bootstrapper=options_bootstrapper)
def test_create_bootstrapped_multiple_config_override(self): # check with multiple config files, the latest values always get taken # in this case worker_count will be overwritten, while fruit stays the same with temporary_file() as fp: fp.write(dedent(""" [compile.apt] worker_count: 1 [fruit] apple: red """)) fp.close() args = ['--config-override={}'.format(fp.name)] + self._config_path(fp.name) bootstrapper_single_config = OptionsBootstrapper(args=args) opts_single_config = bootstrapper_single_config.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('compile.apt', ScopeInfo.TASK), ScopeInfo('fruit', ScopeInfo.TASK), ]) # So we don't choke on these on the cmd line. opts_single_config.register('', '--pants-config-files') opts_single_config.register('', '--config-override', type=list) opts_single_config.register('compile.apt', '--worker-count') opts_single_config.register('fruit', '--apple') self.assertEquals('1', opts_single_config.for_scope('compile.apt').worker_count) self.assertEquals('red', opts_single_config.for_scope('fruit').apple) with temporary_file() as fp2: fp2.write(dedent(""" [compile.apt] worker_count: 2 """)) fp2.close() args = ['--config-override={}'.format(fp.name), '--config-override={}'.format(fp2.name)] + self._config_path(fp.name) bootstrapper_double_config = OptionsBootstrapper(args=args) opts_double_config = bootstrapper_double_config.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('compile.apt', ScopeInfo.TASK), ScopeInfo('fruit', ScopeInfo.TASK), ]) # So we don't choke on these on the cmd line. opts_double_config.register('', '--pants-config-files') opts_double_config.register('', '--config-override', type=list) opts_double_config.register('compile.apt', '--worker-count') opts_double_config.register('fruit', '--apple') self.assertEquals('2', opts_double_config.for_scope('compile.apt').worker_count) self.assertEquals('red', opts_double_config.for_scope('fruit').apple)
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() return self._run(is_remote=global_bootstrap_options.enable_pantsd, exiter=self._exiter, args=self._args, env=self._env, process_metadata_dir=global_bootstrap_options.pants_subprocessdir, options_bootstrapper=options_bootstrapper)
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) global_bootstrap_options = options_bootstrapper.get_bootstrap_options( ).for_global_scope() return self._run(global_bootstrap_options.enable_pantsd, exiter=self._exiter, args=self._args, env=self._env, options_bootstrapper=options_bootstrapper)
def prepare_task(self, config=None, args=None, targets=None, build_graph=None, build_file_parser=None, address_mapper=None, console_outstream=None, workspace=None): """Prepares a Task for execution. task_type: The class of the Task to create. config: An optional string representing the contents of a pants.ini config. args: optional list of command line flags, these should be prefixed with '--test-'. targets: optional list of Target objects passed on the command line. Returns a new Task ready to execute. """ task_type = self.task_type() assert issubclass(task_type, Task), 'task_type must be a Task subclass, got %s' % task_type config = create_config(config or '') workdir = os.path.join(config.getdefault('pants_workdir'), 'test', task_type.__name__) bootstrap_options = OptionsBootstrapper().get_bootstrap_options() options = Options(env={}, config=config, known_scopes=['', 'test'], args=args or []) # A lot of basic code uses these options, so always register them. register_bootstrap_options(options.register_global) # We need to wrap register_global (can't set .bootstrap attr on the bound instancemethod). def register_global_wrapper(*args, **kwargs): return options.register_global(*args, **kwargs) register_global_wrapper.bootstrap = bootstrap_options.for_global_scope() register_global_options(register_global_wrapper) task_type.options_scope = 'test' task_type.register_options_on_scope(options) run_tracker = create_run_tracker() context = Context(config, options, run_tracker, targets or [], build_graph=build_graph, build_file_parser=build_file_parser, address_mapper=address_mapper, console_outstream=console_outstream, workspace=workspace) return task_type(context, workdir)
def test_setting_pants_config_in_config(self): # Test that setting pants_config in the config file has no effect. with temporary_dir() as tmpdir: config1 = os.path.join(tmpdir, "config1") config2 = os.path.join(tmpdir, "config2") with open(config1, "w") as out1: out1.write(b"[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2)) with open(config2, "w") as out2: out2.write(b"[DEFAULT]\nlogdir: logdir2\n") ob = OptionsBootstrapper(env={}, args=["--pants-config-files=['{}']".format(config1)]) logdir = ob.get_bootstrap_options().for_global_scope().logdir self.assertEqual("logdir1", logdir)
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) global_bootstrap_options = options_bootstrapper.get_bootstrap_options( ).for_global_scope() return self._run( is_remote=global_bootstrap_options.enable_pantsd, exiter=self._exiter, args=self._args, env=self._env, process_metadata_dir=global_bootstrap_options.pants_subprocessdir, options_bootstrapper=options_bootstrapper)
def _test_bootstrap_options(self, config, env, args, **expected_entries): with temporary_file() as fp: fp.write('[DEFAULT]\n') if config: for k, v in config.items(): fp.write('{0}: {1}\n'.format(k, v)) fp.close() bootstrapper = OptionsBootstrapper(env=env, configpath=fp.name, args=args) vals = bootstrapper.get_bootstrap_options().for_global_scope() vals_dict = {k: getattr(vals, k) for k in expected_entries} self.assertEquals(expected_entries, vals_dict)
def test_setting_pants_config_in_config(self): # Test that setting pants_config in the config file has no effect. with temporary_dir() as tmpdir: config1 = os.path.join(tmpdir, 'config1') config2 = os.path.join(tmpdir, 'config2') with open(config1, 'w') as out1: out1.write(b"[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2)) with open(config2, 'w') as out2: out2.write(b'[DEFAULT]\nlogdir: logdir2\n') ob = OptionsBootstrapper(env={}, args=["--pants-config-files=['{}']".format(config1)]) logdir = ob.get_bootstrap_options().for_global_scope().logdir self.assertEqual('logdir1', logdir)
def test_create_bootstrapped_multiple_config_override(self): # check with multiple config files, the latest values always get taken # in this case strategy will be overwritten, while fruit stays the same with temporary_file() as fp: fp.write(dedent(""" [compile.apt] strategy: global [fruit] apple: red """)) fp.close() bootstrapper_single_config = OptionsBootstrapper(configpath=fp.name, args=['--config-override={}'.format(fp.name)]) opts_single_config = bootstrapper_single_config.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('compile.apt', ScopeInfo.TASK), ScopeInfo('fruit', ScopeInfo.TASK), ]) opts_single_config.register('', '--config-override') # So we don't choke on it on the cmd line. opts_single_config.register('compile.apt', '--strategy') opts_single_config.register('fruit', '--apple') self.assertEquals('global', opts_single_config.for_scope('compile.apt').strategy) self.assertEquals('red', opts_single_config.for_scope('fruit').apple) with temporary_file() as fp2: fp2.write(dedent(""" [compile.apt] strategy: isolated """)) fp2.close() bootstrapper_double_config = OptionsBootstrapper( configpath=fp.name, args=['--config-override={}'.format(fp.name), '--config-override={}'.format(fp2.name)]) opts_double_config = bootstrapper_double_config.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('compile.apt', ScopeInfo.TASK), ScopeInfo('fruit', ScopeInfo.TASK), ]) opts_double_config.register('', '--config-override') # So we don't choke on it on the cmd line. opts_double_config.register('compile.apt', '--strategy') opts_double_config.register('fruit', '--apple') self.assertEquals('isolated', opts_double_config.for_scope('compile.apt').strategy) self.assertEquals('red', opts_double_config.for_scope('fruit').apple)
def _test_bootstrap_options(self, config, env, args, **expected_entries): with temporary_file() as fp: fp.write('[DEFAULT]\n') if config: for k, v in config.items(): fp.write('{0}: {1}\n'.format(k, v)) fp.close() args = args + self._config_path(fp.name) bootstrapper = OptionsBootstrapper(env=env, args=args) vals = bootstrapper.get_bootstrap_options().for_global_scope() vals_dict = {k: getattr(vals, k) for k in expected_entries} self.assertEquals(expected_entries, vals_dict)
def test_file_spec_args(self): with tempfile.NamedTemporaryFile() as tmp: tmp.write(dedent( ''' foo bar ''' )) tmp.flush() cmdline = './pants --target-spec-file={filename} compile morx fleem'.format(filename=tmp.name) bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline)) bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope() options = self._parse(cmdline, bootstrap_option_values=bootstrap_options) sorted_specs = sorted(options.target_specs) self.assertEqual(['bar', 'fleem', 'foo', 'morx'], sorted_specs)
def select(argv): # Parse positional arguments to the script. args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:]) # Resolve bootstrap options with a fake empty command line. options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]]) subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory) known_scope_infos = reduce(set.union, (ss.known_scope_infos() for ss in subsystems), set()) options = options_bootstrapper.get_full_options(known_scope_infos) # Initialize Subsystems. Subsystem.set_options(options) # If the filename provided ends in a known archive extension (such as ".tar.gz"), then we get the # appropriate Archiver to pass to BinaryUtil. archiver_for_current_binary = None filename = args.filename or args.util_name try: archiver_for_current_binary = archiver_for_path(filename) # BinaryRequest requires the `name` field to be provided without an extension, as it appends the # archiver's extension if one is provided, so we have to remove it here. filename = filename[:-(len(archiver_for_current_binary.extension) + 1)] except ValueError: pass binary_util = BinaryUtil.Factory.create() binary_request = BinaryRequest( supportdir='bin/{}'.format(args.util_name), version=args.version, name=filename, platform_dependent=True, external_url_generator=None, archiver=archiver_for_current_binary) return binary_util.select(binary_request)
def test_full_options_caching(self): with temporary_file_path() as config: args = self._config_path(config) bootstrapper = OptionsBootstrapper.create(env={}, args=args) opts1 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK) ]) opts2 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('', ScopeInfo.GLOBAL) ]) self.assertIs(opts1, opts2) opts3 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('', ScopeInfo.GLOBAL) ]) self.assertIs(opts1, opts3) opts4 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIsNot(opts1, opts4) opts5 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIs(opts4, opts5) self.assertIsNot(opts1, opts5)
def setup_graph( options_bootstrapper: OptionsBootstrapper, build_configuration: BuildConfiguration, env: CompleteEnvironment, executor: Optional[PyExecutor] = None, local_only: bool = False, ) -> GraphScheduler: native = Native() build_root = get_buildroot() bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) options = options_bootstrapper.full_options(build_configuration) assert bootstrap_options is not None executor = executor or PyExecutor( *GlobalOptions.compute_executor_arguments(bootstrap_options)) execution_options = ExecutionOptions.from_options( options, env, local_only=local_only) return EngineInitializer.setup_graph_extended( build_configuration, execution_options, native=native, executor=executor, pants_ignore_patterns=GlobalOptions.compute_pants_ignore( build_root, bootstrap_options), use_gitignore=bootstrap_options.pants_ignore_use_gitignore, local_store_dir=bootstrap_options.local_store_dir, local_execution_root_dir=bootstrap_options. local_execution_root_dir, named_caches_dir=bootstrap_options.named_caches_dir, ca_certs_path=bootstrap_options.ca_certs_path, build_root=build_root, include_trace_on_error=bootstrap_options.print_stacktrace, native_engine_visualize_to=bootstrap_options. native_engine_visualize_to, )
def create_options_bootstrapper( *config_paths: str) -> OptionsBootstrapper: return OptionsBootstrapper.create( env={}, args=[f"--pantsrc-files={cp}" for cp in config_paths], allow_pantsrc=True, )
def get_bootstrap_options(self, cli_options=()): """Retrieves bootstrap options. :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. """ args = tuple(['--pants-config-files=[]']) + tuple(cli_options) return OptionsBootstrapper.create(args=args).bootstrap_options.for_global_scope()
def _init_engine(self, local_store_dir: Optional[str] = None) -> None: if self._scheduler is not None: return options_bootstrapper = OptionsBootstrapper.create( args=["--pants-config-files=[]"]) local_store_dir = (local_store_dir or options_bootstrapper.bootstrap_options. for_global_scope().local_store_dir) # NB: This uses the long form of initialization because it needs to directly specify # `cls.alias_groups` rather than having them be provided by bootstrap options. graph_session = EngineInitializer.setup_legacy_graph_extended( pants_ignore_patterns=[], local_store_dir=local_store_dir, build_file_imports_behavior=BuildFileImportsBehavior.error, native=init_native(), options_bootstrapper=options_bootstrapper, build_root=self.build_root, build_configuration=self.build_config(), build_ignore_patterns=None, ).new_session(zipkin_trace_v2=False, build_id="buildid_for_test") self._scheduler = graph_session.scheduler_session self._build_graph, self._address_mapper = graph_session.create_build_graph( Specs(address_specs=AddressSpecs([]), filesystem_specs=FilesystemSpecs([])), self._build_root(), )
def execute(self): try: pantsd = PantsDaemon.Factory.create(OptionsBootstrapper.create(), full_init=False) with pantsd.lifecycle_lock: pantsd.terminate() except ProcessManager.NonResponsiveProcess as e: raise TaskError('failure while terminating pantsd: {}'.format(e))
def test_options_parsing_request(self): parse_request = OptionsParseRequest.create( ['./pants', '-ldebug', '--python-setup-wheel-version=3.13.37', 'binary', 'src/python::'], dict(PANTS_ENABLE_PANTSD='True', PANTS_BINARIES_BASEURLS='["https://bins.com"]') ) # TODO: Once we have the ability to get FileContent for arbitrary # paths outside of the buildroot, we can move the construction of # OptionsBootstrapper into an @rule by cooperatively calling # OptionsBootstrapper.produce_and_set_bootstrap_options() which # will yield lists of file paths for use as subject values and permit # us to avoid the direct file I/O that this currently requires. options_bootstrapper = OptionsBootstrapper.from_options_parse_request(parse_request) build_config = BuildConfigInitializer.get(options_bootstrapper) options = run_rule(parse_options, options_bootstrapper, build_config)[0] self.assertIn('binary', options.goals) global_options = options.for_global_scope() self.assertEquals(global_options.level, 'debug') self.assertEquals(global_options.enable_pantsd, True) self.assertEquals(global_options.binaries_baseurls, ['https://bins.com']) python_setup_options = options.for_scope('python-setup') self.assertEquals(python_setup_options.wheel_version, '3.13.37')
def setUp(self): super(BaseTest, self).setUp() Goal.clear() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root Subsystem.reset() self.create_file('pants.ini') build_configuration = BuildConfiguration() build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) self.bootstrap_option_values = OptionsBootstrapper().get_bootstrap_options().for_global_scope()
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper.create(args=['--loop', '--v1']) build_config = BuildConfigInitializer.get(ob) with self.assertRaises(OptionsError) as exc: OptionsInitializer.create(ob, build_config) self.assertIn('loop option only works with', str(exc.exception))
def runner_factory(sock, arguments, environment): exiter = self._exiter_class(sock) graph_helper = None deferred_exc = None # Capture the size of the graph prior to any warming, for stats. preceding_graph_size = self._scheduler_service.product_graph_len() self._logger.debug('resident graph size: %s', preceding_graph_size) self._logger.debug('execution commandline: %s', arguments) options, _ = OptionsInitializer( OptionsBootstrapper(args=arguments)).setup(init_logging=False) target_roots = self._target_roots_calculator.create( options, change_calculator=self._scheduler_service.change_calculator) try: self._logger.debug('warming the product graph via %s', self._scheduler_service) # N.B. This call is made in the pre-fork daemon context for reach and reuse of the # resident scheduler. graph_helper = self._scheduler_service.warm_product_graph( target_roots) except Exception: deferred_exc = sys.exc_info() self._logger.warning( 'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s', ''.join(traceback.format_exception(*deferred_exc))) return self._runner_class(sock, exiter, arguments, environment, target_roots, graph_helper, self.fork_lock, preceding_graph_size, deferred_exc)
def plugin_resolution(chroot=None, plugins=None): @contextmanager def provide_chroot(existing): if existing: yield existing, False else: with temporary_dir() as new_chroot: yield new_chroot, True with provide_chroot(chroot) as (root_dir, create_artifacts): env = {'PANTS_BOOTSTRAPDIR': root_dir} repo_dir = None if plugins: repo_dir = os.path.join(root_dir, 'repo') env.update(PANTS_PYTHON_REPOS_REPOS='[{!r}]'.format(repo_dir), PANTS_PYTHON_REPOS_INDEXES='[]', PANTS_PYTHON_SETUP_RESOLVER_CACHE_TTL='1') plugin_list = [] for plugin in plugins: version = None if isinstance(plugin, tuple): plugin, version = plugin plugin_list.append('{}=={}'.format(plugin, version) if version else plugin) if create_artifacts: create_plugin(repo_dir, plugin, version) env['PANTS_PLUGINS'] = '[{}]'.format(','.join(map(repr, plugin_list))) configpath = os.path.join(root_dir, 'pants.ini') if create_artifacts: touch(configpath) options_bootstrapper = OptionsBootstrapper(env=env, configpath=configpath, args=[]) plugin_resolver = PluginResolver(options_bootstrapper) cache_dir = plugin_resolver.plugin_cache_dir yield plugin_resolver.resolve(WorkingSet(entries=[])), root_dir, repo_dir, cache_dir
def run(self, start_time: float) -> ExitCode: self.scrub_pythonpath() options_bootstrapper = OptionsBootstrapper.create(env=self.env, args=self.args, allow_pantsrc=True) with warnings.catch_warnings(record=True): bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() # We enable logging here, and everything before it will be routed through regular # Python logging. setup_logging(global_bootstrap_options, stderr_logging=True) if self._should_run_with_pantsd(global_bootstrap_options): try: remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper) return remote_runner.run() except RemotePantsRunner.Fallback as e: logger.warning( "Client exception: {!r}, falling back to non-daemon mode". format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner # We only install signal handling via ExceptionSink if the run will execute in this process. ExceptionSink.install( log_location=init_workdir(global_bootstrap_options), pantsd_instance=False) runner = LocalPantsRunner.create( env=self.env, options_bootstrapper=options_bootstrapper) return runner.run(start_time)
def test_full_options_caching(self) -> None: with temporary_file_path() as config: args = self._config_path(config) bootstrapper = OptionsBootstrapper.create(env={}, args=args, allow_pantsrc=False) opts1 = bootstrapper.full_options_for_scopes( known_scope_infos=[ ScopeInfo(""), ScopeInfo("foo"), ] ) opts2 = bootstrapper.full_options_for_scopes( known_scope_infos=[ ScopeInfo("foo"), ScopeInfo(""), ] ) assert opts1 is opts2 opts3 = bootstrapper.full_options_for_scopes( known_scope_infos=[ ScopeInfo(""), ScopeInfo("foo"), ScopeInfo(""), ] ) assert opts1 is opts3 opts4 = bootstrapper.full_options_for_scopes(known_scope_infos=[ScopeInfo("")]) assert opts1 is not opts4 opts5 = bootstrapper.full_options_for_scopes(known_scope_infos=[ScopeInfo("")]) assert opts4 is opts5 assert opts1 is not opts5
def _run(exiter): # Place the registration of the unhandled exception hook as early as possible in the code. sys.excepthook = exiter.unhandled_exception_hook # We want to present warnings to the user, set this up early to ensure all warnings are seen. # The "default" action displays a warning for a particular file and line number exactly once. # See https://docs.python.org/2/library/warnings.html#the-warnings-filter for the complete action # list. warnings.simplefilter("default") # The GoalRunner will setup final logging below in `.setup()`, but span the gap until then. logging.basicConfig(level=logging.INFO) # This routes the warnings we enabled above through our loggers instead of straight to stderr raw. logging.captureWarnings(True) root_dir = get_buildroot() if not os.path.exists(root_dir): exiter.exit_and_fail( 'PANTS_BUILD_ROOT does not point to a valid path: {}'.format( root_dir)) options_bootstrapper = OptionsBootstrapper() plugin_resolver = PluginResolver(options_bootstrapper) working_set = plugin_resolver.resolve() goal_runner = GoalRunner(root_dir) goal_runner.setup(options_bootstrapper, working_set) exiter.apply_options(goal_runner.options) result = goal_runner.run() exiter.do_exit(result)
def _default_build_config(self, build_file_aliases=None): # TODO: Get default BuildFileAliases by extending BaseTest post # https://github.com/pantsbuild/pants/issues/4401 build_config = BuildConfigInitializer.get(OptionsBootstrapper()) if build_file_aliases: build_config.register_aliases(build_file_aliases) return build_config
def execute(self): try: pantsd_client = PantsDaemonClient(OptionsBootstrapper.create().bootstrap_options) with pantsd_client.lifecycle_lock: pantsd_client.terminate() except ProcessManager.NonResponsiveProcess as e: raise TaskError("failure while terminating pantsd: {}".format(e))
def run(self, start_time: float) -> ExitCode: self.scrub_pythonpath() options_bootstrapper = OptionsBootstrapper.create( env=self.env, args=self.args, allow_pantsrc=True ) with warnings.catch_warnings(record=True): bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() # Initialize the workdir early enough to ensure that logging has a destination. workdir_src = init_workdir(global_bootstrap_options) ExceptionSink.reset_log_location(workdir_src) setup_warning_filtering(global_bootstrap_options.ignore_pants_warnings or []) # We enable logging here, and everything before it will be routed through regular # Python logging. setup_logging(global_bootstrap_options, stderr_logging=True) if self._should_run_with_pantsd(global_bootstrap_options): try: remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper) return remote_runner.run() except RemotePantsRunner.Fallback as e: logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper) return runner.run(start_time)
def test_create_bootstrapped_options(self) -> None: # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate # correctly into regular config. with temporary_file(binary_mode=False) as fp: fp.write( dedent(""" [foo] bar: %(pants_workdir)s/baz [fruit] apple: %(pants_supportdir)s/banana """)) fp.close() args = ["--pants-workdir=/qux"] + self._config_path(fp.name) bootstrapper = OptionsBootstrapper.create( env={"PANTS_SUPPORTDIR": "/pear"}, args=args) opts = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo("", ScopeInfo.GLOBAL), ScopeInfo("foo", ScopeInfo.TASK), ScopeInfo("fruit", ScopeInfo.TASK), ]) # So we don't choke on these on the cmd line. opts.register("", "--pants-workdir") opts.register("", "--pants-config-files") opts.register("foo", "--bar") opts.register("fruit", "--apple") self.assertEqual("/qux/baz", opts.for_scope("foo").bar) self.assertEqual("/pear/banana", opts.for_scope("fruit").apple)
def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper( args=['--pants-version=99.99.9999']) build_config = BuildConfigInitializer.get(options_bootstrapper) with self.assertRaises(BuildConfigurationError): OptionsInitializer.create(options_bootstrapper, build_config)
def test_alias_pyupgrade(self) -> None: with temporary_dir() as tmpdir: config = os.path.join(tmpdir, "config") with open(config, "w") as out: out.write( dedent( """\ [cli.alias] pyupgrade = "--backend-packages=pants.backend.python.lint.pyupgrade fmt" """ ) ) config_arg = f"--pants-config-files=['{config}']" ob = OptionsBootstrapper.create( env={}, args=[config_arg, "pyupgrade"], allow_pantsrc=False ) self.assertEqual( ( config_arg, "--backend-packages=pants.backend.python.lint.pyupgrade", "fmt", ), ob.args, ) self.assertEqual( ( "./pants", config_arg, "--backend-packages=pants.backend.python.lint.pyupgrade", ), ob.bootstrap_args, )
def _init_engine(self, local_store_dir: Optional[str] = None) -> None: if self._scheduler is not None: return options_bootstrapper = OptionsBootstrapper.create( env={}, args=["--pants-config-files=[]", *self.additional_options], allow_pantsrc=False) global_options = options_bootstrapper.bootstrap_options.for_global_scope( ) local_store_dir = local_store_dir or global_options.local_store_dir local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir graph_session = EngineInitializer.setup_graph_extended( pants_ignore_patterns=[], use_gitignore=False, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, native=Native(), options_bootstrapper=options_bootstrapper, build_root=self.build_root, build_configuration=self.build_config(), execution_options=ExecutionOptions.from_bootstrap_options( global_options), ).new_session(build_id="buildid_for_test", should_report_workunits=True) self._scheduler = graph_session.scheduler_session
def test_full_options_caching(self) -> None: with temporary_file_path() as config: args = self._config_path(config) bootstrapper = OptionsBootstrapper.create(env={}, args=args) opts1 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo("", ScopeInfo.GLOBAL), ScopeInfo("foo", ScopeInfo.TASK), ]) opts2 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo("foo", ScopeInfo.TASK), ScopeInfo("", ScopeInfo.GLOBAL), ]) assert opts1 is opts2 opts3 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo("", ScopeInfo.GLOBAL), ScopeInfo("foo", ScopeInfo.TASK), ScopeInfo("", ScopeInfo.GLOBAL), ]) assert opts1 is opts3 opts4 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)]) assert opts1 is not opts4 opts5 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)]) assert opts4 is opts5 assert opts1 is not opts5
def parse_options(*args: str) -> OptionValueContainer: full_args = [*args, *self._config_path(None)] return ( OptionsBootstrapper.create(env={}, args=full_args, allow_pantsrc=False) .get_bootstrap_options() .for_global_scope() )
def select(argv): # Parse positional arguments to the script. args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:]) # Resolve bootstrap options with a fake empty command line. options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]]) subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory) known_scope_infos = reduce(set.union, (ss.known_scope_infos() for ss in subsystems), set()) options = options_bootstrapper.get_full_options(known_scope_infos) # Initialize Subsystems. Subsystem.set_options(options) # If the filename provided ends in a known archive extension (such as ".tar.gz"), then we get the # appropriate Archiver to pass to BinaryUtil. archiver_for_current_binary = None filename = args.filename or args.util_name try: archiver_for_current_binary = archiver_for_path(filename) # BinaryRequest requires the `name` field to be provided without an extension, as it appends the # archiver's extension if one is provided, so we have to remove it here. filename = filename[:-(len(archiver_for_current_binary.extension) + 1)] except ValueError: pass binary_util = BinaryUtil.Factory.create() binary_request = BinaryRequest(supportdir='bin/{}'.format(args.util_name), version=args.version, name=filename, platform_dependent=True, external_url_generator=None, archiver=archiver_for_current_binary) return binary_util.select(binary_request)
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper(args=['--loop', '--v1']) build_config = BuildConfigInitializer.get(ob) with self.assertRaises(OptionsError) as exc: OptionsInitializer.create(ob, build_config) self.assertIn('loop option only works with', exc.exception.message)
def test_options_parsing_request(self): parse_request = OptionsParseRequest.create( ['./pants', '-ldebug', '--python-setup-wheel-version=3.13.37', 'binary', 'src/python::'], dict(PANTS_ENABLE_PANTSD='True', PANTS_BINARIES_BASEURLS='["https://bins.com"]') ) # TODO: Once we have the ability to get FileContent for arbitrary # paths outside of the buildroot, we can move the construction of # OptionsBootstrapper into an @rule by cooperatively calling # OptionsBootstrapper.produce_and_set_bootstrap_options() which # will yield lists of file paths for use as subject values and permit # us to avoid the direct file I/O that this currently requires. options_bootstrapper = OptionsBootstrapper.from_options_parse_request(parse_request) build_config = BuildConfigInitializer.get(options_bootstrapper) options = run_rule(parse_options, options_bootstrapper, build_config)[0] self.assertIn('binary', options.goals) global_options = options.for_global_scope() self.assertEqual(global_options.level, 'debug') self.assertEqual(global_options.enable_pantsd, True) self.assertEqual(global_options.binaries_baseurls, ['https://bins.com']) python_setup_options = options.for_scope('python-setup') self.assertEqual(python_setup_options.wheel_version, '3.13.37')
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) bootstrap_options = options_bootstrapper.get_bootstrap_options() if bootstrap_options.for_global_scope().enable_pantsd: try: return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run() except RemotePantsRunner.Fallback as e: logger.debug('caught client exception: {!r}, falling back to non-daemon mode'.format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner return LocalPantsRunner(self._exiter, self._args, self._env, options_bootstrapper=options_bootstrapper).run()
def test_file_spec_args(self): with tempfile.NamedTemporaryFile() as tmp: tmp.write(dedent( """ foo bar """ )) tmp.flush() # Note that we prevent loading a real pants.ini during get_bootstrap_options(). cmdline = './pants --target-spec-file={filename} --pants-config-files="[]" ' \ 'compile morx:tgt fleem:tgt'.format( filename=tmp.name) bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline)) bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope() options = self._parse(cmdline, bootstrap_option_values=bootstrap_options) sorted_specs = sorted(options.target_specs) self.assertEqual(['bar', 'fleem:tgt', 'foo', 'morx:tgt'], sorted_specs)
def parse_options(args, env, setup_logging=False, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() if setup_logging: # Bootstrap logging and then fully initialize options. setup_logging_from_options(bootstrap_options) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def _ob(self, args=tuple(), env=tuple()): self.create_file('pants.ini') options_bootstrap = OptionsBootstrapper.create( args=tuple(args), env=dict(env), ) # NB: BuildConfigInitializer has sideeffects on first-run: in actual usage, these # sideeffects will happen during setup. We force them here. BuildConfigInitializer.get(options_bootstrap) return options_bootstrap
def test_bootstrapped_options_ignore_irrelevant_env(self): included = 'PANTS_SUPPORTDIR' excluded = 'NON_PANTS_ENV' bootstrapper = OptionsBootstrapper.create( env={ excluded: 'pear', included: 'banana', } ) self.assertIn(included, bootstrapper.env) self.assertNotIn(excluded, bootstrapper.env)
def test_full_options_caching(self): with temporary_file_path() as config: args = self._config_path(config) bootstrapper = OptionsBootstrapper(env={}, args=args) opts1 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL), ScopeInfo("foo", ScopeInfo.TASK)] ) opts2 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo("foo", ScopeInfo.TASK), ScopeInfo("", ScopeInfo.GLOBAL)] ) self.assertIs(opts1, opts2) opts3 = bootstrapper.get_full_options( known_scope_infos=[ ScopeInfo("", ScopeInfo.GLOBAL), ScopeInfo("foo", ScopeInfo.TASK), ScopeInfo("", ScopeInfo.GLOBAL), ] ) self.assertIs(opts1, opts3) opts4 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)]) self.assertIsNot(opts1, opts4) opts5 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)]) self.assertIs(opts4, opts5) self.assertIsNot(opts1, opts5)
def _init_engine(cls): if cls._scheduler is not None: return cls._local_store_dir = os.path.realpath(safe_mkdtemp()) safe_mkdir(cls._local_store_dir) # NB: This uses the long form of initialization because it needs to directly specify # `cls.alias_groups` rather than having them be provided by bootstrap options. graph_session = EngineInitializer.setup_legacy_graph_extended( pants_ignore_patterns=None, workdir=cls._pants_workdir(), local_store_dir=cls._local_store_dir, build_file_imports_behavior='allow', native=init_native(), options_bootstrapper=OptionsBootstrapper.create(args=['--pants-config-files=[]']), build_configuration=cls.build_config(), build_ignore_patterns=None, ).new_session(zipkin_trace_v2=False) cls._scheduler = graph_session.scheduler_session cls._build_graph, cls._address_mapper = graph_session.create_build_graph( TargetRoots([]), cls._build_root() )
def run(self): # Register our exiter at the beginning of the run() method so that any code in this process from # this point onwards will use that exiter in the case of a fatal error. ExceptionSink.reset_exiter(self._exiter) options_bootstrapper = OptionsBootstrapper.create(env=self._env, args=self._args) bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() # We enable Rust logging here, # and everything before it will be routed through regular Python logging. self._enable_rust_logging(global_bootstrap_options) ExceptionSink.reset_should_print_backtrace_to_terminal(global_bootstrap_options.print_exception_stacktrace) ExceptionSink.reset_log_location(global_bootstrap_options.pants_workdir) for message_regexp in global_bootstrap_options.ignore_pants_warnings: warnings.filterwarnings(action='ignore', message=message_regexp) if global_bootstrap_options.enable_pantsd: try: return RemotePantsRunner(self._exiter, self._args, self._env, options_bootstrapper).run() except RemotePantsRunner.Fallback as e: logger.warn('caught client exception: {!r}, falling back to non-daemon mode'.format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner runner = LocalPantsRunner.create( self._exiter, self._args, self._env, options_bootstrapper=options_bootstrapper ) runner.set_start_time(self._start_time) return runner.run()
def plugin_resolution(self, chroot=None, plugins=None, packager_cls=None): @contextmanager def provide_chroot(existing): if existing: yield existing, False else: with temporary_dir() as new_chroot: yield new_chroot, True with provide_chroot(chroot) as (root_dir, create_artifacts): env = {'PANTS_BOOTSTRAPDIR': root_dir} repo_dir = None if plugins: repo_dir = os.path.join(root_dir, 'repo') env.update(PANTS_PYTHON_REPOS_REPOS='[{!r}]'.format(repo_dir), PANTS_PYTHON_REPOS_INDEXES='[]', PANTS_PYTHON_SETUP_RESOLVER_CACHE_TTL='1') plugin_list = [] for plugin in plugins: version = None if isinstance(plugin, tuple): plugin, version = plugin plugin_list.append('{}=={}'.format(plugin, version) if version else plugin) if create_artifacts: self.create_plugin(repo_dir, plugin, version, packager_cls=packager_cls) env['PANTS_PLUGINS'] = '[{}]'.format(','.join(map(repr, plugin_list))) configpath = os.path.join(root_dir, 'pants.ini') if create_artifacts: touch(configpath) args = ["--pants-config-files=['{}']".format(configpath)] options_bootstrapper = OptionsBootstrapper.create(env=env, args=args) plugin_resolver = PluginResolver(options_bootstrapper) cache_dir = plugin_resolver.plugin_cache_dir yield plugin_resolver.resolve(WorkingSet(entries=[])), root_dir, repo_dir, cache_dir
def test_full_options_caching(self): with temporary_file_path() as config: bootstrapper = OptionsBootstrapper(env={}, configpath=config, args=[]) opts1 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK)]) opts2 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIs(opts1, opts2) opts3 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIs(opts1, opts3) opts4 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIsNot(opts1, opts4) opts5 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIs(opts4, opts5) self.assertIsNot(opts1, opts5)
def parse_options(args, env, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def config_path(*args, **env): return OptionsBootstrapper.get_config_file_paths(env, args)
def launch(): """An external entrypoint that spawns a new pantsd instance.""" PantsDaemon.Factory.create(OptionsBootstrapper.create()).run_sync()