def runner_factory(sock, arguments, environment): exiter = self._exiter_class(sock) graph_helper = None deferred_exc = None # Capture the size of the graph prior to any warming, for stats. preceding_graph_size = self._scheduler_service.product_graph_len() self._logger.debug('resident graph size: %s', preceding_graph_size) self._logger.debug('execution commandline: %s', arguments) options, _ = OptionsInitializer( OptionsBootstrapper(args=arguments)).setup(init_logging=False) target_roots = self._target_roots_calculator.create( options, change_calculator=self._scheduler_service.change_calculator) try: self._logger.debug('warming the product graph via %s', self._scheduler_service) # N.B. This call is made in the pre-fork daemon context for reach and reuse of the # resident scheduler. graph_helper = self._scheduler_service.warm_product_graph( target_roots) except Exception: deferred_exc = sys.exc_info() self._logger.warning( 'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s', ''.join(traceback.format_exception(*deferred_exc))) return self._runner_class(sock, exiter, arguments, environment, target_roots, graph_helper, self.fork_lock, preceding_graph_size, deferred_exc)
def setup(options=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper()).setup() build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] storage = Storage.create(debug=False) project_tree = FileSystemProjectTree(build_root) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, storage, project_tree), storage, options, spec_roots, symbol_table_cls )
def runner_factory(sock, arguments, environment): exiter = self._exiter_class(sock) graph_helper = None deferred_exc = None self._logger.debug('execution commandline: %s', arguments) options_bootstrapper = OptionsBootstrapper(args=arguments) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) graph_helper, target_roots = None, None try: self._logger.debug('warming the product graph via %s', self._scheduler_service) # N.B. This call is made in the pre-fork daemon context for reach and reuse of the # resident scheduler. graph_helper, target_roots = self._scheduler_service.warm_product_graph( options, self._target_roots_calculator) except Exception: deferred_exc = sys.exc_info() self._logger.warning( 'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s', ''.join(traceback.format_exception(*deferred_exc))) return self._runner_class(sock, exiter, arguments, environment, target_roots, graph_helper, self.fork_lock, deferred_exc)
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) bootstrap_options = options_bootstrapper.get_bootstrap_options() ExceptionSink.set_destination( bootstrap_options.for_global_scope().pants_workdir) if bootstrap_options.for_global_scope().enable_pantsd: try: return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run() except RemotePantsRunner.Fallback as e: logger.warn( 'caught client exception: {!r}, falling back to non-daemon mode' .format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner runner = LocalPantsRunner.create( self._exiter, self._args, self._env, options_bootstrapper=options_bootstrapper) runner.set_start_time(self._start_time) return runner.run()
def _default_build_config(self, build_file_aliases=None): # TODO: Get default BuildFileAliases by extending BaseTest post # https://github.com/pantsbuild/pants/issues/4401 build_config = BuildConfigInitializer.get(OptionsBootstrapper()) if build_file_aliases: build_config.register_aliases(build_file_aliases) return build_config
def prepare_task(self, config=None, args=None, targets=None, build_graph=None, build_file_parser=None, address_mapper=None, console_outstream=None, workspace=None): """Prepares a Task for execution. task_type: The class of the Task to create. config: An optional string representing the contents of a pants.ini config. args: optional list of command line flags, these should be prefixed with '--test-'. targets: optional list of Target objects passed on the command line. Returns a new Task ready to execute. """ task_type = self.task_type() assert issubclass( task_type, Task), 'task_type must be a Task subclass, got %s' % task_type config = create_config(config or '') workdir = os.path.join(config.getdefault('pants_workdir'), 'test', task_type.__name__) bootstrap_options = OptionsBootstrapper().get_bootstrap_options() options = Options(env={}, config=config, known_scopes=['', 'test'], args=args or []) # A lot of basic code uses these options, so always register them. register_bootstrap_options(options.register_global) # We need to wrap register_global (can't set .bootstrap attr on the bound instancemethod). def register_global_wrapper(*args, **kwargs): return options.register_global(*args, **kwargs) register_global_wrapper.bootstrap = bootstrap_options.for_global_scope( ) register_global_options(register_global_wrapper) task_type.options_scope = 'test' task_type.register_options_on_scope(options) run_tracker = create_run_tracker() context = Context(config, options, run_tracker, targets or [], build_graph=build_graph, build_file_parser=build_file_parser, address_mapper=address_mapper, console_outstream=console_outstream, workspace=workspace) return task_type(context, workdir)
def test_create_bootstrapped_options(self): # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate # correctly into regular config. with temporary_file() as fp: fp.write( dedent(""" [foo] bar: %(pants_workdir)s/baz [fruit] apple: %(pants_supportdir)s/banana """)) fp.close() bootstrapper = OptionsBootstrapper( env={'PANTS_SUPPORTDIR': '/pear'}, configpath=fp.name, args=['--pants-workdir=/qux']) opts = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('fruit', ScopeInfo.TASK) ]) opts.register( '', '--pants-workdir') # So we don't choke on it on the cmd line. opts.register('foo', '--bar') opts.register('fruit', '--apple') self.assertEquals('/qux/baz', opts.for_scope('foo').bar) self.assertEquals('/pear/banana', opts.for_scope('fruit').apple)
def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper( args=['--pants-version=99.99.9999']) build_config = BuildConfigInitializer.get(options_bootstrapper) with self.assertRaises(BuildConfigurationError): OptionsInitializer.create(options_bootstrapper, build_config)
def test_full_options_caching(self): with temporary_file_path() as config: bootstrapper = OptionsBootstrapper(env={}, configpath=config, args=[]) opts1 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK) ]) opts2 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('', ScopeInfo.GLOBAL) ]) self.assertIs(opts1, opts2) opts3 = bootstrapper.get_full_options(known_scope_infos=[ ScopeInfo('', ScopeInfo.GLOBAL), ScopeInfo('foo', ScopeInfo.TASK), ScopeInfo('', ScopeInfo.GLOBAL) ]) self.assertIs(opts1, opts3) opts4 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIsNot(opts1, opts4) opts5 = bootstrapper.get_full_options( known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)]) self.assertIs(opts4, opts5) self.assertIsNot(opts1, opts5)
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper(args=['--loop', '--v1']) build_config = BuildConfigInitializer.get(ob) with self.assertRaises(OptionsError) as exc: OptionsInitializer.create(ob, build_config) self.assertIn('loop option only works with', exc.exception.message)
def run(self): # Register our exiter at the beginning of the run() method so that any code in this process from # this point onwards will use that exiter in the case of a fatal error. ExceptionSink.reset_exiter(self._exiter) options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() ExceptionSink.reset_should_print_backtrace_to_terminal( global_bootstrap_options.print_exception_stacktrace) ExceptionSink.reset_log_location( global_bootstrap_options.pants_workdir) if global_bootstrap_options.enable_pantsd: try: return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run() except RemotePantsRunner.Fallback as e: logger.warn( 'caught client exception: {!r}, falling back to non-daemon mode' .format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner runner = LocalPantsRunner.create( self._exiter, self._args, self._env, options_bootstrapper=options_bootstrapper) runner.set_start_time(self._start_time) return runner.run()
def plugin_resolution(chroot=None, plugins=None): @contextmanager def provide_chroot(existing): if existing: yield existing, False else: with temporary_dir() as new_chroot: yield new_chroot, True with provide_chroot(chroot) as (root_dir, create_artifacts): env = {'PANTS_BOOTSTRAPDIR': root_dir} repo_dir = None if plugins: repo_dir = os.path.join(root_dir, 'repo') env.update(PANTS_PYTHON_REPOS_REPOS='[{!r}]'.format(repo_dir), PANTS_PYTHON_REPOS_INDEXES='[]', PANTS_PYTHON_SETUP_RESOLVER_CACHE_TTL='1') plugin_list = [] for plugin in plugins: version = None if isinstance(plugin, tuple): plugin, version = plugin plugin_list.append('{}=={}'.format(plugin, version) if version else plugin) if create_artifacts: create_plugin(repo_dir, plugin, version) env['PANTS_PLUGINS'] = '[{}]'.format(','.join(map(repr, plugin_list))) configpath = os.path.join(root_dir, 'pants.ini') if create_artifacts: touch(configpath) options_bootstrapper = OptionsBootstrapper(env=env, configpath=configpath, args=[]) plugin_resolver = PluginResolver(options_bootstrapper) cache_dir = plugin_resolver.plugin_cache_dir yield plugin_resolver.resolve(WorkingSet(entries=[])), root_dir, repo_dir, cache_dir
def create(cls, options=None, args=None, build_root=None, change_calculator=None): """ :param Options options: An `Options` instance to use, if available. :param string args: Raw cli args to use for parsing if an `Options` instance isn't available. :param string build_root: The build root. :param ChangeCalculator change_calculator: A `ChangeCalculator` for calculating changes. """ if not options: assert args is not None, 'must pass `args` if not passing `options`' options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False) # Determine the literal target roots. spec_roots = cls.parse_specs(options.target_specs, build_root) # Determine `Changed` arguments directly from options to support pre-`Subsystem` initialization paths. changed_options = options.for_scope('changed') changed_request = ChangedRequest.from_options(changed_options) logger.debug('args are: %s', args) logger.debug('spec_roots are: %s', spec_roots) logger.debug('changed_request is: %s', changed_request) if change_calculator and changed_request.is_actionable(): if spec_roots: # We've been provided spec roots (e.g. `./pants list ::`) AND a changed request. Error out. raise InvalidSpecConstraint('cannot provide changed parameters and target specs!') # We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute # alternate target roots. changed_addresses = change_calculator.changed_target_addresses(changed_request) logger.debug('changed addresses: %s', changed_addresses) return ChangedTargetRoots(tuple(SingleAddress(a.spec_path, a.target_name) for a in changed_addresses)) return LiteralTargetRoots(spec_roots)
def _run(exiter): # Place the registration of the unhandled exception hook as early as possible in the code. sys.excepthook = exiter.unhandled_exception_hook # We want to present warnings to the user, set this up early to ensure all warnings are seen. # The "default" action displays a warning for a particular file and line number exactly once. # See https://docs.python.org/2/library/warnings.html#the-warnings-filter for the complete action # list. warnings.simplefilter("default") # The GoalRunner will setup final logging below in `.setup()`, but span the gap until then. logging.basicConfig(level=logging.INFO) # This routes the warnings we enabled above through our loggers instead of straight to stderr raw. logging.captureWarnings(True) root_dir = get_buildroot() if not os.path.exists(root_dir): exiter.exit_and_fail( 'PANTS_BUILD_ROOT does not point to a valid path: {}'.format( root_dir)) options_bootstrapper = OptionsBootstrapper() plugin_resolver = PluginResolver(options_bootstrapper) working_set = plugin_resolver.resolve() goal_runner = GoalRunner(root_dir) goal_runner.setup(options_bootstrapper, working_set) exiter.apply_options(goal_runner.options) result = goal_runner.run() exiter.do_exit(result)
def get_bootstrap_options(self, cli_options=()): """Retrieves bootstrap options. :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. """ # Can't parse any options without a pants.ini. self.create_file('pants.ini') return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope()
def parse_options(args, env, setup_logging=False, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper(args=args, env=env) bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() if setup_logging: # Bootstrap logging and then fully initialize options. setup_logging_from_options(bootstrap_options) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper( env=self._env, args=self._args) options, build_config = OptionsInitializer( options_bootstrapper, exiter=self._exiter).setup() global_options = options.for_global_scope() # Apply exiter options. self._exiter.apply_options(options) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify the configs here. if global_options.verify_config: options_bootstrapper.verify_configs_against_options(options) # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() reporting.initialize(run_tracker, self._run_start_time) try: # Determine the build root dir. root_dir = get_buildroot() # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) # Record the preceding product graph size. run_tracker.pantsd_stats.set_preceding_graph_size( self._preceding_graph_size) # Setup and run GoalRunner. goal_runner = GoalRunner.Factory(root_dir, options, build_config, run_tracker, reporting, self._target_roots, self._daemon_build_graph, self._exiter).setup() goal_runner_result = goal_runner.run() if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker_result = run_tracker.end() # Take the exit code with higher abs value in case of negative values. final_exit_code = goal_runner_result if abs(goal_runner_result) > abs( run_tracker_result) else run_tracker_result self._exiter.exit(final_exit_code)
def __init__(self, options_bootstrapper=None, working_set=None, exiter=sys.exit): """ :param OptionsBootStrapper options_bootstrapper: An options bootstrapper instance (Optional). :param pkg_resources.WorkingSet working_set: The working set of the current run as returned by PluginResolver.resolve() (Optional). :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._options_bootstrapper = options_bootstrapper or OptionsBootstrapper() self._working_set = working_set or PluginResolver(self._options_bootstrapper).resolve() self._exiter = exiter
def bootstrap_option_values(): try: return OptionsBootstrapper(buildroot='<buildroot>').get_bootstrap_options().for_global_scope() finally: # Today, the OptionsBootstrapper mutates global state upon construction in the form of: # Config.reset_default_bootstrap_option_values(...) # As such bootstrap options that use the buildroot get contaminated globally here. We only # need the contaminated values locally though for doc display, thus the reset of global state. # TODO(John Sirois): remove this hack when mutable Config._defaults is killed. Config.reset_default_bootstrap_option_values()
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) global_bootstrap_options = options_bootstrapper.get_bootstrap_options( ).for_global_scope() return self._run(global_bootstrap_options.enable_pantsd, exiter=self._exiter, args=self._args, env=self._env, options_bootstrapper=options_bootstrapper)
def parse_commandline_to_spec_roots(options=None, args=None, build_root=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper(args=args), init_logging=False).setup() cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot()) spec_roots = [ cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs ] return spec_roots
def test_setting_pants_config_in_config(self): # Test that setting pants_config in the config file has no effect. with temporary_dir() as tmpdir: config1 = os.path.join(tmpdir, 'config1') config2 = os.path.join(tmpdir, 'config2') with open(config1, 'w') as out1: out1.write(b"[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2)) with open(config2, 'w') as out2: out2.write(b'[DEFAULT]\nlogdir: logdir2\n') ob = OptionsBootstrapper(env={}, args=["--pants-config-files=['{}']".format(config1)]) logdir = ob.get_bootstrap_options().for_global_scope().logdir self.assertEqual('logdir1', logdir)
def run(self): options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) global_bootstrap_options = options_bootstrapper.get_bootstrap_options( ).for_global_scope() return self._run( is_remote=global_bootstrap_options.enable_pantsd, exiter=self._exiter, args=self._args, env=self._env, process_metadata_dir=global_bootstrap_options.pants_subprocessdir, options_bootstrapper=options_bootstrapper)
def _test_bootstrap_options(self, config, env, args, **expected_entries): with temporary_file() as fp: fp.write('[DEFAULT]\n') if config: for k, v in config.items(): fp.write('{0}: {1}\n'.format(k, v)) fp.close() args = args + self._config_path(fp.name) bootstrapper = OptionsBootstrapper(env=env, args=args) vals = bootstrapper.get_bootstrap_options().for_global_scope() vals_dict = {k: getattr(vals, k) for k in expected_entries} self.assertEquals(expected_entries, vals_dict)
def test_file_spec_args(self): with tempfile.NamedTemporaryFile() as tmp: tmp.write(dedent( ''' foo bar ''' )) tmp.flush() cmdline = './pants --target-spec-file={filename} compile morx fleem'.format(filename=tmp.name) bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline)) bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope() options = self._parse(cmdline, bootstrap_option_values=bootstrap_options) sorted_specs = sorted(options.target_specs) self.assertEqual(['bar', 'fleem', 'foo', 'morx'], sorted_specs)
def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper( env=self._env, args=self._args) options, build_config = OptionsInitializer( options_bootstrapper, exiter=self._exiter).setup() # Apply exiter options. self._exiter.apply_options(options) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify the configs here. if options.for_global_scope().verify_config: options_bootstrapper.verify_configs_against_options(options) # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker, reporting = ReportingInitializer().setup() try: # Determine the build root dir. root_dir = get_buildroot() # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) # Setup and run GoalRunner. goal_runner = GoalRunner.Factory(root_dir, options, build_config, run_tracker, reporting, exiter=self._exiter).setup() result = goal_runner.run() if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker.end() self._exiter.exit(result)
def test_file_spec_args(self): with tempfile.NamedTemporaryFile() as tmp: tmp.write(dedent( """ foo bar """ )) tmp.flush() # Note that we prevent loading a real pants.ini during get_bootstrap_options(). cmdline = './pants --target-spec-file={filename} --pants-config-files="[]" ' \ 'compile morx:tgt fleem:tgt'.format( filename=tmp.name) bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline)) bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope() options = self._parse(cmdline, bootstrap_option_values=bootstrap_options) sorted_specs = sorted(options.target_specs) self.assertEqual(['bar', 'fleem:tgt', 'foo', 'morx:tgt'], sorted_specs)
def test_version_request(version_flag): class ExitException(Exception): def __init__(self, exit_code): self.exit_code = exit_code with temporary_dir() as build_root: def exiter(exit_code): raise ExitException(exit_code) options_bootstrapper = OptionsBootstrapper(args=[version_flag]) with pytest.raises(ExitException) as excinfo: OptionsInitializer(options_bootstrapper, WorkingSet(), exiter=exiter).setup() assert 0 == excinfo.value.exit_code
def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman native = None build_root = None services = None port_map = None if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) options_bootstrapper = OptionsBootstrapper() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native, bootstrap_options_values, build_config) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman ) return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, socket_map=port_map, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options )
def _do_test(self, expected_vals, config, env, args): expected_vals_dict = { 'pants_workdir': expected_vals[0], 'pants_supportdir': expected_vals[1], 'pants_distdir': expected_vals[2] } with temporary_file() as fp: fp.write('[DEFAULT]\n') if config: for k, v in config.items(): fp.write('{0}: {1}\n'.format(k, v)) fp.close() vals = OptionsBootstrapper( env=env, configpath=fp.name, args=args, buildroot='/buildroot' ).get_bootstrap_options().for_global_scope() vals_dict = {k: getattr(vals, k) for k in expected_vals_dict} self.assertEquals(expected_vals_dict, vals_dict)