def _ob(self, args=tuple(), env=tuple()): self.create_file("pants.toml") options_bootstrap = OptionsBootstrapper.create(args=tuple(args), env=dict(env),) # NB: BuildConfigInitializer has sideeffects on first-run: in actual usage, these # sideeffects will happen during setup. We force them here. BuildConfigInitializer.get(options_bootstrap) return options_bootstrap
def execute_rule(self, args=tuple(), env=tuple(), exit_code=0): """Executes the @console_rule for this test class. :API: public Returns the text output of the task. """ # Create an OptionsBootstrapper for these args/env, and a captured Console instance. args = self._implicit_args + (self.goal_cls.name, ) + tuple(args) env = dict(env) options_bootstrapper = OptionsBootstrapper.create(args=args, env=env) BuildConfigInitializer.get(options_bootstrapper) full_options = options_bootstrapper.get_full_options( list(self.goal_cls.Options.known_scope_infos())) stdout, stderr = StringIO(), StringIO() console = Console(stdout=stdout, stderr=stderr) # Run for the target specs parsed from the args. specs = TargetRootsCalculator.parse_specs(full_options.target_specs, self.build_root) params = Params(specs, console, options_bootstrapper) actual_exit_code = self.scheduler.run_console_rule( self.goal_cls, params) # Flush and capture console output. console.flush() stdout = stdout.getvalue() stderr = stderr.getvalue() self.assertEqual( exit_code, actual_exit_code, "Exited with {} (expected {}):\nstdout:\n{}\nstderr:\n{}".format( actual_exit_code, exit_code, stdout, stderr)) return stdout
def _ob(self, args=tuple(), env=tuple()): self.create_file('pants.ini') options_bootstrap = OptionsBootstrapper.create( args=tuple(args), env=dict(env), ) # NB: BuildConfigInitializer has sideeffects on first-run: in actual usage, these # sideeffects will happen during setup. We force them here. BuildConfigInitializer.get(options_bootstrap) return options_bootstrap
def clean_global_runtime_state(reset_subsystem=False): """Resets the global runtime state of a pants runtime for cleaner forking. :param bool reset_subsystem: Whether or not to clean Subsystem global state. """ if reset_subsystem: # Reset subsystem state. Subsystem.reset() # Reset global plugin state. BuildConfigInitializer.reset()
def clean_global_runtime_state(reset_subsystem=False): """Resets the global runtime state of a pants runtime for cleaner forking. :param bool reset_subsystem: Whether or not to clean Subsystem global state. """ if reset_subsystem: # Reset subsystem state. Subsystem.reset() # Reset Goals and Tasks. Goal.clear() # Reset global plugin state. BuildConfigInitializer.reset()
def execute_rule( self, args: Optional[Iterable[str]] = None, global_args: Optional[Iterable[str]] = None, env: Optional[Dict[str, str]] = None, exit_code: int = 0, additional_params: Optional[Iterable[Any]] = None, ) -> str: """Executes the @goal_rule for this test class. :API: public Returns the text output of the task. """ # Create an OptionsBootstrapper for these args/env, and a captured Console instance. options_bootstrapper = create_options_bootstrapper( args=(*(global_args or []), self.goal_cls.name, *(args or [])), env=env, ) BuildConfigInitializer.get(options_bootstrapper) full_options = options_bootstrapper.get_full_options( [*GlobalOptions.known_scope_infos(), *self.goal_cls.subsystem_cls.known_scope_infos()] ) stdout, stderr = StringIO(), StringIO() console = Console(stdout=stdout, stderr=stderr) scheduler = self.scheduler workspace = Workspace(scheduler) # Run for the target specs parsed from the args. specs = SpecsCalculator.parse_specs(full_options.specs, self.build_root) params = Params( specs.provided_specs, console, options_bootstrapper, workspace, *(additional_params or []), ) actual_exit_code = self.scheduler.run_goal_rule(self.goal_cls, params) # Flush and capture console output. console.flush() stdout_val = stdout.getvalue() stderr_val = stderr.getvalue() assert ( exit_code == actual_exit_code ), f"Exited with {actual_exit_code} (expected {exit_code}):\nstdout:\n{stdout_val}\nstderr:\n{stderr_val}" return stdout_val
def test_options_parsing_request(self): parse_request = OptionsParseRequest.create( ['./pants', '-ldebug', '--python-setup-wheel-version=3.13.37', 'binary', 'src/python::'], dict(PANTS_ENABLE_PANTSD='True', PANTS_BINARIES_BASEURLS='["https://bins.com"]') ) # TODO: Once we have the ability to get FileContent for arbitrary # paths outside of the buildroot, we can move the construction of # OptionsBootstrapper into an @rule by cooperatively calling # OptionsBootstrapper.produce_and_set_bootstrap_options() which # will yield lists of file paths for use as subject values and permit # us to avoid the direct file I/O that this currently requires. options_bootstrapper = OptionsBootstrapper.from_options_parse_request(parse_request) build_config = BuildConfigInitializer.get(options_bootstrapper) options = run_rule(parse_options, options_bootstrapper, build_config)[0] self.assertIn('binary', options.goals) global_options = options.for_global_scope() self.assertEquals(global_options.level, 'debug') self.assertEquals(global_options.enable_pantsd, True) self.assertEquals(global_options.binaries_baseurls, ['https://bins.com']) python_setup_options = options.for_scope('python-setup') self.assertEquals(python_setup_options.wheel_version, '3.13.37')
def runner_factory(sock, arguments, environment): exiter = self._exiter_class(sock) graph_helper = None deferred_exc = None self._logger.debug('execution commandline: %s', arguments) options_bootstrapper = OptionsBootstrapper(args=arguments) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) graph_helper, target_roots = None, None try: self._logger.debug('warming the product graph via %s', self._scheduler_service) # N.B. This call is made in the pre-fork daemon context for reach and reuse of the # resident scheduler. graph_helper, target_roots = self._scheduler_service.warm_product_graph( options, self._target_roots_calculator) except Exception: deferred_exc = sys.exc_info() self._logger.warning( 'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s', ''.join(traceback.format_exception(*deferred_exc))) return self._runner_class(sock, exiter, arguments, environment, target_roots, graph_helper, self.fork_lock, deferred_exc)
def _default_build_config(self, build_file_aliases=None): # TODO: Get default BuildFileAliases by extending BaseTest post # https://github.com/pantsbuild/pants/issues/4401 build_config = BuildConfigInitializer.get(OptionsBootstrapper()) if build_file_aliases: build_config.register_aliases(build_file_aliases) return build_config
def _default_build_config(self, build_file_aliases=None): # TODO: Get default BuildFileAliases by extending BaseTest post # https://github.com/pantsbuild/pants/issues/4401 build_config = BuildConfigInitializer.get(OptionsBootstrapper()) if build_file_aliases: build_config.register_aliases(build_file_aliases) return build_config
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper.create(args=['--loop', '--v1']) build_config = BuildConfigInitializer.get(ob) with self.assertRaises(OptionsError) as exc: OptionsInitializer.create(ob, build_config) self.assertIn('loop option only works with', str(exc.exception))
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper(args=['--loop', '--v1']) build_config = BuildConfigInitializer.get(ob) with self.assertRaises(OptionsError) as exc: OptionsInitializer.create(ob, build_config) self.assertIn('loop option only works with', exc.exception.message)
def test_options_parsing_request(self): parse_request = OptionsParseRequest.create( ['./pants', '-ldebug', '--python-setup-wheel-version=3.13.37', 'binary', 'src/python::'], dict(PANTS_ENABLE_PANTSD='True', PANTS_BINARIES_BASEURLS='["https://bins.com"]') ) # TODO: Once we have the ability to get FileContent for arbitrary # paths outside of the buildroot, we can move the construction of # OptionsBootstrapper into an @rule by cooperatively calling # OptionsBootstrapper.produce_and_set_bootstrap_options() which # will yield lists of file paths for use as subject values and permit # us to avoid the direct file I/O that this currently requires. options_bootstrapper = OptionsBootstrapper.from_options_parse_request(parse_request) build_config = BuildConfigInitializer.get(options_bootstrapper) options = run_rule(parse_options, options_bootstrapper, build_config)[0] self.assertIn('binary', options.goals) global_options = options.for_global_scope() self.assertEqual(global_options.level, 'debug') self.assertEqual(global_options.enable_pantsd, True) self.assertEqual(global_options.binaries_baseurls, ['https://bins.com']) python_setup_options = options.for_scope('python-setup') self.assertEqual(python_setup_options.wheel_version, '3.13.37')
def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper( args=['--pants-version=99.99.9999']) build_config = BuildConfigInitializer.get(options_bootstrapper) with self.assertRaises(BuildConfigurationError): OptionsInitializer.create(options_bootstrapper, build_config)
def _init_scheduler(self, options_fingerprint: str, options_bootstrapper: OptionsBootstrapper) -> None: """(Re-)Initialize the scheduler. Must be called under the lifecycle lock. """ try: if self._scheduler: logger.info( "initialization options changed: reinitializing pantsd...") else: logger.info("initializing pantsd...") if self._services: self._services.shutdown() with OptionsInitializer.handle_unknown_flags(options_bootstrapper, raise_=True): build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) self._scheduler = EngineInitializer.setup_graph( options, build_config, executor=self._executor) bootstrap_options_values = options.bootstrap_option_values() assert bootstrap_options_values is not None self._services = self._services_constructor( bootstrap_options_values, self._scheduler) self._fingerprint = options_fingerprint logger.info("pantsd initialized.") except Exception as e: self._kill_switch.set() self._scheduler = None raise e
def _handle_unknown_flags(err: UnknownFlagsError, options_bootstrapper: OptionsBootstrapper): build_config = BuildConfigInitializer.get(options_bootstrapper) # We need an options instance in order to get "did you mean" suggestions, but we know # there are bad flags in the args, so we generate options with no flags. no_arg_bootstrapper = replace(options_bootstrapper, args=("dummy_first_arg",)) options = OptionsInitializer.create(no_arg_bootstrapper, build_config) FlagErrorHelpPrinter(options).handle_unknown_flags(err)
def _init_scheduler(self, options_fingerprint: str, options_bootstrapper: OptionsBootstrapper) -> None: """(Re-)Initialize the scheduler. Must be called under the lifecycle lock. """ try: if self._scheduler: logger.info( "initialization options changed: reinitializing pantsd...") else: logger.info("initializing pantsd...") if self._services: self._services.shutdown() build_config = BuildConfigInitializer.get(options_bootstrapper) self._scheduler = EngineInitializer.setup_legacy_graph( options_bootstrapper, build_config) bootstrap_options_values = options_bootstrapper.bootstrap_options.for_global_scope( ) self._services = self._services_constructor( bootstrap_options_values, self._scheduler) self._fingerprint = options_fingerprint logger.info("pantsd initialized.") except Exception as e: self._kill_switch.set() self._scheduler = None raise e
def single_daemonized_run( self, working_dir: str, cancellation_latch: PySessionCancellationLatch) -> ExitCode: """Run a single daemonized run of Pants. All aspects of the `sys` global should already have been replaced in `__call__`, so this method should not need any special handling for the fact that it's running in a proxied environment. """ # Capture the client's start time, which we propagate here in order to get an accurate # view of total time. env_start_time = os.environ.get("PANTSD_RUNTRACKER_CLIENT_START_TIME", None) start_time = float(env_start_time) if env_start_time else time.time() # Clear global mutable state before entering `LocalPantsRunner`. Note that we use # `sys.argv` and `os.environ`, since they have been mutated to maintain the illusion # of a local run: once we allow for concurrent runs, this information should be # propagated down from the caller. # see https://github.com/pantsbuild/pants/issues/7654 BuildConfigInitializer.reset() options_bootstrapper = OptionsBootstrapper.create(env=os.environ, args=sys.argv, allow_pantsrc=True) bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() # Run using the pre-warmed Session. with self._stderr_logging(global_bootstrap_options): try: scheduler = self._core.prepare_scheduler(options_bootstrapper) runner = LocalPantsRunner.create( os.environ, options_bootstrapper, scheduler=scheduler, cancellation_latch=cancellation_latch, ) return runner.run(start_time) except Exception as e: logger.exception(e) return PANTS_FAILED_EXIT_CODE except KeyboardInterrupt: print("Interrupted by user.\n", file=sys.stderr) return PANTS_FAILED_EXIT_CODE
def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env, args=self._args) bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() setup_logging_from_options(bootstrap_options) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) global_options = options.for_global_scope() # Apply exiter options. self._exiter.apply_options(options) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify the configs here. if global_options.verify_config: options_bootstrapper.verify_configs_against_options(options) # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() reporting.initialize(run_tracker, self._run_start_time) try: # Determine the build root dir. root_dir = get_buildroot() # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) # Setup and run GoalRunner. goal_runner = GoalRunner.Factory(root_dir, options, build_config, run_tracker, reporting, self._target_roots, self._daemon_build_graph, self._exiter).setup() goal_runner_result = goal_runner.run() if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker_result = run_tracker.end() # Take the exit code with higher abs value in case of negative values. final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result self._exiter.exit(final_exit_code)
def parse_options( args: List[str], env: Dict[str, str], options_bootstrapper: Optional[OptionsBootstrapper] = None, ) -> Tuple[Options, BuildConfiguration, OptionsBootstrapper]: options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def parse_options(args, env, setup_logging=False, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper(args=args, env=env) bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() if setup_logging: # Bootstrap logging and then fully initialize options. setup_logging_from_options(bootstrap_options) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def parse_options(options_bootstrapper: OptionsBootstrapper) -> _Options: # TODO: Because _OptionsBootstapper is currently provided as a Param, this @rule relies on options # remaining relatively stable in order to be efficient. See #6845 for a discussion of how to make # minimize the size of that value. build_config = BuildConfigInitializer.get(options_bootstrapper) return _Options( OptionsInitializer.create(options_bootstrapper, build_config, init_subsystems=False))
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper.create( args=['--backend-packages=[]', '--backend-packages2=[]', '--v2', '--no-v1', '--loop', '--no-enable-pantsd']) build_config = BuildConfigInitializer.get(ob) with self.assertRaises(OptionsError) as exc: OptionsInitializer.create(ob, build_config) self.assertIn('The `--loop` option requires `--enable-pantsd`', str(exc.exception))
def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper( env=self._env, args=self._args) bootstrap_options = options_bootstrapper.get_bootstrap_options( ).for_global_scope() setup_logging_from_options(bootstrap_options) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) global_options = options.for_global_scope() # Apply exiter options. self._exiter.apply_options(options) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify the configs here. if global_options.verify_config: options_bootstrapper.verify_configs_against_options(options) # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() reporting.initialize(run_tracker, self._run_start_time) try: # Determine the build root dir. root_dir = get_buildroot() # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) # Setup and run GoalRunner. goal_runner = GoalRunner.Factory(root_dir, options, build_config, run_tracker, reporting, self._target_roots, self._daemon_build_graph, self._exiter).setup() goal_runner_result = goal_runner.run() if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker_result = run_tracker.end() # Take the exit code with higher abs value in case of negative values. final_exit_code = goal_runner_result if abs(goal_runner_result) > abs( run_tracker_result) else run_tracker_result self._exiter.exit(final_exit_code)
def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper.create( env={}, args=["--backend-packages=[]", "--pants-version=99.99.9999"], allow_pantsrc=False, ) build_config = BuildConfigInitializer.get(options_bootstrapper) with self.assertRaises(BuildConfigurationError): OptionsInitializer.create(options_bootstrapper, build_config)
def parse_options( cls, options_bootstrapper: OptionsBootstrapper, ) -> Tuple[Options, BuildConfiguration]: build_config = BuildConfigInitializer.get(options_bootstrapper) try: options = OptionsInitializer.create(options_bootstrapper, build_config) except UnknownFlagsError as err: cls._handle_unknown_flags(err, options_bootstrapper) raise return options, build_config
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper.create( env={}, args=["--backend-packages=[]", "--remote-execution"], allow_pantsrc=False) build_config = BuildConfigInitializer.get(ob) with self.assertRaises(OptionsError) as exc: OptionsInitializer.create(ob, build_config) self.assertIn("The `--remote-execution` option requires", str(exc.exception))
def execute_rule(self, args=tuple(), env=tuple(), exit_code=0, additional_params: Iterable[Any] = tuple()): """Executes the @console_rule for this test class. :API: public Returns the text output of the task. """ # Create an OptionsBootstrapper for these args/env, and a captured Console instance. args = self._implicit_args + (self.goal_cls.name, ) + tuple(args) env = dict(env) options_bootstrapper = OptionsBootstrapper.create(args=args, env=env) BuildConfigInitializer.get(options_bootstrapper) full_options = options_bootstrapper.get_full_options( list(self.goal_cls.subsystem_cls.known_scope_infos())) stdout, stderr = StringIO(), StringIO() console = Console(stdout=stdout, stderr=stderr) scheduler = self.scheduler workspace = Workspace(scheduler) # Run for the target specs parsed from the args. specs = TargetRootsCalculator.parse_specs(full_options.positional_args, self.build_root) params = Params(specs, console, options_bootstrapper, workspace, *additional_params) actual_exit_code = self.scheduler.run_console_rule( self.goal_cls, params) # Flush and capture console output. console.flush() stdout_val = stdout.getvalue() stderr_val = stderr.getvalue() assert exit_code == actual_exit_code, \ f"Exited with {actual_exit_code} (expected {exit_code}):\nstdout:\n{stdout_val}\nstderr:\n{stderr_val}" return stdout_val
def create(cls, options_bootstrapper, full_init=True) -> "PantsDaemon": """ :param OptionsBootstrapper options_bootstrapper: The bootstrap options. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = options_bootstrapper.bootstrap_options bootstrap_options_values = bootstrap_options.for_global_scope() native: Optional[Native] = None build_root: Optional[str] = None if full_init: build_root = get_buildroot() native = Native() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create( bootstrap_options_values).watchman services = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, native, watchman, union_membership=UnionMembership( build_config.union_rules()), ) else: services = PantsServices() return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level, services=services, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options, )
def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman native = None build_root = None services = None port_map = None if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) options_bootstrapper = OptionsBootstrapper() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native, bootstrap_options_values, build_config) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman ) return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, socket_map=port_map, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options )
def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman native = None build_root = None services = None port_map = None if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) options_bootstrapper = OptionsBootstrapper() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native, bootstrap_options_values, build_config) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman ) return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, socket_map=port_map, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options )
def runner_factory(sock, arguments, environment): exiter = self._exiter_class(sock) graph_helper = None deferred_exc = None self._logger.debug('execution commandline: %s', arguments) options_bootstrapper = OptionsBootstrapper(args=arguments) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) graph_helper, target_roots = None, None try: self._logger.debug('warming the product graph via %s', self._scheduler_service) # N.B. This call is made in the pre-fork daemon context for reach and reuse of the # resident scheduler. graph_helper, target_roots = self._scheduler_service.warm_product_graph( options, self._target_roots_calculator ) except Exception: deferred_exc = sys.exc_info() self._logger.warning( 'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s', ''.join(traceback.format_exception(*deferred_exc)) ) return self._runner_class( sock, exiter, arguments, environment, target_roots, graph_helper, self.fork_lock, deferred_exc )
def parse_options(args, env, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def parse_options(options_bootstrapper): # TODO: Because _OptionsBootstapper is currently provided as a Param, this @rule relies on options # remaining relatively stable in order to be efficient. See #6845 for a discussion of how to make # minimize the size of that value. build_config = BuildConfigInitializer.get(options_bootstrapper) return _Options(OptionsInitializer.create(options_bootstrapper, build_config, init_subsystems=False))
def setup_legacy_graph_extended( pants_ignore_patterns: List[str], use_gitignore: bool, local_store_dir: str, local_execution_root_dir: str, named_caches_dir: str, build_file_prelude_globs: Tuple[str, ...], options_bootstrapper: OptionsBootstrapper, build_configuration: BuildConfiguration, execution_options: ExecutionOptions, build_root: Optional[str] = None, native: Optional[Native] = None, glob_match_error_behavior: GlobMatchErrorBehavior = GlobMatchErrorBehavior.warn, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error: bool = True, ) -> LegacyGraphScheduler: """Construct and return the components necessary for LegacyBuildGraph construction. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param local_execution_root_dir: The directory to use for local execution sandboxes. :param named_caches_dir: The base directory for named cache storage. :param build_file_prelude_globs: Globs to match files to be prepended to all BUILD files. :param build_root: A path to be used as the build root. If None, then default is used. :param native: An instance of the native-engine subsystem. :param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options. :param build_configuration: The `BuildConfiguration` object to get build file aliases from. :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :param execution_options: Option values for (remote) process execution. """ build_root = build_root or get_buildroot() build_configuration = build_configuration or BuildConfigInitializer.get( options_bootstrapper) bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) build_file_aliases = build_configuration.registered_aliases() rules = build_configuration.rules() registered_target_types = RegisteredTargetTypes.create( build_configuration.target_types()) symbol_table = _legacy_symbol_table(build_file_aliases, registered_target_types) execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS # Register "literal" subjects required for these rules. parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases) address_mapper = AddressMapper( parser=parser, prelude_glob_patterns=build_file_prelude_globs, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots, ) @rule def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior: return glob_match_error_behavior @rule def build_configuration_singleton() -> BuildConfiguration: return build_configuration @rule def symbol_table_singleton() -> SymbolTable: return symbol_table @rule def registered_target_types_singleton() -> RegisteredTargetTypes: return registered_target_types @rule def union_membership_singleton() -> UnionMembership: return UnionMembership(build_configuration.union_rules()) @rule def build_root_singleton() -> BuildRoot: return cast(BuildRoot, BuildRoot.instance) # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( RootRule(Console), glob_match_error_behavior_singleton, build_configuration_singleton, symbol_table_singleton, registered_target_types_singleton, union_membership_singleton, build_root_singleton, *interactive_runner.rules(), *graph.rules(), *options_parsing.rules(), *process.rules(), *target.rules(), *create_legacy_graph_tasks(), *create_fs_rules(), *create_platform_rules(), *create_graph_rules(address_mapper), *structs_rules(), *changed_rules(), *binary_tool_rules(), *binary_util_rules(), *rules, ) goal_map = EngineInitializer._make_goal_map_from_rules(rules) union_rules = build_configuration.union_rules() scheduler = Scheduler( native=native, ignore_patterns=pants_ignore_patterns, use_gitignore=use_gitignore, build_root=build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, rules=rules, union_rules=union_rules, execution_options=execution_options, include_trace_on_error=include_trace_on_error, visualize_to_dir=bootstrap_options.native_engine_visualize_to, ) return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def parse_options(args, env, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def setup_legacy_graph_extended( pants_ignore_patterns, workdir, local_store_dir, build_file_imports_behavior, options_bootstrapper, build_configuration, build_root=None, native=None, glob_match_error_behavior=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True, execution_options=None, ): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use import statements. Valid values: "allow", "warn", "error". :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options. :type options_bootstrapper: :class:`pants.options.options_bootstrapper.OptionsBootstrapper` :param build_configuration: The `BuildConfiguration` object to get build file aliases from. :type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration` :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :param execution_options: Option values for (remote) process execution. :type execution_options: :class:`pants.option.global_options.ExecutionOptions` :returns: A LegacyGraphScheduler. """ build_root = build_root or get_buildroot() build_configuration = build_configuration or BuildConfigInitializer.get(options_bootstrapper) bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() build_file_aliases = build_configuration.registered_aliases() rules = build_configuration.rules() symbol_table = _legacy_symbol_table(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS # Register "literal" subjects required for these rules. parser = LegacyPythonCallbacksParser( symbol_table, build_file_aliases, build_file_imports_behavior ) address_mapper = AddressMapper(parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) @rule(GlobMatchErrorBehavior, []) def glob_match_error_behavior_singleton(): return glob_match_error_behavior or GlobMatchErrorBehavior.ignore @rule(BuildConfiguration, []) def build_configuration_singleton(): return build_configuration @rule(SymbolTable, []) def symbol_table_singleton(): return symbol_table # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( [ RootRule(Console), glob_match_error_behavior_singleton, build_configuration_singleton, symbol_table_singleton, ] + create_legacy_graph_tasks() + create_fs_rules() + create_process_rules() + create_graph_rules(address_mapper) + create_options_parsing_rules() + structs_rules() + # TODO: This should happen automatically, but most tests (e.g. tests/python/pants_test/auth) fail if it's not here: python_test_runner.rules() + rules ) goal_map = EngineInitializer._make_goal_map_from_rules(rules) union_rules = build_configuration.union_rules() scheduler = Scheduler( native, project_tree, workdir, local_store_dir, rules, union_rules, execution_options, include_trace_on_error=include_trace_on_error, visualize_to_dir=bootstrap_options.native_engine_visualize_to, ) return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def setup_legacy_graph_extended( pants_ignore_patterns, local_store_dir, build_file_imports_behavior, options_bootstrapper, build_configuration, build_root=None, native=None, glob_match_error_behavior=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True, execution_options=None, ): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use import statements. Valid values: "allow", "warn", "error". :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options. :type options_bootstrapper: :class:`pants.options.options_bootstrapper.OptionsBootstrapper` :param build_configuration: The `BuildConfiguration` object to get build file aliases from. :type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration` :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :param execution_options: Option values for (remote) process execution. :type execution_options: :class:`pants.option.global_options.ExecutionOptions` :returns: A LegacyGraphScheduler. """ build_root = build_root or get_buildroot() build_configuration = build_configuration or BuildConfigInitializer.get(options_bootstrapper) bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() build_file_aliases = build_configuration.registered_aliases() rules = build_configuration.rules() symbol_table = _legacy_symbol_table(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS # Register "literal" subjects required for these rules. parser = LegacyPythonCallbacksParser( symbol_table, build_file_aliases, build_file_imports_behavior ) address_mapper = AddressMapper(parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) @rule def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior: return glob_match_error_behavior or GlobMatchErrorBehavior.ignore @rule def build_configuration_singleton() -> BuildConfiguration: return build_configuration @rule def symbol_table_singleton() -> SymbolTable: return symbol_table @rule def union_membership_singleton() -> UnionMembership: return UnionMembership(build_configuration.union_rules()) @rule def build_root_singleton() -> BuildRoot: return BuildRoot.instance @rule async def single_build_file_address(specs: Specs) -> BuildFileAddress: build_file_addresses = await Get(BuildFileAddresses, Specs, specs) if len(build_file_addresses.dependencies) == 0: raise ResolveError("No targets were matched") if len(build_file_addresses.dependencies) > 1: potential_addresses = await Get(BuildFileAddresses, Specs, specs) targets = [bfa.to_address() for bfa in potential_addresses] output = '\n '.join(str(target) for target in targets) raise ResolveError( "Expected a single target, but was given multiple targets:\n" f"Did you mean one of:\n {output}" ) return build_file_addresses.dependencies[0] # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( [ RootRule(Console), glob_match_error_behavior_singleton, build_configuration_singleton, symbol_table_singleton, union_membership_singleton, build_root_singleton, single_build_file_address, ] + create_legacy_graph_tasks() + create_fs_rules() + create_interactive_runner_rules() + create_process_rules() + create_platform_rules() + create_graph_rules(address_mapper) + create_options_parsing_rules() + structs_rules() + rules ) goal_map = EngineInitializer._make_goal_map_from_rules(rules) union_rules = build_configuration.union_rules() scheduler = Scheduler( native, project_tree, local_store_dir, rules, union_rules, execution_options, include_trace_on_error=include_trace_on_error, visualize_to_dir=bootstrap_options.native_engine_visualize_to, ) return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def setup_legacy_graph_extended( pants_ignore_patterns, workdir, local_store_dir, build_file_imports_behavior, options_bootstrapper, build_configuration, build_root=None, native=None, glob_match_error_behavior=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True, execution_options=None, ): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use import statements. Valid values: "allow", "warn", "error". :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options. :type options_bootstrapper: :class:`pants.options.options_bootstrapper.OptionsBootstrapper` :param build_configuration: The `BuildConfiguration` object to get build file aliases from. :type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration` :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :param execution_options: Option values for (remote) process execution. :type execution_options: :class:`pants.option.global_options.ExecutionOptions` :returns: A LegacyGraphScheduler. """ build_root = build_root or get_buildroot() build_configuration = build_configuration or BuildConfigInitializer.get( options_bootstrapper) build_file_aliases = build_configuration.registered_aliases() rules = build_configuration.rules() console = Console() symbol_table = LegacySymbolTable(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS # Register "literal" subjects required for these rules. parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases, build_file_imports_behavior) address_mapper = AddressMapper( parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) # Load the native backend. native = native or Native.create() # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( [ SingletonRule.from_instance(console), SingletonRule.from_instance( GlobMatchErrorBehavior.create(glob_match_error_behavior)), SingletonRule.from_instance(build_configuration), ] + create_legacy_graph_tasks(symbol_table) + create_fs_rules() + create_process_rules() + create_graph_rules(address_mapper, symbol_table) + create_options_parsing_rules() + create_core_rules() + # TODO: This should happen automatically, but most tests (e.g. tests/python/pants_test/auth) fail if it's not here: [run_python_test] + rules) goal_map = EngineInitializer._make_goal_map_from_rules(rules) scheduler = Scheduler( native, project_tree, workdir, local_store_dir, rules, execution_options, include_trace_on_error=include_trace_on_error, ) return LegacyGraphScheduler(scheduler, symbol_table, console, goal_map)
def create( cls, env: Mapping[str, str], options_bootstrapper: OptionsBootstrapper, scheduler: Optional[GraphScheduler] = None, cancellation_latch: Optional[PySessionCancellationLatch] = None, ) -> LocalPantsRunner: """Creates a new LocalPantsRunner instance by parsing options. By the time this method runs, logging will already have been initialized in either PantsRunner or DaemonPantsRunner. :param env: The environment (e.g. os.environ) for this run. :param options_bootstrapper: The OptionsBootstrapper instance to reuse. :param scheduler: If being called from the daemon, a warmed scheduler to use. """ build_root = get_buildroot() global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() build_config = BuildConfigInitializer.get(options_bootstrapper) try: options = OptionsInitializer.create(options_bootstrapper, build_config) except UnknownFlagsError as err: cls._handle_unknown_flags(err, options_bootstrapper) raise run_tracker = RunTracker(options) union_membership = UnionMembership.from_rules(build_config.union_rules) # If we're running with the daemon, we'll be handed a warmed Scheduler, which we use # to initialize a session here. graph_session = cls._init_graph_session( options_bootstrapper, build_config, run_tracker.run_id, options, scheduler, cancellation_latch, ) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): try: options.for_scope(scope) except UnknownFlagsError as err: cls._handle_unknown_flags(err, options_bootstrapper) raise # Verify configs. if global_bootstrap_options.verify_config: options.verify_configs(options_bootstrapper.config) specs = calculate_specs( options_bootstrapper=options_bootstrapper, options=options, build_root=build_root, session=graph_session.scheduler_session, ) profile_path = env.get("PANTS_PROFILE") return cls( build_root=build_root, options=options, build_config=build_config, run_tracker=run_tracker, specs=specs, graph_session=graph_session, union_membership=union_membership, profile_path=profile_path, )
def clean_global_runtime_state() -> None: """Resets the global runtime state of a pants runtime.""" # Reset global plugin state. BuildConfigInitializer.reset()
def setup_legacy_graph_extended( pants_ignore_patterns, workdir, build_file_imports_behavior, build_configuration, build_root=None, native=None, glob_match_error_behavior=None, rules=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True, remote_store_server=None, remote_execution_server=None ): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use import statements. Valid values: "allow", "warn", "error". :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param build_configuration: The `BuildConfiguration` object to get build file aliases from. :type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration` :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :returns: A LegacyGraphScheduler. """ build_root = build_root or get_buildroot() build_configuration = build_configuration or BuildConfigInitializer.get(OptionsBootstrapper()) build_file_aliases = build_configuration.registered_aliases() rules = rules or build_configuration.rules() or [] symbol_table = LegacySymbolTable(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these rules. parser = LegacyPythonCallbacksParser( symbol_table, build_file_aliases, build_file_imports_behavior ) address_mapper = AddressMapper(parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) # Load the native backend. native = native or Native.create() # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( [ SingletonRule.from_instance(GlobMatchErrorBehavior.create(glob_match_error_behavior)), SingletonRule.from_instance(build_configuration), ] + create_legacy_graph_tasks(symbol_table) + create_fs_rules() + create_process_rules() + create_graph_rules(address_mapper, symbol_table) + create_options_parsing_rules() + rules ) scheduler = Scheduler( native, project_tree, workdir, rules, remote_store_server, remote_execution_server, include_trace_on_error=include_trace_on_error, ) return LegacyGraphScheduler(scheduler, symbol_table)
def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper.create(args=['--pants-version=99.99.9999']) build_config = BuildConfigInitializer.get(options_bootstrapper) with self.assertRaises(BuildConfigurationError): OptionsInitializer.create(options_bootstrapper, build_config)