def create_execution_options( *, initial_headers: dict[str, str], token_path: str | None = None, plugin: str | None = None, remote_store_address: str = "grpc://fake.url:10", remote_execution_address: str = "grpc://fake.url:10", local_only: bool = False, ) -> ExecutionOptions: args = [ "--remote-cache-read", f"--remote-execution-address={remote_execution_address}", f"--remote-store-address={remote_store_address}", f"--remote-store-headers={initial_headers}", f"--remote-execution-headers={initial_headers}", "--remote-instance-name=main", ] if token_path: args.append(f"--remote-oauth-bearer-token-path={token_path}") if plugin: args.append(f"--remote-auth-plugin={plugin}") ob = create_options_bootstrapper(args) env = CompleteEnvironment({}) _build_config, options = OptionsInitializer( ob, env).build_config_and_options(ob, env, raise_=False) return ExecutionOptions.from_options(options, env, local_only=local_only)
def create_dynamic_remote_options( *, initial_headers: dict[str, str] | None = None, address: str | None = "grpc://fake.url:10", token_path: str | None = None, plugin: str | None = None, ) -> DynamicRemoteOptions: if initial_headers is None: initial_headers = {} args = [ "--remote-cache-read", f"--remote-execution-address={address}", f"--remote-store-address={address}", f"--remote-store-headers={initial_headers}", f"--remote-execution-headers={initial_headers}", "--remote-instance-name=main", ] if token_path: args.append(f"--remote-oauth-bearer-token-path={token_path}") if plugin: args.append(f"--remote-auth-plugin={plugin}") ob = create_options_bootstrapper(args) env = CompleteEnvironment({}) _build_config, options = OptionsInitializer(ob).build_config_and_options(ob, env, raise_=False) return DynamicRemoteOptions.from_options(options, env)[0]
def runner_factory(sock, arguments, environment): exiter = self._exiter_class(sock) graph_helper = None deferred_exc = None # Capture the size of the graph prior to any warming, for stats. preceding_graph_size = self._scheduler_service.product_graph_len() self._logger.debug('resident graph size: %s', preceding_graph_size) self._logger.debug('execution commandline: %s', arguments) options, _ = OptionsInitializer( OptionsBootstrapper(args=arguments)).setup(init_logging=False) target_roots = self._target_roots_calculator.create( options, change_calculator=self._scheduler_service.change_calculator) try: self._logger.debug('warming the product graph via %s', self._scheduler_service) # N.B. This call is made in the pre-fork daemon context for reach and reuse of the # resident scheduler. graph_helper = self._scheduler_service.warm_product_graph( target_roots) except Exception: deferred_exc = sys.exc_info() self._logger.warning( 'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s', ''.join(traceback.format_exception(*deferred_exc))) return self._runner_class(sock, exiter, arguments, environment, target_roots, graph_helper, self.fork_lock, preceding_graph_size, deferred_exc)
def test_global_options_validation(self): # Specify an invalid combination of options. ob = OptionsBootstrapper.create( env={}, args=["--backend-packages=[]", "--remote-execution"], allow_pantsrc=False ) env = CompleteEnvironment({}) with self.assertRaises(OptionsError) as exc: OptionsInitializer(ob, env).build_config_and_options(ob, env, raise_=True) self.assertIn("The `--remote-execution` option requires", str(exc.exception))
def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper( env=self._env, args=self._args) options, build_config = OptionsInitializer( options_bootstrapper, exiter=self._exiter).setup() global_options = options.for_global_scope() # Apply exiter options. self._exiter.apply_options(options) # Option values are usually computed lazily on demand, # but command line options are eagerly computed for validation. for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify the configs here. if global_options.verify_config: options_bootstrapper.verify_configs_against_options(options) # Launch RunTracker as early as possible (just after Subsystem options are initialized). run_tracker = RunTracker.global_instance() reporting = Reporting.global_instance() reporting.initialize(run_tracker, self._run_start_time) try: # Determine the build root dir. root_dir = get_buildroot() # Capture a repro of the 'before' state for this build, if needed. repro = Reproducer.global_instance().create_repro() if repro: repro.capture(run_tracker.run_info.get_as_dict()) # Record the preceding product graph size. run_tracker.pantsd_stats.set_preceding_graph_size( self._preceding_graph_size) # Setup and run GoalRunner. goal_runner = GoalRunner.Factory(root_dir, options, build_config, run_tracker, reporting, self._target_roots, self._daemon_build_graph, self._exiter).setup() goal_runner_result = goal_runner.run() if repro: # TODO: Have Repro capture the 'after' state (as a diff) as well? repro.log_location_of_repro_file() finally: run_tracker_result = run_tracker.end() # Take the exit code with higher abs value in case of negative values. final_exit_code = goal_runner_result if abs(goal_runner_result) > abs( run_tracker_result) else run_tracker_result self._exiter.exit(final_exit_code)
def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper.create( env={}, args=["--backend-packages=[]", "--pants-version=99.99.9999"], allow_pantsrc=False, ) with self.assertRaises(BuildConfigurationError): OptionsInitializer(options_bootstrapper).build_config_and_options( options_bootstrapper, raise_=True)
def test_invalid_version(self) -> None: options_bootstrapper = OptionsBootstrapper.create( env={}, args=["--backend-packages=[]", "--pants-version=99.99.9999"], allow_pantsrc=False, ) env = CompleteEnvironment({}) with self.assertRaises(ExecutionError): OptionsInitializer(options_bootstrapper).build_config_and_options( options_bootstrapper, env, raise_=True)
def __init__( self, options_bootstrapper: OptionsBootstrapper, executor: PyExecutor, services_constructor: PantsServicesConstructor, ): self._options_initializer = OptionsInitializer(options_bootstrapper, executor=executor) self._executor = executor self._services_constructor = services_constructor self._lifecycle_lock = threading.RLock() # N.B. This Event is used as nothing more than an atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._scheduler: Optional[GraphScheduler] = None self._services: Optional[PantsServices] = None self._fingerprint: Optional[str] = None
def create(cls, options=None, args=None, build_root=None, change_calculator=None): """ :param Options options: An `Options` instance to use, if available. :param string args: Raw cli args to use for parsing if an `Options` instance isn't available. :param string build_root: The build root. :param ChangeCalculator change_calculator: A `ChangeCalculator` for calculating changes. """ if not options: assert args is not None, 'must pass `args` if not passing `options`' options, _ = OptionsInitializer( OptionsBootstrapper(args=args)).setup(init_logging=False) # Determine the literal target roots. spec_roots = cls.parse_specs(options.target_specs, build_root) # Determine `Changed` arguments directly from options to support pre-`Subsystem` # initialization paths. changed_options = options.for_scope('changed') changed_request = ChangedRequest.from_options(changed_options) logger.debug('args are: %s', args) logger.debug('spec_roots are: %s', spec_roots) logger.debug('changed_request is: %s', changed_request) if change_calculator and changed_request.is_actionable(): if spec_roots: # We've been provided spec roots (e.g. `./pants list ::`) AND a changed request. Error out. raise InvalidSpecConstraint( 'cannot provide changed parameters and target specs!') # We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute # alternate target roots. changed_addresses = change_calculator.changed_target_addresses( changed_request) logger.debug('changed addresses: %s', changed_addresses) return ChangedTargetRoots( tuple( SingleAddress(a.spec_path, a.target_name) for a in changed_addresses)) return LiteralTargetRoots(spec_roots)
def __init__( self, options_bootstrapper: OptionsBootstrapper, executor: PyExecutor, services_constructor: PantsServicesConstructor, ): self._options_initializer = OptionsInitializer(options_bootstrapper, executor) self._executor = executor self._services_constructor = services_constructor self._lifecycle_lock = threading.RLock() # N.B. This Event is used as nothing more than an atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._scheduler: GraphScheduler | None = None self._services: PantsServices | None = None self._fingerprint: str | None = None self._prior_dynamic_remote_options: DynamicRemoteOptions | None = None self._prior_auth_plugin_result: AuthPluginResult | None = None
def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options( ) bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create( bootstrap_options_values).watchman native = None build_root = None services = None port_map = None if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) _, build_config = OptionsInitializer( OptionsBootstrapper()).setup(init_logging=False) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph( native, bootstrap_options_values, build_config) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman) return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, socket_map=port_map, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options)
def aliases(cls): """TODO: This is a nasty escape hatch to pass aliases to LegacyPythonCallbacksParser.""" _, build_config = OptionsInitializer( OptionsBootstrapper()).setup(init_logging=False) return build_config.registered_aliases()
def _default_build_file_aliases(self): # TODO: Get default BuildFileAliases by extending BaseTest post # https://github.com/pantsbuild/pants/issues/4401 _, build_config = OptionsInitializer( OptionsBootstrapper()).setup(init_logging=False) return build_config.registered_aliases()
def get_default_build_file_aliases(): _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) return build_config.registered_aliases()
def setup_legacy_graph(pants_ignore_patterns, workdir, build_file_imports_behavior, build_root=None, native=None, build_file_aliases=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use import statements. Valid values: "allow", "warn", "error". :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param build_file_aliases: BuildFileAliases to register. :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :returns: A tuple of (scheduler, engine, symbol_table, build_graph_cls). """ build_root = build_root or get_buildroot() scm = get_scm() if not build_file_aliases: _, build_config = OptionsInitializer( OptionsBootstrapper()).setup(init_logging=False) build_file_aliases = build_config.registered_aliases() symbol_table = LegacySymbolTable(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these tasks. parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases, build_file_imports_behavior) address_mapper = AddressMapper( parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) # Load the native backend. native = native or Native.create() # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = (create_legacy_graph_tasks(symbol_table) + create_fs_rules() + create_graph_rules(address_mapper, symbol_table) + create_process_rules()) scheduler = LocalScheduler( workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error) change_calculator = EngineChangeCalculator(scheduler, symbol_table, scm) if scm else None return LegacyGraphHelper(scheduler, symbol_table, change_calculator)
def create( cls, env: CompleteEnvironment, options_bootstrapper: OptionsBootstrapper, options_initializer: Optional[OptionsInitializer] = None, scheduler: Optional[GraphScheduler] = None, cancellation_latch: Optional[PySessionCancellationLatch] = None, ) -> LocalPantsRunner: """Creates a new LocalPantsRunner instance by parsing options. By the time this method runs, logging will already have been initialized in either PantsRunner or DaemonPantsRunner. :param env: The environment for this run. :param options_bootstrapper: The OptionsBootstrapper instance to reuse. :param scheduler: If being called from the daemon, a warmed scheduler to use. """ options_initializer = options_initializer or OptionsInitializer(options_bootstrapper, env) build_config, options = options_initializer.build_config_and_options( options_bootstrapper, env, raise_=True ) run_tracker = RunTracker(options) union_membership = UnionMembership.from_rules(build_config.union_rules) # If we're running with the daemon, we'll be handed a warmed Scheduler, which we use # to initialize a session here. graph_session = cls._init_graph_session( options_initializer, options_bootstrapper, build_config, env, run_tracker.run_id, options, scheduler, cancellation_latch, ) # Option values are usually computed lazily on demand, but command line options are # eagerly computed for validation. with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True): for scope in options.scope_to_flags.keys(): options.for_scope(scope) # Verify configs. global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() if global_bootstrap_options.verify_config: options.verify_configs(options_bootstrapper.config) specs = calculate_specs( options_bootstrapper=options_bootstrapper, options=options, build_root=get_buildroot(), session=graph_session.scheduler_session, ) profile_path = env.get("PANTS_PROFILE") return cls( options=options, options_bootstrapper=options_bootstrapper, build_config=build_config, run_tracker=run_tracker, specs=specs, graph_session=graph_session, union_membership=union_membership, profile_path=profile_path, )
def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper(args=['--pants-version=99.99.9999']) initializer = OptionsInitializer(options_bootstrapper, WorkingSet()) with self.assertRaises(BuildConfigurationError): initializer.setup()