def setup_legacy_graph( native: Native, options_bootstrapper: OptionsBootstrapper, build_configuration: BuildConfiguration, ) -> LegacyGraphScheduler: """Construct and return the components necessary for LegacyBuildGraph construction.""" build_root = get_buildroot() bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) use_gitignore = bootstrap_options.pants_ignore_use_gitignore return EngineInitializer.setup_legacy_graph_extended( OptionsInitializer.compute_pants_ignore(build_root, bootstrap_options), use_gitignore, bootstrap_options.local_store_dir, bootstrap_options.local_execution_root_dir, bootstrap_options.named_caches_dir, bootstrap_options.build_file_prelude_globs, options_bootstrapper, build_configuration, build_root=build_root, native=native, glob_match_error_behavior=( bootstrap_options.files_not_found_behavior. to_glob_match_error_behavior()), build_ignore_patterns=bootstrap_options.build_ignore, exclude_target_regexps=bootstrap_options.exclude_target_regexp, subproject_roots=bootstrap_options.subproject_roots, include_trace_on_error=bootstrap_options. print_exception_stacktrace, execution_options=ExecutionOptions.from_bootstrap_options( bootstrap_options), )
def _init_engine(self, local_store_dir: Optional[str] = None) -> None: if self._scheduler is not None: return options_bootstrapper = OptionsBootstrapper.create( env={}, args=["--pants-config-files=[]", *self.additional_options], allow_pantsrc=False) global_options = options_bootstrapper.bootstrap_options.for_global_scope( ) local_store_dir = local_store_dir or global_options.local_store_dir local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir graph_session = EngineInitializer.setup_graph_extended( pants_ignore_patterns=[], use_gitignore=False, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, native=Native(), options_bootstrapper=options_bootstrapper, build_root=self.build_root, build_configuration=self.build_config(), execution_options=ExecutionOptions.from_bootstrap_options( global_options), ).new_session(build_id="buildid_for_test", should_report_workunits=True) self._scheduler = graph_session.scheduler_session
def setup_graph( options_bootstrapper: OptionsBootstrapper, build_configuration: BuildConfiguration, env: CompleteEnvironment, executor: Optional[PyExecutor] = None, local_only: bool = False, ) -> GraphScheduler: native = Native() build_root = get_buildroot() bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) options = options_bootstrapper.full_options(build_configuration) assert bootstrap_options is not None executor = executor or PyExecutor( *GlobalOptions.compute_executor_arguments(bootstrap_options)) execution_options = ExecutionOptions.from_options( options, env, local_only=local_only) return EngineInitializer.setup_graph_extended( build_configuration, execution_options, native=native, executor=executor, pants_ignore_patterns=GlobalOptions.compute_pants_ignore( build_root, bootstrap_options), use_gitignore=bootstrap_options.pants_ignore_use_gitignore, local_store_dir=bootstrap_options.local_store_dir, local_execution_root_dir=bootstrap_options. local_execution_root_dir, named_caches_dir=bootstrap_options.named_caches_dir, ca_certs_path=bootstrap_options.ca_certs_path, build_root=build_root, include_trace_on_error=bootstrap_options.print_stacktrace, native_engine_visualize_to=bootstrap_options. native_engine_visualize_to, )
def create_execution_options( *, initial_headers: dict[str, str], token_path: str | None = None, plugin: str | None = None, remote_store_address: str = "grpc://fake.url:10", remote_execution_address: str = "grpc://fake.url:10", local_only: bool = False, ) -> ExecutionOptions: args = [ "--remote-cache-read", f"--remote-execution-address={remote_execution_address}", f"--remote-store-address={remote_store_address}", f"--remote-store-headers={initial_headers}", f"--remote-execution-headers={initial_headers}", "--remote-instance-name=main", ] if token_path: args.append(f"--remote-oauth-bearer-token-path={token_path}") if plugin: args.append(f"--remote-auth-plugin={plugin}") ob = create_options_bootstrapper(args) env = CompleteEnvironment({}) _build_config, options = OptionsInitializer( ob, env).build_config_and_options(ob, env, raise_=False) return ExecutionOptions.from_options(options, env, local_only=local_only)
def setup_graph( options_bootstrapper: OptionsBootstrapper, build_configuration: BuildConfiguration, ) -> GraphScheduler: native = Native() build_root = get_buildroot() bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) return EngineInitializer.setup_graph_extended( options_bootstrapper, build_configuration, ExecutionOptions.from_bootstrap_options(bootstrap_options), pants_ignore_patterns=OptionsInitializer.compute_pants_ignore( build_root, bootstrap_options), use_gitignore=bootstrap_options.pants_ignore_use_gitignore, local_store_dir=bootstrap_options.local_store_dir, local_execution_root_dir=bootstrap_options. local_execution_root_dir, named_caches_dir=bootstrap_options.named_caches_dir, ca_certs_path=bootstrap_options.ca_certs_path, build_root=build_root, native=native, include_trace_on_error=bootstrap_options. print_exception_stacktrace, )
def setup_graph( options: Options, build_configuration: BuildConfiguration, executor: Optional[PyExecutor] = None, ) -> GraphScheduler: native = Native() build_root = get_buildroot() bootstrap_options = options.bootstrap_option_values() assert bootstrap_options is not None executor = executor or PyExecutor( *OptionsInitializer.compute_executor_arguments(bootstrap_options)) return EngineInitializer.setup_graph_extended( build_configuration, ExecutionOptions.from_options(options), native=native, executor=executor, pants_ignore_patterns=OptionsInitializer.compute_pants_ignore( build_root, bootstrap_options), use_gitignore=bootstrap_options.pants_ignore_use_gitignore, local_store_dir=bootstrap_options.local_store_dir, local_execution_root_dir=bootstrap_options. local_execution_root_dir, named_caches_dir=bootstrap_options.named_caches_dir, ca_certs_path=bootstrap_options.ca_certs_path, build_root=build_root, include_trace_on_error=bootstrap_options.print_stacktrace, native_engine_visualize_to=bootstrap_options. native_engine_visualize_to, )
def _init_engine(self, local_store_dir: Optional[str] = None) -> None: if self._scheduler is not None: return options_bootstrapper = OptionsBootstrapper.create( env={}, args=["--pants-config-files=[]", *self.additional_options], allow_pantsrc=False ) global_options = options_bootstrapper.bootstrap_options.for_global_scope() local_store_dir = local_store_dir or global_options.local_store_dir local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir # NB: This uses the long form of initialization because it needs to directly specify # `cls.alias_groups` rather than having them be provided by bootstrap options. graph_session = EngineInitializer.setup_legacy_graph_extended( pants_ignore_patterns=[], use_gitignore=False, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, build_file_prelude_globs=(), glob_match_error_behavior=GlobMatchErrorBehavior.error, native=init_native(), options_bootstrapper=options_bootstrapper, build_root=self.build_root, build_configuration=self.build_config(), execution_options=ExecutionOptions.from_bootstrap_options(global_options), ).new_session(build_id="buildid_for_test", should_report_workunits=True) self._scheduler = graph_session.scheduler_session
def setup_graph( bootstrap_options: OptionValueContainer, build_configuration: BuildConfiguration, dynamic_remote_options: DynamicRemoteOptions, executor: PyExecutor | None = None, ) -> GraphScheduler: build_root = get_buildroot() executor = executor or GlobalOptions.create_py_executor( bootstrap_options) execution_options = ExecutionOptions.from_options( bootstrap_options, dynamic_remote_options) local_store_options = LocalStoreOptions.from_options(bootstrap_options) return EngineInitializer.setup_graph_extended( build_configuration, execution_options, executor=executor, pants_ignore_patterns=GlobalOptions.compute_pants_ignore( build_root, bootstrap_options), use_gitignore=bootstrap_options.pants_ignore_use_gitignore, local_store_options=local_store_options, local_execution_root_dir=bootstrap_options. local_execution_root_dir, named_caches_dir=bootstrap_options.named_caches_dir, ca_certs_path=bootstrap_options.ca_certs_path, build_root=build_root, include_trace_on_error=bootstrap_options.print_stacktrace, engine_visualize_to=bootstrap_options.engine_visualize_to, watch_filesystem=bootstrap_options.watch_filesystem, use_deprecated_python_macros=bootstrap_options. use_deprecated_python_macros, )
def mk_scheduler( self, rules=None, union_rules=None, project_tree=None, work_dir=None, include_trace_on_error=True, should_report_workunits=False, execution_options=None, ): """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" rules = rules or [] work_dir = work_dir or self._create_work_dir() project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir) local_store_dir = os.path.realpath(safe_mkdtemp()) if execution_options is not None: eo = asdict(DEFAULT_EXECUTION_OPTIONS) eo.update(execution_options) execution_options = ExecutionOptions(**eo) scheduler = Scheduler( native=self._native, ignore_patterns=project_tree.ignore_patterns, use_gitignore=False, build_root=project_tree.build_root, local_store_dir=local_store_dir, rules=rules, union_rules=union_rules, execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS, include_trace_on_error=include_trace_on_error, ) return scheduler.new_session( zipkin_trace_v2=False, build_id="buildid_for_test", should_report_workunits=should_report_workunits, )
def __init__( self, *, rules: Optional[Iterable] = None, target_types: Optional[Iterable[Type[Target]]] = None, objects: Optional[Dict[str, Any]] = None, context_aware_object_factories: Optional[Dict[str, Any]] = None, ) -> None: self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT")) safe_mkdir(self.build_root, clean=True) safe_mkdir(self.pants_workdir) BuildRoot().path = self.build_root # TODO: Redesign rule registration for tests to be more ergonomic and to make this less # special-cased. all_rules = ( *(rules or ()), *source_root.rules(), *pants_environment.rules(), QueryRule(WrappedTarget, (Address,)), ) build_config_builder = BuildConfiguration.Builder() build_config_builder.register_aliases( BuildFileAliases( objects=objects, context_aware_object_factories=context_aware_object_factories ) ) build_config_builder.register_rules(all_rules) build_config_builder.register_target_types(target_types or ()) self.build_config = build_config_builder.create() options_bootstrapper = create_options_bootstrapper() global_options = options_bootstrapper.bootstrap_options.for_global_scope() local_store_dir = global_options.local_store_dir local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir graph_session = EngineInitializer.setup_graph_extended( pants_ignore_patterns=[], use_gitignore=False, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, native=Native(), options_bootstrapper=options_bootstrapper, build_root=self.build_root, build_configuration=self.build_config, execution_options=ExecutionOptions.from_bootstrap_options(global_options), ).new_session( build_id="buildid_for_test", session_values=SessionValues( {OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment()} ), should_report_workunits=True, ) self.scheduler = graph_session.scheduler_session
def setup_legacy_graph(native, bootstrap_options, build_configuration): """Construct and return the components necessary for LegacyBuildGraph construction.""" return EngineInitializer.setup_legacy_graph_extended( bootstrap_options.pants_ignore, bootstrap_options.pants_workdir, bootstrap_options.build_file_imports, build_configuration, native=native, glob_match_error_behavior=bootstrap_options.glob_expansion_failure, build_ignore_patterns=bootstrap_options.build_ignore, exclude_target_regexps=bootstrap_options.exclude_target_regexp, subproject_roots=bootstrap_options.subproject_roots, include_trace_on_error=bootstrap_options.print_exception_stacktrace, execution_options=ExecutionOptions.from_bootstrap_options(bootstrap_options), )
def setup_legacy_graph(native, options_bootstrapper, build_configuration): """Construct and return the components necessary for LegacyBuildGraph construction.""" build_root = get_buildroot() bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() return EngineInitializer.setup_legacy_graph_extended( OptionsInitializer.compute_pants_ignore(build_root, bootstrap_options), bootstrap_options.local_store_dir, bootstrap_options.build_file_imports, options_bootstrapper, build_configuration, build_root=build_root, native=native, glob_match_error_behavior=bootstrap_options.glob_expansion_failure, build_ignore_patterns=bootstrap_options.build_ignore, exclude_target_regexps=bootstrap_options.exclude_target_regexp, subproject_roots=bootstrap_options.subproject_roots, include_trace_on_error=bootstrap_options.print_exception_stacktrace, execution_options=ExecutionOptions.from_bootstrap_options(bootstrap_options), )
def mk_scheduler( self, rules=None, project_tree=None, work_dir=None, include_trace_on_error=True, execution_options=None, ca_certs_path=None, ) -> SchedulerSession: """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" rules = rules or [] work_dir = work_dir or self._create_work_dir() project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir) local_store_dir = os.path.realpath(safe_mkdtemp()) local_execution_root_dir = os.path.realpath(safe_mkdtemp()) named_caches_dir = os.path.realpath(safe_mkdtemp()) if execution_options is not None: eo = asdict(DEFAULT_EXECUTION_OPTIONS) eo.update(execution_options) execution_options = ExecutionOptions(**eo) scheduler = Scheduler( native=self._native, ignore_patterns=project_tree.ignore_patterns, use_gitignore=False, build_root=project_tree.build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, ca_certs_path=ca_certs_path, rules=rules, union_membership=UnionMembership({}), executor=self._executor, execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS, include_trace_on_error=include_trace_on_error, ) return scheduler.new_session(build_id="buildid_for_test", )
def __init__( self, *, rules: Iterable | None = None, target_types: Iterable[type[Target]] | None = None, objects: dict[str, Any] | None = None, context_aware_object_factories: dict[str, Any] | None = None, isolated_local_store: bool = False, ca_certs_path: str | None = None, ) -> None: self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT")) safe_mkdir(self.build_root, clean=True) safe_mkdir(self.pants_workdir) BuildRoot().path = self.build_root # TODO: Redesign rule registration for tests to be more ergonomic and to make this less # special-cased. all_rules = ( *(rules or ()), *source_root.rules(), *pants_environment.rules(), QueryRule(WrappedTarget, [Address]), QueryRule(UnionMembership, []), ) build_config_builder = BuildConfiguration.Builder() build_config_builder.register_aliases( BuildFileAliases( objects=objects, context_aware_object_factories=context_aware_object_factories)) build_config_builder.register_rules(all_rules) build_config_builder.register_target_types(target_types or ()) self.build_config = build_config_builder.create() self.options_bootstrapper = create_options_bootstrapper() options = self.options_bootstrapper.full_options(self.build_config) global_options = self.options_bootstrapper.bootstrap_options.for_global_scope( ) local_store_dir = (os.path.realpath(safe_mkdtemp()) if isolated_local_store else global_options.local_store_dir) local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir graph_session = EngineInitializer.setup_graph_extended( pants_ignore_patterns=GlobalOptions.compute_pants_ignore( self.build_root, global_options), use_gitignore=False, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, native=Native(), build_root=self.build_root, build_configuration=self.build_config, executor=_EXECUTOR, execution_options=ExecutionOptions.from_options(options), ca_certs_path=ca_certs_path, native_engine_visualize_to=None, ).new_session( build_id="buildid_for_test", session_values=SessionValues({ OptionsBootstrapper: self.options_bootstrapper, PantsEnvironment: PantsEnvironment(), }), ) self.scheduler = graph_session.scheduler_session
def __init__( self, *, rules: Iterable | None = None, target_types: Iterable[type[Target]] | None = None, objects: dict[str, Any] | None = None, context_aware_object_factories: dict[str, Any] | None = None, isolated_local_store: bool = False, preserve_tmpdirs: bool = False, ca_certs_path: str | None = None, bootstrap_args: Iterable[str] = (), ) -> None: bootstrap_args = [*bootstrap_args] root_dir: Path | None = None if preserve_tmpdirs: root_dir = Path(mkdtemp(prefix="RuleRunner.")) print( f"Preserving rule runner temporary directories at {root_dir}.", file=sys.stderr) bootstrap_args.extend([ "--no-process-execution-local-cleanup", f"--local-execution-root-dir={root_dir}", ]) build_root = (root_dir / "BUILD_ROOT").resolve() build_root.mkdir() self.build_root = str(build_root) else: self.build_root = os.path.realpath( safe_mkdtemp(prefix="_BUILD_ROOT")) safe_mkdir(self.pants_workdir) BuildRoot().path = self.build_root # TODO: Redesign rule registration for tests to be more ergonomic and to make this less # special-cased. all_rules = ( *(rules or ()), *source_root.rules(), QueryRule(WrappedTarget, [Address]), QueryRule(UnionMembership, []), ) build_config_builder = BuildConfiguration.Builder() build_config_builder.register_aliases( BuildFileAliases( objects=objects, context_aware_object_factories=context_aware_object_factories)) build_config_builder.register_rules("_dummy_for_test_", all_rules) build_config_builder.register_target_types("_dummy_for_test_", target_types or ()) self.build_config = build_config_builder.create() self.environment = CompleteEnvironment({}) self.options_bootstrapper = create_options_bootstrapper( args=bootstrap_args) options = self.options_bootstrapper.full_options(self.build_config) global_options = self.options_bootstrapper.bootstrap_options.for_global_scope( ) dynamic_remote_options, _ = DynamicRemoteOptions.from_options( options, self.environment) local_store_options = LocalStoreOptions.from_options(global_options) if isolated_local_store: if root_dir: lmdb_store_dir = root_dir / "lmdb_store" lmdb_store_dir.mkdir() store_dir = str(lmdb_store_dir) else: store_dir = safe_mkdtemp(prefix="lmdb_store.") local_store_options = dataclasses.replace(local_store_options, store_dir=store_dir) local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir graph_session = EngineInitializer.setup_graph_extended( pants_ignore_patterns=GlobalOptions.compute_pants_ignore( self.build_root, global_options), use_gitignore=False, local_store_options=local_store_options, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, build_root=self.build_root, build_configuration=self.build_config, executor=_EXECUTOR, execution_options=ExecutionOptions.from_options( global_options, dynamic_remote_options), ca_certs_path=ca_certs_path, engine_visualize_to=None, ).new_session( build_id="buildid_for_test", session_values=SessionValues({ OptionsBootstrapper: self.options_bootstrapper, CompleteEnvironment: self.environment, }), ) self.scheduler = graph_session.scheduler_session
def setup_legacy_graph( native: Native, options_bootstrapper: OptionsBootstrapper, build_configuration: BuildConfiguration, ) -> LegacyGraphScheduler: """Construct and return the components necessary for LegacyBuildGraph construction.""" build_root = get_buildroot() bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) glob_expansion_failure_configured = not bootstrap_options.is_default( "glob_expansion_failure") files_not_found_behavior_configured = not bootstrap_options.is_default( "files_not_found_behavior") if glob_expansion_failure_configured and files_not_found_behavior_configured: raise ValueError( "Conflicting options used. You used the new, preferred `--files-not-found-behavior`, but " "also used the deprecated `--glob-expansion-failure`.\n\nPlease " "specify only one of these (preferably `--files-not-found-behavior`)." ) glob_match_error_behavior = (bootstrap_options.files_not_found_behavior .to_glob_match_error_behavior() if files_not_found_behavior_configured else bootstrap_options.glob_expansion_failure) deprecated_conditional( lambda: cast( bool, bootstrap_options.build_file_imports == BuildFileImportsBehavior.allow), removal_version="1.27.0.dev0", entity_description="Using `--build-file-imports=allow`", hint_message= ("Import statements should be avoided in BUILD files because they can easily break Pants " "caching and lead to stale results. It is not safe to ignore warnings of imports, so the " "`allow` option is being removed.\n\nTo prepare for this change, either set " "`--build-file-imports=warn` or `--build-file-imports=error` (we recommend using `error`)." "\n\nIf you still need to keep the functionality you have from the import statement, " "consider rewriting your code into a Pants plugin: " "https://www.pantsbuild.org/howto_plugin.html")) deprecated_conditional( lambda: bootstrap_options.is_default("build_file_imports"), removal_version="1.27.0.dev0", entity_description="Defaulting to `--build-file-imports=warn`", hint_message= ("Import statements should be avoided in BUILD files because they can easily break Pants " "caching and lead to stale results. The default behavior will change from warning to " "erroring in 1.27.0.dev0, and the option will be removed in 1.29.0.dev0.\n\nTo prepare for " "this change, please explicitly set the option `--build-file-imports=warn` or " "`--build-file-imports=error` (we recommend using `error`).\n\nIf you still need to keep " "the functionality you have from import statements, consider rewriting your code into a " "Pants plugin: https://www.pantsbuild.org/howto_plugin.html")) return EngineInitializer.setup_legacy_graph_extended( OptionsInitializer.compute_pants_ignore(build_root, bootstrap_options), bootstrap_options.local_store_dir, bootstrap_options.build_file_imports, options_bootstrapper, build_configuration, build_root=build_root, native=native, glob_match_error_behavior=glob_match_error_behavior, build_ignore_patterns=bootstrap_options.build_ignore, exclude_target_regexps=bootstrap_options.exclude_target_regexp, subproject_roots=bootstrap_options.subproject_roots, include_trace_on_error=bootstrap_options. print_exception_stacktrace, execution_options=ExecutionOptions.from_bootstrap_options( bootstrap_options), )