def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native_engine.maybe_set_panic_handler()
if scheduler is None:
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return scheduler.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
use_colors=global_options.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def setup_graph(
options: Options,
build_configuration: BuildConfiguration,
executor: Optional[PyExecutor] = None,
) -> GraphScheduler:
native = Native()
build_root = get_buildroot()
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
executor = executor or PyExecutor(
*OptionsInitializer.compute_executor_arguments(bootstrap_options))
return EngineInitializer.setup_graph_extended(
build_configuration,
ExecutionOptions.from_options(options),
native=native,
executor=executor,
pants_ignore_patterns=OptionsInitializer.compute_pants_ignore(
build_root, bootstrap_options),
use_gitignore=bootstrap_options.pants_ignore_use_gitignore,
local_store_dir=bootstrap_options.local_store_dir,
local_execution_root_dir=bootstrap_options.
local_execution_root_dir,
named_caches_dir=bootstrap_options.named_caches_dir,
ca_certs_path=bootstrap_options.ca_certs_path,
build_root=build_root,
include_trace_on_error=bootstrap_options.print_stacktrace,
native_engine_visualize_to=bootstrap_options.
native_engine_visualize_to,
)
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: GraphScheduler | None = None,
cancellation_latch: PySessionCancellationLatch | None = None,
) -> GraphSession:
native_engine.maybe_set_panic_handler()
if scheduler is None:
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return scheduler.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
ui_use_prodash=global_options.dynamic_ui_renderer
== DynamicUIRenderer.experimental_prodash,
use_colors=global_options.get("colors", True),
max_workunit_level=max(
global_options.streaming_workunits_level,
global_options.level,
*(
LogLevel[level.upper()]
for level in global_options.log_levels_by_target.values()
),
),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def from_options(cls, options: Options) -> ExecutionOptions:
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
# Possibly insert some headers and disable remote execution/caching.
remote_execution_headers = cast(
Dict[str, str], bootstrap_options.remote_execution_headers)
remote_store_headers = cast(Dict[str, str],
bootstrap_options.remote_store_headers)
remote_execution = cast(bool, bootstrap_options.remote_execution)
remote_cache_read = cast(bool, bootstrap_options.remote_cache_read)
remote_cache_write = cast(bool, bootstrap_options.remote_cache_write)
if bootstrap_options.remote_oauth_bearer_token_path:
oauth_token = (Path(
bootstrap_options.remote_oauth_bearer_token_path).resolve().
read_text().strip())
if set(oauth_token).intersection({"\n", "\r"}):
raise OptionsError(
f"OAuth bearer token path {bootstrap_options.remote_oauth_bearer_token_path} "
"must not contain multiple lines.")
token_header = {"authorization": f"Bearer {oauth_token}"}
remote_execution_headers.update(token_header)
remote_store_headers.update(token_header)
if bootstrap_options.remote_auth_plugin and (remote_execution
or remote_cache_read
or remote_cache_write):
if ":" not in bootstrap_options.remote_auth_plugin:
raise OptionsError(
"Invalid value for `--remote-auth-plugin`: "
f"{bootstrap_options.remote_auth_plugin}. Please use the format "
f"`path.to.module:my_func`.")
auth_plugin_path, auth_plugin_func = bootstrap_options.remote_auth_plugin.split(
":")
auth_plugin_module = importlib.import_module(auth_plugin_path)
auth_plugin_func = getattr(auth_plugin_module, auth_plugin_func)
auth_plugin_result = cast(
AuthPluginResult,
auth_plugin_func(
initial_execution_headers=remote_execution_headers,
initial_store_headers=remote_store_headers,
options=options,
),
)
if not auth_plugin_result.is_available:
# NB: This is debug because we expect plugins to log more informative messages.
logger.debug(
"Disabling remote caching and remote execution because authentication was not "
"available via the plugin from `--remote-auth-plugin`.")
remote_execution = False
remote_cache_read = False
remote_cache_write = False
else:
remote_execution_headers = auth_plugin_result.execution_headers
remote_store_headers = auth_plugin_result.store_headers
return cls(
# Remote execution strategy.
remote_execution=remote_execution,
remote_cache_read=remote_cache_read,
remote_cache_write=remote_cache_write,
# General remote setup.
remote_instance_name=bootstrap_options.remote_instance_name,
remote_ca_certs_path=bootstrap_options.remote_ca_certs_path,
# Process execution setup.
process_execution_local_parallelism=bootstrap_options.
process_execution_local_parallelism,
process_execution_remote_parallelism=bootstrap_options.
process_execution_remote_parallelism,
process_execution_cleanup_local_dirs=bootstrap_options.
process_execution_cleanup_local_dirs,
process_execution_use_local_cache=bootstrap_options.
process_execution_use_local_cache,
process_execution_cache_namespace=bootstrap_options.
process_execution_cache_namespace,
process_execution_local_enable_nailgun=bootstrap_options.
process_execution_local_enable_nailgun,
# Remote store setup.
remote_store_server=bootstrap_options.remote_store_server,
remote_store_headers=remote_store_headers,
remote_store_thread_count=bootstrap_options.
remote_store_thread_count,
remote_store_chunk_bytes=bootstrap_options.
remote_store_chunk_bytes,
remote_store_chunk_upload_timeout_seconds=bootstrap_options.
remote_store_chunk_upload_timeout_seconds,
remote_store_rpc_retries=bootstrap_options.
remote_store_rpc_retries,
remote_store_connection_limit=bootstrap_options.
remote_store_connection_limit,
remote_store_initial_timeout=bootstrap_options.
remote_store_initial_timeout,
remote_store_timeout_multiplier=bootstrap_options.
remote_store_timeout_multiplier,
remote_store_maximum_timeout=bootstrap_options.
remote_store_maximum_timeout,
# Remote cache setup.
remote_cache_eager_fetch=bootstrap_options.
remote_cache_eager_fetch,
# Remote execution setup.
remote_execution_server=bootstrap_options.remote_execution_server,
remote_execution_extra_platform_properties=bootstrap_options.
remote_execution_extra_platform_properties,
remote_execution_headers=remote_execution_headers,
remote_execution_overall_deadline_secs=bootstrap_options.
remote_execution_overall_deadline_secs,
)
