def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
if self._scheduler is not None:
return
options_bootstrapper = OptionsBootstrapper.create(
env={},
args=["--pants-config-files=[]", *self.additional_options],
allow_pantsrc=False)
global_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
local_store_dir = local_store_dir or global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config(),
execution_options=ExecutionOptions.from_bootstrap_options(
global_options),
).new_session(build_id="buildid_for_test",
should_report_workunits=True)
self._scheduler = graph_session.scheduler_session
def _initialize(
self,
options_fingerprint: str,
bootstrap_options: OptionValueContainer,
build_config: BuildConfiguration,
dynamic_remote_options: DynamicRemoteOptions,
scheduler_restart_explanation: str | None,
) -> None:
"""(Re-)Initialize the scheduler.
Must be called under the lifecycle lock.
"""
try:
logger.info(
f"{scheduler_restart_explanation}: reinitializing scheduler..."
if scheduler_restart_explanation else
"Initializing scheduler...")
if self._services:
self._services.shutdown()
self._scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options,
self._executor)
self._services = self._services_constructor(
bootstrap_options, self._scheduler)
self._fingerprint = options_fingerprint
logger.info("Scheduler initialized.")
except Exception as e:
self._kill_switch.set()
self._scheduler = None
raise e
def _init_scheduler(self, options_fingerprint: str,
options_bootstrapper: OptionsBootstrapper) -> None:
"""(Re-)Initialize the scheduler.
Must be called under the lifecycle lock.
"""
try:
if self._scheduler:
logger.info(
"initialization options changed: reinitializing pantsd...")
else:
logger.info("initializing pantsd...")
if self._services:
self._services.shutdown()
build_config = BuildConfigInitializer.get(options_bootstrapper)
self._scheduler = EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
bootstrap_options_values = options_bootstrapper.bootstrap_options.for_global_scope(
)
self._services = self._services_constructor(
bootstrap_options_values, self._scheduler)
self._fingerprint = options_fingerprint
logger.info("pantsd initialized.")
except Exception as e:
self._kill_switch.set()
self._scheduler = None
raise e
def _init_graph(self, target_roots, graph_helper, exclude_target_regexps, tags):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param TargetRoots target_roots: The existing `TargetRoots` object, if any.
:param LegacyGraphSession graph_helper: A LegacyGraphSession to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, SchedulerSession, TargetRoots).
"""
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(native,
self._global_options,
self._build_config)
graph_helper = graph_scheduler_helper.new_session()
target_roots = target_roots or TargetRootsCalculator.create(
options=self._options,
session=graph_helper.scheduler_session,
build_root=self._root_dir,
symbol_table=graph_helper.symbol_table,
exclude_patterns=tuple(exclude_target_regexps),
tags=tuple(tags)
)
graph, address_mapper = graph_helper.create_build_graph(target_roots,
self._root_dir)
return graph, address_mapper, graph_helper.scheduler_session, target_roots
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
v2_ui,
should_report_workunits=stream_workunits,
)
def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
if self._scheduler is not None:
return
options_bootstrapper = OptionsBootstrapper.create(
env={}, args=["--pants-config-files=[]", *self.additional_options], allow_pantsrc=False
)
global_options = options_bootstrapper.bootstrap_options.for_global_scope()
local_store_dir = local_store_dir or global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
build_file_prelude_globs=(),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
native=init_native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config(),
execution_options=ExecutionOptions.from_bootstrap_options(global_options),
).new_session(build_id="buildid_for_test", should_report_workunits=True)
self._scheduler = graph_session.scheduler_session
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native_engine.maybe_set_panic_handler()
if scheduler is None:
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return scheduler.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
use_colors=global_options.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config
)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
return graph_scheduler_helper.new_session(
run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
PantsEnvironment: PantsEnvironment(os.environ),
}
),
cancellation_latch=cancellation_latch,
)
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config, env
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return graph_scheduler_helper.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
use_colors=global_options.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
if self._scheduler is not None:
return
options_bootstrapper = OptionsBootstrapper.create(
args=["--pants-config-files=[]"])
local_store_dir = (local_store_dir
or options_bootstrapper.bootstrap_options.
for_global_scope().local_store_dir)
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=[],
local_store_dir=local_store_dir,
build_file_imports_behavior=BuildFileImportsBehavior.error,
native=init_native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config(),
build_ignore_patterns=None,
).new_session(zipkin_trace_v2=False, build_id="buildid_for_test")
self._scheduler = graph_session.scheduler_session
self._build_graph, self._address_mapper = graph_session.create_build_graph(
Specs(address_specs=AddressSpecs([]),
filesystem_specs=FilesystemSpecs([])),
self._build_root(),
)
def _init_scheduler(self, options_fingerprint: str,
options_bootstrapper: OptionsBootstrapper) -> None:
"""(Re-)Initialize the scheduler.
Must be called under the lifecycle lock.
"""
try:
if self._scheduler:
logger.info(
"initialization options changed: reinitializing pantsd...")
else:
logger.info("initializing pantsd...")
if self._services:
self._services.shutdown()
with OptionsInitializer.handle_unknown_flags(options_bootstrapper,
raise_=True):
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper,
build_config)
self._scheduler = EngineInitializer.setup_graph(
options, build_config, executor=self._executor)
bootstrap_options_values = options.bootstrap_option_values()
assert bootstrap_options_values is not None
self._services = self._services_constructor(
bootstrap_options_values, self._scheduler)
self._fingerprint = options_fingerprint
logger.info("pantsd initialized.")
except Exception as e:
self._kill_switch.set()
self._scheduler = None
raise e
def _initialize(self, options_fingerprint: str,
options_bootstrapper: OptionsBootstrapper) -> None:
"""(Re-)Initialize the scheduler.
Must be called under the lifecycle lock.
"""
try:
if self._scheduler:
logger.info(
"initialization options changed: reinitializing scheduler..."
)
else:
logger.info("initializing scheduler...")
if self._services:
self._services.shutdown()
build_config, options = self._options_initializer.build_config_and_options(
options_bootstrapper, raise_=True)
self._scheduler = EngineInitializer.setup_graph(
options_bootstrapper, build_config, executor=self._executor)
bootstrap_options_values = options.bootstrap_option_values()
assert bootstrap_options_values is not None
self._services = self._services_constructor(
bootstrap_options_values, self._scheduler)
self._fingerprint = options_fingerprint
logger.info("scheduler initialized.")
except Exception as e:
self._kill_switch.set()
self._scheduler = None
raise e
def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[GraphScheduler] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
should_report_workunits=stream_workunits,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(os.environ),
}),
)
def create_address_mapper(self, build_root):
work_dir = os.path.join(build_root, '.pants.d')
scheduler = EngineInitializer.setup_legacy_graph(
[],
work_dir,
build_file_imports_behavior='allow',
build_root=build_root,
native=self._native).scheduler
return LegacyAddressMapper(scheduler.new_session(), build_root)
def _maybe_init_graph_session(graph_session, global_options, build_config):
if graph_session:
return graph_session
native = Native.create(global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, global_options, build_config)
return graph_scheduler_helper.new_session()
def __init__(
self,
*,
rules: Optional[Iterable] = None,
target_types: Optional[Iterable[Type[Target]]] = None,
objects: Optional[Dict[str, Any]] = None,
context_aware_object_factories: Optional[Dict[str, Any]] = None,
) -> None:
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
safe_mkdir(self.build_root, clean=True)
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
*pants_environment.rules(),
QueryRule(WrappedTarget, (Address,)),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects, context_aware_object_factories=context_aware_object_factories
)
)
build_config_builder.register_rules(all_rules)
build_config_builder.register_target_types(target_types or ())
self.build_config = build_config_builder.create()
options_bootstrapper = create_options_bootstrapper()
global_options = options_bootstrapper.bootstrap_options.for_global_scope()
local_store_dir = global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config,
execution_options=ExecutionOptions.from_bootstrap_options(global_options),
).new_session(
build_id="buildid_for_test",
session_values=SessionValues(
{OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment()}
),
should_report_workunits=True,
)
self.scheduler = graph_session.scheduler_session
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config, global_options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
return graph_scheduler_helper.new_session(global_options.v2_ui)
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config):
if graph_session:
return graph_session
native = Native.create(
options_bootstrapper.bootstrap_options.for_global_scope())
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
return graph_scheduler_helper.new_session()
def _maybe_init_graph_session(graph_session, global_options, build_config):
if graph_session:
return graph_session
native = Native.create(global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
global_options,
build_config
)
return graph_scheduler_helper.new_session()
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config, options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def _setup_legacy_graph_scheduler(native, bootstrap_options):
"""Initializes a `LegacyGraphScheduler` instance."""
return EngineInitializer.setup_legacy_graph(
bootstrap_options.pants_ignore,
bootstrap_options.pants_workdir,
bootstrap_options.build_file_imports,
native=native,
build_ignore_patterns=bootstrap_options.build_ignore,
exclude_target_regexps=bootstrap_options.exclude_target_regexp,
subproject_roots=bootstrap_options.subproject_roots,
remote_store_server=bootstrap_options.remote_store_server,
remote_execution_server=bootstrap_options.remote_execution_server,
)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
options_bootstrapper,
build_config
)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def _init_engine(cls):
if cls._scheduler is not None:
return
graph_session = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
build_file_imports_behavior='allow',
native=init_native(),
build_file_aliases=cls.alias_groups(),
build_ignore_patterns=None,
).new_session()
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root()
)
def _init_engine(cls):
if cls._scheduler is not None:
return
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
build_file_imports_behavior='allow',
native=init_native(),
build_configuration=cls.build_config(),
build_ignore_patterns=None,
).new_session()
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root())
def graph_helper(self,
build_file_aliases=None,
build_file_imports_behavior='allow',
include_trace_on_error=True,
path_ignore_patterns=None):
with temporary_dir() as work_dir:
path_ignore_patterns = path_ignore_patterns or []
graph_helper = EngineInitializer.setup_legacy_graph(
path_ignore_patterns,
work_dir,
build_file_imports_behavior,
build_file_aliases=build_file_aliases,
native=self._native,
include_trace_on_error=include_trace_on_error
)
yield graph_helper
def create(cls, options_bootstrapper, full_init=True) -> "PantsDaemon":
"""
:param OptionsBootstrapper options_bootstrapper: The bootstrap options.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
native: Optional[Native] = None
build_root: Optional[str] = None
if full_init:
build_root = get_buildroot()
native = Native()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(
bootstrap_options_values).watchman
services = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
native,
watchman,
union_membership=UnionMembership(
build_config.union_rules()),
)
else:
services = PantsServices()
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
services=services,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def _init_engine(cls):
if cls._scheduler is not None:
return
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
build_file_imports_behavior='allow',
native=init_native(),
build_configuration=cls.build_config(),
build_ignore_patterns=None,
).new_session()
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root()
)
def create_bootstrap_scheduler(
options_bootstrapper: OptionsBootstrapper,
executor: PyExecutor | None = None) -> BootstrapScheduler:
bc_builder = BuildConfiguration.Builder()
# To load plugins, we only need access to the Python/PEX rules.
load_build_configuration_from_source(bc_builder, ["pants.backend.python"])
# And to plugin-loading-specific rules.
bc_builder.register_rules("_dummy_for_bootstrapping_",
plugin_resolver_rules())
# We allow unrecognized options to defer any option error handling until post-bootstrap.
bc_builder.allow_unknown_options()
return BootstrapScheduler(
EngineInitializer.setup_graph(
options_bootstrapper.bootstrap_options.for_global_scope(),
bc_builder.create(),
DynamicRemoteOptions.disabled(),
executor,
).scheduler)
def graph_helper(self,
build_configuration=None,
build_file_imports_behavior='allow',
include_trace_on_error=True,
path_ignore_patterns=None):
with temporary_dir() as work_dir:
path_ignore_patterns = path_ignore_patterns or []
build_config = build_configuration or self._default_build_config()
# TODO: This test should be swapped to using TestBase.
graph_helper = EngineInitializer.setup_legacy_graph_extended(
path_ignore_patterns,
work_dir,
build_file_imports_behavior,
build_configuration=build_config,
native=self._native,
include_trace_on_error=include_trace_on_error)
yield graph_helper
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
native = None
build_root = None
services = None
port_map = None
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
options_bootstrapper = OptionsBootstrapper()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
bootstrap_options_values,
build_config)
services, port_map = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman
)
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
socket_map=port_map,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options
)
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
native = None
build_root = None
services = None
port_map = None
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
options_bootstrapper = OptionsBootstrapper()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
bootstrap_options_values,
build_config)
services, port_map = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman
)
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
socket_map=port_map,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options
)
def graph_helper(self,
build_configuration=None,
build_file_imports_behavior='allow',
include_trace_on_error=True,
path_ignore_patterns=None):
with temporary_dir() as work_dir:
path_ignore_patterns = path_ignore_patterns or []
build_config = build_configuration or self._default_build_config()
# TODO: This test should be swapped to using TestBase.
graph_helper = EngineInitializer.setup_legacy_graph_extended(
path_ignore_patterns,
work_dir,
build_file_imports_behavior,
build_configuration=build_config,
native=self._native,
include_trace_on_error=include_trace_on_error
)
yield graph_helper
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: GraphScheduler | None = None,
cancellation_latch: PySessionCancellationLatch | None = None,
) -> GraphSession:
native_engine.maybe_set_panic_handler()
if scheduler is None:
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return scheduler.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
ui_use_prodash=global_options.dynamic_ui_renderer
== DynamicUIRenderer.experimental_prodash,
use_colors=global_options.get("colors", True),
max_workunit_level=max(
global_options.streaming_workunits_level,
global_options.level,
*(
LogLevel[level.upper()]
for level in global_options.log_levels_by_target.values()
),
),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def create_bootstrap_scheduler(
options_bootstrapper: OptionsBootstrapper,
executor: Optional[PyExecutor] = None,
) -> BootstrapScheduler:
bc_builder = BuildConfiguration.Builder()
# To load plugins, we only need access to the Python/PEX rules.
load_build_configuration_from_source(bc_builder, ["pants.backend.python"])
# And to plugin-loading-specific rules.
bc_builder.register_rules(plugin_resolver_rules())
# We allow unrecognized options to defer any option error handling until post-bootstrap.
bc_builder.allow_unknown_options()
return BootstrapScheduler(
EngineInitializer.setup_graph(
options_bootstrapper,
bc_builder.create(),
executor=executor,
# TODO: We use the default execution options to avoid invoking remote execution auth
# plugins. They should be loaded via rules using the bootstrap Scheduler in the future.
execution_options=DEFAULT_EXECUTION_OPTIONS,
).scheduler)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options):
if not graph_session:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
options_bootstrapper,
build_config
)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
#TODO(gregorys) This should_report_workunits flag must be set to True for
# AsyncWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on async workunit reporting.
should_report_workunits = False
graph_session = graph_scheduler_helper.new_session(zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits)
return graph_session, graph_session.scheduler_session
def _init_engine(cls):
if cls._scheduler is not None:
return
cls._local_store_dir = os.path.realpath(safe_mkdtemp())
safe_mkdir(cls._local_store_dir)
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
local_store_dir=cls._local_store_dir,
build_file_imports_behavior='allow',
native=init_native(),
options_bootstrapper=OptionsBootstrapper.create(args=['--pants-config-files=[]']),
build_configuration=cls.build_config(),
build_ignore_patterns=None,
).new_session(zipkin_trace_v2=False)
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root()
)
