def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
v2_ui,
should_report_workunits=stream_workunits,
)
def _init_graph(self, target_roots, graph_helper, exclude_target_regexps, tags):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param TargetRoots target_roots: The existing `TargetRoots` object, if any.
:param LegacyGraphSession graph_helper: A LegacyGraphSession to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, SchedulerSession, TargetRoots).
"""
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(native,
self._global_options,
self._build_config)
graph_helper = graph_scheduler_helper.new_session()
target_roots = target_roots or TargetRootsCalculator.create(
options=self._options,
session=graph_helper.scheduler_session,
build_root=self._root_dir,
symbol_table=graph_helper.symbol_table,
exclude_patterns=tuple(exclude_target_regexps),
tags=tuple(tags)
)
graph, address_mapper = graph_helper.create_build_graph(target_roots,
self._root_dir)
return graph, address_mapper, graph_helper.scheduler_session, target_roots
def _init_scheduler(self, options_fingerprint: str,
options_bootstrapper: OptionsBootstrapper) -> None:
"""(Re-)Initialize the scheduler.
Must be called under the lifecycle lock.
"""
try:
if self._scheduler:
logger.info(
"initialization options changed: reinitializing pantsd...")
else:
logger.info("initializing pantsd...")
if self._services:
self._services.shutdown()
build_config = BuildConfigInitializer.get(options_bootstrapper)
self._scheduler = EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
bootstrap_options_values = options_bootstrapper.bootstrap_options.for_global_scope(
)
self._services = self._services_constructor(
bootstrap_options_values, self._scheduler)
self._fingerprint = options_fingerprint
logger.info("pantsd initialized.")
except Exception as e:
self._kill_switch.set()
self._scheduler = None
raise e
def _maybe_init_graph_session(graph_session, global_options, build_config):
if graph_session:
return graph_session
native = Native.create(global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, global_options, build_config)
return graph_scheduler_helper.new_session()
def create_address_mapper(self, build_root):
work_dir = os.path.join(build_root, '.pants.d')
scheduler = EngineInitializer.setup_legacy_graph(
[],
work_dir,
build_file_imports_behavior='allow',
build_root=build_root,
native=self._native).scheduler
return LegacyAddressMapper(scheduler.new_session(), build_root)
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config):
if graph_session:
return graph_session
native = Native.create(
options_bootstrapper.bootstrap_options.for_global_scope())
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
return graph_scheduler_helper.new_session()
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config, global_options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
return graph_scheduler_helper.new_session(global_options.v2_ui)
def _maybe_init_graph_session(graph_session, global_options, build_config):
if graph_session:
return graph_session
native = Native.create(global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
global_options,
build_config
)
return graph_scheduler_helper.new_session()
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config, options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def _setup_legacy_graph_scheduler(native, bootstrap_options):
"""Initializes a `LegacyGraphScheduler` instance."""
return EngineInitializer.setup_legacy_graph(
bootstrap_options.pants_ignore,
bootstrap_options.pants_workdir,
bootstrap_options.build_file_imports,
native=native,
build_ignore_patterns=bootstrap_options.build_ignore,
exclude_target_regexps=bootstrap_options.exclude_target_regexp,
subproject_roots=bootstrap_options.subproject_roots,
remote_store_server=bootstrap_options.remote_store_server,
remote_execution_server=bootstrap_options.remote_execution_server,
)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
options_bootstrapper,
build_config
)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def _init_engine(cls):
if cls._scheduler is not None:
return
graph_session = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
build_file_imports_behavior='allow',
native=init_native(),
build_file_aliases=cls.alias_groups(),
build_ignore_patterns=None,
).new_session()
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root()
)
def graph_helper(self,
build_file_aliases=None,
build_file_imports_behavior='allow',
include_trace_on_error=True,
path_ignore_patterns=None):
with temporary_dir() as work_dir:
path_ignore_patterns = path_ignore_patterns or []
graph_helper = EngineInitializer.setup_legacy_graph(
path_ignore_patterns,
work_dir,
build_file_imports_behavior,
build_file_aliases=build_file_aliases,
native=self._native,
include_trace_on_error=include_trace_on_error
)
yield graph_helper
def create(cls, options_bootstrapper, full_init=True) -> "PantsDaemon":
"""
:param OptionsBootstrapper options_bootstrapper: The bootstrap options.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
native: Optional[Native] = None
build_root: Optional[str] = None
if full_init:
build_root = get_buildroot()
native = Native()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(
bootstrap_options_values).watchman
services = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
native,
watchman,
union_membership=UnionMembership(
build_config.union_rules()),
)
else:
services = PantsServices()
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
services=services,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
native = None
build_root = None
services = None
port_map = None
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
options_bootstrapper = OptionsBootstrapper()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
bootstrap_options_values,
build_config)
services, port_map = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman
)
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
socket_map=port_map,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options
)
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
native = None
build_root = None
services = None
port_map = None
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
options_bootstrapper = OptionsBootstrapper()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
bootstrap_options_values,
build_config)
services, port_map = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman
)
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
socket_map=port_map,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options
)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options):
if not graph_session:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
options_bootstrapper,
build_config
)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
#TODO(gregorys) This should_report_workunits flag must be set to True for
# AsyncWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on async workunit reporting.
should_report_workunits = False
graph_session = graph_scheduler_helper.new_session(zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits)
return graph_session, graph_session.scheduler_session
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
if global_scope.v2:
dynamic_ui = resolve_conflicting_options(
old_option="v2_ui",
new_option="dynamic_ui",
old_scope=GLOBAL_SCOPE,
new_scope=GLOBAL_SCOPE,
old_container=global_scope,
new_container=global_scope,
)
else:
dynamic_ui = False
use_colors = global_scope.get("colors", True)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
dynamic_ui = global_scope.dynamic_ui if global_scope.v2 else False
use_colors = global_scope.get("colors", True)
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
