def _setup_services(
bootstrap_options: OptionValueContainer,
graph_scheduler: GraphScheduler,
):
"""Initialize pantsd services.
:returns: A PantsServices instance.
"""
build_root = get_buildroot()
invalidation_globs = GlobalOptions.compute_pantsd_invalidation_globs(
build_root,
bootstrap_options,
)
scheduler_service = SchedulerService(
graph_scheduler=graph_scheduler,
build_root=build_root,
invalidation_globs=invalidation_globs,
pidfile=PantsDaemon.metadata_file_path(
"pantsd", "pid", bootstrap_options.pants_subprocessdir),
pid=os.getpid(),
max_memory_usage_in_bytes=bootstrap_options.
pantsd_max_memory_usage,
)
store_gc_service = StoreGCService(graph_scheduler.scheduler)
return PantsServices(services=(scheduler_service, store_gc_service))
def _setup_services(
build_root,
bootstrap_options,
legacy_graph_scheduler,
watchman,
union_membership: UnionMembership,
):
"""Initialize pantsd services.
:returns: A PantsServices instance.
"""
should_shutdown_after_run = bootstrap_options.shutdown_pantsd_after_run
fs_event_service = (FSEventService(
watchman,
build_root,
) if bootstrap_options.watchman_enable else None)
pidfile_absolute = PantsDaemon.metadata_file_path(
"pantsd", "pid", bootstrap_options.pants_subprocessdir)
if pidfile_absolute.startswith(build_root):
pidfile = os.path.relpath(pidfile_absolute, build_root)
else:
pidfile = None
logging.getLogger(__name__).warning(
"Not watching pantsd pidfile because subprocessdir is outside of buildroot. Having "
"subprocessdir be a child of buildroot (as it is by default) may help avoid stray "
"pantsd processes.")
# TODO make SchedulerService handle fs_event_service_being None
scheduler_service = SchedulerService(
fs_event_service=fs_event_service,
legacy_graph_scheduler=legacy_graph_scheduler,
build_root=build_root,
invalidation_globs=OptionsInitializer.
compute_pantsd_invalidation_globs(build_root,
bootstrap_options),
pantsd_pidfile=pidfile,
union_membership=union_membership,
)
pailgun_service = PailgunService(
(bootstrap_options.pantsd_pailgun_host,
bootstrap_options.pantsd_pailgun_port),
DaemonPantsRunner,
scheduler_service,
should_shutdown_after_run,
)
store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler)
return PantsServices(
services=tuple(service for service in (
fs_event_service,
scheduler_service,
pailgun_service,
store_gc_service,
) if service is not None),
port_map=dict(pailgun=pailgun_service.pailgun_port),
)
def test_run_services_startupfailure(self, mock_shutdown, mock_thread):
mock_thread.return_value.start.side_effect = RuntimeError('oops!')
with self.assertRaises(PantsDaemon.StartupFailure):
self.pantsd._run_services(
PantsServices(services=(self.mock_service, )))
self.assertGreater(mock_shutdown.call_count, 0)
def setUp(self):
super(PantsDaemonTest, self).setUp()
mock_options = mock.Mock()
mock_options.pants_subprocessdir = 'non_existent_dir'
self.pantsd = PantsDaemon(None, 'test_buildroot', 'test_work_dir',
logging.INFO, PantsServices(),
'/tmp/pants_test_metadata_dir', mock_options)
self.mock_killswitch = mock.Mock()
self.pantsd._kill_switch = self.mock_killswitch
self.mock_service = mock.create_autospec(PantsService, spec_set=True)
def test_run_services_runtimefailure(self, mock_fp, mock_shutdown,
mock_thread):
self.mock_killswitch.is_set.side_effect = [False, False, True]
mock_thread.return_value.is_alive.side_effect = [True, False]
mock_fp.return_value = 'some_sha'
with self.assertRaises(PantsDaemon.RuntimeFailure):
self.pantsd._run_services(
PantsServices(services=(self.mock_service, )))
self.assertGreater(mock_shutdown.call_count, 0)
def _setup_services(
build_root,
bootstrap_options,
legacy_graph_scheduler,
native,
watchman,
union_membership: UnionMembership,
):
"""Initialize pantsd services.
:returns: A PantsServices instance.
"""
native.override_thread_logging_destination_to_just_pantsd()
fs_event_service = (
FSEventService(
watchman, scheduler=legacy_graph_scheduler.scheduler, build_root=build_root
)
if bootstrap_options.watchman_enable
else None
)
invalidation_globs = OptionsInitializer.compute_pantsd_invalidation_globs(
build_root, bootstrap_options
)
scheduler_service = SchedulerService(
fs_event_service=fs_event_service,
legacy_graph_scheduler=legacy_graph_scheduler,
build_root=build_root,
invalidation_globs=invalidation_globs,
union_membership=union_membership,
)
pailgun_service = PailgunService(
bootstrap_options.pantsd_pailgun_port,
DaemonPantsRunner(scheduler_service),
scheduler_service,
)
store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler)
return PantsServices(
services=tuple(
service
for service in (
fs_event_service,
scheduler_service,
pailgun_service,
store_gc_service,
)
if service is not None
),
port_map=dict(pailgun=pailgun_service.pailgun_port()),
)
def _setup_services(build_root, bootstrap_options,
legacy_graph_scheduler, watchman):
"""Initialize pantsd services.
:returns: A PantsServices instance.
"""
should_shutdown_after_run = bootstrap_options.shutdown_pantsd_after_run
fs_event_service = FSEventService(
watchman,
build_root,
)
pidfile_absolute = PantsDaemon.metadata_file_path(
'pantsd', 'pid', bootstrap_options.pants_subprocessdir)
if pidfile_absolute.startswith(build_root):
pidfile = os.path.relpath(pidfile_absolute, build_root)
else:
pidfile = None
logging.getLogger(__name__).warning(
'Not watching pantsd pidfile because subprocessdir is outside of buildroot. Having '
'subprocessdir be a child of buildroot (as it is by default) may help avoid stray '
'pantsd processes.')
scheduler_service = SchedulerService(
fs_event_service,
legacy_graph_scheduler,
build_root,
OptionsInitializer.compute_pantsd_invalidation_globs(
build_root, bootstrap_options),
pidfile,
)
pailgun_service = PailgunService(
(bootstrap_options.pantsd_pailgun_host,
bootstrap_options.pantsd_pailgun_port),
DaemonPantsRunner,
scheduler_service,
should_shutdown_after_run,
)
store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler)
return PantsServices(
services=(fs_event_service, scheduler_service, pailgun_service,
store_gc_service),
port_map=dict(pailgun=pailgun_service.pailgun_port),
)
def setUp(self):
super().setUp()
mock_options = unittest.mock.Mock()
mock_options.pants_subprocessdir = "non_existent_dir"
self.pantsd = PantsDaemon(
None,
"test_buildroot",
"test_work_dir",
logging.INFO,
PantsServices(),
"/tmp/pants_test_metadata_dir",
mock_options,
)
self.mock_killswitch = unittest.mock.Mock()
self.pantsd._kill_switch = self.mock_killswitch
self.mock_service = unittest.mock.create_autospec(PantsService,
spec_set=True)
def create(cls, options_bootstrapper, full_init=True) -> "PantsDaemon":
"""
:param OptionsBootstrapper options_bootstrapper: The bootstrap options.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
native: Optional[Native] = None
build_root: Optional[str] = None
if full_init:
build_root = get_buildroot()
native = Native()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(
bootstrap_options_values).watchman
services = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
native,
watchman,
union_membership=UnionMembership(
build_config.union_rules()),
)
else:
services = PantsServices()
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
services=services,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def _setup_services(
bootstrap_options: OptionValueContainer,
legacy_graph_scheduler: LegacyGraphScheduler,
):
"""Initialize pantsd services.
:returns: A PantsServices instance.
"""
build_root = get_buildroot()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman_launcher = WatchmanLauncher.create(bootstrap_options)
watchman_launcher.maybe_launch()
watchman = watchman_launcher.watchman
fs_event_service = (FSEventService(
watchman,
scheduler=legacy_graph_scheduler.scheduler,
build_root=build_root)
if bootstrap_options.watchman_enable else None)
invalidation_globs = OptionsInitializer.compute_pantsd_invalidation_globs(
build_root,
bootstrap_options,
PantsDaemon.metadata_file_path(
"pantsd", "pid", bootstrap_options.pants_subprocessdir),
)
scheduler_service = SchedulerService(
fs_event_service=fs_event_service,
legacy_graph_scheduler=legacy_graph_scheduler,
build_root=build_root,
invalidation_globs=invalidation_globs,
max_memory_usage_pid=os.getpid(),
max_memory_usage_in_bytes=bootstrap_options.
pantsd_max_memory_usage,
)
store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler)
return PantsServices(services=tuple(service for service in (
fs_event_service,
scheduler_service,
store_gc_service,
) if service is not None), )
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
options_bootstrapper = OptionsBootstrapper()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
bootstrap_options_values,
build_config)
services = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman
)
else:
build_root = None
native = None
services = PantsServices()
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options
)
def test_run_services_no_services(self):
self.pantsd._run_services(PantsServices())
def create_services(bootstrap_options, legacy_graph_scheduler):
return PantsServices()