def _setup_services( bootstrap_options: OptionValueContainer, graph_scheduler: GraphScheduler, ): """Initialize pantsd services. :returns: A PantsServices instance. """ build_root = get_buildroot() invalidation_globs = GlobalOptions.compute_pantsd_invalidation_globs( build_root, bootstrap_options, ) scheduler_service = SchedulerService( graph_scheduler=graph_scheduler, build_root=build_root, invalidation_globs=invalidation_globs, pidfile=PantsDaemon.metadata_file_path( "pantsd", "pid", bootstrap_options.pants_subprocessdir), pid=os.getpid(), max_memory_usage_in_bytes=bootstrap_options. pantsd_max_memory_usage, ) store_gc_service = StoreGCService(graph_scheduler.scheduler) return PantsServices(services=(scheduler_service, store_gc_service))
def _setup_services( build_root, bootstrap_options, legacy_graph_scheduler, watchman, union_membership: UnionMembership, ): """Initialize pantsd services. :returns: A PantsServices instance. """ should_shutdown_after_run = bootstrap_options.shutdown_pantsd_after_run fs_event_service = (FSEventService( watchman, build_root, ) if bootstrap_options.watchman_enable else None) pidfile_absolute = PantsDaemon.metadata_file_path( "pantsd", "pid", bootstrap_options.pants_subprocessdir) if pidfile_absolute.startswith(build_root): pidfile = os.path.relpath(pidfile_absolute, build_root) else: pidfile = None logging.getLogger(__name__).warning( "Not watching pantsd pidfile because subprocessdir is outside of buildroot. Having " "subprocessdir be a child of buildroot (as it is by default) may help avoid stray " "pantsd processes.") # TODO make SchedulerService handle fs_event_service_being None scheduler_service = SchedulerService( fs_event_service=fs_event_service, legacy_graph_scheduler=legacy_graph_scheduler, build_root=build_root, invalidation_globs=OptionsInitializer. compute_pantsd_invalidation_globs(build_root, bootstrap_options), pantsd_pidfile=pidfile, union_membership=union_membership, ) pailgun_service = PailgunService( (bootstrap_options.pantsd_pailgun_host, bootstrap_options.pantsd_pailgun_port), DaemonPantsRunner, scheduler_service, should_shutdown_after_run, ) store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler) return PantsServices( services=tuple(service for service in ( fs_event_service, scheduler_service, pailgun_service, store_gc_service, ) if service is not None), port_map=dict(pailgun=pailgun_service.pailgun_port), )
def test_run_services_startupfailure(self, mock_shutdown, mock_thread): mock_thread.return_value.start.side_effect = RuntimeError('oops!') with self.assertRaises(PantsDaemon.StartupFailure): self.pantsd._run_services( PantsServices(services=(self.mock_service, ))) self.assertGreater(mock_shutdown.call_count, 0)
def setUp(self): super(PantsDaemonTest, self).setUp() mock_options = mock.Mock() mock_options.pants_subprocessdir = 'non_existent_dir' self.pantsd = PantsDaemon(None, 'test_buildroot', 'test_work_dir', logging.INFO, PantsServices(), '/tmp/pants_test_metadata_dir', mock_options) self.mock_killswitch = mock.Mock() self.pantsd._kill_switch = self.mock_killswitch self.mock_service = mock.create_autospec(PantsService, spec_set=True)
def test_run_services_runtimefailure(self, mock_fp, mock_shutdown, mock_thread): self.mock_killswitch.is_set.side_effect = [False, False, True] mock_thread.return_value.is_alive.side_effect = [True, False] mock_fp.return_value = 'some_sha' with self.assertRaises(PantsDaemon.RuntimeFailure): self.pantsd._run_services( PantsServices(services=(self.mock_service, ))) self.assertGreater(mock_shutdown.call_count, 0)
def _setup_services( build_root, bootstrap_options, legacy_graph_scheduler, native, watchman, union_membership: UnionMembership, ): """Initialize pantsd services. :returns: A PantsServices instance. """ native.override_thread_logging_destination_to_just_pantsd() fs_event_service = ( FSEventService( watchman, scheduler=legacy_graph_scheduler.scheduler, build_root=build_root ) if bootstrap_options.watchman_enable else None ) invalidation_globs = OptionsInitializer.compute_pantsd_invalidation_globs( build_root, bootstrap_options ) scheduler_service = SchedulerService( fs_event_service=fs_event_service, legacy_graph_scheduler=legacy_graph_scheduler, build_root=build_root, invalidation_globs=invalidation_globs, union_membership=union_membership, ) pailgun_service = PailgunService( bootstrap_options.pantsd_pailgun_port, DaemonPantsRunner(scheduler_service), scheduler_service, ) store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler) return PantsServices( services=tuple( service for service in ( fs_event_service, scheduler_service, pailgun_service, store_gc_service, ) if service is not None ), port_map=dict(pailgun=pailgun_service.pailgun_port()), )
def _setup_services(build_root, bootstrap_options, legacy_graph_scheduler, watchman): """Initialize pantsd services. :returns: A PantsServices instance. """ should_shutdown_after_run = bootstrap_options.shutdown_pantsd_after_run fs_event_service = FSEventService( watchman, build_root, ) pidfile_absolute = PantsDaemon.metadata_file_path( 'pantsd', 'pid', bootstrap_options.pants_subprocessdir) if pidfile_absolute.startswith(build_root): pidfile = os.path.relpath(pidfile_absolute, build_root) else: pidfile = None logging.getLogger(__name__).warning( 'Not watching pantsd pidfile because subprocessdir is outside of buildroot. Having ' 'subprocessdir be a child of buildroot (as it is by default) may help avoid stray ' 'pantsd processes.') scheduler_service = SchedulerService( fs_event_service, legacy_graph_scheduler, build_root, OptionsInitializer.compute_pantsd_invalidation_globs( build_root, bootstrap_options), pidfile, ) pailgun_service = PailgunService( (bootstrap_options.pantsd_pailgun_host, bootstrap_options.pantsd_pailgun_port), DaemonPantsRunner, scheduler_service, should_shutdown_after_run, ) store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler) return PantsServices( services=(fs_event_service, scheduler_service, pailgun_service, store_gc_service), port_map=dict(pailgun=pailgun_service.pailgun_port), )
def setUp(self): super().setUp() mock_options = unittest.mock.Mock() mock_options.pants_subprocessdir = "non_existent_dir" self.pantsd = PantsDaemon( None, "test_buildroot", "test_work_dir", logging.INFO, PantsServices(), "/tmp/pants_test_metadata_dir", mock_options, ) self.mock_killswitch = unittest.mock.Mock() self.pantsd._kill_switch = self.mock_killswitch self.mock_service = unittest.mock.create_autospec(PantsService, spec_set=True)
def create(cls, options_bootstrapper, full_init=True) -> "PantsDaemon": """ :param OptionsBootstrapper options_bootstrapper: The bootstrap options. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = options_bootstrapper.bootstrap_options bootstrap_options_values = bootstrap_options.for_global_scope() native: Optional[Native] = None build_root: Optional[str] = None if full_init: build_root = get_buildroot() native = Native() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create( bootstrap_options_values).watchman services = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, native, watchman, union_membership=UnionMembership( build_config.union_rules()), ) else: services = PantsServices() return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level, services=services, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options, )
def _setup_services( bootstrap_options: OptionValueContainer, legacy_graph_scheduler: LegacyGraphScheduler, ): """Initialize pantsd services. :returns: A PantsServices instance. """ build_root = get_buildroot() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman_launcher = WatchmanLauncher.create(bootstrap_options) watchman_launcher.maybe_launch() watchman = watchman_launcher.watchman fs_event_service = (FSEventService( watchman, scheduler=legacy_graph_scheduler.scheduler, build_root=build_root) if bootstrap_options.watchman_enable else None) invalidation_globs = OptionsInitializer.compute_pantsd_invalidation_globs( build_root, bootstrap_options, PantsDaemon.metadata_file_path( "pantsd", "pid", bootstrap_options.pants_subprocessdir), ) scheduler_service = SchedulerService( fs_event_service=fs_event_service, legacy_graph_scheduler=legacy_graph_scheduler, build_root=build_root, invalidation_globs=invalidation_globs, max_memory_usage_pid=os.getpid(), max_memory_usage_in_bytes=bootstrap_options. pantsd_max_memory_usage, ) store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler) return PantsServices(services=tuple(service for service in ( fs_event_service, scheduler_service, store_gc_service, ) if service is not None), )
def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) options_bootstrapper = OptionsBootstrapper() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native, bootstrap_options_values, build_config) services = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman ) else: build_root = None native = None services = PantsServices() return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options )
def test_run_services_no_services(self): self.pantsd._run_services(PantsServices())
def create_services(bootstrap_options, legacy_graph_scheduler): return PantsServices()