def main(): exiter = Exiter() exiter.set_except_hook() try: PantsRunner(exiter).run() except KeyboardInterrupt: exiter.exit_and_fail(b'Interrupted by user.')
def main(): exiter = Exiter() exiter.set_except_hook() with maybe_profiled(os.environ.get('PANTSC_PROFILE')): try: PantsRunner(exiter).run() except KeyboardInterrupt: exiter.exit_and_fail(b'Interrupted by user.')
class PantsDaemon(FingerprintedProcessManager): """A daemon that manages PantsService instances.""" JOIN_TIMEOUT_SECONDS = 1 LOG_NAME = 'pantsd.log' class StartupFailure(Exception): """Represents a failure to start pantsd.""" class RuntimeFailure(Exception): """Represents a pantsd failure at runtime, usually from an underlying service failure.""" class Factory(object): @classmethod def maybe_launch(cls, bootstrap_options=None): """Creates and launches a daemon instance if one does not already exist. :param Options bootstrap_options: The bootstrap options, if available. :returns: The pailgun port number of the running pantsd instance. :rtype: int """ stub_pantsd = cls.create(bootstrap_options, full_init=False) with stub_pantsd.lifecycle_lock: if stub_pantsd.needs_restart(stub_pantsd.options_fingerprint): # Once we determine we actually need to launch, recreate with full initialization. pantsd = cls.create(bootstrap_options) return pantsd.launch() else: return stub_pantsd.read_named_socket('pailgun', int) @classmethod def restart(cls, bootstrap_options=None): """Restarts a running daemon instance. :param Options bootstrap_options: The bootstrap options, if available. :returns: The pailgun port number of the new pantsd instance. :rtype: int """ pantsd = cls.create(bootstrap_options) with pantsd.lifecycle_lock: # N.B. This will call `pantsd.terminate()` before starting. return pantsd.launch() @classmethod def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options( ) bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create( bootstrap_options_values).watchman native = None build_root = None services = None port_map = None if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) options_bootstrapper = OptionsBootstrapper() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph( native, bootstrap_options_values, build_config) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman) return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, socket_map=port_map, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options) @staticmethod def _parse_bootstrap_options(): return OptionsBootstrapper().get_bootstrap_options() @staticmethod def _setup_services(build_root, bootstrap_options, legacy_graph_scheduler, watchman): """Initialize pantsd services. :returns: A tuple of (`tuple` service_instances, `dict` port_map). """ fs_event_service = FSEventService( watchman, build_root, bootstrap_options.pantsd_fs_event_workers) pidfile_absolute = PantsDaemon.metadata_file_path( 'pantsd', 'pid', bootstrap_options.pants_subprocessdir) if pidfile_absolute.startswith(build_root): pidfile = os.path.relpath(pidfile_absolute, build_root) else: pidfile = None logging.getLogger(__name__).warning( 'Not watching pantsd pidfile because subprocessdir is outside of buildroot. Having ' 'subprocessdir be a child of buildroot (as it is by default) may help avoid stray ' 'pantsd processes.') scheduler_service = SchedulerService( fs_event_service, legacy_graph_scheduler, build_root, bootstrap_options.pantsd_invalidation_globs, pidfile, ) pailgun_service = PailgunService( bind_addr=(bootstrap_options.pantsd_pailgun_host, bootstrap_options.pantsd_pailgun_port), exiter_class=DaemonExiter, runner_class=DaemonPantsRunner, target_roots_calculator=TargetRootsCalculator, scheduler_service=scheduler_service) store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler) return ( # Services. (fs_event_service, scheduler_service, pailgun_service, store_gc_service), # Port map. dict(pailgun=pailgun_service.pailgun_port)) def __init__(self, native, build_root, work_dir, log_level, services, socket_map, metadata_base_dir, bootstrap_options=None): """ :param Native native: A `Native` instance. :param string build_root: The pants build root. :param string work_dir: The pants work directory. :param string log_level: The log level to use for daemon logging. :param string metadata_base_dir: The ProcessManager metadata base dir. :param Options bootstrap_options: The bootstrap options, if available. """ super(PantsDaemon, self).__init__(name='pantsd', metadata_base_dir=metadata_base_dir) self._native = native self._build_root = build_root self._work_dir = work_dir self._log_level = log_level self._services = services self._socket_map = socket_map self._bootstrap_options = bootstrap_options self._log_dir = os.path.join(work_dir, self.name) self._logger = logging.getLogger(__name__) # A lock to guard the service thread lifecycles. This can be used by individual services # to safeguard daemon-synchronous sections that should be protected from abrupt teardown. self._lifecycle_lock = threading.RLock() # A lock to guard pantsd->runner forks. This can be used by services to safeguard resources # held by threads at fork time, so that we can fork without deadlocking. self._fork_lock = threading.RLock() # N.B. This Event is used as nothing more than a convenient atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._exiter = Exiter() @memoized_property def watchman_launcher(self): return WatchmanLauncher.create( self._bootstrap_options.for_global_scope()) @property def is_killed(self): return self._kill_switch.is_set() @property def options_fingerprint(self): return OptionsFingerprinter.combined_options_fingerprint_for_scope( GLOBAL_SCOPE, self._bootstrap_options, fingerprint_key='daemon', invert=True) def shutdown(self, service_thread_map): """Gracefully terminate all services and kill the main PantsDaemon loop.""" with self._lifecycle_lock: for service, service_thread in service_thread_map.items(): self._logger.info( 'terminating pantsd service: {}'.format(service)) service.terminate() service_thread.join(self.JOIN_TIMEOUT_SECONDS) self._logger.info('terminating pantsd') self._kill_switch.set() @staticmethod def _close_stdio(): """Close stdio streams to avoid output in the tty that launched pantsd.""" for fd in (sys.stdin, sys.stdout, sys.stderr): file_no = fd.fileno() fd.flush() fd.close() os.close(file_no) @contextmanager def _pantsd_logging(self): """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( 'pantsd logging cannot initialize while stdio is open: {}'. format(fd)) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. result = setup_logging(self._log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler) self._logger.debug('logging initialized') yield result.log_handler.stream def _setup_services(self, services): assert self._lifecycle_lock is not None, 'PantsDaemon lock has not been set!' assert self._fork_lock is not None, 'PantsDaemon fork lock has not been set!' for service in services: self._logger.info('setting up service {}'.format(service)) service.setup(self._lifecycle_lock, self._fork_lock) @staticmethod def _make_thread(target): t = threading.Thread(target=target) t.daemon = True return t def _run_services(self, services): """Service runner main loop.""" if not services: self._logger.critical('no services to run, bailing!') return service_thread_map = { service: self._make_thread(service.run) for service in services } # Start services. for service, service_thread in service_thread_map.items(): self._logger.info('starting service {}'.format(service)) try: service_thread.start() except (RuntimeError, service.ServiceError): self.shutdown(service_thread_map) raise self.StartupFailure( 'service {} failed to start, shutting down!'.format( service)) # Once all services are started, write our pid. self.write_pid() self.write_metadata_by_name('pantsd', self.FINGERPRINT_KEY, self.options_fingerprint) # Monitor services. while not self.is_killed: for service, service_thread in service_thread_map.items(): if not service_thread.is_alive(): self.shutdown(service_thread_map) raise self.RuntimeFailure( 'service failure for {}, shutting down!'.format( service)) else: # Avoid excessive CPU utilization. service_thread.join(self.JOIN_TIMEOUT_SECONDS) def _write_named_sockets(self, socket_map): """Write multiple named sockets using a socket mapping.""" for socket_name, socket_info in socket_map.items(): self.write_named_socket(socket_name, socket_info) def run_sync(self): """Synchronously run pantsd.""" # Switch log output to the daemon's log stream from here forward. self._close_stdio() with self._pantsd_logging() as log_stream: self._exiter.set_except_hook(log_stream) self._logger.info('pantsd starting, log level is {}'.format( self._log_level)) self._native.set_panic_handler() # Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'. set_process_title('pantsd [{}]'.format(self._build_root)) # Write service socket information to .pids. self._write_named_sockets(self._socket_map) # Enter the main service runner loop. self._setup_services(self._services) self._run_services(self._services) def post_fork_child(self): """Post-fork() child callback for ProcessManager.daemon_spawn().""" entry_point = '{}:launch'.format(__name__) exec_env = combined_dict(os.environ, dict(PANTS_ENTRYPOINT=entry_point)) # Pass all of sys.argv so that we can proxy arg flags e.g. `-ldebug`. cmd = [sys.executable] + sys.argv self._logger.debug('cmd is: PANTS_ENTRYPOINT={} {}'.format( entry_point, ' '.join(cmd))) # TODO: Improve error handling on launch failures. os.spawnve(os.P_NOWAIT, sys.executable, cmd, env=exec_env) def needs_launch(self): """Determines if pantsd needs to be launched. N.B. This should always be called under care of `self.lifecycle_lock`. :returns: True if the daemon needs launching, False otherwise. :rtype: bool """ new_fingerprint = self.options_fingerprint self._logger.debug( 'pantsd: is_alive={} new_fingerprint={} current_fingerprint={}'. format(self.is_alive(), new_fingerprint, self.fingerprint)) return self.needs_restart(new_fingerprint) def launch(self): """Launches pantsd in a subprocess. N.B. This should always be called under care of `self.lifecycle_lock`. :returns: The port that pantsd is listening on. :rtype: int """ self.terminate(include_watchman=False) self.watchman_launcher.maybe_launch() self._logger.debug('launching pantsd') self.daemon_spawn() # Wait up to 60 seconds for pantsd to write its pidfile. self.await_pid(60) listening_port = self.read_named_socket('pailgun', int) self._logger.debug( 'pantsd is running at pid {}, pailgun port is {}'.format( self.pid, listening_port)) return listening_port def terminate(self, include_watchman=True): """Terminates pantsd and watchman. N.B. This should always be called under care of `self.lifecycle_lock`. """ super(PantsDaemon, self).terminate() if include_watchman: self.watchman_launcher.terminate()
class PantsDaemon(FingerprintedProcessManager): """A daemon that manages PantsService instances.""" JOIN_TIMEOUT_SECONDS = 1 LOG_NAME = 'pantsd.log' class StartupFailure(Exception): """Represents a failure to start pantsd.""" class RuntimeFailure(Exception): """Represents a pantsd failure at runtime, usually from an underlying service failure.""" class Factory(object): @classmethod def create(cls, bootstrap_options=None): """ :param Options bootstrap_options: The bootstrap options, if available. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() build_root = get_buildroot() native = Native.create(bootstrap_options_values) # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman legacy_graph_helper = cls._setup_legacy_graph_helper(native, bootstrap_options_values) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_helper, watchman ) return PantsDaemon( native, build_root, bootstrap_options_values.pants_workdir, bootstrap_options_values.level.upper(), services, port_map, bootstrap_options_values.pants_subprocessdir, bootstrap_options ) @staticmethod def _parse_bootstrap_options(): return OptionsBootstrapper().get_bootstrap_options() @staticmethod def _setup_legacy_graph_helper(native, bootstrap_options): """Initializes a `LegacyGraphHelper` instance.""" return EngineInitializer.setup_legacy_graph( bootstrap_options.pants_ignore, bootstrap_options.pants_workdir, bootstrap_options.build_file_imports, native=native, build_ignore_patterns=bootstrap_options.build_ignore, exclude_target_regexps=bootstrap_options.exclude_target_regexp, subproject_roots=bootstrap_options.subproject_roots, ) @staticmethod def _setup_services(build_root, bootstrap_options, legacy_graph_helper, watchman): """Initialize pantsd services. :returns: A tuple of (`tuple` service_instances, `dict` port_map). """ fs_event_service = FSEventService(watchman, build_root, bootstrap_options.pantsd_fs_event_workers) scheduler_service = SchedulerService(fs_event_service, legacy_graph_helper) pailgun_service = PailgunService( bind_addr=(bootstrap_options.pantsd_pailgun_host, bootstrap_options.pantsd_pailgun_port), exiter_class=DaemonExiter, runner_class=DaemonPantsRunner, target_roots_class=TargetRoots, scheduler_service=scheduler_service ) store_gc_service = StoreGCService(legacy_graph_helper.scheduler) return ( # Services. (fs_event_service, scheduler_service, pailgun_service, store_gc_service), # Port map. dict(pailgun=pailgun_service.pailgun_port) ) def __init__(self, native, build_root, work_dir, log_level, services, socket_map, metadata_base_dir, bootstrap_options=None): """ :param Native native: A `Native` instance. :param string build_root: The pants build root. :param string work_dir: The pants work directory. :param string log_level: The log level to use for daemon logging. :param string metadata_base_dir: The ProcessManager metadata base dir. :param Options bootstrap_options: The bootstrap options, if available. """ super(PantsDaemon, self).__init__(name='pantsd', metadata_base_dir=metadata_base_dir) self._native = native self._build_root = build_root self._work_dir = work_dir self._log_level = log_level self._services = services self._socket_map = socket_map self._bootstrap_options = bootstrap_options self._log_dir = os.path.join(work_dir, self.name) self._logger = logging.getLogger(__name__) # A lock to guard the service thread lifecycles. This can be used by individual services # to safeguard daemon-synchronous sections that should be protected from abrupt teardown. self._lifecycle_lock = threading.RLock() # A lock to guard pantsd->runner forks. This can be used by services to safeguard resources # held by threads at fork time, so that we can fork without deadlocking. self._fork_lock = threading.RLock() # N.B. This Event is used as nothing more than a convenient atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._exiter = Exiter() @memoized_property def watchman_launcher(self): return WatchmanLauncher.create(self._bootstrap_options.for_global_scope()) @property def is_killed(self): return self._kill_switch.is_set() @property def options_fingerprint(self): return OptionsFingerprinter.combined_options_fingerprint_for_scope( GLOBAL_SCOPE, self._bootstrap_options, fingerprint_key='daemon', invert=True ) def shutdown(self, service_thread_map): """Gracefully terminate all services and kill the main PantsDaemon loop.""" with self._lifecycle_lock: for service, service_thread in service_thread_map.items(): self._logger.info('terminating pantsd service: {}'.format(service)) service.terminate() service_thread.join(self.JOIN_TIMEOUT_SECONDS) self._logger.info('terminating pantsd') self._kill_switch.set() @staticmethod def _close_stdio(): """Close stdio streams to avoid output in the tty that launched pantsd.""" for fd in (sys.stdin, sys.stdout, sys.stderr): file_no = fd.fileno() fd.flush() fd.close() os.close(file_no) @contextmanager def _pantsd_logging(self): """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( 'pantsd logging cannot initialize while stdio is open: {}'.format(fd)) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. result = setup_logging(self._log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler) self._logger.debug('logging initialized') yield result.log_handler.stream def _setup_services(self, services): assert self._lifecycle_lock is not None, 'PantsDaemon lock has not been set!' assert self._fork_lock is not None, 'PantsDaemon fork lock has not been set!' for service in services: self._logger.info('setting up service {}'.format(service)) service.setup(self._lifecycle_lock, self._fork_lock) @staticmethod def _make_thread(target): t = threading.Thread(target=target) t.daemon = True return t def _run_services(self, services): """Service runner main loop.""" if not services: self._logger.critical('no services to run, bailing!') return service_thread_map = {service: self._make_thread(service.run) for service in services} # Start services. for service, service_thread in service_thread_map.items(): self._logger.info('starting service {}'.format(service)) try: service_thread.start() except (RuntimeError, service.ServiceError): self.shutdown(service_thread_map) raise self.StartupFailure('service {} failed to start, shutting down!'.format(service)) # Once all services are started, write our pid. self.write_pid() self.write_metadata_by_name('pantsd', self.FINGERPRINT_KEY, self.options_fingerprint) # Monitor services. while not self.is_killed: for service, service_thread in service_thread_map.items(): if not service_thread.is_alive(): self.shutdown(service_thread_map) raise self.RuntimeFailure('service failure for {}, shutting down!'.format(service)) else: # Avoid excessive CPU utilization. service_thread.join(self.JOIN_TIMEOUT_SECONDS) def _write_named_sockets(self, socket_map): """Write multiple named sockets using a socket mapping.""" for socket_name, socket_info in socket_map.items(): self.write_named_socket(socket_name, socket_info) def run_sync(self): """Synchronously run pantsd.""" # Switch log output to the daemon's log stream from here forward. self._close_stdio() with self._pantsd_logging() as log_stream: self._exiter.set_except_hook(log_stream) self._logger.info('pantsd starting, log level is {}'.format(self._log_level)) self._native.set_panic_handler() # Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'. set_process_title('pantsd [{}]'.format(self._build_root)) # Write service socket information to .pids. self._write_named_sockets(self._socket_map) # Enter the main service runner loop. self._setup_services(self._services) self._run_services(self._services) def post_fork_child(self): """Post-fork() child callback for ProcessManager.daemon_spawn().""" entry_point = '{}:launch'.format(__name__) exec_env = combined_dict(os.environ, dict(PANTS_ENTRYPOINT=entry_point)) # Pass all of sys.argv so that we can proxy arg flags e.g. `-ldebug`. cmd = [sys.executable] + sys.argv self._logger.debug('cmd is: PANTS_ENTRYPOINT={} {}'.format(entry_point, ' '.join(cmd))) # TODO: Improve error handling on launch failures. os.spawnve(os.P_NOWAIT, sys.executable, cmd, env=exec_env) def maybe_launch(self): """Launches pantsd (if not already running) in a subprocess. :returns: The port that pantsd is listening on. :rtype: int """ self.watchman_launcher.maybe_launch() self._logger.debug('acquiring lock: {}'.format(self.process_lock)) with self.process_lock: new_fingerprint = self.options_fingerprint self._logger.debug('pantsd: is_alive={} new_fingerprint={} current_fingerprint={}' .format(self.is_alive(), new_fingerprint, self.fingerprint)) if self.needs_restart(new_fingerprint): self.terminate(include_watchman=False) self._logger.debug('launching pantsd') self.daemon_spawn() # Wait up to 60 seconds for pantsd to write its pidfile. self.await_pid(60) listening_port = self.read_named_socket('pailgun', int) pantsd_pid = self.pid self._logger.debug('released lock: {}'.format(self.process_lock)) self._logger.debug('pantsd is running at pid {}, pailgun port is {}' .format(pantsd_pid, listening_port)) return listening_port def terminate(self, include_watchman=True): """Terminates pantsd and watchman.""" super(PantsDaemon, self).terminate() if include_watchman: self.watchman_launcher.terminate()
class PantsDaemon(ProcessManager): """A daemon that manages PantsService instances.""" JOIN_TIMEOUT_SECONDS = 1 LOG_NAME = 'pantsd.log' class StartupFailure(Exception): pass class RuntimeFailure(Exception): pass def __init__(self, build_root, work_dir, log_level, native, log_dir=None, services=None, metadata_base_dir=None, reset_func=None): """ :param string build_root: The pants build root. :param string work_dir: The pants work directory. :param string log_level: The log level to use for daemon logging. :param string log_dir: The directory to use for file-based logging via the daemon. (Optional) :param tuple services: A tuple of PantsService instances to launch/manage. (Optional) :param callable reset_func: Called after the daemon is forked to reset any state inherited from the parent process. (Optional) """ super(PantsDaemon, self).__init__(name='pantsd', metadata_base_dir=metadata_base_dir) self._logger = logging.getLogger(__name__) self._build_root = build_root self._work_dir = work_dir self._log_level = log_level self._native = native self._log_dir = log_dir or os.path.join(work_dir, self.name) self._services = services or () self._reset_func = reset_func self._socket_map = {} # N.B. This Event is used as nothing more than a convenient atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._exiter = Exiter() @property def is_killed(self): return self._kill_switch.is_set() def set_services(self, services): self._services = services def set_socket_map(self, socket_map): self._socket_map = socket_map def shutdown(self, service_thread_map): """Gracefully terminate all services and kill the main PantsDaemon loop.""" for service, service_thread in service_thread_map.items(): self._logger.info('terminating pantsd service: {}'.format(service)) service.terminate() service_thread.join() self._logger.info('terminating pantsd') self._kill_switch.set() @staticmethod def _close_fds(): """Close pre-fork stdio streams to avoid output in the pants process that launched pantsd.""" for fd in (sys.stdin, sys.stdout, sys.stderr): file_no = fd.fileno() fd.flush() fd.close() os.close(file_no) def _setup_logging(self, log_level): """Reinitialize logging post-fork to clear all handlers, file descriptors, locks etc. This must happen first thing post-fork, before any further logging is emitted. """ # Re-initialize the childs logging locks post-fork to avoid potential deadlocks if pre-fork # threads have any locks acquired at the time of fork. logging._lock = threading.RLock() if logging.thread else None for handler in logging.getLogger().handlers: handler.createLock() # Invoke a global teardown for all logging handlers created before now. logging.shutdown() # Reinitialize logging for the daemon context. result = setup_logging(log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Close out pre-fork file descriptors. self._close_fds() # Redirect stdio to the root logger. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_stream) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_stream) self._logger.debug('logging initialized') return result.log_stream def _setup_services(self, services): for service in services: self._logger.info('setting up service {}'.format(service)) service.setup() def _run_services(self, services): """Service runner main loop.""" if not services: self._logger.critical('no services to run, bailing!') return service_thread_map = { service: threading.Thread(target=service.run) for service in services } # Start services. for service, service_thread in service_thread_map.items(): self._logger.info('starting service {}'.format(service)) try: service_thread.start() except (RuntimeError, service.ServiceError): self.shutdown(service_thread_map) raise self.StartupFailure( 'service {} failed to start, shutting down!'.format( service)) # Monitor services. while not self.is_killed: for service, service_thread in service_thread_map.items(): if not service_thread.is_alive(): self.shutdown(service_thread_map) raise self.RuntimeFailure( 'service failure for {}, shutting down!'.format( service)) else: # Avoid excessive CPU utilization. service_thread.join(self.JOIN_TIMEOUT_SECONDS) def _write_named_sockets(self, socket_map): """Write multiple named sockets using a socket mapping.""" for socket_name, socket_info in socket_map.items(): self.write_named_socket(socket_name, socket_info) def _run(self): """Synchronously run pantsd.""" # Switch log output to the daemon's log stream from here forward. log_stream = self._setup_logging(self._log_level) self._exiter.set_except_hook(log_stream) self._logger.info('pantsd starting, log level is {}'.format( self._log_level)) # Purge as much state as possible from the pants run that launched us. if self._reset_func: self._reset_func() # Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'. set_process_title('pantsd [{}]'.format(self._build_root)) # Write service socket information to .pids. self._write_named_sockets(self._socket_map) # Enter the main service runner loop. self._setup_services(self._services) self._run_services(self._services) def pre_fork(self): """Pre-fork() callback for ProcessManager.daemonize().""" for service in self._services: service.pre_fork() # Teardown the RunTracker's SubprocPool pre-fork. RunTracker.global_instance().shutdown_worker_pool() # TODO(kwlzn): This currently aborts tracking of the remainder of the pants run that launched # pantsd. def post_fork_child(self): """Post-fork() child callback for ProcessManager.daemonize().""" self._native.set_panic_handler() self._run()
class PantsDaemon(ProcessManager): """A daemon that manages PantsService instances.""" JOIN_TIMEOUT_SECONDS = 1 LOG_NAME = 'pantsd.log' class StartupFailure(Exception): """Represents a failure to start pantsd.""" class RuntimeFailure(Exception): """Represents a pantsd failure at runtime, usually from an underlying service failure.""" class Factory(object): @classmethod def create(cls, bootstrap_options=None): """ :param Options bootstrap_options: The bootstrap options, if available. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options( ) build_root = get_buildroot() native = Native.create(bootstrap_options) # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options).watchman legacy_graph_helper = cls._setup_legacy_graph_helper( native, bootstrap_options) services, port_map = cls._setup_services(build_root, bootstrap_options, legacy_graph_helper, watchman) return PantsDaemon(native, build_root, bootstrap_options.pants_workdir, bootstrap_options.level.upper(), legacy_graph_helper.scheduler.lock, services, port_map, bootstrap_options.pants_subprocessdir, bootstrap_options) @staticmethod def _parse_bootstrap_options(): options_bootstrapper = OptionsBootstrapper() return options_bootstrapper.get_bootstrap_options( ).for_global_scope() @staticmethod def _setup_legacy_graph_helper(native, bootstrap_options): """Initializes a `LegacyGraphHelper` instance.""" return EngineInitializer.setup_legacy_graph( bootstrap_options.pants_ignore, bootstrap_options.pants_workdir, native=native, build_ignore_patterns=bootstrap_options.build_ignore, exclude_target_regexps=bootstrap_options.exclude_target_regexp, subproject_roots=bootstrap_options.subproject_roots, ) @staticmethod def _setup_services(build_root, bootstrap_options, legacy_graph_helper, watchman): """Initialize pantsd services. :returns: A tuple of (`tuple` service_instances, `dict` port_map). """ fs_event_service = FSEventService( watchman, build_root, bootstrap_options.pantsd_fs_event_workers) scheduler_service = SchedulerService(fs_event_service, legacy_graph_helper) pailgun_service = PailgunService( bind_addr=(bootstrap_options.pantsd_pailgun_host, bootstrap_options.pantsd_pailgun_port), exiter_class=DaemonExiter, runner_class=DaemonPantsRunner, target_roots_class=TargetRoots, scheduler_service=scheduler_service) return ( # Services. (fs_event_service, scheduler_service, pailgun_service), # Port map. dict(pailgun=pailgun_service.pailgun_port)) def __init__(self, native, build_root, work_dir, log_level, lock, services, socket_map, metadata_base_dir, bootstrap_options=None): """ :param Native native: A `Native` instance. :param string build_root: The pants build root. :param string work_dir: The pants work directory. :param string log_level: The log level to use for daemon logging. :param string metadata_base_dir: The ProcessManager metadata base dir. :param Options bootstrap_options: The bootstrap options, if available. """ super(PantsDaemon, self).__init__(name='pantsd', metadata_base_dir=metadata_base_dir) self._native = native self._build_root = build_root self._work_dir = work_dir self._log_level = log_level self._lock = lock self._services = services self._socket_map = socket_map self._bootstrap_options = bootstrap_options self._log_dir = os.path.join(work_dir, self.name) self._logger = logging.getLogger(__name__) # N.B. This Event is used as nothing more than a convenient atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._exiter = Exiter() @memoized_property def watchman_launcher(self): return WatchmanLauncher.create(self._bootstrap_options) @property def is_killed(self): return self._kill_switch.is_set() def shutdown(self, service_thread_map): """Gracefully terminate all services and kill the main PantsDaemon loop.""" with self._lock: for service, service_thread in service_thread_map.items(): self._logger.info( 'terminating pantsd service: {}'.format(service)) service.terminate() service_thread.join() self._logger.info('terminating pantsd') self._kill_switch.set() @staticmethod def _close_fds(): """Close stdio streams to avoid output in the tty that launched pantsd.""" for fd in (sys.stdin, sys.stdout, sys.stderr): file_no = fd.fileno() fd.flush() fd.close() os.close(file_no) def _setup_logging(self, log_level): """Initializes logging.""" # Reinitialize logging for the daemon context. result = setup_logging(log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Close out tty file descriptors. self._close_fds() # Redirect stdio to the root logger. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_stream) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_stream) self._logger.debug('logging initialized') return result.log_stream def _setup_services(self, services): assert self._lock is not None, 'PantsDaemon lock has not been set!' for service in services: self._logger.info('setting up service {}'.format(service)) service.setup(self._lock) def _run_services(self, services): """Service runner main loop.""" if not services: self._logger.critical('no services to run, bailing!') return service_thread_map = { service: threading.Thread(target=service.run) for service in services } # Start services. for service, service_thread in service_thread_map.items(): self._logger.info('starting service {}'.format(service)) try: service_thread.start() except (RuntimeError, service.ServiceError): self.shutdown(service_thread_map) raise self.StartupFailure( 'service {} failed to start, shutting down!'.format( service)) # Once all services are started, write our pid. self.write_pid() # Monitor services. while not self.is_killed: for service, service_thread in service_thread_map.items(): if not service_thread.is_alive(): self.shutdown(service_thread_map) raise self.RuntimeFailure( 'service failure for {}, shutting down!'.format( service)) else: # Avoid excessive CPU utilization. service_thread.join(self.JOIN_TIMEOUT_SECONDS) def _write_named_sockets(self, socket_map): """Write multiple named sockets using a socket mapping.""" for socket_name, socket_info in socket_map.items(): self.write_named_socket(socket_name, socket_info) def run_sync(self): """Synchronously run pantsd.""" # Switch log output to the daemon's log stream from here forward. log_stream = self._setup_logging(self._log_level) self._exiter.set_except_hook(log_stream) self._logger.info('pantsd starting, log level is {}'.format( self._log_level)) self._native.set_panic_handler() # Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'. set_process_title('pantsd [{}]'.format(self._build_root)) # Write service socket information to .pids. self._write_named_sockets(self._socket_map) # Enter the main service runner loop. self._setup_services(self._services) self._run_services(self._services) def post_fork_child(self): """Post-fork() child callback for ProcessManager.daemon_spawn().""" entry_point = '{}:launch'.format(__name__) exec_env = combined_dict(os.environ, dict(PANTS_ENTRYPOINT=entry_point)) # Pass all of sys.argv so that we can proxy arg flags e.g. `-ldebug`. cmd = [sys.executable] + sys.argv self._logger.debug('cmd is: PANTS_ENTRYPOINT={} {}'.format( entry_point, ' '.join(cmd))) # TODO: Improve error handling on launch failures. os.spawnve(os.P_NOWAIT, sys.executable, cmd, env=exec_env) def maybe_launch(self): """Launches pantsd (if not already running) in a subprocess. :returns: The port that pantsd is listening on. :rtype: int """ self.watchman_launcher.maybe_launch() self._logger.debug('acquiring lock: {}'.format(self.process_lock)) with self.process_lock: if not self.is_alive(): self._logger.debug('launching pantsd') self.daemon_spawn() # Wait up to 10 seconds for pantsd to write its pidfile so we can display the pid to the user. self.await_pid(10) listening_port = self.read_named_socket('pailgun', int) pantsd_pid = self.pid self._logger.debug('released lock: {}'.format(self.process_lock)) self._logger.debug( 'pantsd is running at pid {}, pailgun port is {}'.format( pantsd_pid, listening_port)) return listening_port def terminate(self): """Terminates pantsd and watchman.""" super(PantsDaemon, self).terminate() self.watchman_launcher.terminate()
class PantsDaemon(ProcessManager): """A daemon that manages PantsService instances.""" JOIN_TIMEOUT_SECONDS = 1 LOG_NAME = 'pantsd.log' class StartupFailure(Exception): pass class RuntimeFailure(Exception): pass def __init__(self, build_root, work_dir, log_level, native, log_dir=None, services=None, metadata_base_dir=None, reset_func=None): """ :param string build_root: The pants build root. :param string work_dir: The pants work directory. :param string log_level: The log level to use for daemon logging. :param string log_dir: The directory to use for file-based logging via the daemon. (Optional) :param tuple services: A tuple of PantsService instances to launch/manage. (Optional) :param callable reset_func: Called after the daemon is forked to reset any state inherited from the parent process. (Optional) """ super(PantsDaemon, self).__init__(name='pantsd', metadata_base_dir=metadata_base_dir) self._logger = logging.getLogger(__name__) self._build_root = build_root self._work_dir = work_dir self._log_level = log_level self._native = native self._log_dir = log_dir or os.path.join(work_dir, self.name) self._services = services or () self._reset_func = reset_func self._socket_map = {} # N.B. This Event is used as nothing more than a convenient atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._exiter = Exiter() @property def is_killed(self): return self._kill_switch.is_set() def set_services(self, services): self._services = services def set_socket_map(self, socket_map): self._socket_map = socket_map def shutdown(self, service_thread_map): """Gracefully terminate all services and kill the main PantsDaemon loop.""" for service, service_thread in service_thread_map.items(): self._logger.info('terminating pantsd service: {}'.format(service)) service.terminate() service_thread.join() self._logger.info('terminating pantsd') self._kill_switch.set() @staticmethod def _close_fds(): """Close pre-fork stdio streams to avoid output in the pants process that launched pantsd.""" for fd in (sys.stdin, sys.stdout, sys.stderr): file_no = fd.fileno() fd.flush() fd.close() os.close(file_no) def _setup_logging(self, log_level): """Reinitialize logging post-fork to clear all handlers, file descriptors, locks etc. This must happen first thing post-fork, before any further logging is emitted. """ # Re-initialize the childs logging locks post-fork to avoid potential deadlocks if pre-fork # threads have any locks acquired at the time of fork. logging._lock = threading.RLock() if logging.thread else None for handler in logging.getLogger().handlers: handler.createLock() # Invoke a global teardown for all logging handlers created before now. logging.shutdown() # Reinitialize logging for the daemon context. result = setup_logging(log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Close out pre-fork file descriptors. self._close_fds() # Redirect stdio to the root logger. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_stream) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_stream) self._logger.debug('logging initialized') return result.log_stream def _setup_services(self, services): for service in services: self._logger.info('setting up service {}'.format(service)) service.setup() def _run_services(self, services): """Service runner main loop.""" if not services: self._logger.critical('no services to run, bailing!') return service_thread_map = {service: threading.Thread(target=service.run) for service in services} # Start services. for service, service_thread in service_thread_map.items(): self._logger.info('starting service {}'.format(service)) try: service_thread.start() except (RuntimeError, service.ServiceError): self.shutdown(service_thread_map) raise self.StartupFailure('service {} failed to start, shutting down!'.format(service)) # Monitor services. while not self.is_killed: for service, service_thread in service_thread_map.items(): if not service_thread.is_alive(): self.shutdown(service_thread_map) raise self.RuntimeFailure('service failure for {}, shutting down!'.format(service)) else: # Avoid excessive CPU utilization. service_thread.join(self.JOIN_TIMEOUT_SECONDS) def _write_named_sockets(self, socket_map): """Write multiple named sockets using a socket mapping.""" for socket_name, socket_info in socket_map.items(): self.write_named_socket(socket_name, socket_info) def _run(self): """Synchronously run pantsd.""" # Switch log output to the daemon's log stream from here forward. log_stream = self._setup_logging(self._log_level) self._exiter.set_except_hook(log_stream) self._logger.info('pantsd starting, log level is {}'.format(self._log_level)) # Purge as much state as possible from the pants run that launched us. if self._reset_func: self._reset_func() # Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'. set_process_title('pantsd [{}]'.format(self._build_root)) # Write service socket information to .pids. self._write_named_sockets(self._socket_map) # Enter the main service runner loop. self._setup_services(self._services) self._run_services(self._services) def pre_fork(self): """Pre-fork() callback for ProcessManager.daemonize().""" for service in self._services: service.pre_fork() # Teardown the RunTracker's SubprocPool pre-fork. RunTracker.global_instance().shutdown_worker_pool() # TODO(kwlzn): This currently aborts tracking of the remainder of the pants run that launched # pantsd. def post_fork_child(self): """Post-fork() child callback for ProcessManager.daemonize().""" self._native.set_panic_handler() self._run()