def start(self): try: self.connection = transport_utils.get_connection() self._updates_thread = concurrency.spawn(self.run) self._load_thread = concurrency.spawn(self._load_triggers_from_db) except: LOG.exception('Failed to start watcher.') self.connection.release()
def _run_worker(): LOG.info("(PID=%s) TimerEngine started.", os.getpid()) timer = None try: timer_thread = None if cfg.CONF.timer.enable or cfg.CONF.timersengine.enable: local_tz = (cfg.CONF.timer.local_timezone or cfg.CONF.timersengine.local_timezone) timer = St2Timer(local_timezone=local_tz) timer_thread = concurrency.spawn(_kickoff_timer, timer) LOG.info(TIMER_ENABLED_LOG_LINE) return timer_thread.wait() else: LOG.info(TIMER_DISABLED_LOG_LINE) except (KeyboardInterrupt, SystemExit): LOG.info("(PID=%s) TimerEngine stopped.", os.getpid()) except: LOG.exception("(PID:%s) TimerEngine quit due to exception.", os.getpid()) return 1 finally: if timer: timer.cleanup() return 0
def start(self): try: self.connection = transport_utils.get_connection() self._updates_thread = concurrency.spawn(self.run) except: LOG.exception("Failed to start sensor_watcher.") self.connection.release()
def _spin_container_and_wait(self, sensors): exit_code = 0 try: self._sensor_container = ProcessSensorContainer( sensors=sensors, single_sensor_mode=self._single_sensor_mode) self._container_thread = concurrency.spawn( self._sensor_container.run) LOG.debug('Starting sensor CUD watcher...') self._sensors_watcher.start() exit_code = self._container_thread.wait() LOG.error('Process container quit with exit_code %d.', exit_code) LOG.error('(PID:%s) SensorContainer stopped.', os.getpid()) except (KeyboardInterrupt, SystemExit): self._sensor_container.shutdown() self._sensors_watcher.stop() LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(), sys.exc_info()[0].__name__) concurrency.kill(self._container_thread) self._container_thread = None return exit_code return exit_code
def test_no_sensors_dont_quit(self): process_container = ProcessSensorContainer(None, poll_interval=0.1) process_container_thread = concurrency.spawn(process_container.run) concurrency.sleep(0.5) self.assertEqual(process_container.running(), 0) self.assertEqual(process_container.stopped(), False) process_container.shutdown() process_container_thread.kill()
def _poll_sensors_for_results(self, sensor_ids): """ Main loop which polls sensor for results and detects dead sensors. """ for sensor_id in sensor_ids: now = int(time.time()) process = self._processes[sensor_id] status = process.poll() if status is not None: # Dead process detected LOG.info("Process for sensor %s has exited with code %s", sensor_id, status) sensor = self._sensors[sensor_id] self._delete_sensor(sensor_id) self._dispatch_trigger_for_sensor_exit(sensor=sensor, exit_code=status) # Try to respawn a dead process (maybe it was a simple failure which can be # resolved with a restart) concurrency.spawn( self._respawn_sensor, sensor_id=sensor_id, sensor=sensor, exit_code=status, ) else: sensor_start_time = self._sensor_start_times[sensor_id] sensor_respawn_count = self._sensor_respawn_counts[sensor_id] successfully_started = (now - sensor_start_time ) >= SENSOR_SUCCESSFUL_START_THRESHOLD if successfully_started and sensor_respawn_count >= 1: # Sensor has been successfully running more than threshold seconds, clear the # respawn counter so we can try to restart the sensor if it dies later on self._sensor_respawn_counts[sensor_id] = 0
def start(self, wait=False): LOG.info('Starting %s...', self.__class__.__name__) self._consumer_thread = concurrency.spawn(self._queue_consumer.run) if wait: self.wait()
def run_command( cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, cwd=None, env=None, timeout=60, preexec_func=None, kill_func=None, read_stdout_func=None, read_stderr_func=None, read_stdout_buffer=None, read_stderr_buffer=None, stdin_value=None, bufsize=0, ): """ Run the provided command in a subprocess and wait until it completes. :param cmd: Command to run. :type cmd: ``str`` or ``list`` :param stdin: Process stdin. :type stdin: ``object`` :param stdout: Process stdout. :type stdout: ``object`` :param stderr: Process stderr. :type stderr: ``object`` :param shell: True to use a shell. :type shell ``boolean`` :param cwd: Optional working directory. :type cwd: ``str`` :param env: Optional environment to use with the command. If not provided, environment from the current process is inherited. :type env: ``dict`` :param timeout: How long to wait before timing out. :type timeout: ``float`` :param preexec_func: Optional pre-exec function. :type preexec_func: ``callable`` :param kill_func: Optional function which will be called on timeout to kill the process. If not provided, it defaults to `process.kill` :type kill_func: ``callable`` :param read_stdout_func: Function which is responsible for reading process stdout when using live read mode. :type read_stdout_func: ``func`` :param read_stdout_func: Function which is responsible for reading process stderr when using live read mode. :type read_stdout_func: ``func`` :param bufsize: Buffer size argument to pass to subprocess.popen function. :type bufsize: ``int`` :rtype: ``tuple`` (exit_code, stdout, stderr, timed_out) """ LOG.debug("Entering st2common.util.green.run_command.") if not isinstance(cmd, (list, tuple) + six.string_types): raise TypeError( f"Command must be a type of list, tuple, or string, not '{type(cmd)}'." ) if (read_stdout_func and not read_stderr_func) or ( read_stderr_func and not read_stdout_func ): raise ValueError( "Both read_stdout_func and read_stderr_func arguments need " "to be provided." ) if read_stdout_func and not (read_stdout_buffer or read_stderr_buffer): raise ValueError( "read_stdout_buffer and read_stderr_buffer arguments need to be provided " "when read_stdout_func is provided" ) if not env: LOG.debug("env argument not provided. using process env (os.environ).") env = os.environ.copy() subprocess = concurrency.get_subprocess_module() # Note: We are using eventlet / gevent friendly implementation of subprocess which uses # GreenPipe so it doesn't block LOG.debug("Creating subprocess.") process = concurrency.subprocess_popen( args=cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=env, cwd=cwd, shell=shell, preexec_fn=preexec_func, bufsize=bufsize, ) if read_stdout_func: LOG.debug("Spawning read_stdout_func function") read_stdout_thread = concurrency.spawn( read_stdout_func, process.stdout, read_stdout_buffer ) if read_stderr_func: LOG.debug("Spawning read_stderr_func function") read_stderr_thread = concurrency.spawn( read_stderr_func, process.stderr, read_stderr_buffer ) def on_timeout_expired(timeout): global timed_out try: LOG.debug("Starting process wait inside timeout handler.") process.wait(timeout=timeout) except subprocess.TimeoutExpired: # Command has timed out, kill the process and propagate the error. # Note: We explicitly set the returncode to indicate the timeout. LOG.debug("Command execution timeout reached.") # NOTE: It's important we set returncode twice - here and below to avoid race in this # function because "kill_func()" is async and "process.kill()" is not. process.returncode = TIMEOUT_EXIT_CODE if kill_func: LOG.debug("Calling kill_func.") kill_func(process=process) else: LOG.debug("Killing process.") process.kill() # NOTE: It's imporant to set returncode here as well, since call to process.kill() sets # it and overwrites it if we set it earlier. process.returncode = TIMEOUT_EXIT_CODE if read_stdout_func and read_stderr_func: LOG.debug("Killing read_stdout_thread and read_stderr_thread") concurrency.kill(read_stdout_thread) concurrency.kill(read_stderr_thread) LOG.debug("Spawning timeout handler thread.") timeout_thread = concurrency.spawn(on_timeout_expired, timeout) LOG.debug("Attaching to process.") if stdin_value: if six.PY3: stdin_value = stdin_value.encode("utf-8") process.stdin.write(stdin_value) if read_stdout_func and read_stderr_func: LOG.debug("Using real-time stdout and stderr read mode, calling process.wait()") process.wait() else: LOG.debug( "Using delayed stdout and stderr read mode, calling process.communicate()" ) stdout, stderr = process.communicate() concurrency.cancel(timeout_thread) exit_code = process.returncode if read_stdout_func and read_stderr_func: # Wait on those green threads to finish reading from stdout and stderr before continuing concurrency.wait(read_stdout_thread) concurrency.wait(read_stderr_thread) stdout = read_stdout_buffer.getvalue() stderr = read_stderr_buffer.getvalue() if exit_code == TIMEOUT_EXIT_CODE: LOG.debug("Timeout.") timed_out = True else: LOG.debug("No timeout.") timed_out = False LOG.debug("Returning.") return (exit_code, stdout, stderr, timed_out)