def start(self):
try:
self.connection = transport_utils.get_connection()
self._updates_thread = concurrency.spawn(self.run)
self._load_thread = concurrency.spawn(self._load_triggers_from_db)
except:
LOG.exception('Failed to start watcher.')
self.connection.release()
def _run_worker():
LOG.info("(PID=%s) TimerEngine started.", os.getpid())
timer = None
try:
timer_thread = None
if cfg.CONF.timer.enable or cfg.CONF.timersengine.enable:
local_tz = (cfg.CONF.timer.local_timezone
or cfg.CONF.timersengine.local_timezone)
timer = St2Timer(local_timezone=local_tz)
timer_thread = concurrency.spawn(_kickoff_timer, timer)
LOG.info(TIMER_ENABLED_LOG_LINE)
return timer_thread.wait()
else:
LOG.info(TIMER_DISABLED_LOG_LINE)
except (KeyboardInterrupt, SystemExit):
LOG.info("(PID=%s) TimerEngine stopped.", os.getpid())
except:
LOG.exception("(PID:%s) TimerEngine quit due to exception.",
os.getpid())
return 1
finally:
if timer:
timer.cleanup()
return 0
def start(self):
try:
self.connection = transport_utils.get_connection()
self._updates_thread = concurrency.spawn(self.run)
except:
LOG.exception("Failed to start sensor_watcher.")
self.connection.release()
def _spin_container_and_wait(self, sensors):
exit_code = 0
try:
self._sensor_container = ProcessSensorContainer(
sensors=sensors, single_sensor_mode=self._single_sensor_mode)
self._container_thread = concurrency.spawn(
self._sensor_container.run)
LOG.debug('Starting sensor CUD watcher...')
self._sensors_watcher.start()
exit_code = self._container_thread.wait()
LOG.error('Process container quit with exit_code %d.', exit_code)
LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
except (KeyboardInterrupt, SystemExit):
self._sensor_container.shutdown()
self._sensors_watcher.stop()
LOG.info('(PID:%s) SensorContainer stopped. Reason - %s',
os.getpid(),
sys.exc_info()[0].__name__)
concurrency.kill(self._container_thread)
self._container_thread = None
return exit_code
return exit_code
def test_no_sensors_dont_quit(self):
process_container = ProcessSensorContainer(None, poll_interval=0.1)
process_container_thread = concurrency.spawn(process_container.run)
concurrency.sleep(0.5)
self.assertEqual(process_container.running(), 0)
self.assertEqual(process_container.stopped(), False)
process_container.shutdown()
process_container_thread.kill()
def _poll_sensors_for_results(self, sensor_ids):
"""
Main loop which polls sensor for results and detects dead sensors.
"""
for sensor_id in sensor_ids:
now = int(time.time())
process = self._processes[sensor_id]
status = process.poll()
if status is not None:
# Dead process detected
LOG.info("Process for sensor %s has exited with code %s",
sensor_id, status)
sensor = self._sensors[sensor_id]
self._delete_sensor(sensor_id)
self._dispatch_trigger_for_sensor_exit(sensor=sensor,
exit_code=status)
# Try to respawn a dead process (maybe it was a simple failure which can be
# resolved with a restart)
concurrency.spawn(
self._respawn_sensor,
sensor_id=sensor_id,
sensor=sensor,
exit_code=status,
)
else:
sensor_start_time = self._sensor_start_times[sensor_id]
sensor_respawn_count = self._sensor_respawn_counts[sensor_id]
successfully_started = (now - sensor_start_time
) >= SENSOR_SUCCESSFUL_START_THRESHOLD
if successfully_started and sensor_respawn_count >= 1:
# Sensor has been successfully running more than threshold seconds, clear the
# respawn counter so we can try to restart the sensor if it dies later on
self._sensor_respawn_counts[sensor_id] = 0
def start(self, wait=False):
LOG.info('Starting %s...', self.__class__.__name__)
self._consumer_thread = concurrency.spawn(self._queue_consumer.run)
if wait:
self.wait()
def run_command(
cmd,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
cwd=None,
env=None,
timeout=60,
preexec_func=None,
kill_func=None,
read_stdout_func=None,
read_stderr_func=None,
read_stdout_buffer=None,
read_stderr_buffer=None,
stdin_value=None,
bufsize=0,
):
"""
Run the provided command in a subprocess and wait until it completes.
:param cmd: Command to run.
:type cmd: ``str`` or ``list``
:param stdin: Process stdin.
:type stdin: ``object``
:param stdout: Process stdout.
:type stdout: ``object``
:param stderr: Process stderr.
:type stderr: ``object``
:param shell: True to use a shell.
:type shell ``boolean``
:param cwd: Optional working directory.
:type cwd: ``str``
:param env: Optional environment to use with the command. If not provided,
environment from the current process is inherited.
:type env: ``dict``
:param timeout: How long to wait before timing out.
:type timeout: ``float``
:param preexec_func: Optional pre-exec function.
:type preexec_func: ``callable``
:param kill_func: Optional function which will be called on timeout to kill the process.
If not provided, it defaults to `process.kill`
:type kill_func: ``callable``
:param read_stdout_func: Function which is responsible for reading process stdout when
using live read mode.
:type read_stdout_func: ``func``
:param read_stdout_func: Function which is responsible for reading process stderr when
using live read mode.
:type read_stdout_func: ``func``
:param bufsize: Buffer size argument to pass to subprocess.popen function.
:type bufsize: ``int``
:rtype: ``tuple`` (exit_code, stdout, stderr, timed_out)
"""
LOG.debug("Entering st2common.util.green.run_command.")
if not isinstance(cmd, (list, tuple) + six.string_types):
raise TypeError(
f"Command must be a type of list, tuple, or string, not '{type(cmd)}'."
)
if (read_stdout_func and not read_stderr_func) or (
read_stderr_func and not read_stdout_func
):
raise ValueError(
"Both read_stdout_func and read_stderr_func arguments need "
"to be provided."
)
if read_stdout_func and not (read_stdout_buffer or read_stderr_buffer):
raise ValueError(
"read_stdout_buffer and read_stderr_buffer arguments need to be provided "
"when read_stdout_func is provided"
)
if not env:
LOG.debug("env argument not provided. using process env (os.environ).")
env = os.environ.copy()
subprocess = concurrency.get_subprocess_module()
# Note: We are using eventlet / gevent friendly implementation of subprocess which uses
# GreenPipe so it doesn't block
LOG.debug("Creating subprocess.")
process = concurrency.subprocess_popen(
args=cmd,
stdin=stdin,
stdout=stdout,
stderr=stderr,
env=env,
cwd=cwd,
shell=shell,
preexec_fn=preexec_func,
bufsize=bufsize,
)
if read_stdout_func:
LOG.debug("Spawning read_stdout_func function")
read_stdout_thread = concurrency.spawn(
read_stdout_func, process.stdout, read_stdout_buffer
)
if read_stderr_func:
LOG.debug("Spawning read_stderr_func function")
read_stderr_thread = concurrency.spawn(
read_stderr_func, process.stderr, read_stderr_buffer
)
def on_timeout_expired(timeout):
global timed_out
try:
LOG.debug("Starting process wait inside timeout handler.")
process.wait(timeout=timeout)
except subprocess.TimeoutExpired:
# Command has timed out, kill the process and propagate the error.
# Note: We explicitly set the returncode to indicate the timeout.
LOG.debug("Command execution timeout reached.")
# NOTE: It's important we set returncode twice - here and below to avoid race in this
# function because "kill_func()" is async and "process.kill()" is not.
process.returncode = TIMEOUT_EXIT_CODE
if kill_func:
LOG.debug("Calling kill_func.")
kill_func(process=process)
else:
LOG.debug("Killing process.")
process.kill()
# NOTE: It's imporant to set returncode here as well, since call to process.kill() sets
# it and overwrites it if we set it earlier.
process.returncode = TIMEOUT_EXIT_CODE
if read_stdout_func and read_stderr_func:
LOG.debug("Killing read_stdout_thread and read_stderr_thread")
concurrency.kill(read_stdout_thread)
concurrency.kill(read_stderr_thread)
LOG.debug("Spawning timeout handler thread.")
timeout_thread = concurrency.spawn(on_timeout_expired, timeout)
LOG.debug("Attaching to process.")
if stdin_value:
if six.PY3:
stdin_value = stdin_value.encode("utf-8")
process.stdin.write(stdin_value)
if read_stdout_func and read_stderr_func:
LOG.debug("Using real-time stdout and stderr read mode, calling process.wait()")
process.wait()
else:
LOG.debug(
"Using delayed stdout and stderr read mode, calling process.communicate()"
)
stdout, stderr = process.communicate()
concurrency.cancel(timeout_thread)
exit_code = process.returncode
if read_stdout_func and read_stderr_func:
# Wait on those green threads to finish reading from stdout and stderr before continuing
concurrency.wait(read_stdout_thread)
concurrency.wait(read_stderr_thread)
stdout = read_stdout_buffer.getvalue()
stderr = read_stderr_buffer.getvalue()
if exit_code == TIMEOUT_EXIT_CODE:
LOG.debug("Timeout.")
timed_out = True
else:
LOG.debug("No timeout.")
timed_out = False
LOG.debug("Returning.")
return (exit_code, stdout, stderr, timed_out)
