def _read_out(self):
stream = PluginIPCStream()
while self._process_running:
exit_code = self._proc.poll()
if exit_code is not None:
self.logger('[Runner] Stopped with exit code {0}'.format(exit_code))
self._process_running = False
break
line = ''
while line == '':
line = self._proc.stdout.readline().strip()
try:
response = stream.feed(line)
if response is None:
continue
except Exception as ex:
self.logger('[Runner] Exception while parsing output: {0}'.format(ex))
continue
if response['cid'] == 0:
self._handle_async_response(response)
elif response['cid'] == self._cid:
self._response_queue.put(response)
else:
self.logger('[Runner] Received message with unknown cid: {0}'.format(response))
def _wait_and_read_command():
stream = PluginIPCStream()
line = ''
while line == '':
line = sys.stdin.readline().strip()
try:
response = stream.feed(line)
if response is not None:
return response
except Exception as ex:
IO._log(
'Exception in _wait_and_read_command: Could not decode stdin: {0}'
.format(ex))
def start(self):
if self._running:
raise Exception('PluginRunner is already running')
self.logger('[Runner] Starting')
python_executable = sys.executable
if python_executable is None or len(python_executable) == 0:
python_executable = '/usr/bin/python'
self._proc = subprocess.Popen(
[python_executable, "runtime.py", "start", self.plugin_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=None,
cwd=self.runtime_path)
self._process_running = True
self._commands_executed = 0
self._commands_failed = 0
self._stream = PluginIPCStream(stream=self._proc.stdout,
logger=lambda message, ex: self.logger(
'{0}: {1}'.format(message, ex)),
command_receiver=self._process_command)
self._stream.start()
start_out = self._do_command('start', timeout=180)
self.name = start_out['name']
self.version = start_out['version']
self.interfaces = start_out['interfaces']
self._receivers = start_out['receivers']
self._exposes = start_out['exposes']
self._metric_collectors = start_out['metric_collectors']
self._metric_receivers = start_out['metric_receivers']
exception = start_out.get('exception')
if exception is not None:
raise RuntimeError(exception)
self._async_command_queue = Queue(1000)
self._async_command_thread = Thread(
target=self._perform_async_commands,
name='PluginRunner {0} async thread'.format(self.plugin_path))
self._async_command_thread.daemon = True
self._async_command_thread.start()
self._running = True
self.logger('[Runner] Started')
def _do_command(self, action, fields=None, timeout=None):
if fields is None:
fields = {}
self._commands_executed += 1
if timeout is None:
timeout = self.command_timeout
if not self._process_running:
raise Exception('Plugin was stopped')
with self._command_lock:
command = self._create_command(action, fields)
self._proc.stdin.write(PluginIPCStream.encode(command))
self._proc.stdin.flush()
try:
response = self._response_queue.get(block=True, timeout=timeout)
while response['cid'] != self._cid:
response = self._response_queue.get(block=False)
exception = response.get('_exception')
if exception is not None:
raise RuntimeError(exception)
return response
except Empty:
self.logger('[Runner] No response within {0}s (action={1}, fields={2})'.format(timeout, action, fields))
self._commands_failed += 1
raise Exception('Plugin did not respond')
def __init__(self, path):
self._stopped = False
self._path = path.rstrip('/')
self._input_status_receivers = []
self._output_status_receivers = []
self._shutter_status_receivers = []
self._event_receivers = []
self._name = None
self._version = None
self._interfaces = []
self._receivers = []
self._exposes = []
self._metric_definitions = []
self._metric_collectors = []
self._metric_receivers = []
self._plugin = None
self._stream = PluginIPCStream(sys.stdin, IO._log_exception)
self._webinterface = WebInterfaceDispatcher(IO._log)
def _write(msg):
sys.stdout.write(PluginIPCStream.write(msg))
sys.stdout.flush()
class PluginRuntime:
def __init__(self, path):
self._stopped = False
self._path = path.rstrip('/')
self._input_status_receivers = []
self._output_status_receivers = []
self._shutter_status_receivers = []
self._event_receivers = []
self._name = None
self._version = None
self._interfaces = []
self._receivers = []
self._exposes = []
self._metric_definitions = []
self._metric_collectors = []
self._metric_receivers = []
self._plugin = None
self._stream = PluginIPCStream(sys.stdin, IO._log_exception)
self._webinterface = WebInterfaceDispatcher(IO._log)
def _init_plugin(self):
plugin_root = os.path.dirname(self._path)
plugin_dir = os.path.basename(self._path)
# Add the plugin and it's eggs to the python path
sys.path.insert(0, plugin_root)
for egg_file in os.listdir(self._path):
if egg_file.endswith('.egg'):
sys.path.append(os.path.join(self._path, egg_file))
# Expose plugins.base to the plugin
sys.modules['plugins'] = sys.modules['__main__']
sys.modules["plugins.base"] = base
# Instanciate the plugin class
plugin_class = get_plugin_class(plugin_dir)
check_plugin(plugin_class)
# Set the name, version, interfaces
self._name = plugin_class.name
self._version = plugin_class.version
self._interfaces = plugin_class.interfaces
# Initialze the plugin
self._plugin = plugin_class(self._webinterface, IO._log)
# Set the receivers
receiver_mapping = {
'input_status': self._input_status_receivers,
'output_status': self._output_status_receivers,
'shutter_status': self._shutter_status_receivers,
'receive_events': self._event_receivers
}
for method_attribute, target in receiver_mapping.iteritems():
for method in get_special_methods(self._plugin, method_attribute):
target.append(method)
if len(target) > 0:
self._receivers.append(method_attribute)
# Set the exposed methods
for method in get_special_methods(self._plugin, 'om_expose'):
self._exposes.append({
'name':
method.__name__,
'auth':
method.om_expose['auth'],
'content_type':
method.om_expose['content_type']
})
# Set the metric definitions
if has_interface(plugin_class, 'metrics', '1.0'):
if hasattr(plugin_class, 'metric_definitions'):
self._metric_definitions = plugin_class.metric_definitions
# Set the metric collectors
for method in get_special_methods(self._plugin, 'om_metric_data'):
self._metric_collectors.append({
'name':
method.__name__,
'interval':
method.om_metric_data['interval']
})
# Set the metric receivers
for method in get_special_methods(self._plugin, 'om_metric_receive'):
self._metric_receivers.append({
'name':
method.__name__,
'source':
method.om_metric_receive['source'],
'metric_type':
method.om_metric_receive['metric_type'],
'interval':
method.om_metric_receive['interval']
})
def _start_background_tasks(self):
""" Start all background tasks. """
tasks = get_special_methods(self._plugin, 'background_task')
for task in tasks:
thread = Thread(target=PluginRuntime._run_background_task,
args=(task, ))
thread.name = 'Background thread ({0})'.format(task.__name__)
thread.daemon = True
thread.start()
@staticmethod
def _run_background_task(task):
running = True
while running:
try:
task()
running = False # Stop execution if the task returns without exception
except Exception as exception:
IO._log_exception('background task', exception)
time.sleep(30)
def process_stdin(self):
self._stream.start()
while not self._stopped:
command = self._stream.get(block=True)
if command is None:
continue
action = command['action']
response = {'cid': command['cid'], 'action': action}
try:
ret = None
if action == 'start':
ret = self._handle_start()
elif action == 'stop':
ret = self._handle_stop()
elif action == 'input_status':
ret = self._handle_input_status(command['event'])
elif action == 'output_status':
ret = self._handle_output_status(command['status'])
elif action == 'shutter_status':
ret = self._handle_shutter_status(command)
elif action == 'receive_events':
ret = self._handle_receive_events(command['code'])
elif action == 'get_metric_definitions':
ret = self._handle_get_metric_definitions()
elif action == 'collect_metrics':
ret = self._handle_collect_metrics(command['name'])
elif action == 'distribute_metrics':
ret = self._handle_distribute_metrics(
command['name'], command['metrics'])
elif action == 'request':
ret = self._handle_request(command['method'],
command['args'],
command['kwargs'])
elif action == 'remove_callback':
ret = self._handle_remove_callback()
else:
raise RuntimeError('Unknown action: {0}'.format(action))
if ret is not None:
response.update(ret)
except Exception as exception:
response['_exception'] = str(exception)
IO._write(response)
def _handle_start(self):
""" Handles the start command. Cover exceptions manually to make sure as much metadata is returned as possible. """
data = {}
try:
self._init_plugin()
self._start_background_tasks()
except Exception as exception:
data['exception'] = str(exception)
data.update({
'name': self._name,
'version': self._version,
'receivers': self._receivers,
'exposes': self._exposes,
'interfaces': self._interfaces,
'metric_collectors': self._metric_collectors,
'metric_receivers': self._metric_receivers
})
return data
def _handle_stop(self):
def delayed_stop():
time.sleep(2)
os._exit(0)
stop_thread = Thread(target=delayed_stop)
stop_thread.daemon = True
stop_thread.start()
self._stream.stop()
self._stopped = True
def _handle_input_status(self, event_json):
event = Event.deserialize(event_json)
# get relevant event details
input_id = event.data['id']
status = event.data['status']
for receiver in self._input_status_receivers:
version = receiver.input_status.get('version', 1)
if version == 1:
# Backwards compatibility: only send rising edges of the input (no input releases)
if status:
IO._with_catch('input status', receiver,
[(input_id, None)])
elif version == 2:
# Version 2 will send ALL input status changes AND in a dict format
data = {'input_id': input_id, 'status': status}
IO._with_catch('input status', receiver, [data])
else:
error = NotImplementedError(
'Version {} is not supported for input status decorators'.
format(version))
IO._log_exception('input status', error)
def _handle_output_status(self, status):
for receiver in self._output_status_receivers:
IO._with_catch('output status', receiver, [status])
def _handle_shutter_status(self, status):
for receiver in self._shutter_status_receivers:
if receiver.shutter_status['add_detail']:
IO._with_catch('shutter status', receiver,
[status['status'], status['detail']])
else:
IO._with_catch('shutter status', receiver, [status['status']])
def _handle_receive_events(self, code):
for receiver in self._event_receivers:
IO._with_catch('process event', receiver, [code])
def _handle_get_metric_definitions(self):
return {'metric_definitions': self._metric_definitions}
def _handle_collect_metrics(self, name):
metrics = []
collect = getattr(self._plugin, name)
try:
metrics.extend(list(collect()))
except Exception as exception:
IO._log_exception('collect metrics', exception)
return {'metrics': metrics}
def _handle_distribute_metrics(self, name, metrics):
receive = getattr(self._plugin, name)
for metric in metrics:
IO._with_catch('distribute metric', receive, [metric])
def _handle_request(self, method, args, kwargs):
func = getattr(self._plugin, method)
try:
return {'success': True, 'response': func(*args, **kwargs)}
except Exception as exception:
return {
'success': False,
'exception': str(exception),
'stacktrace': traceback.format_exc()
}
def _handle_remove_callback(self):
for method in get_special_methods(self._plugin, 'on_remove'):
try:
method()
except Exception as exception:
IO._log_exception('on remove', exception)
class PluginRunner:
def __init__(self,
name,
runtime_path,
plugin_path,
logger,
command_timeout=5):
self.runtime_path = runtime_path
self.plugin_path = plugin_path
self.command_timeout = command_timeout
self._logger = logger
self._cid = 0
self._proc = None
self._running = False
self._process_running = False
self._command_lock = Lock()
self._response_queue = Queue()
self._stream = None
self.name = name
self.version = None
self.interfaces = None
self._receivers = []
self._exposes = []
self._metric_collectors = []
self._metric_receivers = []
self._async_command_thread = None
self._async_command_queue = None
self._commands_executed = 0
self._commands_failed = 0
self.__collector_runs = {}
def start(self):
if self._running:
raise Exception('PluginRunner is already running')
self.logger('[Runner] Starting')
python_executable = sys.executable
if python_executable is None or len(python_executable) == 0:
python_executable = '/usr/bin/python'
self._proc = subprocess.Popen(
[python_executable, "runtime.py", "start", self.plugin_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=None,
cwd=self.runtime_path)
self._process_running = True
self._commands_executed = 0
self._commands_failed = 0
self._stream = PluginIPCStream(stream=self._proc.stdout,
logger=lambda message, ex: self.logger(
'{0}: {1}'.format(message, ex)),
command_receiver=self._process_command)
self._stream.start()
start_out = self._do_command('start', timeout=180)
self.name = start_out['name']
self.version = start_out['version']
self.interfaces = start_out['interfaces']
self._receivers = start_out['receivers']
self._exposes = start_out['exposes']
self._metric_collectors = start_out['metric_collectors']
self._metric_receivers = start_out['metric_receivers']
exception = start_out.get('exception')
if exception is not None:
raise RuntimeError(exception)
self._async_command_queue = Queue(1000)
self._async_command_thread = Thread(
target=self._perform_async_commands,
name='PluginRunner {0} async thread'.format(self.plugin_path))
self._async_command_thread.daemon = True
self._async_command_thread.start()
self._running = True
self.logger('[Runner] Started')
def logger(self, message):
self._logger(message)
logger.info('Plugin {0} - {1}'.format(self.name, message))
def get_webservice(self, webinterface):
class Service:
def __init__(self, runner):
self.runner = runner
# Set the user controller, required to check the auth token
self._user_controller = webinterface._user_controller
def _cp_dispatch(self, vpath):
method = vpath.pop()
for exposed in self.runner._exposes:
if exposed['name'] == method:
cherrypy.request.params['method'] = method
cherrypy.response.headers['Content-Type'] = exposed[
'content_type']
if exposed['auth'] is True:
cherrypy.request.hooks.attach(
'before_handler',
cherrypy.tools.authenticated.callable)
return self
return None
@cherrypy.expose
def index(self, method, *args, **kwargs):
try:
return self.runner.request(method,
args=args,
kwargs=kwargs)
except Exception as ex:
cherrypy.response.headers[
"Content-Type"] = "application/json"
cherrypy.response.status = 500
return json.dumps({"success": False, "msg": str(ex)})
return Service(self)
def is_running(self):
return self._running
def stop(self):
if self._process_running:
self._running = False
self.logger('[Runner] Sending stop command')
try:
self._do_command('stop')
except Exception as exception:
self.logger(
'[Runner] Exception during stopping plugin: {0}'.format(
exception))
time.sleep(0.1)
self._stream.stop()
self._process_running = False
if self._proc.poll() is None:
self.logger('[Runner] Terminating process')
try:
self._proc.terminate()
except Exception as exception:
self.logger(
'[Runner] Exception during terminating plugin: {0}'.
format(exception))
time.sleep(0.5)
if self._proc.poll() is None:
self.logger('[Runner] Killing process')
try:
self._proc.kill()
except Exception as exception:
self.logger(
'[Runner] Exception during killing plugin: {0}'.
format(exception))
self.logger('[Runner] Stopped')
def process_input_status(self, input_event):
event_json = input_event.serialize()
self._do_async('input_status', {'event': event_json},
should_filter=True)
def process_output_status(self, status):
self._do_async('output_status', {'status': status}, should_filter=True)
def process_shutter_status(self, status):
self._do_async('shutter_status', status, should_filter=True)
def process_event(self, code):
self._do_async('receive_events', {'code': code}, should_filter=True)
def collect_metrics(self):
for mc in self._metric_collectors:
try:
now = time.time()
(name, interval) = (mc['name'], mc['interval'])
if self.__collector_runs.get(name, 0) < now - interval:
self.__collector_runs[name] = now
metrics = self._do_command('collect_metrics',
{'name': name})['metrics']
for metric in metrics:
if metric is None:
continue
metric['source'] = self.name
yield metric
except Exception as exception:
self.logger(
'[Runner] Exception while collecting metrics {0}: {1}'.
format(exception, traceback.format_exc()))
def get_metric_receivers(self):
return self._metric_receivers
def distribute_metrics(self, method, metrics):
self._do_async('distribute_metrics', {
'name': method,
'metrics': metrics
})
def get_metric_definitions(self):
return self._do_command('get_metric_definitions')['metric_definitions']
def request(self, method, args=None, kwargs=None):
if args is None:
args = []
if kwargs is None:
kwargs = {}
ret = self._do_command('request', {
'method': method,
'args': args,
'kwargs': kwargs
})
if ret['success']:
return ret['response']
else:
raise Exception('{0}: {1}'.format(ret['exception'],
ret['stacktrace']))
def remove_callback(self):
self._do_command('remove_callback')
def _process_command(self, response):
if not self._process_running:
return
exit_code = self._proc.poll()
if exit_code is not None:
self.logger(
'[Runner] Stopped with exit code {0}'.format(exit_code))
self._process_running = False
return
if response['cid'] == 0:
self._handle_async_response(response)
elif response['cid'] == self._cid:
self._response_queue.put(response)
else:
self.logger(
'[Runner] Received message with unknown cid: {0}'.format(
response))
def _handle_async_response(self, response):
if response['action'] == 'logs':
self.logger(response['logs'])
else:
self.logger('[Runner] Unkown async message: {0}'.format(response))
def _do_async(self, action, fields, should_filter=False):
if (should_filter and action
not in self._receivers) or not self._process_running:
return
try:
self._async_command_queue.put({
'action': action,
'fields': fields
},
block=False)
except Full:
self.logger('Async action cannot be queued, queue is full')
def _perform_async_commands(self):
while self._process_running:
try:
# Give it a timeout in order to check whether the plugin is not stopped.
command = self._async_command_queue.get(block=True, timeout=10)
self._do_command(command['action'], command['fields'])
except Empty:
self._do_async('ping', {})
except Exception as exception:
self.logger(
'[Runner] Failed to perform async command: {0}'.format(
exception))
def _do_command(self, action, fields=None, timeout=None):
if fields is None:
fields = {}
self._commands_executed += 1
if timeout is None:
timeout = self.command_timeout
if not self._process_running:
raise Exception('Plugin was stopped')
with self._command_lock:
try:
command = self._create_command(action, fields)
self._proc.stdin.write(PluginIPCStream.write(command))
self._proc.stdin.flush()
except Exception:
self._commands_failed += 1
raise
try:
response = self._response_queue.get(block=True,
timeout=timeout)
while response['cid'] != self._cid:
response = self._response_queue.get(block=False)
exception = response.get('_exception')
if exception is not None:
raise RuntimeError(exception)
return response
except Empty:
metadata = ''
if action == 'request':
metadata = ' {0}'.format(fields['method'])
self.logger('[Runner] No response within {0}s ({1}{2})'.format(
timeout, action, metadata))
self._commands_failed += 1
raise Exception('Plugin did not respond')
def _create_command(self, action, fields=None):
if fields is None:
fields = {}
self._cid += 1
command = {'cid': self._cid, 'action': action}
command.update(fields)
return command
def error_score(self):
if self._commands_executed == 0:
return 0
else:
score = float(self._commands_failed) / self._commands_executed
self._commands_failed = 0
self._commands_executed = 0
return score
def get_queue_length(self):
if self._async_command_queue is None:
return 0
return self._async_command_queue.qsize()