def _read_out(self): stream = PluginIPCStream() while self._process_running: exit_code = self._proc.poll() if exit_code is not None: self.logger('[Runner] Stopped with exit code {0}'.format(exit_code)) self._process_running = False break line = '' while line == '': line = self._proc.stdout.readline().strip() try: response = stream.feed(line) if response is None: continue except Exception as ex: self.logger('[Runner] Exception while parsing output: {0}'.format(ex)) continue if response['cid'] == 0: self._handle_async_response(response) elif response['cid'] == self._cid: self._response_queue.put(response) else: self.logger('[Runner] Received message with unknown cid: {0}'.format(response))
def _wait_and_read_command(): stream = PluginIPCStream() line = '' while line == '': line = sys.stdin.readline().strip() try: response = stream.feed(line) if response is not None: return response except Exception as ex: IO._log( 'Exception in _wait_and_read_command: Could not decode stdin: {0}' .format(ex))
def start(self): if self._running: raise Exception('PluginRunner is already running') self.logger('[Runner] Starting') python_executable = sys.executable if python_executable is None or len(python_executable) == 0: python_executable = '/usr/bin/python' self._proc = subprocess.Popen( [python_executable, "runtime.py", "start", self.plugin_path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, cwd=self.runtime_path) self._process_running = True self._commands_executed = 0 self._commands_failed = 0 self._stream = PluginIPCStream(stream=self._proc.stdout, logger=lambda message, ex: self.logger( '{0}: {1}'.format(message, ex)), command_receiver=self._process_command) self._stream.start() start_out = self._do_command('start', timeout=180) self.name = start_out['name'] self.version = start_out['version'] self.interfaces = start_out['interfaces'] self._receivers = start_out['receivers'] self._exposes = start_out['exposes'] self._metric_collectors = start_out['metric_collectors'] self._metric_receivers = start_out['metric_receivers'] exception = start_out.get('exception') if exception is not None: raise RuntimeError(exception) self._async_command_queue = Queue(1000) self._async_command_thread = Thread( target=self._perform_async_commands, name='PluginRunner {0} async thread'.format(self.plugin_path)) self._async_command_thread.daemon = True self._async_command_thread.start() self._running = True self.logger('[Runner] Started')
def _do_command(self, action, fields=None, timeout=None): if fields is None: fields = {} self._commands_executed += 1 if timeout is None: timeout = self.command_timeout if not self._process_running: raise Exception('Plugin was stopped') with self._command_lock: command = self._create_command(action, fields) self._proc.stdin.write(PluginIPCStream.encode(command)) self._proc.stdin.flush() try: response = self._response_queue.get(block=True, timeout=timeout) while response['cid'] != self._cid: response = self._response_queue.get(block=False) exception = response.get('_exception') if exception is not None: raise RuntimeError(exception) return response except Empty: self.logger('[Runner] No response within {0}s (action={1}, fields={2})'.format(timeout, action, fields)) self._commands_failed += 1 raise Exception('Plugin did not respond')
def __init__(self, path): self._stopped = False self._path = path.rstrip('/') self._input_status_receivers = [] self._output_status_receivers = [] self._shutter_status_receivers = [] self._event_receivers = [] self._name = None self._version = None self._interfaces = [] self._receivers = [] self._exposes = [] self._metric_definitions = [] self._metric_collectors = [] self._metric_receivers = [] self._plugin = None self._stream = PluginIPCStream(sys.stdin, IO._log_exception) self._webinterface = WebInterfaceDispatcher(IO._log)
def _write(msg): sys.stdout.write(PluginIPCStream.write(msg)) sys.stdout.flush()
class PluginRuntime: def __init__(self, path): self._stopped = False self._path = path.rstrip('/') self._input_status_receivers = [] self._output_status_receivers = [] self._shutter_status_receivers = [] self._event_receivers = [] self._name = None self._version = None self._interfaces = [] self._receivers = [] self._exposes = [] self._metric_definitions = [] self._metric_collectors = [] self._metric_receivers = [] self._plugin = None self._stream = PluginIPCStream(sys.stdin, IO._log_exception) self._webinterface = WebInterfaceDispatcher(IO._log) def _init_plugin(self): plugin_root = os.path.dirname(self._path) plugin_dir = os.path.basename(self._path) # Add the plugin and it's eggs to the python path sys.path.insert(0, plugin_root) for egg_file in os.listdir(self._path): if egg_file.endswith('.egg'): sys.path.append(os.path.join(self._path, egg_file)) # Expose plugins.base to the plugin sys.modules['plugins'] = sys.modules['__main__'] sys.modules["plugins.base"] = base # Instanciate the plugin class plugin_class = get_plugin_class(plugin_dir) check_plugin(plugin_class) # Set the name, version, interfaces self._name = plugin_class.name self._version = plugin_class.version self._interfaces = plugin_class.interfaces # Initialze the plugin self._plugin = plugin_class(self._webinterface, IO._log) # Set the receivers receiver_mapping = { 'input_status': self._input_status_receivers, 'output_status': self._output_status_receivers, 'shutter_status': self._shutter_status_receivers, 'receive_events': self._event_receivers } for method_attribute, target in receiver_mapping.iteritems(): for method in get_special_methods(self._plugin, method_attribute): target.append(method) if len(target) > 0: self._receivers.append(method_attribute) # Set the exposed methods for method in get_special_methods(self._plugin, 'om_expose'): self._exposes.append({ 'name': method.__name__, 'auth': method.om_expose['auth'], 'content_type': method.om_expose['content_type'] }) # Set the metric definitions if has_interface(plugin_class, 'metrics', '1.0'): if hasattr(plugin_class, 'metric_definitions'): self._metric_definitions = plugin_class.metric_definitions # Set the metric collectors for method in get_special_methods(self._plugin, 'om_metric_data'): self._metric_collectors.append({ 'name': method.__name__, 'interval': method.om_metric_data['interval'] }) # Set the metric receivers for method in get_special_methods(self._plugin, 'om_metric_receive'): self._metric_receivers.append({ 'name': method.__name__, 'source': method.om_metric_receive['source'], 'metric_type': method.om_metric_receive['metric_type'], 'interval': method.om_metric_receive['interval'] }) def _start_background_tasks(self): """ Start all background tasks. """ tasks = get_special_methods(self._plugin, 'background_task') for task in tasks: thread = Thread(target=PluginRuntime._run_background_task, args=(task, )) thread.name = 'Background thread ({0})'.format(task.__name__) thread.daemon = True thread.start() @staticmethod def _run_background_task(task): running = True while running: try: task() running = False # Stop execution if the task returns without exception except Exception as exception: IO._log_exception('background task', exception) time.sleep(30) def process_stdin(self): self._stream.start() while not self._stopped: command = self._stream.get(block=True) if command is None: continue action = command['action'] response = {'cid': command['cid'], 'action': action} try: ret = None if action == 'start': ret = self._handle_start() elif action == 'stop': ret = self._handle_stop() elif action == 'input_status': ret = self._handle_input_status(command['event']) elif action == 'output_status': ret = self._handle_output_status(command['status']) elif action == 'shutter_status': ret = self._handle_shutter_status(command) elif action == 'receive_events': ret = self._handle_receive_events(command['code']) elif action == 'get_metric_definitions': ret = self._handle_get_metric_definitions() elif action == 'collect_metrics': ret = self._handle_collect_metrics(command['name']) elif action == 'distribute_metrics': ret = self._handle_distribute_metrics( command['name'], command['metrics']) elif action == 'request': ret = self._handle_request(command['method'], command['args'], command['kwargs']) elif action == 'remove_callback': ret = self._handle_remove_callback() else: raise RuntimeError('Unknown action: {0}'.format(action)) if ret is not None: response.update(ret) except Exception as exception: response['_exception'] = str(exception) IO._write(response) def _handle_start(self): """ Handles the start command. Cover exceptions manually to make sure as much metadata is returned as possible. """ data = {} try: self._init_plugin() self._start_background_tasks() except Exception as exception: data['exception'] = str(exception) data.update({ 'name': self._name, 'version': self._version, 'receivers': self._receivers, 'exposes': self._exposes, 'interfaces': self._interfaces, 'metric_collectors': self._metric_collectors, 'metric_receivers': self._metric_receivers }) return data def _handle_stop(self): def delayed_stop(): time.sleep(2) os._exit(0) stop_thread = Thread(target=delayed_stop) stop_thread.daemon = True stop_thread.start() self._stream.stop() self._stopped = True def _handle_input_status(self, event_json): event = Event.deserialize(event_json) # get relevant event details input_id = event.data['id'] status = event.data['status'] for receiver in self._input_status_receivers: version = receiver.input_status.get('version', 1) if version == 1: # Backwards compatibility: only send rising edges of the input (no input releases) if status: IO._with_catch('input status', receiver, [(input_id, None)]) elif version == 2: # Version 2 will send ALL input status changes AND in a dict format data = {'input_id': input_id, 'status': status} IO._with_catch('input status', receiver, [data]) else: error = NotImplementedError( 'Version {} is not supported for input status decorators'. format(version)) IO._log_exception('input status', error) def _handle_output_status(self, status): for receiver in self._output_status_receivers: IO._with_catch('output status', receiver, [status]) def _handle_shutter_status(self, status): for receiver in self._shutter_status_receivers: if receiver.shutter_status['add_detail']: IO._with_catch('shutter status', receiver, [status['status'], status['detail']]) else: IO._with_catch('shutter status', receiver, [status['status']]) def _handle_receive_events(self, code): for receiver in self._event_receivers: IO._with_catch('process event', receiver, [code]) def _handle_get_metric_definitions(self): return {'metric_definitions': self._metric_definitions} def _handle_collect_metrics(self, name): metrics = [] collect = getattr(self._plugin, name) try: metrics.extend(list(collect())) except Exception as exception: IO._log_exception('collect metrics', exception) return {'metrics': metrics} def _handle_distribute_metrics(self, name, metrics): receive = getattr(self._plugin, name) for metric in metrics: IO._with_catch('distribute metric', receive, [metric]) def _handle_request(self, method, args, kwargs): func = getattr(self._plugin, method) try: return {'success': True, 'response': func(*args, **kwargs)} except Exception as exception: return { 'success': False, 'exception': str(exception), 'stacktrace': traceback.format_exc() } def _handle_remove_callback(self): for method in get_special_methods(self._plugin, 'on_remove'): try: method() except Exception as exception: IO._log_exception('on remove', exception)
class PluginRunner: def __init__(self, name, runtime_path, plugin_path, logger, command_timeout=5): self.runtime_path = runtime_path self.plugin_path = plugin_path self.command_timeout = command_timeout self._logger = logger self._cid = 0 self._proc = None self._running = False self._process_running = False self._command_lock = Lock() self._response_queue = Queue() self._stream = None self.name = name self.version = None self.interfaces = None self._receivers = [] self._exposes = [] self._metric_collectors = [] self._metric_receivers = [] self._async_command_thread = None self._async_command_queue = None self._commands_executed = 0 self._commands_failed = 0 self.__collector_runs = {} def start(self): if self._running: raise Exception('PluginRunner is already running') self.logger('[Runner] Starting') python_executable = sys.executable if python_executable is None or len(python_executable) == 0: python_executable = '/usr/bin/python' self._proc = subprocess.Popen( [python_executable, "runtime.py", "start", self.plugin_path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, cwd=self.runtime_path) self._process_running = True self._commands_executed = 0 self._commands_failed = 0 self._stream = PluginIPCStream(stream=self._proc.stdout, logger=lambda message, ex: self.logger( '{0}: {1}'.format(message, ex)), command_receiver=self._process_command) self._stream.start() start_out = self._do_command('start', timeout=180) self.name = start_out['name'] self.version = start_out['version'] self.interfaces = start_out['interfaces'] self._receivers = start_out['receivers'] self._exposes = start_out['exposes'] self._metric_collectors = start_out['metric_collectors'] self._metric_receivers = start_out['metric_receivers'] exception = start_out.get('exception') if exception is not None: raise RuntimeError(exception) self._async_command_queue = Queue(1000) self._async_command_thread = Thread( target=self._perform_async_commands, name='PluginRunner {0} async thread'.format(self.plugin_path)) self._async_command_thread.daemon = True self._async_command_thread.start() self._running = True self.logger('[Runner] Started') def logger(self, message): self._logger(message) logger.info('Plugin {0} - {1}'.format(self.name, message)) def get_webservice(self, webinterface): class Service: def __init__(self, runner): self.runner = runner # Set the user controller, required to check the auth token self._user_controller = webinterface._user_controller def _cp_dispatch(self, vpath): method = vpath.pop() for exposed in self.runner._exposes: if exposed['name'] == method: cherrypy.request.params['method'] = method cherrypy.response.headers['Content-Type'] = exposed[ 'content_type'] if exposed['auth'] is True: cherrypy.request.hooks.attach( 'before_handler', cherrypy.tools.authenticated.callable) return self return None @cherrypy.expose def index(self, method, *args, **kwargs): try: return self.runner.request(method, args=args, kwargs=kwargs) except Exception as ex: cherrypy.response.headers[ "Content-Type"] = "application/json" cherrypy.response.status = 500 return json.dumps({"success": False, "msg": str(ex)}) return Service(self) def is_running(self): return self._running def stop(self): if self._process_running: self._running = False self.logger('[Runner] Sending stop command') try: self._do_command('stop') except Exception as exception: self.logger( '[Runner] Exception during stopping plugin: {0}'.format( exception)) time.sleep(0.1) self._stream.stop() self._process_running = False if self._proc.poll() is None: self.logger('[Runner] Terminating process') try: self._proc.terminate() except Exception as exception: self.logger( '[Runner] Exception during terminating plugin: {0}'. format(exception)) time.sleep(0.5) if self._proc.poll() is None: self.logger('[Runner] Killing process') try: self._proc.kill() except Exception as exception: self.logger( '[Runner] Exception during killing plugin: {0}'. format(exception)) self.logger('[Runner] Stopped') def process_input_status(self, input_event): event_json = input_event.serialize() self._do_async('input_status', {'event': event_json}, should_filter=True) def process_output_status(self, status): self._do_async('output_status', {'status': status}, should_filter=True) def process_shutter_status(self, status): self._do_async('shutter_status', status, should_filter=True) def process_event(self, code): self._do_async('receive_events', {'code': code}, should_filter=True) def collect_metrics(self): for mc in self._metric_collectors: try: now = time.time() (name, interval) = (mc['name'], mc['interval']) if self.__collector_runs.get(name, 0) < now - interval: self.__collector_runs[name] = now metrics = self._do_command('collect_metrics', {'name': name})['metrics'] for metric in metrics: if metric is None: continue metric['source'] = self.name yield metric except Exception as exception: self.logger( '[Runner] Exception while collecting metrics {0}: {1}'. format(exception, traceback.format_exc())) def get_metric_receivers(self): return self._metric_receivers def distribute_metrics(self, method, metrics): self._do_async('distribute_metrics', { 'name': method, 'metrics': metrics }) def get_metric_definitions(self): return self._do_command('get_metric_definitions')['metric_definitions'] def request(self, method, args=None, kwargs=None): if args is None: args = [] if kwargs is None: kwargs = {} ret = self._do_command('request', { 'method': method, 'args': args, 'kwargs': kwargs }) if ret['success']: return ret['response'] else: raise Exception('{0}: {1}'.format(ret['exception'], ret['stacktrace'])) def remove_callback(self): self._do_command('remove_callback') def _process_command(self, response): if not self._process_running: return exit_code = self._proc.poll() if exit_code is not None: self.logger( '[Runner] Stopped with exit code {0}'.format(exit_code)) self._process_running = False return if response['cid'] == 0: self._handle_async_response(response) elif response['cid'] == self._cid: self._response_queue.put(response) else: self.logger( '[Runner] Received message with unknown cid: {0}'.format( response)) def _handle_async_response(self, response): if response['action'] == 'logs': self.logger(response['logs']) else: self.logger('[Runner] Unkown async message: {0}'.format(response)) def _do_async(self, action, fields, should_filter=False): if (should_filter and action not in self._receivers) or not self._process_running: return try: self._async_command_queue.put({ 'action': action, 'fields': fields }, block=False) except Full: self.logger('Async action cannot be queued, queue is full') def _perform_async_commands(self): while self._process_running: try: # Give it a timeout in order to check whether the plugin is not stopped. command = self._async_command_queue.get(block=True, timeout=10) self._do_command(command['action'], command['fields']) except Empty: self._do_async('ping', {}) except Exception as exception: self.logger( '[Runner] Failed to perform async command: {0}'.format( exception)) def _do_command(self, action, fields=None, timeout=None): if fields is None: fields = {} self._commands_executed += 1 if timeout is None: timeout = self.command_timeout if not self._process_running: raise Exception('Plugin was stopped') with self._command_lock: try: command = self._create_command(action, fields) self._proc.stdin.write(PluginIPCStream.write(command)) self._proc.stdin.flush() except Exception: self._commands_failed += 1 raise try: response = self._response_queue.get(block=True, timeout=timeout) while response['cid'] != self._cid: response = self._response_queue.get(block=False) exception = response.get('_exception') if exception is not None: raise RuntimeError(exception) return response except Empty: metadata = '' if action == 'request': metadata = ' {0}'.format(fields['method']) self.logger('[Runner] No response within {0}s ({1}{2})'.format( timeout, action, metadata)) self._commands_failed += 1 raise Exception('Plugin did not respond') def _create_command(self, action, fields=None): if fields is None: fields = {} self._cid += 1 command = {'cid': self._cid, 'action': action} command.update(fields) return command def error_score(self): if self._commands_executed == 0: return 0 else: score = float(self._commands_failed) / self._commands_executed self._commands_failed = 0 self._commands_executed = 0 return score def get_queue_length(self): if self._async_command_queue is None: return 0 return self._async_command_queue.qsize()