class HTTPServer(threading.Thread): """ Our HTTP server wrapper class. """ name = "SUB-HTTPServer" runner = None def __init__(self, config, runner, vertical_queue): super().__init__() self.config = config self.request_handler = RequestHandler self.runner = runner self.logger = Logger() self.routes = [("/", self.config['docRoot'])] self.vertical_queue = vertical_queue def run(self): """ Runner for http server, uses user defined config for server. """ try: self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START) address = (self.config['listenAddress'], self.config['listenPort']) self.request_handler.routes = self.routes httpd = Server.HTTPServer(address, self.request_handler) httpd.timeout = 2 while self.runner.is_set(): httpd.handle_request() except KeyError: comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(comm)
class HTTPServer(threading.Thread): """ Our HTTP server wrapper class. """ name = "SUB-HTTPServer" runner = None def __init__(self, config, runner, vertical_queue): super().__init__() self.config = config self.request_handler = RequestHandler self.runner = runner self.logger = Logger() self.routes = [ ("/", self.config['docRoot']) ] self.vertical_queue = vertical_queue def run(self): """ Runner for http server, uses user defined config for server. """ try: self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START) address = (self.config['listenAddress'], self.config['listenPort']) self.request_handler.routes = self.routes httpd = Server.HTTPServer(address, self.request_handler) httpd.timeout = 2 while self.runner.is_set(): httpd.handle_request() except KeyError: comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(comm)
class ConfigWatcher(threading.Thread): """ App submodule. This runnable has the responsibility of read the json settings file, save hash information about it in memory and reload it if hash differs at some point of program execution. """ runner = None name = "SUB-ConfigWatcher" def __init__(self, config_path, runner, vertical_queue): super().__init__() self.runner = runner self.vertical_queue = vertical_queue self.logger = Logger() self.interval_secs = 1 self.config_path = config_path self.last_config_checksum = None def set_config(self): """ Sets the parsed config into app config in memory object. """ config_file = open(self.config_path) Config.config_dict = json.load(config_file) def run(self): """ Endless loop execution. """ counter = 0 self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START) while self.runner.is_set(): try: time.sleep(self.interval_secs) hash = hashlib.md5(open(self.config_path, 'rb').read()).hexdigest() if hash != self.last_config_checksum: self.set_config() self.last_config_checksum = hash if counter != 0: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_REBOOT, Strings.CONFIG_CHANGES) self.vertical_queue.put(ipc_msg) counter += 1 except IOError: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_FILE_ERROR) self.vertical_queue.put(ipc_msg)
class WebPanel(threading.Thread): """ This runnable has the responsibility or maintain the webpanel updated. It starts up the web server subsystem and forwards IPC signaling between the main process and the webserver. It consumes the web_panel_queue. """ name = "SUB-WebPanel" runner = None def __init__(self, runner, config, web_panel_queue, vertical_queue): super().__init__() self.runner = runner self.logger = Logger() self.config = config self.web_panel_queue = web_panel_queue self.HTTP_server = HTTPServer self.data = {"config": self.config['frontEndConfig'], "logs": []} self.vertical_queue = vertical_queue def run(self): self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START) web_server = self.HTTP_server(self.config['webServer'], self.runner, self.vertical_queue) web_server.start() while self.runner.is_set(): try: log_message = self.web_panel_queue.get(False) self.logger.info(self.name + " " + Strings.WEBPANEL_PUBLISH) if len(self.data['logs']) >= self.config['maxEntries']: self.data['logs'].pop() self.data['logs'].insert(0, log_message.get_object()) with open(self.config['dataOutput'] + "/" + "logs.json", 'w') as f: json.dump(self.data, f, indent=4) except queue.Empty: time.sleep(1) except KeyError: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(ipc_msg)
class LogHandler(threading.Thread): """ This subsystem receives all log messages that need to be saved or notified. Its the consumer of the logs queue and the producer of the webpanel queue. """ name = "SUB-GenLogHandler" def __init__(self, config, log_queue, runner, web_panel_queue, vertical_queue): super().__init__() self.runner = runner self.log_queue = log_queue self.web_panel_queue = web_panel_queue self.logger = Logger() self.notifier = Notifier() self.config = config['generalHandler'] self.web_panel_active = config['webPanel']['active'] self.vertical_queue = vertical_queue def run(self): """ Consumer from log queue and taking action for each log settings. """ self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START) while self.runner.is_set(): try: log_message = self.log_queue.get(False) log_path = self.calc_log_path(log_message) self.write_log(log_message, log_path) if log_message.system_notifications is True: self.notify_sys(log_message) if self.web_panel_active: self.send_to_webpanel(log_message) self.finish_handling() self.logger.info(self.name + " " + Strings.LOG_PROCESSED) except queue.Empty: time.sleep(1) except KeyError: comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(comm) except IOError: comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.IO_ERROR) self.vertical_queue.put(comm) def send_to_webpanel(self, log_message): """ Send log info to the webpanel subsystem. :param log_message: JSON object """ self.web_panel_queue.put(log_message) def calc_log_path(self, log_message): """ Calculates the name for the log file. :param log_message: :return: string """ log_file_name = log_message.name.replace("\n", "").replace( " ", "_") + ".log".lower() log_write_path = self.config['logsPath'] + "/" + log_file_name return log_write_path def write_log(self, log_message, path): """ Writes log in specifiec path. :param log_message: :param path: """ with open(path, "a") as log_file: log_file.write(log_message.data + "\n") def notify_sys(self, log_message): """ Send info to the notifications subsystem :param log_message: """ self.notifier.send_notify(log_message.name, log_message.data) def finish_handling(self): """ Finish handler jobs. """ self.log_queue.task_done()
class OmniLogD(object): """ This is the main process. It setups config and IPC. It starts all services needed to run this app, and is responsible of communication between them. """ name = "MainProcess" def __init__(self): if len(sys.argv) == 2: if sys.argv[1] == 'skeleton': self.skeleton() else: self.config_path = sys.argv[1] else: self.config_path = "config.json" self.logger = Logger() self.booting = True self.review_interval = 1 """ Create the IPC system -vertical_queue is intended to communicate subsystem -> main process. -log_queue communicates log parsers with the handler -web_panel_queue passes log information from general handler to web panel subsystem. """ self.vertical_queue = Queue() self.log_queue = Queue() self.web_panel_queue = Queue() """ It creates the events for signaling start or shutdown in threads. """ self.main_runner = threading.Event() self.web_panel_runner = threading.Event() self.config_runner = threading.Event() self.gen_log_handler_runner = threading.Event() self.logs_runner = threading.Event() """ Sets by default all events (app services or threads) on. """ self.main_runner.set() self.web_panel_runner.set() self.config_runner.set() self.gen_log_handler_runner.set() self.logs_runner.set() signal.signal(signal.SIGTERM, self.shutdown) signal.signal(signal.SIGINT, self.shutdown) def skeleton(self): try: home = os.path.expanduser("~") github_front_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/public/index.html" github_config_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/docs/config.dist.json" config_example = request.urlopen(github_config_url).read().decode("utf8") front_html = request.urlopen(github_front_url).read().decode("utf8") dirs = collections.OrderedDict() dirs['omnilog_root'] = os.path.join(home, 'omnilog') dirs['logs'] = os.path.join(home, 'omnilog', 'logs') dirs['http'] = os.path.join(home, 'omnilog', 'public') dirs['http_data_source'] = os.path.join(home, 'omnilog', 'public', 'data_source') for k, v in dirs.items(): os.makedirs(v) if k == "http": with open(os.path.join(dirs['http'], "index.html"), "w") as f: f.write(front_html) if k == "omnilog_root": with open(os.path.join(dirs['omnilog_root'], "config.json"), "w") as f: f.write(config_example) exit(Strings.SKELETON_DIR_CREATED) except FileExistsError: exit(Strings.SKELETON_DIR) except URLError: exit(Strings.SKELETON_URL_ERROR) def run(self): """ The Main function of this program. Its a loop that instantiates all runnable services (threading) with review a interval that listens for messages of low level processes (aka threads or app services). Its An intermediary between services. """ threads = list() config_watcher = ConfigWatcher(self.config_path, self.config_runner, self.vertical_queue) config_watcher.start() threads.append(config_watcher) time.sleep(2) while self.main_runner.is_set(): try: time.sleep(self.review_interval) if self.vertical_queue.empty() and self.booting: if self.gen_log_handler_runner.is_set(): gen_log_handler = LogHandler(Config.config_dict, self.log_queue, self.gen_log_handler_runner, self.web_panel_queue, self.vertical_queue) threads.append(gen_log_handler) gen_log_handler.start() if Config.config_dict['webPanel']['active'] and self.web_panel_runner.is_set(): web_panel = WebPanel(self.web_panel_runner, Config.config_dict['webPanel'], self.web_panel_queue, self.vertical_queue) threads.append(web_panel) web_panel.start() if self.logs_runner.is_set(): logs = Config.config_dict['logs'] for l in logs: if l['active'] is True: t = LogParser(l, self.logs_runner, self.log_queue, self.vertical_queue) t.name = l['name'] threads.append(t) t.start() self.logger.info(self.name + " - Started log watcher for " + l['name']) self.booting = False elif not self.vertical_queue.empty(): self.manage_com() except KeyboardInterrupt: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.KEYBOARD_INTERRUPTION) self.vertical_queue.put(ipc_msg) except KeyError or AttributeError: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(ipc_msg) except: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.UNHANDLED_EXCEPTION) self.logger.emergency(self.name + " " + Strings.UNHANDLED_EXCEPTION) self.vertical_queue.put(ipc_msg) for te in threads: te.join() def manage_com(self): """ This function is responsible of handling messages arrived from the different app services. This queue needs to be based on topics because the main process needs to understand from what service come each message. """ com = self.vertical_queue.get(True) self.logger.info(self.name + Strings.IPC_RECEIVED + str(com)) if com.action == IPCActions.ACTION_REBOOT: self.restart() elif com.action == IPCActions.ACTION_SHUTDOWN: self.shutdown() def restart(self, sig_num=None, frame=None): """ The restart operation. It send the event to stop gracefully stop all threads, waits for job ending and then start it again. """ self.logger.info(self.name + Strings.APP_RESTARTING) self.logs_runner.clear() time.sleep(2) self.web_panel_runner.clear() self.gen_log_handler_runner.clear() time.sleep(2) self.web_panel_runner.set() self.gen_log_handler_runner.set() self.logs_runner.set() self.booting = True def shutdown(self, sig_num=None, frame=None): """ The shutdown operation. Same procedure as restart, but this time the main loop will not do more review and exit. """ self.logger.info(self.name + Strings.APP_SHUTDOWN) self.config_runner.clear() self.logs_runner.clear() time.sleep(2) self.web_panel_runner.clear() self.gen_log_handler_runner.clear() self.main_runner.clear()
class LogParser(threading.Thread): """ This subsystem may run for each log instance. Its responsibility is to pull changes from paramiko channel, split it into lines and check user defined regex over them. After all , it queues the result (if valid) into the general log handler queue.SS """ name = "SUB-LogParser" runner = None def __init__(self, log, runner, log_queue, vertical_queue): super().__init__() self.runner = runner self.log_queue = log_queue self.config = log self.ssh = SSHhandler(self.config['ssh']) self.logger = Logger() self.interval_secs = 1 self.recv_buffer = 1024 self.vertical_queue = vertical_queue def run(self): try: self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START) ssh = self.ssh.get_session() transport = ssh.get_transport() transport.set_keepalive(5) channel = transport.open_session() channel.exec_command('tail -f ' + self.config['logReadPath']) while self.runner.is_set() and transport.is_active(): time.sleep(self.interval_secs) rl, wl, xl = select.select([channel], [], [], 0.0) if len(rl) > 0: data = channel.recv(self.recv_buffer) lines = self.get_lines_from_data(data) valid_lines = self.check_patterns(lines) if len(valid_lines) > 0: for line in valid_lines: self.logger.info(self.name + " " + Strings.PARSER_VALID_LOG_REACHED) self.log_queue.put( LogMessage( self.config['name'], line, self.config['systemNotifications'], datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") )) ssh.close() except KeyError: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(ipc_msg) except SSHException: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.SSH_ERROR) self.vertical_queue.put(ipc_msg) def get_lines_from_data(self, data): """ Transform data received from ssh channel to a list of lines. :param data: bytes :return: list """ string = data.decode("unicode_escape", "ignore") lines = string.split("\n") return lines def check_patterns(self, lines): """ Validates log lines against user defined regex (only if defined). Returns the result list. :param lines: list :return: list """ not_black_listed = [] valid_lines = [] if "ignorePatterns" in self.config.keys() and len(self.config['ignorePatterns']) > 0: for l in lines: for p in self.config['ignorePatterns']: if not re.search(p, l): not_black_listed.append(l) else: not_black_listed = lines if "patterns" in self.config.keys() and len(self.config['patterns']) > 0: for l in not_black_listed: for p in self.config['patterns']: if re.search(p, l): valid_lines.append(l) else: valid_lines = not_black_listed result = [v for v in valid_lines if v != ''] return result
class OmniLogD(object): """ This is the main process. It setups config and IPC. It starts all services needed to run this app, and is responsible of communication between them. """ name = "MainProcess" def __init__(self): if len(sys.argv) == 2: if sys.argv[1] == 'skeleton': self.skeleton() else: self.config_path = sys.argv[1] else: self.config_path = "config.json" self.logger = Logger() self.booting = True self.review_interval = 1 """ Create the IPC system -vertical_queue is intended to communicate subsystem -> main process. -log_queue communicates log parsers with the handler -web_panel_queue passes log information from general handler to web panel subsystem. """ self.vertical_queue = Queue() self.log_queue = Queue() self.web_panel_queue = Queue() """ It creates the events for signaling start or shutdown in threads. """ self.main_runner = threading.Event() self.web_panel_runner = threading.Event() self.config_runner = threading.Event() self.gen_log_handler_runner = threading.Event() self.logs_runner = threading.Event() """ Sets by default all events (app services or threads) on. """ self.main_runner.set() self.web_panel_runner.set() self.config_runner.set() self.gen_log_handler_runner.set() self.logs_runner.set() signal.signal(signal.SIGTERM, self.shutdown) signal.signal(signal.SIGINT, self.shutdown) def skeleton(self): try: home = os.path.expanduser("~") github_front_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/public/index.html" github_config_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/docs/config.dist.json" config_example = request.urlopen(github_config_url).read().decode( "utf8") front_html = request.urlopen(github_front_url).read().decode( "utf8") dirs = collections.OrderedDict() dirs['omnilog_root'] = os.path.join(home, 'omnilog') dirs['logs'] = os.path.join(home, 'omnilog', 'logs') dirs['http'] = os.path.join(home, 'omnilog', 'public') dirs['http_data_source'] = os.path.join(home, 'omnilog', 'public', 'data_source') for k, v in dirs.items(): os.makedirs(v) if k == "http": with open(os.path.join(dirs['http'], "index.html"), "w") as f: f.write(front_html) if k == "omnilog_root": with open( os.path.join(dirs['omnilog_root'], "config.json"), "w") as f: f.write(config_example) exit(Strings.SKELETON_DIR_CREATED) except FileExistsError: exit(Strings.SKELETON_DIR) except URLError: exit(Strings.SKELETON_URL_ERROR) def run(self): """ The Main function of this program. Its a loop that instantiates all runnable services (threading) with review a interval that listens for messages of low level processes (aka threads or app services). Its An intermediary between services. """ threads = list() config_watcher = ConfigWatcher(self.config_path, self.config_runner, self.vertical_queue) config_watcher.start() threads.append(config_watcher) time.sleep(2) while self.main_runner.is_set(): try: time.sleep(self.review_interval) if self.vertical_queue.empty() and self.booting: if self.gen_log_handler_runner.is_set(): gen_log_handler = LogHandler( Config.config_dict, self.log_queue, self.gen_log_handler_runner, self.web_panel_queue, self.vertical_queue) threads.append(gen_log_handler) gen_log_handler.start() if Config.config_dict['webPanel'][ 'active'] and self.web_panel_runner.is_set(): web_panel = WebPanel(self.web_panel_runner, Config.config_dict['webPanel'], self.web_panel_queue, self.vertical_queue) threads.append(web_panel) web_panel.start() if self.logs_runner.is_set(): logs = Config.config_dict['logs'] for l in logs: if l['active'] is True: t = LogParser(l, self.logs_runner, self.log_queue, self.vertical_queue) t.name = l['name'] threads.append(t) t.start() self.logger.info( self.name + " - Started log watcher for " + l['name']) self.booting = False elif not self.vertical_queue.empty(): self.manage_com() except KeyboardInterrupt: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.KEYBOARD_INTERRUPTION) self.vertical_queue.put(ipc_msg) except KeyError or AttributeError: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(ipc_msg) except: ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.UNHANDLED_EXCEPTION) self.logger.emergency(self.name + " " + Strings.UNHANDLED_EXCEPTION) self.vertical_queue.put(ipc_msg) for te in threads: te.join() def manage_com(self): """ This function is responsible of handling messages arrived from the different app services. This queue needs to be based on topics because the main process needs to understand from what service come each message. """ com = self.vertical_queue.get(True) self.logger.info(self.name + Strings.IPC_RECEIVED + str(com)) if com.action == IPCActions.ACTION_REBOOT: self.restart() elif com.action == IPCActions.ACTION_SHUTDOWN: self.shutdown() def restart(self, sig_num=None, frame=None): """ The restart operation. It send the event to stop gracefully stop all threads, waits for job ending and then start it again. """ self.logger.info(self.name + Strings.APP_RESTARTING) self.logs_runner.clear() time.sleep(2) self.web_panel_runner.clear() self.gen_log_handler_runner.clear() time.sleep(2) self.web_panel_runner.set() self.gen_log_handler_runner.set() self.logs_runner.set() self.booting = True def shutdown(self, sig_num=None, frame=None): """ The shutdown operation. Same procedure as restart, but this time the main loop will not do more review and exit. """ self.logger.info(self.name + Strings.APP_SHUTDOWN) self.config_runner.clear() self.logs_runner.clear() time.sleep(2) self.web_panel_runner.clear() self.gen_log_handler_runner.clear() self.main_runner.clear()
class LogHandler(threading.Thread): """ This subsystem receives all log messages that need to be saved or notified. Its the consumer of the logs queue and the producer of the webpanel queue. """ name = "SUB-GenLogHandler" def __init__(self, config, log_queue, runner, web_panel_queue, vertical_queue): super().__init__() self.runner = runner self.log_queue = log_queue self.web_panel_queue = web_panel_queue self.logger = Logger() self.notifier = Notifier() self.config = config['generalHandler'] self.web_panel_active = config['webPanel']['active'] self.vertical_queue = vertical_queue def run(self): """ Consumer from log queue and taking action for each log settings. """ self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START) while self.runner.is_set(): try: log_message = self.log_queue.get(False) log_path = self.calc_log_path(log_message) self.write_log(log_message, log_path) if log_message.system_notifications is True: self.notify_sys(log_message) if self.web_panel_active: self.send_to_webpanel(log_message) self.finish_handling() self.logger.info(self.name + " " + Strings.LOG_PROCESSED) except queue.Empty: time.sleep(1) except KeyError: comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR) self.vertical_queue.put(comm) except IOError: comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.IO_ERROR) self.vertical_queue.put(comm) def send_to_webpanel(self, log_message): """ Send log info to the webpanel subsystem. :param log_message: JSON object """ self.web_panel_queue.put(log_message) def calc_log_path(self, log_message): """ Calculates the name for the log file. :param log_message: :return: string """ log_file_name = log_message.name.replace("\n", "").replace(" ", "_") + ".log".lower() log_write_path = self.config['logsPath'] + "/" + log_file_name return log_write_path def write_log(self, log_message, path): """ Writes log in specifiec path. :param log_message: :param path: """ with open(path, "a") as log_file: log_file.write(log_message.data + "\n") def notify_sys(self, log_message): """ Send info to the notifications subsystem :param log_message: """ self.notifier.send_notify(log_message.name, log_message.data) def finish_handling(self): """ Finish handler jobs. """ self.log_queue.task_done()