class HTTPServer(threading.Thread):
"""
Our HTTP server wrapper class.
"""
name = "SUB-HTTPServer"
runner = None
def __init__(self, config, runner, vertical_queue):
super().__init__()
self.config = config
self.request_handler = RequestHandler
self.runner = runner
self.logger = Logger()
self.routes = [("/", self.config['docRoot'])]
self.vertical_queue = vertical_queue
def run(self):
"""
Runner for http server, uses user defined config for server.
"""
try:
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
address = (self.config['listenAddress'], self.config['listenPort'])
self.request_handler.routes = self.routes
httpd = Server.HTTPServer(address, self.request_handler)
httpd.timeout = 2
while self.runner.is_set():
httpd.handle_request()
except KeyError:
comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN,
Strings.CONFIG_ERROR)
self.vertical_queue.put(comm)
class HTTPServer(threading.Thread):
"""
Our HTTP server wrapper class.
"""
name = "SUB-HTTPServer"
runner = None
def __init__(self, config, runner, vertical_queue):
super().__init__()
self.config = config
self.request_handler = RequestHandler
self.runner = runner
self.logger = Logger()
self.routes = [
("/", self.config['docRoot'])
]
self.vertical_queue = vertical_queue
def run(self):
"""
Runner for http server, uses user defined config for server.
"""
try:
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
address = (self.config['listenAddress'], self.config['listenPort'])
self.request_handler.routes = self.routes
httpd = Server.HTTPServer(address, self.request_handler)
httpd.timeout = 2
while self.runner.is_set():
httpd.handle_request()
except KeyError:
comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR)
self.vertical_queue.put(comm)
def __init__(self, config, runner, vertical_queue):
super().__init__()
self.config = config
self.request_handler = RequestHandler
self.runner = runner
self.logger = Logger()
self.routes = [("/", self.config['docRoot'])]
self.vertical_queue = vertical_queue
def __init__(self, runner, config, web_panel_queue, vertical_queue):
super().__init__()
self.runner = runner
self.logger = Logger()
self.config = config
self.web_panel_queue = web_panel_queue
self.HTTP_server = HTTPServer
self.data = {"config": self.config['frontEndConfig'], "logs": []}
self.vertical_queue = vertical_queue
def __init__(self, log, runner, log_queue, vertical_queue):
super().__init__()
self.runner = runner
self.log_queue = log_queue
self.config = log
self.ssh = SSHhandler(self.config['ssh'])
self.logger = Logger()
self.interval_secs = 1
self.recv_buffer = 1024
self.vertical_queue = vertical_queue
class ConfigWatcher(threading.Thread):
"""
App submodule. This runnable has the responsibility of read the
json settings file, save hash information about it in memory and reload
it if hash differs at some point of program execution.
"""
runner = None
name = "SUB-ConfigWatcher"
def __init__(self, config_path, runner, vertical_queue):
super().__init__()
self.runner = runner
self.vertical_queue = vertical_queue
self.logger = Logger()
self.interval_secs = 1
self.config_path = config_path
self.last_config_checksum = None
def set_config(self):
"""
Sets the parsed config into app config in memory object.
"""
config_file = open(self.config_path)
Config.config_dict = json.load(config_file)
def run(self):
"""
Endless loop execution.
"""
counter = 0
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
while self.runner.is_set():
try:
time.sleep(self.interval_secs)
hash = hashlib.md5(open(self.config_path, 'rb').read()).hexdigest()
if hash != self.last_config_checksum:
self.set_config()
self.last_config_checksum = hash
if counter != 0:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_REBOOT, Strings.CONFIG_CHANGES)
self.vertical_queue.put(ipc_msg)
counter += 1
except IOError:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_FILE_ERROR)
self.vertical_queue.put(ipc_msg)
def __init__(self, config, log_queue, runner, web_panel_queue,
vertical_queue):
super().__init__()
self.runner = runner
self.log_queue = log_queue
self.web_panel_queue = web_panel_queue
self.logger = Logger()
self.notifier = Notifier()
self.config = config['generalHandler']
self.web_panel_active = config['webPanel']['active']
self.vertical_queue = vertical_queue
class WebPanel(threading.Thread):
"""
This runnable has the responsibility or maintain the webpanel updated.
It starts up the web server subsystem and forwards IPC signaling between the
main process and the webserver.
It consumes the web_panel_queue.
"""
name = "SUB-WebPanel"
runner = None
def __init__(self, runner, config, web_panel_queue, vertical_queue):
super().__init__()
self.runner = runner
self.logger = Logger()
self.config = config
self.web_panel_queue = web_panel_queue
self.HTTP_server = HTTPServer
self.data = {"config": self.config['frontEndConfig'], "logs": []}
self.vertical_queue = vertical_queue
def run(self):
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
web_server = self.HTTP_server(self.config['webServer'], self.runner,
self.vertical_queue)
web_server.start()
while self.runner.is_set():
try:
log_message = self.web_panel_queue.get(False)
self.logger.info(self.name + " " + Strings.WEBPANEL_PUBLISH)
if len(self.data['logs']) >= self.config['maxEntries']:
self.data['logs'].pop()
self.data['logs'].insert(0, log_message.get_object())
with open(self.config['dataOutput'] + "/" + "logs.json",
'w') as f:
json.dump(self.data, f, indent=4)
except queue.Empty:
time.sleep(1)
except KeyError:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN,
Strings.CONFIG_ERROR)
self.vertical_queue.put(ipc_msg)
def __init__(self, config_path, runner, vertical_queue):
super().__init__()
self.runner = runner
self.vertical_queue = vertical_queue
self.logger = Logger()
self.interval_secs = 1
self.config_path = config_path
self.last_config_checksum = None
def __init__(self, runner, config, web_panel_queue, vertical_queue):
super().__init__()
self.runner = runner
self.logger = Logger()
self.config = config
self.web_panel_queue = web_panel_queue
self.HTTP_server = HTTPServer
self.data = {"config": self.config['frontEndConfig'], "logs": []}
self.vertical_queue = vertical_queue
def __init__(self):
if len(sys.argv) == 2:
if sys.argv[1] == 'skeleton':
self.skeleton()
else:
self.config_path = sys.argv[1]
else:
self.config_path = "config.json"
self.logger = Logger()
self.booting = True
self.review_interval = 1
"""
Create the IPC system
-vertical_queue is intended to communicate subsystem -> main process.
-log_queue communicates log parsers with the handler
-web_panel_queue passes log information from general handler to web panel subsystem.
"""
self.vertical_queue = Queue()
self.log_queue = Queue()
self.web_panel_queue = Queue()
"""
It creates the events for signaling start or shutdown in threads.
"""
self.main_runner = threading.Event()
self.web_panel_runner = threading.Event()
self.config_runner = threading.Event()
self.gen_log_handler_runner = threading.Event()
self.logs_runner = threading.Event()
"""
Sets by default all events (app services or threads) on.
"""
self.main_runner.set()
self.web_panel_runner.set()
self.config_runner.set()
self.gen_log_handler_runner.set()
self.logs_runner.set()
signal.signal(signal.SIGTERM, self.shutdown)
signal.signal(signal.SIGINT, self.shutdown)
def __init__(self, config, runner, vertical_queue):
super().__init__()
self.config = config
self.request_handler = RequestHandler
self.runner = runner
self.logger = Logger()
self.routes = [
("/", self.config['docRoot'])
]
self.vertical_queue = vertical_queue
def __init__(self, config, log_queue, runner, web_panel_queue, vertical_queue):
super().__init__()
self.runner = runner
self.log_queue = log_queue
self.web_panel_queue = web_panel_queue
self.logger = Logger()
self.notifier = Notifier()
self.config = config['generalHandler']
self.web_panel_active = config['webPanel']['active']
self.vertical_queue = vertical_queue
class WebPanel(threading.Thread):
"""
This runnable has the responsibility or maintain the webpanel updated.
It starts up the web server subsystem and forwards IPC signaling between the
main process and the webserver.
It consumes the web_panel_queue.
"""
name = "SUB-WebPanel"
runner = None
def __init__(self, runner, config, web_panel_queue, vertical_queue):
super().__init__()
self.runner = runner
self.logger = Logger()
self.config = config
self.web_panel_queue = web_panel_queue
self.HTTP_server = HTTPServer
self.data = {"config": self.config['frontEndConfig'], "logs": []}
self.vertical_queue = vertical_queue
def run(self):
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
web_server = self.HTTP_server(self.config['webServer'], self.runner, self.vertical_queue)
web_server.start()
while self.runner.is_set():
try:
log_message = self.web_panel_queue.get(False)
self.logger.info(self.name + " " + Strings.WEBPANEL_PUBLISH)
if len(self.data['logs']) >= self.config['maxEntries']:
self.data['logs'].pop()
self.data['logs'].insert(0, log_message.get_object())
with open(self.config['dataOutput'] + "/" + "logs.json", 'w') as f:
json.dump(self.data, f, indent=4)
except queue.Empty:
time.sleep(1)
except KeyError:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR)
self.vertical_queue.put(ipc_msg)
def __init__(self):
if len(sys.argv) == 2:
if sys.argv[1] == 'skeleton':
self.skeleton()
else:
self.config_path = sys.argv[1]
else:
self.config_path = "config.json"
self.logger = Logger()
self.booting = True
self.review_interval = 1
"""
Create the IPC system
-vertical_queue is intended to communicate subsystem -> main process.
-log_queue communicates log parsers with the handler
-web_panel_queue passes log information from general handler to web panel subsystem.
"""
self.vertical_queue = Queue()
self.log_queue = Queue()
self.web_panel_queue = Queue()
"""
It creates the events for signaling start or shutdown in threads.
"""
self.main_runner = threading.Event()
self.web_panel_runner = threading.Event()
self.config_runner = threading.Event()
self.gen_log_handler_runner = threading.Event()
self.logs_runner = threading.Event()
"""
Sets by default all events (app services or threads) on.
"""
self.main_runner.set()
self.web_panel_runner.set()
self.config_runner.set()
self.gen_log_handler_runner.set()
self.logs_runner.set()
signal.signal(signal.SIGTERM, self.shutdown)
signal.signal(signal.SIGINT, self.shutdown)
class LogHandler(threading.Thread):
"""
This subsystem receives all log messages that need to be saved or notified.
Its the consumer of the logs queue and the producer of the webpanel queue.
"""
name = "SUB-GenLogHandler"
def __init__(self, config, log_queue, runner, web_panel_queue, vertical_queue):
super().__init__()
self.runner = runner
self.log_queue = log_queue
self.web_panel_queue = web_panel_queue
self.logger = Logger()
self.notifier = Notifier()
self.config = config['generalHandler']
self.web_panel_active = config['webPanel']['active']
self.vertical_queue = vertical_queue
def run(self):
"""
Consumer from log queue and taking action for each log settings.
"""
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
while self.runner.is_set():
try:
log_message = self.log_queue.get(False)
log_path = self.calc_log_path(log_message)
self.write_log(log_message, log_path)
if log_message.system_notifications is True:
self.notify_sys(log_message)
if self.web_panel_active:
self.send_to_webpanel(log_message)
self.finish_handling()
self.logger.info(self.name + " " + Strings.LOG_PROCESSED)
except queue.Empty:
time.sleep(1)
except KeyError:
comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR)
self.vertical_queue.put(comm)
except IOError:
comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.IO_ERROR)
self.vertical_queue.put(comm)
def send_to_webpanel(self, log_message):
"""
Send log info to the webpanel subsystem.
:param log_message: JSON object
"""
self.web_panel_queue.put(log_message)
def calc_log_path(self, log_message):
"""
Calculates the name for the log file.
:param log_message:
:return: string """
log_file_name = log_message.name.replace("\n", "").replace(" ", "_") + ".log".lower()
log_write_path = self.config['logsPath'] + "/" + log_file_name
return log_write_path
def write_log(self, log_message, path):
"""
Writes log in specifiec path.
:param log_message:
:param path:
"""
with open(path, "a") as log_file:
log_file.write(log_message.data + "\n")
def notify_sys(self, log_message):
"""
Send info to the notifications subsystem
:param log_message:
"""
self.notifier.send_notify(log_message.name, log_message.data)
def finish_handling(self):
"""
Finish handler jobs.
"""
self.log_queue.task_done()
class LogHandler(threading.Thread):
"""
This subsystem receives all log messages that need to be saved or notified.
Its the consumer of the logs queue and the producer of the webpanel queue.
"""
name = "SUB-GenLogHandler"
def __init__(self, config, log_queue, runner, web_panel_queue,
vertical_queue):
super().__init__()
self.runner = runner
self.log_queue = log_queue
self.web_panel_queue = web_panel_queue
self.logger = Logger()
self.notifier = Notifier()
self.config = config['generalHandler']
self.web_panel_active = config['webPanel']['active']
self.vertical_queue = vertical_queue
def run(self):
"""
Consumer from log queue and taking action for each log settings.
"""
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
while self.runner.is_set():
try:
log_message = self.log_queue.get(False)
log_path = self.calc_log_path(log_message)
self.write_log(log_message, log_path)
if log_message.system_notifications is True:
self.notify_sys(log_message)
if self.web_panel_active:
self.send_to_webpanel(log_message)
self.finish_handling()
self.logger.info(self.name + " " + Strings.LOG_PROCESSED)
except queue.Empty:
time.sleep(1)
except KeyError:
comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN,
Strings.CONFIG_ERROR)
self.vertical_queue.put(comm)
except IOError:
comm = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN,
Strings.IO_ERROR)
self.vertical_queue.put(comm)
def send_to_webpanel(self, log_message):
"""
Send log info to the webpanel subsystem.
:param log_message: JSON object
"""
self.web_panel_queue.put(log_message)
def calc_log_path(self, log_message):
"""
Calculates the name for the log file.
:param log_message:
:return: string """
log_file_name = log_message.name.replace("\n", "").replace(
" ", "_") + ".log".lower()
log_write_path = self.config['logsPath'] + "/" + log_file_name
return log_write_path
def write_log(self, log_message, path):
"""
Writes log in specifiec path.
:param log_message:
:param path:
"""
with open(path, "a") as log_file:
log_file.write(log_message.data + "\n")
def notify_sys(self, log_message):
"""
Send info to the notifications subsystem
:param log_message:
"""
self.notifier.send_notify(log_message.name, log_message.data)
def finish_handling(self):
"""
Finish handler jobs.
"""
self.log_queue.task_done()
class OmniLogD(object):
"""
This is the main process. It setups config and IPC.
It starts all services needed to run this app, and is responsible of communication
between them.
"""
name = "MainProcess"
def __init__(self):
if len(sys.argv) == 2:
if sys.argv[1] == 'skeleton':
self.skeleton()
else:
self.config_path = sys.argv[1]
else:
self.config_path = "config.json"
self.logger = Logger()
self.booting = True
self.review_interval = 1
"""
Create the IPC system
-vertical_queue is intended to communicate subsystem -> main process.
-log_queue communicates log parsers with the handler
-web_panel_queue passes log information from general handler to web panel subsystem.
"""
self.vertical_queue = Queue()
self.log_queue = Queue()
self.web_panel_queue = Queue()
"""
It creates the events for signaling start or shutdown in threads.
"""
self.main_runner = threading.Event()
self.web_panel_runner = threading.Event()
self.config_runner = threading.Event()
self.gen_log_handler_runner = threading.Event()
self.logs_runner = threading.Event()
"""
Sets by default all events (app services or threads) on.
"""
self.main_runner.set()
self.web_panel_runner.set()
self.config_runner.set()
self.gen_log_handler_runner.set()
self.logs_runner.set()
signal.signal(signal.SIGTERM, self.shutdown)
signal.signal(signal.SIGINT, self.shutdown)
def skeleton(self):
try:
home = os.path.expanduser("~")
github_front_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/public/index.html"
github_config_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/docs/config.dist.json"
config_example = request.urlopen(github_config_url).read().decode("utf8")
front_html = request.urlopen(github_front_url).read().decode("utf8")
dirs = collections.OrderedDict()
dirs['omnilog_root'] = os.path.join(home, 'omnilog')
dirs['logs'] = os.path.join(home, 'omnilog', 'logs')
dirs['http'] = os.path.join(home, 'omnilog', 'public')
dirs['http_data_source'] = os.path.join(home, 'omnilog', 'public', 'data_source')
for k, v in dirs.items():
os.makedirs(v)
if k == "http":
with open(os.path.join(dirs['http'], "index.html"), "w") as f:
f.write(front_html)
if k == "omnilog_root":
with open(os.path.join(dirs['omnilog_root'], "config.json"), "w") as f:
f.write(config_example)
exit(Strings.SKELETON_DIR_CREATED)
except FileExistsError:
exit(Strings.SKELETON_DIR)
except URLError:
exit(Strings.SKELETON_URL_ERROR)
def run(self):
"""
The Main function of this program. Its a loop that instantiates all runnable services (threading) with review a
interval that listens for messages of low level processes (aka threads or app services).
Its An intermediary between services.
"""
threads = list()
config_watcher = ConfigWatcher(self.config_path, self.config_runner, self.vertical_queue)
config_watcher.start()
threads.append(config_watcher)
time.sleep(2)
while self.main_runner.is_set():
try:
time.sleep(self.review_interval)
if self.vertical_queue.empty() and self.booting:
if self.gen_log_handler_runner.is_set():
gen_log_handler = LogHandler(Config.config_dict, self.log_queue,
self.gen_log_handler_runner, self.web_panel_queue,
self.vertical_queue)
threads.append(gen_log_handler)
gen_log_handler.start()
if Config.config_dict['webPanel']['active'] and self.web_panel_runner.is_set():
web_panel = WebPanel(self.web_panel_runner, Config.config_dict['webPanel'],
self.web_panel_queue, self.vertical_queue)
threads.append(web_panel)
web_panel.start()
if self.logs_runner.is_set():
logs = Config.config_dict['logs']
for l in logs:
if l['active'] is True:
t = LogParser(l, self.logs_runner, self.log_queue, self.vertical_queue)
t.name = l['name']
threads.append(t)
t.start()
self.logger.info(self.name + " - Started log watcher for " + l['name'])
self.booting = False
elif not self.vertical_queue.empty():
self.manage_com()
except KeyboardInterrupt:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.KEYBOARD_INTERRUPTION)
self.vertical_queue.put(ipc_msg)
except KeyError or AttributeError:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR)
self.vertical_queue.put(ipc_msg)
except:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.UNHANDLED_EXCEPTION)
self.logger.emergency(self.name + " " + Strings.UNHANDLED_EXCEPTION)
self.vertical_queue.put(ipc_msg)
for te in threads:
te.join()
def manage_com(self):
"""
This function is responsible of handling messages arrived from the different app services.
This queue needs to be based on topics because the main process needs to understand from what service
come each message.
"""
com = self.vertical_queue.get(True)
self.logger.info(self.name + Strings.IPC_RECEIVED + str(com))
if com.action == IPCActions.ACTION_REBOOT:
self.restart()
elif com.action == IPCActions.ACTION_SHUTDOWN:
self.shutdown()
def restart(self, sig_num=None, frame=None):
"""
The restart operation. It send the event to stop gracefully stop all threads, waits for
job ending and then start it again.
"""
self.logger.info(self.name + Strings.APP_RESTARTING)
self.logs_runner.clear()
time.sleep(2)
self.web_panel_runner.clear()
self.gen_log_handler_runner.clear()
time.sleep(2)
self.web_panel_runner.set()
self.gen_log_handler_runner.set()
self.logs_runner.set()
self.booting = True
def shutdown(self, sig_num=None, frame=None):
"""
The shutdown operation. Same procedure as restart, but this time the main loop will not do more
review and exit.
"""
self.logger.info(self.name + Strings.APP_SHUTDOWN)
self.config_runner.clear()
self.logs_runner.clear()
time.sleep(2)
self.web_panel_runner.clear()
self.gen_log_handler_runner.clear()
self.main_runner.clear()
class LogParser(threading.Thread):
"""
This subsystem may run for each log instance.
Its responsibility is to pull changes from paramiko channel,
split it into lines and check user defined regex over them.
After all , it queues the result (if valid) into the general
log handler queue.SS
"""
name = "SUB-LogParser"
runner = None
def __init__(self, log, runner, log_queue, vertical_queue):
super().__init__()
self.runner = runner
self.log_queue = log_queue
self.config = log
self.ssh = SSHhandler(self.config['ssh'])
self.logger = Logger()
self.interval_secs = 1
self.recv_buffer = 1024
self.vertical_queue = vertical_queue
def run(self):
try:
self.logger.info(self.name + " " + Strings.SUB_SYSTEM_START)
ssh = self.ssh.get_session()
transport = ssh.get_transport()
transport.set_keepalive(5)
channel = transport.open_session()
channel.exec_command('tail -f ' + self.config['logReadPath'])
while self.runner.is_set() and transport.is_active():
time.sleep(self.interval_secs)
rl, wl, xl = select.select([channel], [], [], 0.0)
if len(rl) > 0:
data = channel.recv(self.recv_buffer)
lines = self.get_lines_from_data(data)
valid_lines = self.check_patterns(lines)
if len(valid_lines) > 0:
for line in valid_lines:
self.logger.info(self.name + " " + Strings.PARSER_VALID_LOG_REACHED)
self.log_queue.put(
LogMessage(
self.config['name'],
line,
self.config['systemNotifications'],
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
))
ssh.close()
except KeyError:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.CONFIG_ERROR)
self.vertical_queue.put(ipc_msg)
except SSHException:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN, Strings.SSH_ERROR)
self.vertical_queue.put(ipc_msg)
def get_lines_from_data(self, data):
"""
Transform data received from ssh channel to a list of lines.
:param data: bytes
:return: list
"""
string = data.decode("unicode_escape", "ignore")
lines = string.split("\n")
return lines
def check_patterns(self, lines):
"""
Validates log lines against user defined regex (only if defined).
Returns the result list.
:param lines: list
:return: list
"""
not_black_listed = []
valid_lines = []
if "ignorePatterns" in self.config.keys() and len(self.config['ignorePatterns']) > 0:
for l in lines:
for p in self.config['ignorePatterns']:
if not re.search(p, l):
not_black_listed.append(l)
else:
not_black_listed = lines
if "patterns" in self.config.keys() and len(self.config['patterns']) > 0:
for l in not_black_listed:
for p in self.config['patterns']:
if re.search(p, l):
valid_lines.append(l)
else:
valid_lines = not_black_listed
result = [v for v in valid_lines if v != '']
return result
class OmniLogD(object):
"""
This is the main process. It setups config and IPC.
It starts all services needed to run this app, and is responsible of communication
between them.
"""
name = "MainProcess"
def __init__(self):
if len(sys.argv) == 2:
if sys.argv[1] == 'skeleton':
self.skeleton()
else:
self.config_path = sys.argv[1]
else:
self.config_path = "config.json"
self.logger = Logger()
self.booting = True
self.review_interval = 1
"""
Create the IPC system
-vertical_queue is intended to communicate subsystem -> main process.
-log_queue communicates log parsers with the handler
-web_panel_queue passes log information from general handler to web panel subsystem.
"""
self.vertical_queue = Queue()
self.log_queue = Queue()
self.web_panel_queue = Queue()
"""
It creates the events for signaling start or shutdown in threads.
"""
self.main_runner = threading.Event()
self.web_panel_runner = threading.Event()
self.config_runner = threading.Event()
self.gen_log_handler_runner = threading.Event()
self.logs_runner = threading.Event()
"""
Sets by default all events (app services or threads) on.
"""
self.main_runner.set()
self.web_panel_runner.set()
self.config_runner.set()
self.gen_log_handler_runner.set()
self.logs_runner.set()
signal.signal(signal.SIGTERM, self.shutdown)
signal.signal(signal.SIGINT, self.shutdown)
def skeleton(self):
try:
home = os.path.expanduser("~")
github_front_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/public/index.html"
github_config_url = "https://raw.githubusercontent.com/sandboxwebs/omnilog/master/omnilog/docs/config.dist.json"
config_example = request.urlopen(github_config_url).read().decode(
"utf8")
front_html = request.urlopen(github_front_url).read().decode(
"utf8")
dirs = collections.OrderedDict()
dirs['omnilog_root'] = os.path.join(home, 'omnilog')
dirs['logs'] = os.path.join(home, 'omnilog', 'logs')
dirs['http'] = os.path.join(home, 'omnilog', 'public')
dirs['http_data_source'] = os.path.join(home, 'omnilog', 'public',
'data_source')
for k, v in dirs.items():
os.makedirs(v)
if k == "http":
with open(os.path.join(dirs['http'], "index.html"),
"w") as f:
f.write(front_html)
if k == "omnilog_root":
with open(
os.path.join(dirs['omnilog_root'], "config.json"),
"w") as f:
f.write(config_example)
exit(Strings.SKELETON_DIR_CREATED)
except FileExistsError:
exit(Strings.SKELETON_DIR)
except URLError:
exit(Strings.SKELETON_URL_ERROR)
def run(self):
"""
The Main function of this program. Its a loop that instantiates all runnable services (threading) with review a
interval that listens for messages of low level processes (aka threads or app services).
Its An intermediary between services.
"""
threads = list()
config_watcher = ConfigWatcher(self.config_path, self.config_runner,
self.vertical_queue)
config_watcher.start()
threads.append(config_watcher)
time.sleep(2)
while self.main_runner.is_set():
try:
time.sleep(self.review_interval)
if self.vertical_queue.empty() and self.booting:
if self.gen_log_handler_runner.is_set():
gen_log_handler = LogHandler(
Config.config_dict, self.log_queue,
self.gen_log_handler_runner, self.web_panel_queue,
self.vertical_queue)
threads.append(gen_log_handler)
gen_log_handler.start()
if Config.config_dict['webPanel'][
'active'] and self.web_panel_runner.is_set():
web_panel = WebPanel(self.web_panel_runner,
Config.config_dict['webPanel'],
self.web_panel_queue,
self.vertical_queue)
threads.append(web_panel)
web_panel.start()
if self.logs_runner.is_set():
logs = Config.config_dict['logs']
for l in logs:
if l['active'] is True:
t = LogParser(l, self.logs_runner,
self.log_queue,
self.vertical_queue)
t.name = l['name']
threads.append(t)
t.start()
self.logger.info(
self.name + " - Started log watcher for " +
l['name'])
self.booting = False
elif not self.vertical_queue.empty():
self.manage_com()
except KeyboardInterrupt:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN,
Strings.KEYBOARD_INTERRUPTION)
self.vertical_queue.put(ipc_msg)
except KeyError or AttributeError:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN,
Strings.CONFIG_ERROR)
self.vertical_queue.put(ipc_msg)
except:
ipc_msg = IPCMessage(self.name, IPCActions.ACTION_SHUTDOWN,
Strings.UNHANDLED_EXCEPTION)
self.logger.emergency(self.name + " " +
Strings.UNHANDLED_EXCEPTION)
self.vertical_queue.put(ipc_msg)
for te in threads:
te.join()
def manage_com(self):
"""
This function is responsible of handling messages arrived from the different app services.
This queue needs to be based on topics because the main process needs to understand from what service
come each message.
"""
com = self.vertical_queue.get(True)
self.logger.info(self.name + Strings.IPC_RECEIVED + str(com))
if com.action == IPCActions.ACTION_REBOOT:
self.restart()
elif com.action == IPCActions.ACTION_SHUTDOWN:
self.shutdown()
def restart(self, sig_num=None, frame=None):
"""
The restart operation. It send the event to stop gracefully stop all threads, waits for
job ending and then start it again.
"""
self.logger.info(self.name + Strings.APP_RESTARTING)
self.logs_runner.clear()
time.sleep(2)
self.web_panel_runner.clear()
self.gen_log_handler_runner.clear()
time.sleep(2)
self.web_panel_runner.set()
self.gen_log_handler_runner.set()
self.logs_runner.set()
self.booting = True
def shutdown(self, sig_num=None, frame=None):
"""
The shutdown operation. Same procedure as restart, but this time the main loop will not do more
review and exit.
"""
self.logger.info(self.name + Strings.APP_SHUTDOWN)
self.config_runner.clear()
self.logs_runner.clear()
time.sleep(2)
self.web_panel_runner.clear()
self.gen_log_handler_runner.clear()
self.main_runner.clear()