def log_parser(): db = get_db() end_time = datetime.now() start_time = end_time - timedelta(days=7) lp = LogParser(start_time, end_time) result = lp.parse() return render_template('show_job_entries.html', job_entries=result)
def update_measures(m_id, is_new_measure=False): if m_id is None: raise AttributeError("Measure id is None") else: measures = models.Measures.select().where(models.Measures.measure_id == int(m_id)).order_by(models.Measures.id) measures_lst = [] measures_lst.append(['Id', 'Temperature Outside', 'Engine Temperature', 'Pressure', 'Voltage', 'Fuel level']) for m in measures: measures_lst.append(m.to_list()) lp = LogParser() if int(is_new_measure) == 1: lp.parse(truncate_log=True, new_measure=True) else: lp.parse(truncate_log=True) return jsonify(measures_lst)
def parse_all_sessions(files): logParser = LogParser(); sessions = []; for file in files: session = logParser.parse(file) sessions.append(session) return sessions
def parse_all_sessions(files): logParser = LogParser() sessions = [] for file in files: session = logParser.parse(file) sessions.append(session) return sessions
def test_nginx_log_parsing(self): log_format = '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$request_body"' log_type = 'nginx' parser = LogParser(log_format, log_type) record = '127.0.0.1 - - [08/Jul/2016:08:45:58 -0400] "POST /place HTTP/1.1" 404 233 "-" "curl/7.38.0" "way"' parsed = parser.parse(record) self.assertEqual(parsed, {'request.first_line': 'POST /place HTTP/1.1', 'request.headers.referer': '-', 'request.headers.user-agent': 'curl/7.38.0', 'request.method': 'POST', 'request.protocol': 'HTTP/1.1', 'request.src_ip': '127.0.0.1', 'request.time': '08/Jul/2016:08:45:58 -0400', 'request.uri': '/place', 'request.user': '******', 'request.body': 'way', 'response.body_size': '233', 'response.status': '404'})
def test_apache_log_parsing(self): log_format = '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i"' log_type = 'apache' parser = LogParser(log_format, log_type) record = '127.0.0.1 - - [12/Jul/2016:14:00:22 +0300] "GET /test HTTP/1.1" 200 10956 "-" "curl/7.47.0"' parsed = parser.parse(record) self.assertEqual(parsed, {'request.client_id': '-', 'request.first_line': 'GET /test HTTP/1.1', 'request.headers.referer': '-', 'request.headers.user-agent': 'curl/7.47.0', 'request.method': 'GET', 'request.protocol': 'HTTP/1.1', 'request.src_ip': '127.0.0.1', 'request.time': '[12/Jul/2016:14:00:22 +0300]', 'request.uri': '/test', 'request.user': '******', 'response.size': '10956', 'response.status': '200'})
def main(): parser = argparse.ArgumentParser(description='Convert logs to custom fileformat(now supports json and yaml fileformats)', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('input_file', help='file with input logs', type=argparse.FileType('r')) parser.add_argument('output_file', help='file with parsed data', type=argparse.FileType('w')) # default nginx log_format: '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"' # Note: spaces in log_format is important! parser.add_argument('log_format', help='''logs format. Supports: nginx log_format and apache CustomLog, also you may use custom format''') parser.add_argument('--log_type', help='type of log file.', choices=['nginx', 'apache', 'custom'], default='nginx') parser.add_argument('--result_type', help='type of parsed file', choices=['yaml', 'json'], default='json') parser.add_argument('-f', '--follow', action='store_true', default=False, help='process appended data as the file grows') args = parser.parse_args() parser = LogParser(args.log_format, args.log_type) parsed_logs = [] try: for record in (tailer.follow(args.input_file) if args.follow else args.input_file): parsed_logs.append(parser.parse(record)) except KeyboardInterrupt: print('interrupt received, stopping.') finally: if args.result_type == 'yaml': yaml.dump(parsed_logs, args.output_file, default_flow_style=False) elif args.result_type == 'json': json.dump(parsed_logs, args.output_file)
from log_parser import LogParser from cluster import Cluster, NodeFactory from converter import LogsToEventsConverter from events import EventLoop N = 4 WIDTH = HEIGHT = 600 TITLE = "Fast Paxos" LOGFILE = "../../../logs/test.log" factory = NodeFactory((20, 20), 8) cluster = Cluster(TITLE, N, (WIDTH, HEIGHT), factory) loop = EventLoop() logs = LogParser.parse(LOGFILE) converter = LogsToEventsConverter(cluster, loop) converter.convert(logs) loop.run() cluster.close()
def run(self): """ The main routine which controls everything """ update_notifier = self.check_for_update() framecount = 0 # Create drawing tool to use to draw everything - it'll create its own screen drawing_tool = DrawingTool(self.file_prefix) drawing_tool.set_window_title(update_notifier) parser = LogParser(self.file_prefix, self.tracker_version) opt = Options() log = logging.getLogger("tracker") event_result = None state = None read_from_server = opt.read_from_server write_to_server = opt.write_to_server state_version = -1 twitch_username = None new_states_queue = [] screen_error_message = None while event_result != Event.DONE: # Check for events and handle them event_result = drawing_tool.handle_events() # A change means the user has (de)activated an option if opt.read_from_server != read_from_server\ or opt.twitch_name != twitch_username: twitch_username = opt.twitch_name read_from_server = opt.read_from_server new_states_queue = [] # Also restart version count if we go back and forth from log.txt to server if read_from_server: state_version = -1 state = None # show who we are watching in the title bar drawing_tool.set_window_title(update_notifier, watching_player=twitch_username, updates_queued=len(new_states_queue)) else: drawing_tool.set_window_title(update_notifier) if opt.write_to_server and opt.write_to_server != write_to_server: write_to_server = True drawing_tool.set_window_title(update_notifier, uploading=True) if not opt.write_to_server: write_to_server = False if opt.read_from_server: # Change the delay for polling, as we probably don't want to fetch it every second update_timer = 2 else: update_timer = self.read_timer if event_result == Event.OPTIONS_UPDATE: # By setting the framecount to 0 we ensure we'll refresh the state right away framecount = 0 screen_error_message = None # force updates after changing options if state is not None: state.modified = True # Now we re-process the log file to get anything that might have loaded; # do it every update_timer seconds (making sure to truncate to an integer # or else it might never mod to 0) if (framecount % int(Options().framerate_limit * update_timer) == 0): # Let the parser do his thing and give us a state if opt.read_from_server: base_url = opt.trackerserver_url + "/tracker/api/user/" + opt.twitch_name json_dict = None try: json_version = urllib2.urlopen(base_url + "/version").read() if int(json_version) > state_version: # FIXME better handling of 404 error ? json_state = urllib2.urlopen(base_url).read() json_dict = json.loads(json_state) new_state = TrackerState.from_json(json_dict) if new_state is None: raise Exception state_version = int(json_version) new_states_queue.append((state_version, new_state)) drawing_tool.set_window_title(update_notifier, watching_player=twitch_username, updates_queued=len(new_states_queue), read_delay=opt.read_delay) except Exception: state = None log.error("Couldn't load state from server") import traceback log.error(traceback.format_exc()) if json_dict is not None: their_version = "" if "tracker_version" in json_dict: their_version = json_dict["tracker_version"] else: # this is the only version that can upload to the server but doesn't include a version string their_version = "0.10-beta1" if their_version != self.tracker_version: screen_error_message = "They are using tracker version " + their_version + " but you have " + self.tracker_version else: force_draw = state and state.modified state = parser.parse() if force_draw: state.modified = True if write_to_server and not opt.trackerserver_authkey: screen_error_message = "Your authkey is blank. Get a new authkey in the options menu and paste it into the authkey text field." if state is not None and write_to_server and state.modified and screen_error_message is None: opener = urllib2.build_opener(urllib2.HTTPHandler) put_url = opt.trackerserver_url + "/tracker/api/update/" + opt.trackerserver_authkey json_string = json.dumps(state, cls=TrackerStateEncoder, sort_keys=True) request = urllib2.Request(put_url, data=json_string) request.add_header('Content-Type', 'application/json') request.get_method = lambda: 'PUT' try: result = opener.open(request) result_json = json.loads(result.read()) updated_user = result_json["updated_user"] if updated_user is None: screen_error_message = "The server didn't recognize you. Try getting a new authkey in the options menu." else: screen_error_message = None except Exception as e: import traceback errmsg = traceback.format_exc() log.error("ERROR: Couldn't send item info to server") log.error(errmsg) screen_error_message = "ERROR: Couldn't send item info to server, check tracker_log.txt" # check the new state at the front of the queue to see if it's time to use it if len(new_states_queue) > 0: (state_timestamp, new_state) = new_states_queue[0] current_timestamp = int(time.time()) if current_timestamp - state_timestamp >= opt.read_delay or state is None: state = new_state new_states_queue.pop(0) drawing_tool.set_window_title(update_notifier, watching_player=twitch_username, updates_queued=len(new_states_queue), read_delay=opt.read_delay) if state is None and screen_error_message is None: if read_from_server: screen_error_message = "Unable to read state from server. Please verify your options setup and tracker_log.txt" else: screen_error_message = "log.txt not found. Put the RebirthItemTracker folder inside the isaac folder, next to log.txt" if screen_error_message is not None: drawing_tool.write_error_message(screen_error_message) else: # We got a state, now we draw it drawing_tool.draw_state(state) drawing_tool.tick() framecount += 1 # main loop finished. program is exiting drawing_tool.save_window_position()
def run(self): """ The main routine which controls everything """ framecount = 0 # Create drawing tool to use to draw everything - it'll create its own screen drawing_tool = DrawingTool(wdir_prefix) drawing_tool.set_window_title_info( update_notifier=(" v" + self.tracker_version)) opt = Options() parser = LogParser(wdir_prefix, self.tracker_version, LogFinder()) event_result = None state = None custom_title_enabled = opt.custom_title_enabled read_from_server = opt.read_from_server write_to_server = opt.write_to_server game_version = opt.game_version state_version = -1 twitch_username = None new_states_queue = [] screen_error_message = None retry_in = 0 update_timer = opt.log_file_check_seconds last_game_version = None while event_result != Event.DONE: # Check for events and handle them event_result = drawing_tool.handle_events() # The user checked or unchecked the "Custom Title Enabled" checkbox if opt.custom_title_enabled != custom_title_enabled: custom_title_enabled = opt.custom_title_enabled drawing_tool.update_window_title() # The user started or stopped watching someone from the server (or they started watching a new person from the server) if opt.read_from_server != read_from_server or opt.twitch_name != twitch_username: twitch_username = opt.twitch_name read_from_server = opt.read_from_server new_states_queue = [] # Also restart version count if we go back and forth from log.txt to server if read_from_server: state_version = -1 state = None # Change the delay for polling, as we probably don't want to fetch it every second update_timer_override = 2 # Show who we are watching in the title bar drawing_tool.set_window_title_info( watching=True, watching_player=twitch_username, updates_queued=len(new_states_queue)) else: drawing_tool.set_window_title_info(watching=False) update_timer_override = 0 # The user started or stopped broadcasting to the server if opt.write_to_server != write_to_server: write_to_server = opt.write_to_server drawing_tool.set_window_title_info( uploading=opt.write_to_server) if opt.game_version != game_version: parser.reset() game_version = opt.game_version # Force refresh state if we updated options or if we need to retry # to contact the server. if (event_result == Event.OPTIONS_UPDATE or (screen_error_message is not None and retry_in == 0)): # By setting the framecount to 0 we ensure we'll refresh the state right away framecount = 0 screen_error_message = None retry_in = 0 # Force updates after changing options if state is not None: state.modified = True # normally we check for updates based on how the option is set # when doing network stuff, this can be overridden update_delay = opt.log_file_check_seconds if update_timer_override != 0: update_delay = update_timer_override # Now we re-process the log file to get anything that might have loaded; # do it every update_timer seconds (making sure to truncate to an integer # or else it might never mod to 0) frames_between_checks = int(Options().framerate_limit * update_delay) if frames_between_checks <= 0: frames_between_checks = 1 if framecount % frames_between_checks == 0: if retry_in != 0: retry_in -= 1 # Let the parser do his thing and give us a state if opt.read_from_server: base_url = opt.trackerserver_url + "/tracker/api/user/" + opt.twitch_name json_dict = None try: json_version = urllib.request.urlopen( base_url + "/version").read() if int(json_version) > state_version: # FIXME better handling of 404 error ? json_state = urllib.request.urlopen( base_url).read() json_dict = json.loads(json_state) new_state = TrackerState.from_json(json_dict) if new_state is None: raise Exception("server gave us empty state") state_version = int(json_version) new_states_queue.append((state_version, new_state)) drawing_tool.set_window_title_info( updates_queued=len(new_states_queue)) except Exception: state = None log_error("Couldn't load state from server\n" + traceback.format_exc()) if json_dict is not None: if "tracker_version" in json_dict: their_version = json_dict["tracker_version"] else: # This is the only version that can upload to the server but doesn't include a version string their_version = "0.10-beta1" if their_version != self.tracker_version: screen_error_message = "They are using tracker version " + their_version + " but you have " + self.tracker_version else: force_draw = state and state.modified state = parser.parse() if force_draw and state is not None: state.modified = True if write_to_server and not opt.trackerserver_authkey: screen_error_message = "Your authkey is blank. Get a new authkey in the options menu and paste it into the authkey text field." if state is not None and write_to_server and state.modified and screen_error_message is None: opener = urllib.request.build_opener( urllib.request.HTTPHandler) put_url = opt.trackerserver_url + "/tracker/api/update/" + opt.trackerserver_authkey json_string = json.dumps( state, cls=TrackerStateEncoder, sort_keys=True).encode("utf-8") request = urllib.request.Request(put_url, data=json_string) request.add_header('Content-Type', 'application/json') request.get_method = lambda: 'PUT' try: result = opener.open(request) result_json = json.loads(result.read()) updated_user = result_json["updated_user"] if updated_user is None: screen_error_message = "The server didn't recognize you. Try getting a new authkey in the options menu." else: screen_error_message = None except Exception as e: log_error( "ERROR: Couldn't send item info to server\n" + traceback.format_exc()) screen_error_message = "ERROR: Couldn't send item info to server, check tracker_log.txt" # Retry to write the state in 10*update_timer (aka 10 sec in write mode) retry_in = 10 # Check the new state at the front of the queue to see if it's time to use it if len(new_states_queue) > 0: (state_timestamp, new_state) = new_states_queue[0] current_timestamp = int(time.time()) if current_timestamp - state_timestamp >= opt.read_delay or opt.read_delay == 0 or state is None: state = new_state new_states_queue.pop(0) drawing_tool.set_window_title_info( updates_queued=len(new_states_queue)) if state is None and screen_error_message is None: if read_from_server: screen_error_message = "Unable to read state from server. Please verify your options setup and tracker_log.txt" # Retry to read the state in 5*update_timer (aka 10 sec in read mode) retry_in = 5 else: screen_error_message = "log.txt for " + opt.game_version + " not found. Make sure you have the right game selected in the options." if screen_error_message is not None: drawing_tool.write_error_message(screen_error_message) else: # We got a state, now we draw it drawing_tool.draw_state(state, framecount) # if we're watching someone and they change their game version, it can require us to reset if state and last_game_version != state.game_version: drawing_tool.reset_options() last_game_version = state.game_version drawing_tool.tick() framecount += 1 # Main loop finished; program is exiting drawing_tool.save_window_position() Options().save_options(wdir_prefix + "options.json")
#/usr/bin/python3 #Python script to run cron task daily to create database log entries from cron_monitor import app from log_parser import LogParser from datetime import datetime,timedelta from flask_mysqldb import MySQL from flask import Flask, g import config end_time = datetime.now() start_time = end_time - timedelta(days=1) ctx = app.app_context() ctx.push() app.config['MYSQL_USER'] = config.MYSQL_DATABASE_USER app.config['MYSQL_PASSWORD'] = config.MYSQL_DATABASE_PASSWORD app.config['MYSQL_DB'] = config.MYSQL_DATABASE_DB app.config['MYSQL_HOST'] = config.MYSQL_DATABASE_HOST app.config['MYSQL_CURSORCLASS'] = 'DictCursor' mysql = MySQL(app) g.mysql_db = mysql.connection lp = LogParser(start_time, end_time) result = lp.parse()
from log_parser import LogParser from feature_extractor import FeatureExtractor from clustering import Clustering from idf import IDF parser = LogParser('../../data/HDFS_2K.log') tagged_events, log_sequences = parser.parse() extractor = FeatureExtractor(log_sequences, list(tagged_events.keys())) log_sequences = extractor.extract() clustering = Clustering(log_sequences, tagged_events) cluster_ids, cluster_values, silhouette = clustering.cluster() for cluster_value in cluster_values: if cluster_value['num_possible_abnormal_events'] != 0: print(cluster_value) print() #print ("El coeficiente de silueta es =", silhouette)
def run(self): """ The main routine which controls everything """ framecount = 0 # Create drawing tool to use to draw everything - it'll create its own screen drawing_tool = DrawingTool(wdir_prefix) drawing_tool.set_window_title_info(update_notifier=(" v" + self.tracker_version)) opt = Options() parser = LogParser(wdir_prefix, self.tracker_version, LogFinder()) event_result = None state = None custom_title_enabled = opt.custom_title_enabled read_from_server = opt.read_from_server write_to_server = opt.write_to_server game_version = opt.game_version state_version = -1 twitch_username = None new_states_queue = [] screen_error_message = None retry_in = 0 update_timer = opt.log_file_check_seconds last_game_version = None while event_result != Event.DONE: # Check for events and handle them event_result = drawing_tool.handle_events() # The user checked or unchecked the "Custom Title Enabled" checkbox if opt.custom_title_enabled != custom_title_enabled: custom_title_enabled = opt.custom_title_enabled drawing_tool.update_window_title() # The user started or stopped watching someone from the server (or they started watching a new person from the server) if opt.read_from_server != read_from_server or opt.twitch_name != twitch_username: twitch_username = opt.twitch_name read_from_server = opt.read_from_server new_states_queue = [] # Also restart version count if we go back and forth from log.txt to server if read_from_server: state_version = -1 state = None # Change the delay for polling, as we probably don't want to fetch it every second update_timer_override = 2 # Show who we are watching in the title bar drawing_tool.set_window_title_info(watching=True, watching_player=twitch_username, updates_queued=len(new_states_queue)) else: drawing_tool.set_window_title_info(watching=False) update_timer_override = 0 # The user started or stopped broadcasting to the server if opt.write_to_server != write_to_server: write_to_server = opt.write_to_server drawing_tool.set_window_title_info(uploading=opt.write_to_server) if opt.game_version != game_version: parser.reset() game_version = opt.game_version # Force refresh state if we updated options or if we need to retry # to contact the server. if (event_result == Event.OPTIONS_UPDATE or (screen_error_message is not None and retry_in == 0)): # By setting the framecount to 0 we ensure we'll refresh the state right away framecount = 0 screen_error_message = None retry_in = 0 # Force updates after changing options if state is not None: state.modified = True # normally we check for updates based on how the option is set # when doing network stuff, this can be overridden update_delay = opt.log_file_check_seconds if update_timer_override != 0: update_delay = update_timer_override # Now we re-process the log file to get anything that might have loaded; # do it every update_timer seconds (making sure to truncate to an integer # or else it might never mod to 0) frames_between_checks = int(Options().framerate_limit * update_delay) if frames_between_checks <= 0: frames_between_checks = 1 if framecount % frames_between_checks == 0: if retry_in != 0: retry_in -= 1 # Let the parser do his thing and give us a state if opt.read_from_server: base_url = opt.trackerserver_url + "/tracker/api/user/" + opt.twitch_name json_dict = None try: json_version = urllib.request.urlopen(base_url + "/version").read() if int(json_version) > state_version: # FIXME better handling of 404 error ? json_state = urllib.request.urlopen(base_url).read() json_dict = json.loads(json_state, "utf-8") new_state = TrackerState.from_json(json_dict) if new_state is None: raise Exception("server gave us empty state") state_version = int(json_version) new_states_queue.append((state_version, new_state)) drawing_tool.set_window_title_info(updates_queued=len(new_states_queue)) except Exception: state = None log_error("Couldn't load state from server\n" + traceback.format_exc()) if json_dict is not None: if "tracker_version" in json_dict: their_version = json_dict["tracker_version"] else: # This is the only version that can upload to the server but doesn't include a version string their_version = "0.10-beta1" if their_version != self.tracker_version: screen_error_message = "They are using tracker version " + their_version + " but you have " + self.tracker_version else: force_draw = state and state.modified state = parser.parse() if force_draw: state.modified = True if write_to_server and not opt.trackerserver_authkey: screen_error_message = "Your authkey is blank. Get a new authkey in the options menu and paste it into the authkey text field." if state is not None and write_to_server and state.modified and screen_error_message is None: opener = urllib.request.build_opener(urllib.request.HTTPHandler) put_url = opt.trackerserver_url + "/tracker/api/update/" + opt.trackerserver_authkey json_string = json.dumps(state, cls=TrackerStateEncoder, sort_keys=True) request = urllib.request.Request(put_url, data=json_string) request.add_header('Content-Type', 'application/json') request.get_method = lambda: 'PUT' try: result = opener.open(request) result_json = json.loads(result.read()) updated_user = result_json["updated_user"] if updated_user is None: screen_error_message = "The server didn't recognize you. Try getting a new authkey in the options menu." else: screen_error_message = None except Exception as e: log_error("ERROR: Couldn't send item info to server\n" + traceback.format_exc()) screen_error_message = "ERROR: Couldn't send item info to server, check tracker_log.txt" # Retry to write the state in 10*update_timer (aka 10 sec in write mode) retry_in = 10 # Check the new state at the front of the queue to see if it's time to use it if len(new_states_queue) > 0: (state_timestamp, new_state) = new_states_queue[0] current_timestamp = int(time.time()) if current_timestamp - state_timestamp >= opt.read_delay or opt.read_delay == 0 or state is None: state = new_state new_states_queue.pop(0) drawing_tool.set_window_title_info(updates_queued=len(new_states_queue)) if state is None and screen_error_message is None: if read_from_server: screen_error_message = "Unable to read state from server. Please verify your options setup and tracker_log.txt" # Retry to read the state in 5*update_timer (aka 10 sec in read mode) retry_in = 5 else: screen_error_message = "log.txt for " + opt.game_version + " not found. Make sure you have the right game selected in the options." if screen_error_message is not None: drawing_tool.write_error_message(screen_error_message) else: # We got a state, now we draw it drawing_tool.draw_state(state) # if we're watching someone and they change their game version, it can require us to reset if state and last_game_version != state.game_version: drawing_tool.reset_options() last_game_version = state.game_version drawing_tool.tick() framecount += 1 # Main loop finished; program is exiting drawing_tool.save_window_position() Options().save_options(wdir_prefix + "options.json")
import time from datetime import timedelta from log_parser import LogParser start_time = time.monotonic() parser = LogParser('logs.txt') parser.parse() end_time = time.monotonic() print('Total time:', timedelta(seconds=end_time - start_time))
def run(self): parser = LogParser() while True: for line in Pygtail(self.log_file): record = parser.parse(line) self.stats.record(record)