def log_parser():
db = get_db()
end_time = datetime.now()
start_time = end_time - timedelta(days=7)
lp = LogParser(start_time, end_time)
result = lp.parse()
return render_template('show_job_entries.html', job_entries=result)
def update_measures(m_id, is_new_measure=False):
if m_id is None:
raise AttributeError("Measure id is None")
else:
measures = models.Measures.select().where(models.Measures.measure_id == int(m_id)).order_by(models.Measures.id)
measures_lst = []
measures_lst.append(['Id', 'Temperature Outside', 'Engine Temperature', 'Pressure', 'Voltage', 'Fuel level'])
for m in measures:
measures_lst.append(m.to_list())
lp = LogParser()
if int(is_new_measure) == 1:
lp.parse(truncate_log=True, new_measure=True)
else:
lp.parse(truncate_log=True)
return jsonify(measures_lst)
def parse_all_sessions(files):
logParser = LogParser();
sessions = [];
for file in files:
session = logParser.parse(file)
sessions.append(session)
return sessions
def parse_all_sessions(files):
logParser = LogParser()
sessions = []
for file in files:
session = logParser.parse(file)
sessions.append(session)
return sessions
def test_nginx_log_parsing(self):
log_format = '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$request_body"'
log_type = 'nginx'
parser = LogParser(log_format, log_type)
record = '127.0.0.1 - - [08/Jul/2016:08:45:58 -0400] "POST /place HTTP/1.1" 404 233 "-" "curl/7.38.0" "way"'
parsed = parser.parse(record)
self.assertEqual(parsed, {'request.first_line': 'POST /place HTTP/1.1',
'request.headers.referer': '-',
'request.headers.user-agent': 'curl/7.38.0',
'request.method': 'POST',
'request.protocol': 'HTTP/1.1',
'request.src_ip': '127.0.0.1',
'request.time': '08/Jul/2016:08:45:58 -0400',
'request.uri': '/place',
'request.user': '******',
'request.body': 'way',
'response.body_size': '233',
'response.status': '404'})
def test_apache_log_parsing(self):
log_format = '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i"'
log_type = 'apache'
parser = LogParser(log_format, log_type)
record = '127.0.0.1 - - [12/Jul/2016:14:00:22 +0300] "GET /test HTTP/1.1" 200 10956 "-" "curl/7.47.0"'
parsed = parser.parse(record)
self.assertEqual(parsed, {'request.client_id': '-',
'request.first_line': 'GET /test HTTP/1.1',
'request.headers.referer': '-',
'request.headers.user-agent': 'curl/7.47.0',
'request.method': 'GET',
'request.protocol': 'HTTP/1.1',
'request.src_ip': '127.0.0.1',
'request.time': '[12/Jul/2016:14:00:22 +0300]',
'request.uri': '/test',
'request.user': '******',
'response.size': '10956',
'response.status': '200'})
def main():
parser = argparse.ArgumentParser(description='Convert logs to custom fileformat(now supports json and yaml fileformats)', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('input_file', help='file with input logs', type=argparse.FileType('r'))
parser.add_argument('output_file', help='file with parsed data', type=argparse.FileType('w'))
# default nginx log_format: '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'
# Note: spaces in log_format is important!
parser.add_argument('log_format', help='''logs format. Supports: nginx log_format and apache CustomLog, also you may use custom format''')
parser.add_argument('--log_type', help='type of log file.', choices=['nginx', 'apache', 'custom'], default='nginx')
parser.add_argument('--result_type', help='type of parsed file', choices=['yaml', 'json'], default='json')
parser.add_argument('-f', '--follow', action='store_true', default=False, help='process appended data as the file grows')
args = parser.parse_args()
parser = LogParser(args.log_format, args.log_type)
parsed_logs = []
try:
for record in (tailer.follow(args.input_file) if args.follow else args.input_file):
parsed_logs.append(parser.parse(record))
except KeyboardInterrupt:
print('interrupt received, stopping.')
finally:
if args.result_type == 'yaml':
yaml.dump(parsed_logs, args.output_file, default_flow_style=False)
elif args.result_type == 'json':
json.dump(parsed_logs, args.output_file)
from log_parser import LogParser from cluster import Cluster, NodeFactory from converter import LogsToEventsConverter from events import EventLoop N = 4 WIDTH = HEIGHT = 600 TITLE = "Fast Paxos" LOGFILE = "../../../logs/test.log" factory = NodeFactory((20, 20), 8) cluster = Cluster(TITLE, N, (WIDTH, HEIGHT), factory) loop = EventLoop() logs = LogParser.parse(LOGFILE) converter = LogsToEventsConverter(cluster, loop) converter.convert(logs) loop.run() cluster.close()
def run(self):
""" The main routine which controls everything """
update_notifier = self.check_for_update()
framecount = 0
# Create drawing tool to use to draw everything - it'll create its own screen
drawing_tool = DrawingTool(self.file_prefix)
drawing_tool.set_window_title(update_notifier)
parser = LogParser(self.file_prefix, self.tracker_version)
opt = Options()
log = logging.getLogger("tracker")
event_result = None
state = None
read_from_server = opt.read_from_server
write_to_server = opt.write_to_server
state_version = -1
twitch_username = None
new_states_queue = []
screen_error_message = None
while event_result != Event.DONE:
# Check for events and handle them
event_result = drawing_tool.handle_events()
# A change means the user has (de)activated an option
if opt.read_from_server != read_from_server\
or opt.twitch_name != twitch_username:
twitch_username = opt.twitch_name
read_from_server = opt.read_from_server
new_states_queue = []
# Also restart version count if we go back and forth from log.txt to server
if read_from_server:
state_version = -1
state = None
# show who we are watching in the title bar
drawing_tool.set_window_title(update_notifier, watching_player=twitch_username, updates_queued=len(new_states_queue))
else:
drawing_tool.set_window_title(update_notifier)
if opt.write_to_server and opt.write_to_server != write_to_server:
write_to_server = True
drawing_tool.set_window_title(update_notifier, uploading=True)
if not opt.write_to_server:
write_to_server = False
if opt.read_from_server:
# Change the delay for polling, as we probably don't want to fetch it every second
update_timer = 2
else:
update_timer = self.read_timer
if event_result == Event.OPTIONS_UPDATE:
# By setting the framecount to 0 we ensure we'll refresh the state right away
framecount = 0
screen_error_message = None
# force updates after changing options
if state is not None:
state.modified = True
# Now we re-process the log file to get anything that might have loaded;
# do it every update_timer seconds (making sure to truncate to an integer
# or else it might never mod to 0)
if (framecount % int(Options().framerate_limit * update_timer) == 0):
# Let the parser do his thing and give us a state
if opt.read_from_server:
base_url = opt.trackerserver_url + "/tracker/api/user/" + opt.twitch_name
json_dict = None
try:
json_version = urllib2.urlopen(base_url + "/version").read()
if int(json_version) > state_version:
# FIXME better handling of 404 error ?
json_state = urllib2.urlopen(base_url).read()
json_dict = json.loads(json_state)
new_state = TrackerState.from_json(json_dict)
if new_state is None:
raise Exception
state_version = int(json_version)
new_states_queue.append((state_version, new_state))
drawing_tool.set_window_title(update_notifier, watching_player=twitch_username, updates_queued=len(new_states_queue), read_delay=opt.read_delay)
except Exception:
state = None
log.error("Couldn't load state from server")
import traceback
log.error(traceback.format_exc())
if json_dict is not None:
their_version = ""
if "tracker_version" in json_dict:
their_version = json_dict["tracker_version"]
else:
# this is the only version that can upload to the server but doesn't include a version string
their_version = "0.10-beta1"
if their_version != self.tracker_version:
screen_error_message = "They are using tracker version " + their_version + " but you have " + self.tracker_version
else:
force_draw = state and state.modified
state = parser.parse()
if force_draw:
state.modified = True
if write_to_server and not opt.trackerserver_authkey:
screen_error_message = "Your authkey is blank. Get a new authkey in the options menu and paste it into the authkey text field."
if state is not None and write_to_server and state.modified and screen_error_message is None:
opener = urllib2.build_opener(urllib2.HTTPHandler)
put_url = opt.trackerserver_url + "/tracker/api/update/" + opt.trackerserver_authkey
json_string = json.dumps(state, cls=TrackerStateEncoder, sort_keys=True)
request = urllib2.Request(put_url,
data=json_string)
request.add_header('Content-Type', 'application/json')
request.get_method = lambda: 'PUT'
try:
result = opener.open(request)
result_json = json.loads(result.read())
updated_user = result_json["updated_user"]
if updated_user is None:
screen_error_message = "The server didn't recognize you. Try getting a new authkey in the options menu."
else:
screen_error_message = None
except Exception as e:
import traceback
errmsg = traceback.format_exc()
log.error("ERROR: Couldn't send item info to server")
log.error(errmsg)
screen_error_message = "ERROR: Couldn't send item info to server, check tracker_log.txt"
# check the new state at the front of the queue to see if it's time to use it
if len(new_states_queue) > 0:
(state_timestamp, new_state) = new_states_queue[0]
current_timestamp = int(time.time())
if current_timestamp - state_timestamp >= opt.read_delay or state is None:
state = new_state
new_states_queue.pop(0)
drawing_tool.set_window_title(update_notifier, watching_player=twitch_username, updates_queued=len(new_states_queue), read_delay=opt.read_delay)
if state is None and screen_error_message is None:
if read_from_server:
screen_error_message = "Unable to read state from server. Please verify your options setup and tracker_log.txt"
else:
screen_error_message = "log.txt not found. Put the RebirthItemTracker folder inside the isaac folder, next to log.txt"
if screen_error_message is not None:
drawing_tool.write_error_message(screen_error_message)
else:
# We got a state, now we draw it
drawing_tool.draw_state(state)
drawing_tool.tick()
framecount += 1
# main loop finished. program is exiting
drawing_tool.save_window_position()
def run(self):
""" The main routine which controls everything """
framecount = 0
# Create drawing tool to use to draw everything - it'll create its own screen
drawing_tool = DrawingTool(wdir_prefix)
drawing_tool.set_window_title_info(
update_notifier=(" v" + self.tracker_version))
opt = Options()
parser = LogParser(wdir_prefix, self.tracker_version, LogFinder())
event_result = None
state = None
custom_title_enabled = opt.custom_title_enabled
read_from_server = opt.read_from_server
write_to_server = opt.write_to_server
game_version = opt.game_version
state_version = -1
twitch_username = None
new_states_queue = []
screen_error_message = None
retry_in = 0
update_timer = opt.log_file_check_seconds
last_game_version = None
while event_result != Event.DONE:
# Check for events and handle them
event_result = drawing_tool.handle_events()
# The user checked or unchecked the "Custom Title Enabled" checkbox
if opt.custom_title_enabled != custom_title_enabled:
custom_title_enabled = opt.custom_title_enabled
drawing_tool.update_window_title()
# The user started or stopped watching someone from the server (or they started watching a new person from the server)
if opt.read_from_server != read_from_server or opt.twitch_name != twitch_username:
twitch_username = opt.twitch_name
read_from_server = opt.read_from_server
new_states_queue = []
# Also restart version count if we go back and forth from log.txt to server
if read_from_server:
state_version = -1
state = None
# Change the delay for polling, as we probably don't want to fetch it every second
update_timer_override = 2
# Show who we are watching in the title bar
drawing_tool.set_window_title_info(
watching=True,
watching_player=twitch_username,
updates_queued=len(new_states_queue))
else:
drawing_tool.set_window_title_info(watching=False)
update_timer_override = 0
# The user started or stopped broadcasting to the server
if opt.write_to_server != write_to_server:
write_to_server = opt.write_to_server
drawing_tool.set_window_title_info(
uploading=opt.write_to_server)
if opt.game_version != game_version:
parser.reset()
game_version = opt.game_version
# Force refresh state if we updated options or if we need to retry
# to contact the server.
if (event_result == Event.OPTIONS_UPDATE
or (screen_error_message is not None and retry_in == 0)):
# By setting the framecount to 0 we ensure we'll refresh the state right away
framecount = 0
screen_error_message = None
retry_in = 0
# Force updates after changing options
if state is not None:
state.modified = True
# normally we check for updates based on how the option is set
# when doing network stuff, this can be overridden
update_delay = opt.log_file_check_seconds
if update_timer_override != 0:
update_delay = update_timer_override
# Now we re-process the log file to get anything that might have loaded;
# do it every update_timer seconds (making sure to truncate to an integer
# or else it might never mod to 0)
frames_between_checks = int(Options().framerate_limit *
update_delay)
if frames_between_checks <= 0:
frames_between_checks = 1
if framecount % frames_between_checks == 0:
if retry_in != 0:
retry_in -= 1
# Let the parser do his thing and give us a state
if opt.read_from_server:
base_url = opt.trackerserver_url + "/tracker/api/user/" + opt.twitch_name
json_dict = None
try:
json_version = urllib.request.urlopen(
base_url + "/version").read()
if int(json_version) > state_version:
# FIXME better handling of 404 error ?
json_state = urllib.request.urlopen(
base_url).read()
json_dict = json.loads(json_state)
new_state = TrackerState.from_json(json_dict)
if new_state is None:
raise Exception("server gave us empty state")
state_version = int(json_version)
new_states_queue.append((state_version, new_state))
drawing_tool.set_window_title_info(
updates_queued=len(new_states_queue))
except Exception:
state = None
log_error("Couldn't load state from server\n" +
traceback.format_exc())
if json_dict is not None:
if "tracker_version" in json_dict:
their_version = json_dict["tracker_version"]
else:
# This is the only version that can upload to the server but doesn't include a version string
their_version = "0.10-beta1"
if their_version != self.tracker_version:
screen_error_message = "They are using tracker version " + their_version + " but you have " + self.tracker_version
else:
force_draw = state and state.modified
state = parser.parse()
if force_draw and state is not None:
state.modified = True
if write_to_server and not opt.trackerserver_authkey:
screen_error_message = "Your authkey is blank. Get a new authkey in the options menu and paste it into the authkey text field."
if state is not None and write_to_server and state.modified and screen_error_message is None:
opener = urllib.request.build_opener(
urllib.request.HTTPHandler)
put_url = opt.trackerserver_url + "/tracker/api/update/" + opt.trackerserver_authkey
json_string = json.dumps(
state, cls=TrackerStateEncoder,
sort_keys=True).encode("utf-8")
request = urllib.request.Request(put_url,
data=json_string)
request.add_header('Content-Type', 'application/json')
request.get_method = lambda: 'PUT'
try:
result = opener.open(request)
result_json = json.loads(result.read())
updated_user = result_json["updated_user"]
if updated_user is None:
screen_error_message = "The server didn't recognize you. Try getting a new authkey in the options menu."
else:
screen_error_message = None
except Exception as e:
log_error(
"ERROR: Couldn't send item info to server\n" +
traceback.format_exc())
screen_error_message = "ERROR: Couldn't send item info to server, check tracker_log.txt"
# Retry to write the state in 10*update_timer (aka 10 sec in write mode)
retry_in = 10
# Check the new state at the front of the queue to see if it's time to use it
if len(new_states_queue) > 0:
(state_timestamp, new_state) = new_states_queue[0]
current_timestamp = int(time.time())
if current_timestamp - state_timestamp >= opt.read_delay or opt.read_delay == 0 or state is None:
state = new_state
new_states_queue.pop(0)
drawing_tool.set_window_title_info(
updates_queued=len(new_states_queue))
if state is None and screen_error_message is None:
if read_from_server:
screen_error_message = "Unable to read state from server. Please verify your options setup and tracker_log.txt"
# Retry to read the state in 5*update_timer (aka 10 sec in read mode)
retry_in = 5
else:
screen_error_message = "log.txt for " + opt.game_version + " not found. Make sure you have the right game selected in the options."
if screen_error_message is not None:
drawing_tool.write_error_message(screen_error_message)
else:
# We got a state, now we draw it
drawing_tool.draw_state(state, framecount)
# if we're watching someone and they change their game version, it can require us to reset
if state and last_game_version != state.game_version:
drawing_tool.reset_options()
last_game_version = state.game_version
drawing_tool.tick()
framecount += 1
# Main loop finished; program is exiting
drawing_tool.save_window_position()
Options().save_options(wdir_prefix + "options.json")
#/usr/bin/python3 #Python script to run cron task daily to create database log entries from cron_monitor import app from log_parser import LogParser from datetime import datetime,timedelta from flask_mysqldb import MySQL from flask import Flask, g import config end_time = datetime.now() start_time = end_time - timedelta(days=1) ctx = app.app_context() ctx.push() app.config['MYSQL_USER'] = config.MYSQL_DATABASE_USER app.config['MYSQL_PASSWORD'] = config.MYSQL_DATABASE_PASSWORD app.config['MYSQL_DB'] = config.MYSQL_DATABASE_DB app.config['MYSQL_HOST'] = config.MYSQL_DATABASE_HOST app.config['MYSQL_CURSORCLASS'] = 'DictCursor' mysql = MySQL(app) g.mysql_db = mysql.connection lp = LogParser(start_time, end_time) result = lp.parse()
from log_parser import LogParser
from feature_extractor import FeatureExtractor
from clustering import Clustering
from idf import IDF
parser = LogParser('../../data/HDFS_2K.log')
tagged_events, log_sequences = parser.parse()
extractor = FeatureExtractor(log_sequences, list(tagged_events.keys()))
log_sequences = extractor.extract()
clustering = Clustering(log_sequences, tagged_events)
cluster_ids, cluster_values, silhouette = clustering.cluster()
for cluster_value in cluster_values:
if cluster_value['num_possible_abnormal_events'] != 0:
print(cluster_value)
print()
#print ("El coeficiente de silueta es =", silhouette)
def run(self):
""" The main routine which controls everything """
framecount = 0
# Create drawing tool to use to draw everything - it'll create its own screen
drawing_tool = DrawingTool(wdir_prefix)
drawing_tool.set_window_title_info(update_notifier=(" v" + self.tracker_version))
opt = Options()
parser = LogParser(wdir_prefix, self.tracker_version, LogFinder())
event_result = None
state = None
custom_title_enabled = opt.custom_title_enabled
read_from_server = opt.read_from_server
write_to_server = opt.write_to_server
game_version = opt.game_version
state_version = -1
twitch_username = None
new_states_queue = []
screen_error_message = None
retry_in = 0
update_timer = opt.log_file_check_seconds
last_game_version = None
while event_result != Event.DONE:
# Check for events and handle them
event_result = drawing_tool.handle_events()
# The user checked or unchecked the "Custom Title Enabled" checkbox
if opt.custom_title_enabled != custom_title_enabled:
custom_title_enabled = opt.custom_title_enabled
drawing_tool.update_window_title()
# The user started or stopped watching someone from the server (or they started watching a new person from the server)
if opt.read_from_server != read_from_server or opt.twitch_name != twitch_username:
twitch_username = opt.twitch_name
read_from_server = opt.read_from_server
new_states_queue = []
# Also restart version count if we go back and forth from log.txt to server
if read_from_server:
state_version = -1
state = None
# Change the delay for polling, as we probably don't want to fetch it every second
update_timer_override = 2
# Show who we are watching in the title bar
drawing_tool.set_window_title_info(watching=True, watching_player=twitch_username, updates_queued=len(new_states_queue))
else:
drawing_tool.set_window_title_info(watching=False)
update_timer_override = 0
# The user started or stopped broadcasting to the server
if opt.write_to_server != write_to_server:
write_to_server = opt.write_to_server
drawing_tool.set_window_title_info(uploading=opt.write_to_server)
if opt.game_version != game_version:
parser.reset()
game_version = opt.game_version
# Force refresh state if we updated options or if we need to retry
# to contact the server.
if (event_result == Event.OPTIONS_UPDATE or
(screen_error_message is not None and retry_in == 0)):
# By setting the framecount to 0 we ensure we'll refresh the state right away
framecount = 0
screen_error_message = None
retry_in = 0
# Force updates after changing options
if state is not None:
state.modified = True
# normally we check for updates based on how the option is set
# when doing network stuff, this can be overridden
update_delay = opt.log_file_check_seconds
if update_timer_override != 0:
update_delay = update_timer_override
# Now we re-process the log file to get anything that might have loaded;
# do it every update_timer seconds (making sure to truncate to an integer
# or else it might never mod to 0)
frames_between_checks = int(Options().framerate_limit * update_delay)
if frames_between_checks <= 0:
frames_between_checks = 1
if framecount % frames_between_checks == 0:
if retry_in != 0:
retry_in -= 1
# Let the parser do his thing and give us a state
if opt.read_from_server:
base_url = opt.trackerserver_url + "/tracker/api/user/" + opt.twitch_name
json_dict = None
try:
json_version = urllib.request.urlopen(base_url + "/version").read()
if int(json_version) > state_version:
# FIXME better handling of 404 error ?
json_state = urllib.request.urlopen(base_url).read()
json_dict = json.loads(json_state, "utf-8")
new_state = TrackerState.from_json(json_dict)
if new_state is None:
raise Exception("server gave us empty state")
state_version = int(json_version)
new_states_queue.append((state_version, new_state))
drawing_tool.set_window_title_info(updates_queued=len(new_states_queue))
except Exception:
state = None
log_error("Couldn't load state from server\n" + traceback.format_exc())
if json_dict is not None:
if "tracker_version" in json_dict:
their_version = json_dict["tracker_version"]
else:
# This is the only version that can upload to the server but doesn't include a version string
their_version = "0.10-beta1"
if their_version != self.tracker_version:
screen_error_message = "They are using tracker version " + their_version + " but you have " + self.tracker_version
else:
force_draw = state and state.modified
state = parser.parse()
if force_draw:
state.modified = True
if write_to_server and not opt.trackerserver_authkey:
screen_error_message = "Your authkey is blank. Get a new authkey in the options menu and paste it into the authkey text field."
if state is not None and write_to_server and state.modified and screen_error_message is None:
opener = urllib.request.build_opener(urllib.request.HTTPHandler)
put_url = opt.trackerserver_url + "/tracker/api/update/" + opt.trackerserver_authkey
json_string = json.dumps(state, cls=TrackerStateEncoder, sort_keys=True)
request = urllib.request.Request(put_url,
data=json_string)
request.add_header('Content-Type', 'application/json')
request.get_method = lambda: 'PUT'
try:
result = opener.open(request)
result_json = json.loads(result.read())
updated_user = result_json["updated_user"]
if updated_user is None:
screen_error_message = "The server didn't recognize you. Try getting a new authkey in the options menu."
else:
screen_error_message = None
except Exception as e:
log_error("ERROR: Couldn't send item info to server\n" + traceback.format_exc())
screen_error_message = "ERROR: Couldn't send item info to server, check tracker_log.txt"
# Retry to write the state in 10*update_timer (aka 10 sec in write mode)
retry_in = 10
# Check the new state at the front of the queue to see if it's time to use it
if len(new_states_queue) > 0:
(state_timestamp, new_state) = new_states_queue[0]
current_timestamp = int(time.time())
if current_timestamp - state_timestamp >= opt.read_delay or opt.read_delay == 0 or state is None:
state = new_state
new_states_queue.pop(0)
drawing_tool.set_window_title_info(updates_queued=len(new_states_queue))
if state is None and screen_error_message is None:
if read_from_server:
screen_error_message = "Unable to read state from server. Please verify your options setup and tracker_log.txt"
# Retry to read the state in 5*update_timer (aka 10 sec in read mode)
retry_in = 5
else:
screen_error_message = "log.txt for " + opt.game_version + " not found. Make sure you have the right game selected in the options."
if screen_error_message is not None:
drawing_tool.write_error_message(screen_error_message)
else:
# We got a state, now we draw it
drawing_tool.draw_state(state)
# if we're watching someone and they change their game version, it can require us to reset
if state and last_game_version != state.game_version:
drawing_tool.reset_options()
last_game_version = state.game_version
drawing_tool.tick()
framecount += 1
# Main loop finished; program is exiting
drawing_tool.save_window_position()
Options().save_options(wdir_prefix + "options.json")
import time
from datetime import timedelta
from log_parser import LogParser
start_time = time.monotonic()
parser = LogParser('logs.txt')
parser.parse()
end_time = time.monotonic()
print('Total time:', timedelta(seconds=end_time - start_time))
def run(self):
parser = LogParser()
while True:
for line in Pygtail(self.log_file):
record = parser.parse(line)
self.stats.record(record)
