def collect_user_sessions(self, from_date=None, to_date=None, inclusive_to=False, whitelist=None):
t_start = timeutils.date_str_to_epoch(from_date) or float('-inf')
t_end = timeutils.date_str_to_epoch(to_date or timeutils.latest_log_date_str(self.logs_dir))
# TODO inclusive_to
user_sessions = {} # uuid -> [sessions]
def crop_and_add(uuid, t_from, t_to, name):
if whitelist and uuid not in whitelist:
return
# crop to interval
t_from = max(t_from, t_start)
t_to = min(t_to, t_end)
if t_from >= t_to:
return
# add data to collection
if uuid not in user_sessions:
user_sessions[uuid] = []
user_sessions[uuid].append([uuid, t_from, t_to, name])
times, online = self.collect_data(from_date, to_date, inclusive_to)
for uuid, t_from, t_to, name in times:
crop_and_add(uuid, t_from, t_to, name)
for name, sess_begin in online.items():
uuid, t_from = sess_begin[:2]
crop_and_add(uuid, t_from, t_end, name)
return user_sessions
def get_timeline_data(logs, from_date=None, to_date=None):
lines = list(logs.collect_user_sessions(from_date, to_date).values())
uptimes = list(logs.collect_uptimes(from_date, to_date))
t_start = int(min(uptimes[0][0] if uptimes else float('inf'),
timeutils.date_str_to_epoch(from_date) \
or min(s[1] for sessions in lines for s in sessions)))
t_end = int(max(uptimes[-1][1] if uptimes else float('-inf'),
timeutils.date_str_to_epoch(to_date) \
or max(s[2] for sessions in lines for s in sessions)))
return t_start, t_end, lines, uptimes
def collect_uptimes(self, from_date=None, to_date=None, inclusive_to=False):
from_day, to_day, inclusive_to = self.date_to_log_day(from_date, to_date, inclusive_to)
first_event = None
for log_file in self.read_interval_iter(from_day, to_day, inclusive_to):
if log_file.started and log_file.prev_log and not log_file.prev_log.stopped:
log_file.prev_log.read_log()
if not log_file.prev_log.stopped:
yield (first_event or timeutils.date_str_to_epoch(from_date),
log_file.prev_log.last_event)
first_event = None
if not first_event:
first_event = log_file.first_event if log_file.started \
else timeutils.date_str_to_epoch(from_date)
if log_file.stopped:
yield (first_event, log_file.last_event)
first_event = None
if first_event:
yield (first_event, timeutils.date_str_to_epoch(
to_date or timeutils.latest_log_date_str(self.logs_dir)))
def read_interval_iter(self, from_log=None, to_log=None, inclusive_to=False, force_convert=False):
"""
Returns an iterator over the LogFiles that lie between `from_log` and `to_log`.
These are also read and converted if necessary.
"""
logger.debug('read_interval: from_log=%s to_log=%s', from_log, to_log)
prev_log = None
for name_tuple in self.iter_log_name_tuples_between(from_log, to_log, inclusive_to):
log_file = self.log_files[name_tuple]
log_file.read_log(force_convert)
yield log_file
prev_log = log_file
if not to_log or not prev_log or prev_log.last_event < timeutils.date_str_to_epoch(to_log):
log_file = self.log_files['latest']
log_file.read_log(force_convert)
yield log_file
def convert_log(self, force_convert=False):
self.peek_start()
if self.prev_log:
self.prev_log.read_log(force_convert)
self.online = self.prev_log.online
if self.started:
# TODO crash, update previous yaml instead?
for name in list(self.online.keys())[:]:
self.found_leave(-1, self.prev_log.last_event, name, 'Server Crash')
logger.info('Server started, leaving %s at %s' % (name, self.log_name))
elif not self.started:
raise ValueError('First log and no server start')
if self.log_name == 'latest': logger.debug('Converting latest')
else: logger.info('Converting %s', self.log_name)
if self.log_name == 'latest':
log_file = open(self.log_path, 'rb')
else:
log_file = gzip.open(self.log_path + '.gz', 'rb')
with log_file:
line_no = 0
for line in log_file:
if b' [@' == line[32:35]:
continue # ignore command block activity
logger.debug('[Log %s@%i] %s', self.log_name, line_no, line)
line_no += 1
line = line.decode('latin_1')
time_match = self.RE_TIME.match(line)
if time_match: # only look at lines with a timestamp
seconds = timeutils.date_str_to_epoch(self.day_str, line[1:9])
if self.first_event is None:
self.first_event = seconds
if self.last_event is None or self.last_event < seconds:
self.last_event = seconds
line_after_time = line[11:] # strip off the `[12:34:56] `
for regex, action in log_actions:
match = regex.match(line_after_time)
if match:
args = match.groups()
logger.debug('Action: %s (%2i %i) %s: %s' % (self.log_name, line_no, seconds, action.__name__, args))
action(self, line_no, seconds, *args)
break
if self.stopped:
for name in list(self.online.keys())[:]:
self.found_leave(-1, self.last_event, name, 'Server Stop')
logger.info('Server stopped, leaving %s at %s' % (name, self.log_name))
self.write_yaml()
def peek_start(self):
if self.started is not None:
# already peeked
logger.warn('peek_start: Already peeked')
return self.started
if self.log_name == 'latest':
log_file = open(self.log_path, 'rb')
else:
log_file = gzip.open(self.log_path + '.gz', 'rb')
with log_file:
for line in log_file:
line = line.decode('latin_1')
self.started = bool(self.RE_START.match(line))
if self.started:
self.first_event = timeutils.date_str_to_epoch(self.day_str, line[1:9])
logger.debug('peek_start: In log: %s', self.started)
return self.started
self.started = False # empty log or no start
logger.error('peek_start: Empty log')
return self.started
def test_date_str_to_epoch(self):
self.assertEqual(1426114800, timeutils.date_str_to_epoch('2015-03-12'))
self.assertEqual(1426114800, timeutils.date_str_to_epoch('2015-03-12 00:00:00'))
self.assertEqual(1426114800, timeutils.date_str_to_epoch('2015-03-12', '00:00:00'))
self.assertEqual(1426119001, timeutils.date_str_to_epoch('2015-03-12 01:10:01'))