def get_result_group_names(self, request_datetime):
"""Only today/yesterday/last interval are valid datetime"""
request_date = request_datetime.date()
today = datetime.date.today()
yesterday = datetime.date.today() - datetime.timedelta(days=1)
result = []
if total_seconds(datetime.datetime.now() - request_datetime) < REALTIME_UPDATE_INTERVAL:
result.append('last_interval')
if request_date == today:
result.append(today.isoformat())
elif request_date == yesterday:
result.append(yesterday.isoformat())
return result
def get_result_group_names(self, request_datetime):
"""Only today/yesterday/last interval are valid datetime"""
request_date = request_datetime.date()
today = datetime.date.today()
yesterday = datetime.date.today() - datetime.timedelta(days=1)
result = []
if total_seconds(datetime.datetime.now() -
request_datetime) < REALTIME_UPDATE_INTERVAL:
result.append('last_interval')
if request_date == today:
result.append(today.isoformat())
elif request_date == yesterday:
result.append(yesterday.isoformat())
return result
def start(args):
# Load config file
config = ConfigObj(infile=args.config.name)
data_dir = config['data_dir']
uwsgi_log_path = config['uwsgi_log_path']
min_msecs = int(config.get('min_msecs', DEFAULT_MIN_MSECS))
url_file = config.get('url_file')
# Load custom url rules
url_rules = []
if url_file:
with open(url_file, 'r') as fp:
url_rules = parse_url_rules(fp)
html_dir = os.path.join(data_dir, 'html')
db_dir = os.path.join(data_dir, 'data')
makedir_if_none_exists(html_dir)
makedir_if_none_exists(db_dir)
save_point = SavePoint(db_dir)
last_log_datetime = save_point.get_last_datetime() or \
(datetime.datetime.now() - datetime.timedelta(seconds=REALTIME_UPDATE_INTERVAL))
logger.info('Start from last savepoint, last_log_datetime: %s' % last_log_datetime)
last_update_datetime = None
url_classifier = URLClassifier(user_defined_rules=url_rules)
analyzer = RealtimeLogAnalyzer(url_classifier=url_classifier, min_msecs=min_msecs,
start_from_datetime=last_log_datetime)
file_tailer = Tailer(uwsgi_log_path)
html_render = HTMLRender(html_dir, domain=config.get('domain'))
# Listen INT/TERM signal
def gracefully_exit(*args):
logger.info('Sinal received, exit.')
file_tailer.stop_follow()
signal.signal(signal.SIGINT, gracefully_exit)
for line in file_tailer:
# Analyze line
if line != no_new_line:
analyzer.analyze_line(line)
now = datetime.datetime.now()
if not file_tailer.trailing:
continue
if last_update_datetime and \
total_seconds(now - last_update_datetime) < REALTIME_UPDATE_INTERVAL:
continue
# Render HTML file when:
# - file_tailer reaches end of file.
# - last_update_datetime if over one `interval` from now
# Render latest interval HTML file
html_render.render_requests_data_to_html(analyzer.get_data('last_interval'),
'latest_5mins.html', context={'datetime_range': 'Last 5 minutes'})
analyzer.clean_data_by_key('last_interval')
for date in analyzer.data.keys():
day_requests_data = RequestsData(date, db_dir)
merge_requests_data_to(day_requests_data.data, analyzer.get_data(date))
# Render to HTML file
html_render.render_requests_data_to_html(day_requests_data.data,
'day_%s.html' % date, context={'datetime_range': date})
# Save data to pickle file
day_requests_data.save()
# Reset Everything
analyzer.clean_data_by_key(date)
update_html_symlink(html_dir)
last_update_datetime = now
if analyzer.last_analyzed_datetime:
save_point.set_last_datetime(analyzer.last_analyzed_datetime)
save_point.save()
def start(args):
# Load config file
config = ConfigObj(infile=args.config.name)
data_dir = config['data_dir']
uwsgi_log_path = config['uwsgi_log_path']
min_msecs = int(config.get('min_msecs', DEFAULT_MIN_MSECS))
url_file = config.get('url_file')
# Load custom url rules
url_rules = []
if url_file:
with open(url_file, 'r') as fp:
url_rules = parse_url_rules(fp)
html_dir = os.path.join(data_dir, 'html')
db_dir = os.path.join(data_dir, 'data')
makedir_if_none_exists(html_dir)
makedir_if_none_exists(db_dir)
save_point = SavePoint(db_dir)
last_log_datetime = save_point.get_last_datetime() or \
(datetime.datetime.now() - datetime.timedelta(seconds=REALTIME_UPDATE_INTERVAL))
logger.info('Start from last savepoint, last_log_datetime: %s' %
last_log_datetime)
last_update_datetime = None
url_classifier = URLClassifier(user_defined_rules=url_rules)
analyzer = RealtimeLogAnalyzer(url_classifier=url_classifier,
min_msecs=min_msecs,
start_from_datetime=last_log_datetime)
file_tailer = Tailer(uwsgi_log_path)
html_render = HTMLRender(html_dir, domain=config.get('domain'))
# Listen INT/TERM signal
def gracefully_exit(*args):
logger.info('Sinal received, exit.')
file_tailer.stop_follow()
signal.signal(signal.SIGINT, gracefully_exit)
for line in file_tailer:
# Analyze line
if line != no_new_line:
analyzer.analyze_line(line)
now = datetime.datetime.now()
if not file_tailer.trailing:
continue
if last_update_datetime and \
total_seconds(now - last_update_datetime) < REALTIME_UPDATE_INTERVAL:
continue
# Render HTML file when:
# - file_tailer reaches end of file.
# - last_update_datetime if over one `interval` from now
# Render latest interval HTML file
html_render.render_requests_data_to_html(
analyzer.get_data('last_interval'),
'latest_5mins.html',
context={'datetime_range': 'Last 5 minutes'})
analyzer.clean_data_by_key('last_interval')
for date in analyzer.data.keys():
day_requests_data = RequestsData(date, db_dir)
merge_requests_data_to(day_requests_data.data,
analyzer.get_data(date))
# Render to HTML file
html_render.render_requests_data_to_html(
day_requests_data.data,
'day_%s.html' % date,
context={'datetime_range': date})
# Save data to pickle file
day_requests_data.save()
# Reset Everything
analyzer.clean_data_by_key(date)
update_html_symlink(html_dir)
last_update_datetime = now
if analyzer.last_analyzed_datetime:
save_point.set_last_datetime(analyzer.last_analyzed_datetime)
save_point.save()
