def get_result_group_names(self, request_datetime): """Only today/yesterday/last interval are valid datetime""" request_date = request_datetime.date() today = datetime.date.today() yesterday = datetime.date.today() - datetime.timedelta(days=1) result = [] if total_seconds(datetime.datetime.now() - request_datetime) < REALTIME_UPDATE_INTERVAL: result.append('last_interval') if request_date == today: result.append(today.isoformat()) elif request_date == yesterday: result.append(yesterday.isoformat()) return result
def start(args): # Load config file config = ConfigObj(infile=args.config.name) data_dir = config['data_dir'] uwsgi_log_path = config['uwsgi_log_path'] min_msecs = int(config.get('min_msecs', DEFAULT_MIN_MSECS)) url_file = config.get('url_file') # Load custom url rules url_rules = [] if url_file: with open(url_file, 'r') as fp: url_rules = parse_url_rules(fp) html_dir = os.path.join(data_dir, 'html') db_dir = os.path.join(data_dir, 'data') makedir_if_none_exists(html_dir) makedir_if_none_exists(db_dir) save_point = SavePoint(db_dir) last_log_datetime = save_point.get_last_datetime() or \ (datetime.datetime.now() - datetime.timedelta(seconds=REALTIME_UPDATE_INTERVAL)) logger.info('Start from last savepoint, last_log_datetime: %s' % last_log_datetime) last_update_datetime = None url_classifier = URLClassifier(user_defined_rules=url_rules) analyzer = RealtimeLogAnalyzer(url_classifier=url_classifier, min_msecs=min_msecs, start_from_datetime=last_log_datetime) file_tailer = Tailer(uwsgi_log_path) html_render = HTMLRender(html_dir, domain=config.get('domain')) # Listen INT/TERM signal def gracefully_exit(*args): logger.info('Sinal received, exit.') file_tailer.stop_follow() signal.signal(signal.SIGINT, gracefully_exit) for line in file_tailer: # Analyze line if line != no_new_line: analyzer.analyze_line(line) now = datetime.datetime.now() if not file_tailer.trailing: continue if last_update_datetime and \ total_seconds(now - last_update_datetime) < REALTIME_UPDATE_INTERVAL: continue # Render HTML file when: # - file_tailer reaches end of file. # - last_update_datetime if over one `interval` from now # Render latest interval HTML file html_render.render_requests_data_to_html(analyzer.get_data('last_interval'), 'latest_5mins.html', context={'datetime_range': 'Last 5 minutes'}) analyzer.clean_data_by_key('last_interval') for date in analyzer.data.keys(): day_requests_data = RequestsData(date, db_dir) merge_requests_data_to(day_requests_data.data, analyzer.get_data(date)) # Render to HTML file html_render.render_requests_data_to_html(day_requests_data.data, 'day_%s.html' % date, context={'datetime_range': date}) # Save data to pickle file day_requests_data.save() # Reset Everything analyzer.clean_data_by_key(date) update_html_symlink(html_dir) last_update_datetime = now if analyzer.last_analyzed_datetime: save_point.set_last_datetime(analyzer.last_analyzed_datetime) save_point.save()
def start(args): # Load config file config = ConfigObj(infile=args.config.name) data_dir = config['data_dir'] uwsgi_log_path = config['uwsgi_log_path'] min_msecs = int(config.get('min_msecs', DEFAULT_MIN_MSECS)) url_file = config.get('url_file') # Load custom url rules url_rules = [] if url_file: with open(url_file, 'r') as fp: url_rules = parse_url_rules(fp) html_dir = os.path.join(data_dir, 'html') db_dir = os.path.join(data_dir, 'data') makedir_if_none_exists(html_dir) makedir_if_none_exists(db_dir) save_point = SavePoint(db_dir) last_log_datetime = save_point.get_last_datetime() or \ (datetime.datetime.now() - datetime.timedelta(seconds=REALTIME_UPDATE_INTERVAL)) logger.info('Start from last savepoint, last_log_datetime: %s' % last_log_datetime) last_update_datetime = None url_classifier = URLClassifier(user_defined_rules=url_rules) analyzer = RealtimeLogAnalyzer(url_classifier=url_classifier, min_msecs=min_msecs, start_from_datetime=last_log_datetime) file_tailer = Tailer(uwsgi_log_path) html_render = HTMLRender(html_dir, domain=config.get('domain')) # Listen INT/TERM signal def gracefully_exit(*args): logger.info('Sinal received, exit.') file_tailer.stop_follow() signal.signal(signal.SIGINT, gracefully_exit) for line in file_tailer: # Analyze line if line != no_new_line: analyzer.analyze_line(line) now = datetime.datetime.now() if not file_tailer.trailing: continue if last_update_datetime and \ total_seconds(now - last_update_datetime) < REALTIME_UPDATE_INTERVAL: continue # Render HTML file when: # - file_tailer reaches end of file. # - last_update_datetime if over one `interval` from now # Render latest interval HTML file html_render.render_requests_data_to_html( analyzer.get_data('last_interval'), 'latest_5mins.html', context={'datetime_range': 'Last 5 minutes'}) analyzer.clean_data_by_key('last_interval') for date in analyzer.data.keys(): day_requests_data = RequestsData(date, db_dir) merge_requests_data_to(day_requests_data.data, analyzer.get_data(date)) # Render to HTML file html_render.render_requests_data_to_html( day_requests_data.data, 'day_%s.html' % date, context={'datetime_range': date}) # Save data to pickle file day_requests_data.save() # Reset Everything analyzer.clean_data_by_key(date) update_html_symlink(html_dir) last_update_datetime = now if analyzer.last_analyzed_datetime: save_point.set_last_datetime(analyzer.last_analyzed_datetime) save_point.save()