def test_main(self):
config = dict(REPORT_SIZE=1000,
REPORT_DIR="./test/reports",
LOG_DIR="./test",
LEVEL_PARSE=50,
LOGGING_LEVEL=logging.INFO,
LOGGING_TO_FILE='./test/log_analyzer.log')
main(config)
res1 = os.path.exists('./test/log_analyzer.log')
with open('./test/log_analyzer.log', 'r') as f:
res2 = f.readlines()[-1][28:]
res3 = os.path.exists('./test/reports/report-2017.06.30.html')
main(config)
with open('./test/log_analyzer.log', 'r') as f:
res4 = f.readlines()[-1][28:]
shutil.rmtree('./test/reports')
self.assertEquals(res1, True)
path_report = os.path.join(os.path.abspath(os.path.dirname(__file__)),
config['REPORT_DIR'])
self.assertEquals(
res2,
'render_report: OK. Report file: ./test/reports/report-2017.06.30.html\n'
)
self.assertEquals(res3, True)
self.assertEquals(
res4, 'the file:nginx-access-ui.log-20170630 already processed\n')
def test_conf_not_found(self):
"""Если файла конфигурации не существует, то падает с ошибкой"""
config_file = os.path.join(self.config_dir, "not_existing_config.json")
with self.assertRaises(FileNotFoundError) as file_error:
with self.assertRaises(SystemExit) as sys_exit:
log_analyzer.main(TEST_CONFIG, config_file)
self.assertNotEqual(sys_exit.exception.code, OK_EXIT_CODE)
def test_config_is_not_valid(self):
"""Если файл конфигурации не парсится, то падает с ошибкой"""
cfg_path = os.path.join(self.config_dir, "cfg.json")
with open(cfg_path, "w") as cfg:
cfg.write("invalid config format")
with self.assertRaises(json.decoder.JSONDecodeError) as file_error:
with self.assertRaises(SystemExit) as sys_exit:
log_analyzer.main(TEST_CONFIG, cfg_path)
self.assertNotEqual(sys_exit.exception.code, OK_EXIT_CODE)
def test_case_1_log_file_created(self):
case_name, config = self.cases[0]
# close previous log handler
log = logging.getLogger()
for hdlr in log.handlers:
hdlr.close()
log.removeHandler(hdlr)
log_analyzer.setup_logging(config['LOG_FILE'])
log_analyzer.main(config)
# check that log file created
self.assertTrue(os.path.isfile(config['LOG_FILE']),
msg="Log file not been created in {0}".format(
config['LOG_FILE']))
def test_when_failure_main(self):
"""
Проверка выходных результатов при неудачном завершении скрипта, вызванное
низким значением FAIL_PERC в файле конфига test_failure_config.txt. Проверяется
создание выходного лога, а также его содержания.
"""
main(['--config', './tests/test_failure_config.txt'])
current_path = os.path.realpath(__file__)
os.chdir(os.path.dirname(current_path))
self.assertTrue(
os.path.exists(r'./test_failure_logs/test_failure_log.log'))
with open(r'./test_failure_logs/test_failure_log.log') as f:
log_text = f.readlines()
self.assertTrue('что выше допустимого' in log_text[-2])
def msghndlr_statistics(msg):
chat_id = msg.chat.id
log(chat_id, "statistics")
username = msg.from_user.username
if username in admin_usernames:
try:
log_analyzer.main(plot=True, pretty_print=False)
with open(log_analyze_addr, "rb") as f:
bot.send_chat_action(chat_id, "upload_photo")
bot.send_photo(chat_id, f)
except Exception as ex:
log(None, ex)
bot.send_message(chat_id, ui.get_message("err500"))
else:
bot.send_message(chat_id, ui.get_message("err403"))
start(chat_id)
def test_case_1_report_file_created_with_correct_date(self):
case_name, config = self.cases[0]
# close previous log handler
log = logging.getLogger()
for hdlr in log.handlers:
hdlr.close()
log.removeHandler(hdlr)
log_analyzer.setup_logging(config['LOG_FILE'])
log_analyzer.main(config)
# check that report file created
self.assertTrue(os.path.isfile(
os.path.join(config['REPORT_DIR'], 'report-2017.06.29.html')),
msg="Report file not been created in {0}".format(
config['REPORT_DIR']))
def test_case_2_rise_not_founded_log_files(self):
case_name, config = self.cases[1]
# close previous log handler
log = logging.getLogger()
for hdlr in log.handlers:
hdlr.close()
log.removeHandler(hdlr)
log_analyzer.setup_logging(config['LOG_FILE'])
log_analyzer.main(config)
with open(config['LOG_FILE']) as f:
data = f.read()
# check that log file has info msg
self.assertTrue(data.find('Log file was not founded') != -1,
msg="Not rises info msg: 'Log file was not founded'")
def test_main1(self):
config = dict(REPORT_SIZE=1000,
REPORT_DIR="./test/reports",
LOG_DIR="./test/test_logs_nginx/",
LEVEL_PARSE=50,
LOGGING_LEVEL=logging.INFO,
LOGGING_TO_FILE='./test/log_analyzer.log')
main(config)
res1 = os.path.exists('./test/log_analyzer.log')
with open('./test/log_analyzer.log', 'r') as f:
res2 = f.readlines()[-1][28:]
shutil.rmtree('./test/reports')
os.remove('./test/log_analyzer.log')
self.assertEquals(res1, True)
self.assertEquals(
res2,
'Could not parse more 50% in ./test/test_logs_nginx/nginx-access-ui.log-20200413.gz. \
Try to check log format.\n')
def test_case_3_rise_current_report_is_uptodate(self):
case_name, config = self.cases[2]
# close previous log handler
log = logging.getLogger()
for hdlr in log.handlers:
hdlr.close()
log.removeHandler(hdlr)
log_analyzer.setup_logging(config['LOG_FILE'])
log_analyzer.main(config)
with open(config['LOG_FILE']) as f:
data = f.read()
# check that log file has info msg
self.assertTrue(
data.find('Current report is up-to-date') != -1,
msg="Not rises info msg: 'Current report is up-to-date'")
def test_when_success_main(self):
"""
Проверка выходных результатов при успешном завершении скрипта, настроенном
на входном конфиге test_config.txt: создание выходной
директории/файла отчета и его содержания; наличие выходного лога.
"""
main(['--config', './tests/test_success_config.txt'])
current_path = os.path.realpath(__file__)
os.chdir(os.path.dirname(current_path))
self.assertTrue(
os.path.exists(r'./test_success_logs/test_success_log.log'))
self.assertTrue(
os.path.exists(r'./test_success_reports/report-2019.11.11.html'))
with open(r'./test_success_reports/report-2019.11.11.html', 'r') as f:
report_text = f.read()
list_len = len(
re.findall(r"((\{('\w+': [^\{\}]*(, ){0,1})\})(, ){0,1}){1}",
report_text))
self.assertTrue(list_len == 42)
def test_integral(self):
"""Ultimate integral test. It may takes several minutes. Check result in browser (10 rows)."""
# print("\n"
# )
sys.argv = sys.argv[:1]
sys.argv.append('--config=tests/test_integral.cfg')
logger_file = 'tests/log_integral/log_analyzer.log'
output_file = 'tests/log_integral_report/report-2017-06-30.html'
output_url = 'file:///' + os.path.abspath(output_file)
if exists(output_file):
os.remove(output_file)
if exists(logger_file):
os.remove(logger_file)
# main will raises exception on errors
try:
la.main(la.config)
except Exception as ex:
self.fail("log_analyzer main() function failed with %s(%s)" %
(type(ex).__name__, ex))
self.assertTrue(exists(output_file),
msg="Checks report file generated")
self.assertTrue(exists(logger_file),
msg="Checks logger file was created")
# check do not process when report is already present
start = time.time()
try:
la.main(la.config)
except Exception as ex:
self.fail("log_analyzer main() function failed with %s(%s)" %
(type(ex).__name__, ex))
timeout = time.time() - start
self.assertTrue(
timeout < 1,
"If report is ready, program shouldn't parse log again")
webbrowser.open(output_url, new=2)
def test_case_1_report_file_contain_correct_data(self):
case_name, config = self.cases[0]
# close previous log handler
log = logging.getLogger()
for hdlr in log.handlers:
hdlr.close()
log.removeHandler(hdlr)
log_analyzer.setup_logging(config['LOG_FILE'])
log_analyzer.main(config)
# read result from report file
report_file = os.path.join(config['REPORT_DIR'],
'report-2017.06.29.html')
with open(report_file) as f:
data = f.read()
data = json.loads(data.split('= ')[1][:-1])
# check that report result is correct
self.assertCountEqual(data,
test_answer_1,
msg="Report file contain not correct data")
def test_when_success_main(self):
"""
Проверка выходных результатов при успешном завершении скрипта, настроенном
на входном конфиге test_config.txt: создание выходной
директории/файла отчета и его содержания; наличие выходного лога.
"""
main(['--config', './tests/test_success_config.txt'])
current_path = os.path.realpath(__file__)
os.chdir(os.path.dirname(current_path))
self.assertTrue(
os.path.exists(r'./test_success_logs/test_success_log.log'))
self.assertTrue(
os.path.exists(r'./test_success_reports/report-2019.11.11.html'))
with open(r'./test_success_reports/report-2019.11.11.html', 'r') as f:
report_text = f.read()
list_len = len(
re.findall(r"((\{('\w+': [^\{\}]*(, ){0,1})\})(, ){0,1}){1}",
report_text))
self.assertTrue(list_len == 42)
"""
Проверка выходных результатов при неудачном завершении скрипта, вызванное
низким значением FAIL_PERC в файле конфига test_failure_config.txt. Проверяется
создание выходного лога, а также его содержания.
"""
main(['--config', './tests/test_failure_config.txt'])
current_path = os.path.realpath(__file__)
os.chdir(os.path.dirname(current_path))
self.assertTrue(
os.path.exists(r'./test_failure_logs/test_failure_log.log'))
with open(r'./test_failure_logs/test_failure_log.log') as f:
log_text = f.readlines()
self.assertTrue('что выше допустимого' in log_text[-2])
def test_run(self):
la.main(self.test_args)
self.check_files(self.config)
def test_main(self):
main(self.default_config)
import glob
import log_analyzer
if __name__ == '__main__':
args = ['--batch', '--html', '--xxlocal', "--dns", "-o", "/users/fy/www/test.html"]
# args = ['--batch', '--by-time', '--xxlocal', "-o", "/tmp/1"]
# args = ['--xxfake', '--xxlocal', '--dns', '-o', '/tmp/realtime']
# args = ['--slug-summary', '-o', '/tmp/1']
# args.append('--realtime')
# args.append("--dns")
# args.extend(glob.glob("/users/fy/SETI/logs/tools.pds_access_log-201?-??-??"))
# args.extend(glob.glob("/users/fy/SETI/logs/tools.pds_access_log-*"))
args.extend(glob.glob("/users/fy/SETI/logs/tools.pds_access_log-2018-12-2[4567]"))
log_analyzer.main(args)
