def test_parse_log(self): """ Проверяем правильность разбора лога """ # Передаём лог с корректными строками log_gen = log_analyzer.parse_log( test_config, LogParams(path='test_not_empty_log-20170802', date=NOT_EMPTY_LOG_DATE, ext='')) self.assertIsNotNone(log_gen, 'Could not parse latest log') first_parsed_line = next(log_gen) self.assertEqual(first_parsed_line.url, '/api/v2/banner/25019354', 'Wrong url is parsed in first log row') self.assertEqual(first_parsed_line.request_time, '0.390', 'Wrong request_time is parsed in first log row') # Передаём лог с корректными строками в архиве .gz log_gen = log_analyzer.parse_log( test_config, LogParams(path='test_not_empty_log-20170802.gz', date=NOT_EMPTY_LOG_DATE, ext='.gz')) self.assertIsNotNone(log_gen, 'Could not parse latest log') first_parsed_line = next(log_gen) self.assertEqual(first_parsed_line.url, '/api/v2/banner/25019354', 'Wrong url is parsed in first log row') self.assertEqual(first_parsed_line.request_time, '0.390', 'Wrong request_time is parsed in first log row') # Передаём пустой лог empty_log_gen = log_analyzer.parse_log( test_config, LogParams(path='test_empty_log-20161209', date=EMPTY_LOG_DATE, ext='')) with self.assertRaises(StopIteration, msg='Empty log returned not empty results'): next(empty_log_gen) # Передаём лог с неправильными строками error_log_gen = log_analyzer.parse_log( test_config, LogParams(path='test_with_errors_log-20170801', date=ERROR_LOG_DATE, ext='')) with self.assertRaises( RuntimeError, msg='Log with wrong rows has processed without errors'): sum(1 for _ in error_log_gen)
def test_create_report_with_another_regex(self): """ Проверяем правильность создания отчёта """ # Передаём лог c демо данными log_analyzer.create_report( test_config, log_analyzer.parse_log( { "LOG_DIR": "../src/log", "ALLOWED_ERRORS_PERCENT": 15, "LOG_ENCODING": 'utf-8', "LOGGING_LEVEL": 'DEBUG', "LOG_REGEXP": ANOTHER_REGEXP }, LogParams(path='test_not_empty_log-20170802', date=NOT_EMPTY_LOG_DATE, ext='')), LogParams(path='test_not_empty_log-20170802', date=NOT_EMPTY_LOG_DATE, ext='')) expected_path = os.path.join(test_config['REPORT_DIR'], NOT_EMPTY_LOG_REPORT) with open(expected_path, 'r', encoding='UTF-8') as f: expected = f.read() self.assertIsNotNone(expected, 'Generated report is empty') actual_path = os.path.join(test_config['REPORT_DIR'], 'report-2017.08.02.html') with open(actual_path, 'r', encoding='UTF-8') as f: actual = f.read() self.assertEqual(expected, actual, 'Generated report is not equal to sample') os.remove(actual_path)
def test_create_report(self): """ Проверяем правильность создания отчёта """ # Передаём лог c демо данными log_analyzer.create_report( test_config, log_analyzer.parse_log( test_config, LogParams(path='test_not_empty_log-20170802', date=NOT_EMPTY_LOG_DATE, ext='')), LogParams(path='test_not_empty_log-20170802', date=NOT_EMPTY_LOG_DATE, ext='')) expected_path = os.path.join(test_config['REPORT_DIR'], NOT_EMPTY_LOG_REPORT) with open(expected_path, 'r', encoding='UTF-8') as f: expected = f.read() self.assertIsNotNone(expected, 'Generated report is empty') actual_path = os.path.join(test_config['REPORT_DIR'], 'report-2017.08.02.html') with open(actual_path, 'r', encoding='UTF-8') as f: actual = f.read() self.assertEqual(expected, actual, 'Generated report is not equal to sample') os.remove(actual_path)
def test_parse_log_gz_file(self): res = parse_log( './test/test_logs_nginx/nginx-access-ui.log-20200413.gz', 'gz') parse = ({ '/api/v2/banner/25019354': [0.390], '/api/1/photogenic_banners/list/?server_name=WIN7RB4': [0.133], '/api/v2/banner/16852664': [0.199] }, 3, 0.722, Fraction(3, 35)) self.assertEquals(res, parse)
def test_parse_log_with_error(self): result = la.parse_log( os.path.join(config["LOG_DIR"], "nginx-access-ui.log-20170730"), config) total_sum = 0 for el in result[2]: total_sum += sum(result[2][el]) self.assertEqual(result[0], 4) self.assertEqual(result[1], total_sum) self.assertEqual(result[1], 1.1) self.assertEqual(len(result[2]), 3)
def test_process_data(self): """test log_analyzer.process_data""" last_log = la.get_last_log_file(config) data = la.parse_log(last_log.fullname, config) result = la.process_data(data, config) self.assertEqual(result[0]["count"], 2) self.assertEqual(result[0]["count_perc"], 40.000) self.assertEqual(result[0]["time_avg"], 0.450) self.assertEqual(result[0]["time_max"], 0.500) self.assertEqual(result[0]["time_sum"], 0.900) self.assertEqual(result[0]["time_perc"], 60.000)
def test_parse_log(self): """test log_analyzer.parce_log""" last_log = la.get_last_log_file(config) result = la.parse_log(last_log.fullname, config) self.assertEqual(result[0], 5) total_sum = 0 for el in result[2]: total_sum += sum(result[2][el]) self.assertEqual(result[1], total_sum) self.assertEqual(result[1], 1.5) self.assertEqual(len(result[2]), 4)
def test_parse_log_plain(self): try: access_log = list( parse_log( log_path='./tests/log/test_nginx-access-ui.log-20170630', parser=parse_line)) except Exception as e: print(f"Something's wrong: {e}") access_log = list() # 1. Checks if log is parsed at all. self.assertTrue(access_log) # 2. Checks if parsed access_log is list of dicts. self.assertIs(type(access_log[0]), dict)
def test_make_report_table(self): latest_log = find_latest_log(log_dir='./tests/log/') access_logs = parse_log(log_path=f'./tests/log/{latest_log.log_name}', parser=parse_line) try: report_table = make_report_table(access_logs, report_length=10) except Exception as e: print(f"Something is wrong: {e}") report_table = None # 1. Checks if report table is constructed. self.assertIsNotNone(report_table) # 2. Checks if report table is a list. self.assertIs(type(report_table), list) # 3. Checks if report table is a list of dicts. self.assertIs(type(report_table[0]), dict) # 4. Checks if report table is sorted properly. self.assertGreater(report_table[0]['time_sum'], report_table[1]['time_sum'])
def test_parse_log(self): records = [ '0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0', '0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0', '0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0', '0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0' ] urls = { '/api/test': { 'count': 4, 'time_sum': 4.0, 'times': [1.0, 1.0, 1.0, 1.0], 'time_max': 1.0, } } sum_req_time = 4.0 req_count = 4 error_count = 0 self.assertEqual(parse_log(records), (urls, sum_req_time, req_count, error_count))
def test_render_html_report(self): latest_log = find_latest_log(log_dir='./tests/log/') access_log = parse_log(log_path=f'./tests/log/{latest_log.log_name}', parser=parse_line) report_length = 10 report_table = make_report_table(access_log, report_length=report_length) new_report_date = latest_log.log_date.strftime("%Y.%m.%d") try: os.remove(f"./tests/reports/report-{new_report_date}.html") except OSError: pass render_result = render_html_report(table=report_table, report_path="./tests/reports/", latest_log_date=latest_log.log_date) # 1. Checks if report is created. self.assertTrue(render_result in os.listdir("./tests/reports/")) # 2. Checks if report has desired length. self.assertEqual(len(report_table), report_length)
def test_parse_log_empty_file(self): res = parse_log('./test/test_configs/empty_file', None) self.assertEquals(res, ({}, 0, 0, 0))
def test_parse_log(self): parsed = [ x for x in log_analyzer.parse_log(self.config, self.log_info) ] self.assertListEqual(parsed, self.parsed)
def test_parse_log_with_bad_requests(self): result = la.parse_log( os.path.join(config["LOG_DIR"], "nginx-access-ui.log-20170330"), config) self.assertEqual(result, None)
def test_parse_log_with_error_treshold(self): result = la.parse_log( os.path.join(config["LOG_DIR"], "nginx-access-ui.log-20170430"), config) self.assertEqual(result, None)