def test_parse_log(self):
"""
Проверяем правильность разбора лога
"""
# Передаём лог с корректными строками
log_gen = log_analyzer.parse_log(
test_config,
LogParams(path='test_not_empty_log-20170802',
date=NOT_EMPTY_LOG_DATE,
ext=''))
self.assertIsNotNone(log_gen, 'Could not parse latest log')
first_parsed_line = next(log_gen)
self.assertEqual(first_parsed_line.url, '/api/v2/banner/25019354',
'Wrong url is parsed in first log row')
self.assertEqual(first_parsed_line.request_time, '0.390',
'Wrong request_time is parsed in first log row')
# Передаём лог с корректными строками в архиве .gz
log_gen = log_analyzer.parse_log(
test_config,
LogParams(path='test_not_empty_log-20170802.gz',
date=NOT_EMPTY_LOG_DATE,
ext='.gz'))
self.assertIsNotNone(log_gen, 'Could not parse latest log')
first_parsed_line = next(log_gen)
self.assertEqual(first_parsed_line.url, '/api/v2/banner/25019354',
'Wrong url is parsed in first log row')
self.assertEqual(first_parsed_line.request_time, '0.390',
'Wrong request_time is parsed in first log row')
# Передаём пустой лог
empty_log_gen = log_analyzer.parse_log(
test_config,
LogParams(path='test_empty_log-20161209',
date=EMPTY_LOG_DATE,
ext=''))
with self.assertRaises(StopIteration,
msg='Empty log returned not empty results'):
next(empty_log_gen)
# Передаём лог с неправильными строками
error_log_gen = log_analyzer.parse_log(
test_config,
LogParams(path='test_with_errors_log-20170801',
date=ERROR_LOG_DATE,
ext=''))
with self.assertRaises(
RuntimeError,
msg='Log with wrong rows has processed without errors'):
sum(1 for _ in error_log_gen)
def test_create_report_with_another_regex(self):
"""
Проверяем правильность создания отчёта
"""
# Передаём лог c демо данными
log_analyzer.create_report(
test_config,
log_analyzer.parse_log(
{
"LOG_DIR": "../src/log",
"ALLOWED_ERRORS_PERCENT": 15,
"LOG_ENCODING": 'utf-8',
"LOGGING_LEVEL": 'DEBUG',
"LOG_REGEXP": ANOTHER_REGEXP
},
LogParams(path='test_not_empty_log-20170802',
date=NOT_EMPTY_LOG_DATE,
ext='')),
LogParams(path='test_not_empty_log-20170802',
date=NOT_EMPTY_LOG_DATE,
ext=''))
expected_path = os.path.join(test_config['REPORT_DIR'],
NOT_EMPTY_LOG_REPORT)
with open(expected_path, 'r', encoding='UTF-8') as f:
expected = f.read()
self.assertIsNotNone(expected, 'Generated report is empty')
actual_path = os.path.join(test_config['REPORT_DIR'],
'report-2017.08.02.html')
with open(actual_path, 'r', encoding='UTF-8') as f:
actual = f.read()
self.assertEqual(expected, actual,
'Generated report is not equal to sample')
os.remove(actual_path)
def test_create_report(self):
"""
Проверяем правильность создания отчёта
"""
# Передаём лог c демо данными
log_analyzer.create_report(
test_config,
log_analyzer.parse_log(
test_config,
LogParams(path='test_not_empty_log-20170802',
date=NOT_EMPTY_LOG_DATE,
ext='')),
LogParams(path='test_not_empty_log-20170802',
date=NOT_EMPTY_LOG_DATE,
ext=''))
expected_path = os.path.join(test_config['REPORT_DIR'],
NOT_EMPTY_LOG_REPORT)
with open(expected_path, 'r', encoding='UTF-8') as f:
expected = f.read()
self.assertIsNotNone(expected, 'Generated report is empty')
actual_path = os.path.join(test_config['REPORT_DIR'],
'report-2017.08.02.html')
with open(actual_path, 'r', encoding='UTF-8') as f:
actual = f.read()
self.assertEqual(expected, actual,
'Generated report is not equal to sample')
os.remove(actual_path)
def test_parse_log_gz_file(self):
res = parse_log(
'./test/test_logs_nginx/nginx-access-ui.log-20200413.gz', 'gz')
parse = ({
'/api/v2/banner/25019354': [0.390],
'/api/1/photogenic_banners/list/?server_name=WIN7RB4': [0.133],
'/api/v2/banner/16852664': [0.199]
}, 3, 0.722, Fraction(3, 35))
self.assertEquals(res, parse)
def test_parse_log_with_error(self):
result = la.parse_log(
os.path.join(config["LOG_DIR"], "nginx-access-ui.log-20170730"),
config)
total_sum = 0
for el in result[2]:
total_sum += sum(result[2][el])
self.assertEqual(result[0], 4)
self.assertEqual(result[1], total_sum)
self.assertEqual(result[1], 1.1)
self.assertEqual(len(result[2]), 3)
def test_process_data(self):
"""test log_analyzer.process_data"""
last_log = la.get_last_log_file(config)
data = la.parse_log(last_log.fullname, config)
result = la.process_data(data, config)
self.assertEqual(result[0]["count"], 2)
self.assertEqual(result[0]["count_perc"], 40.000)
self.assertEqual(result[0]["time_avg"], 0.450)
self.assertEqual(result[0]["time_max"], 0.500)
self.assertEqual(result[0]["time_sum"], 0.900)
self.assertEqual(result[0]["time_perc"], 60.000)
def test_parse_log(self):
"""test log_analyzer.parce_log"""
last_log = la.get_last_log_file(config)
result = la.parse_log(last_log.fullname, config)
self.assertEqual(result[0], 5)
total_sum = 0
for el in result[2]:
total_sum += sum(result[2][el])
self.assertEqual(result[1], total_sum)
self.assertEqual(result[1], 1.5)
self.assertEqual(len(result[2]), 4)
def test_parse_log_plain(self):
try:
access_log = list(
parse_log(
log_path='./tests/log/test_nginx-access-ui.log-20170630',
parser=parse_line))
except Exception as e:
print(f"Something's wrong: {e}")
access_log = list()
# 1. Checks if log is parsed at all.
self.assertTrue(access_log)
# 2. Checks if parsed access_log is list of dicts.
self.assertIs(type(access_log[0]), dict)
def test_make_report_table(self):
latest_log = find_latest_log(log_dir='./tests/log/')
access_logs = parse_log(log_path=f'./tests/log/{latest_log.log_name}',
parser=parse_line)
try:
report_table = make_report_table(access_logs, report_length=10)
except Exception as e:
print(f"Something is wrong: {e}")
report_table = None
# 1. Checks if report table is constructed.
self.assertIsNotNone(report_table)
# 2. Checks if report table is a list.
self.assertIs(type(report_table), list)
# 3. Checks if report table is a list of dicts.
self.assertIs(type(report_table[0]), dict)
# 4. Checks if report table is sorted properly.
self.assertGreater(report_table[0]['time_sum'],
report_table[1]['time_sum'])
def test_parse_log(self):
records = [
'0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0',
'0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0',
'0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0',
'0.0.0.0 - - [29/Jun/2017:03:50:22 +0300] "GET /api/test" 200 407 "-" "-" "-" "1498697422-2539198130-4709-9928846" "89f7f1be37d" 1.0'
]
urls = {
'/api/test': {
'count': 4,
'time_sum': 4.0,
'times': [1.0, 1.0, 1.0, 1.0],
'time_max': 1.0,
}
}
sum_req_time = 4.0
req_count = 4
error_count = 0
self.assertEqual(parse_log(records),
(urls, sum_req_time, req_count, error_count))
def test_render_html_report(self):
latest_log = find_latest_log(log_dir='./tests/log/')
access_log = parse_log(log_path=f'./tests/log/{latest_log.log_name}',
parser=parse_line)
report_length = 10
report_table = make_report_table(access_log,
report_length=report_length)
new_report_date = latest_log.log_date.strftime("%Y.%m.%d")
try:
os.remove(f"./tests/reports/report-{new_report_date}.html")
except OSError:
pass
render_result = render_html_report(table=report_table,
report_path="./tests/reports/",
latest_log_date=latest_log.log_date)
# 1. Checks if report is created.
self.assertTrue(render_result in os.listdir("./tests/reports/"))
# 2. Checks if report has desired length.
self.assertEqual(len(report_table), report_length)
def test_parse_log_empty_file(self):
res = parse_log('./test/test_configs/empty_file', None)
self.assertEquals(res, ({}, 0, 0, 0))
def test_parse_log(self):
parsed = [
x for x in log_analyzer.parse_log(self.config, self.log_info)
]
self.assertListEqual(parsed, self.parsed)
def test_parse_log_with_bad_requests(self):
result = la.parse_log(
os.path.join(config["LOG_DIR"], "nginx-access-ui.log-20170330"),
config)
self.assertEqual(result, None)
def test_parse_log_with_error_treshold(self):
result = la.parse_log(
os.path.join(config["LOG_DIR"], "nginx-access-ui.log-20170430"),
config)
self.assertEqual(result, None)
