def test_errors_filtered():
_t = """\
aaa:AAA\tbbb:BBB\tccc:CCC\r
aaa:AAA1\tbbb:BBB1
c#c:CCC
"""
records = list(reader(StringIO(_t), labels=('bbb','c#c',)))
assert len(records) == 3
assert records[0] == [['bbb', 'BBB']]
assert records[1] == [['bbb', 'BBB1']]
assert records[2] == []
def test_simple_message(self):
""" Testing format with a common message """
msg = "testing logging format"
self.logger.info(msg)
ltsv_object = ltsv.reader(self.buffer.getvalue().splitlines())
for line in ltsv_object:
cell = dict(line)
self.assertEqual(cell["msg"], msg)
self.assertEqual(cell["levelname"], "INFO")
self.assertEqual(cell["level"], "6")
self.assertEqual(cell["app"], "djehouty-ltsv")
self.assertEqual(cell["funcName"], "test_simple_message")
def test_simple_message(self):
""" Testing format with a common message """
msg = "testing logging format"
self.logger.info(msg)
ltsv_object = ltsv.reader(self.buffer.getvalue().splitlines())
for line in ltsv_object:
cell = dict(line)
self.assertEqual(cell["msg"], msg)
self.assertEqual(cell["levelname"], "INFO")
self.assertEqual(cell["level"], "6")
self.assertEqual(cell["app"], "djehouty-ltsv")
self.assertEqual(cell["funcName"], "test_simple_message")
def test_complex_message(self):
""" Testing format with extra values """
msg = "testing logging format"
self.logger.info(msg, extra={"lang": 'en', "env": 'test'})
ltsv_object = ltsv.reader(self.buffer.getvalue().splitlines())
for line in ltsv_object:
cell = dict(line)
self.assertEqual(cell["msg"], msg)
self.assertEqual(cell["levelname"], "INFO")
self.assertEqual(cell["level"], "6")
self.assertEqual(cell["app"], "djehouty-ltsv")
self.assertEqual(cell["funcName"], "test_complex_message")
self.assertEqual(cell["lang"], "en")
self.assertEqual(cell["env"], "test")
def test_complex_message(self):
""" Testing format with extra values """
msg = "testing logging format"
self.logger.info(msg, extra={"lang": 'en', "env": 'test'})
ltsv_object = ltsv.reader(self.buffer.getvalue().splitlines())
for line in ltsv_object:
cell = dict(line)
self.assertEqual(cell["msg"], msg)
self.assertEqual(cell["levelname"], "INFO")
self.assertEqual(cell["level"], "6")
self.assertEqual(cell["app"], "djehouty-ltsv")
self.assertEqual(cell["funcName"], "test_complex_message")
self.assertEqual(cell["lang"], "en")
self.assertEqual(cell["env"], "test")
def test_errors():
_t = """\
aaa:AAA\tbbb:BBB\tccc:CCC\r
This is broken line.\tAnd next line is empty (valid).
\r
aaa:AAA1\tbbb:BBB1
a#a:AAA
aaa:AAA1\tc#c:CCC\tbbb:BBB1
"""
records = list(reader(StringIO(_t)))
assert len(records) == 6
assert records[0] == _expected[0]
assert records[1] == []
assert records[2] == []
assert records[3] == _expected[1]
assert records[4] == []
assert records[5] == _expected[1]
def read_nginx_log(self, start_pos, last_line):
"""
checks that file is valid and get new data
:param start_pos: int, start position in log file
:param last_line: str, last line value
:return: None
elastic_metric = {
'url': {
'index': {
'count': 0,
'errors': 0,
'req_time': [],
'up_time': [],
},
'refresh': {
'count': 0,
'errors': 0,
'req_time': [],
'up_time': [],
}
},
}
"""
if not os.path.isfile(self.log_file):
return
with open(self.log_file, 'r') as log_file:
log_file.seek(start_pos, 0)
if log_file.readline().strip() != last_line:
print('starting from the beginning')
log_file.seek(0, 0)
"""
python repesentation of ltsv
[['host', '127.0.0.1'], ['user', '-'], ['time', '[22/Sep/2017:14:36:39 +0000]'],
['request', 'GET / HTTP/1.1'], ['status', '200'], ['size', '323'], ['referer', '-'],
['user_agent', 'curl/7.47.0'], ['req_time', '0.060'],
['upstream_res_time', '0.060'], ['upstream_addr', '127.0.0.1:9200']]
"""
start_parsing_time = datetime.now()
last_tell = start_pos
cur_tell = log_file.tell()
line = log_file.readline()
if line:
start_params = next(ltsv.reader(StringIO(line)))
start_time = datetime.strptime(
start_params[2][1].split(' ')[0], '[%d/%b/%Y:%H:%M:%S')
else:
self.write_cur_pos_n_line(last_tell, last_line)
return last_tell, last_line
while line:
last_line = line
last_tell = cur_tell
line_params = next(ltsv.reader(StringIO(line)))
line_time = datetime.strptime(line_params[2][1].split(' ')[0],
'[%d/%b/%Y:%H:%M:%S')
if line_time >= start_parsing_time:
timestamp = int(mktime(start_time.timetuple()))
self.send_data(timestamp)
self.write_cur_pos_n_line(cur_tell, line)
return cur_tell, line
if line_time - start_time > timedelta(minutes=1):
timestamp = int(mktime(start_time.timetuple()))
self.send_data(timestamp)
start_time = line_time
url = line_params[3][1].split()[1]
req_times = map(float, line_params[8][1].split(','))
up_times = map(float, line_params[9][1].split(','))
if re.match('/_bulk', url):
self.elastic_metric['bulk']['index']['count'] += 1
for req_time in req_times:
self.elastic_metric['bulk']['index'][
'req_times'].append(req_time)
for up_time in up_times:
self.elastic_metric['bulk']['index'][
'up_times'].append(up_time)
elif re.match('^(/[^_]\w+[?/]+)', url):
index = re.findall('^/([^_]\w+)[/?]', url)[0]
if not index:
continue
self.elastic_metric[index]['index']['count'] += 1
for req_time in req_times:
self.elastic_metric[index]['index'][
'req_times'].append(req_time)
for up_time in up_times:
self.elastic_metric[index]['index'][
'req_times'].append(up_time)
if is_error_code(line_params[4][1]):
self.elastic_metric[index]['index']['errors'] += 1
if re.match('^(/[^_]?\w+/_refresh)', url):
self.elastic_metric[index]['refresh']['count'] += 1
for req_time in req_times:
self.elastic_metric[index]['refresh'][
'req_times'].append(req_time)
for up_time in up_times:
self.elastic_metric[index]['refresh'][
'up_times'].append(up_time)
if is_error_code(line_params[4][1]):
self.elastic_metric[index]['refresh'][
'errors'] += 1
cur_tell = log_file.tell()
line = log_file.readline()
timestamp = int(mktime(start_time.timetuple()))
self.send_data(timestamp)
self.write_cur_pos_n_line(last_tell, last_line)
return last_tell, last_line
def test_reader_filtered(file):
rec1, rec2 = reader(file, labels=('ccc',))
assert rec1 == [['ccc', 'CCC']]
assert rec2 == []
def test_reader(file):
assert list(reader(file)) == _expected
