def test_errors_filtered(): _t = """\ aaa:AAA\tbbb:BBB\tccc:CCC\r aaa:AAA1\tbbb:BBB1 c#c:CCC """ records = list(reader(StringIO(_t), labels=('bbb','c#c',))) assert len(records) == 3 assert records[0] == [['bbb', 'BBB']] assert records[1] == [['bbb', 'BBB1']] assert records[2] == []
def test_simple_message(self): """ Testing format with a common message """ msg = "testing logging format" self.logger.info(msg) ltsv_object = ltsv.reader(self.buffer.getvalue().splitlines()) for line in ltsv_object: cell = dict(line) self.assertEqual(cell["msg"], msg) self.assertEqual(cell["levelname"], "INFO") self.assertEqual(cell["level"], "6") self.assertEqual(cell["app"], "djehouty-ltsv") self.assertEqual(cell["funcName"], "test_simple_message")
def test_complex_message(self): """ Testing format with extra values """ msg = "testing logging format" self.logger.info(msg, extra={"lang": 'en', "env": 'test'}) ltsv_object = ltsv.reader(self.buffer.getvalue().splitlines()) for line in ltsv_object: cell = dict(line) self.assertEqual(cell["msg"], msg) self.assertEqual(cell["levelname"], "INFO") self.assertEqual(cell["level"], "6") self.assertEqual(cell["app"], "djehouty-ltsv") self.assertEqual(cell["funcName"], "test_complex_message") self.assertEqual(cell["lang"], "en") self.assertEqual(cell["env"], "test")
def test_errors(): _t = """\ aaa:AAA\tbbb:BBB\tccc:CCC\r This is broken line.\tAnd next line is empty (valid). \r aaa:AAA1\tbbb:BBB1 a#a:AAA aaa:AAA1\tc#c:CCC\tbbb:BBB1 """ records = list(reader(StringIO(_t))) assert len(records) == 6 assert records[0] == _expected[0] assert records[1] == [] assert records[2] == [] assert records[3] == _expected[1] assert records[4] == [] assert records[5] == _expected[1]
def read_nginx_log(self, start_pos, last_line): """ checks that file is valid and get new data :param start_pos: int, start position in log file :param last_line: str, last line value :return: None elastic_metric = { 'url': { 'index': { 'count': 0, 'errors': 0, 'req_time': [], 'up_time': [], }, 'refresh': { 'count': 0, 'errors': 0, 'req_time': [], 'up_time': [], } }, } """ if not os.path.isfile(self.log_file): return with open(self.log_file, 'r') as log_file: log_file.seek(start_pos, 0) if log_file.readline().strip() != last_line: print('starting from the beginning') log_file.seek(0, 0) """ python repesentation of ltsv [['host', '127.0.0.1'], ['user', '-'], ['time', '[22/Sep/2017:14:36:39 +0000]'], ['request', 'GET / HTTP/1.1'], ['status', '200'], ['size', '323'], ['referer', '-'], ['user_agent', 'curl/7.47.0'], ['req_time', '0.060'], ['upstream_res_time', '0.060'], ['upstream_addr', '127.0.0.1:9200']] """ start_parsing_time = datetime.now() last_tell = start_pos cur_tell = log_file.tell() line = log_file.readline() if line: start_params = next(ltsv.reader(StringIO(line))) start_time = datetime.strptime( start_params[2][1].split(' ')[0], '[%d/%b/%Y:%H:%M:%S') else: self.write_cur_pos_n_line(last_tell, last_line) return last_tell, last_line while line: last_line = line last_tell = cur_tell line_params = next(ltsv.reader(StringIO(line))) line_time = datetime.strptime(line_params[2][1].split(' ')[0], '[%d/%b/%Y:%H:%M:%S') if line_time >= start_parsing_time: timestamp = int(mktime(start_time.timetuple())) self.send_data(timestamp) self.write_cur_pos_n_line(cur_tell, line) return cur_tell, line if line_time - start_time > timedelta(minutes=1): timestamp = int(mktime(start_time.timetuple())) self.send_data(timestamp) start_time = line_time url = line_params[3][1].split()[1] req_times = map(float, line_params[8][1].split(',')) up_times = map(float, line_params[9][1].split(',')) if re.match('/_bulk', url): self.elastic_metric['bulk']['index']['count'] += 1 for req_time in req_times: self.elastic_metric['bulk']['index'][ 'req_times'].append(req_time) for up_time in up_times: self.elastic_metric['bulk']['index'][ 'up_times'].append(up_time) elif re.match('^(/[^_]\w+[?/]+)', url): index = re.findall('^/([^_]\w+)[/?]', url)[0] if not index: continue self.elastic_metric[index]['index']['count'] += 1 for req_time in req_times: self.elastic_metric[index]['index'][ 'req_times'].append(req_time) for up_time in up_times: self.elastic_metric[index]['index'][ 'req_times'].append(up_time) if is_error_code(line_params[4][1]): self.elastic_metric[index]['index']['errors'] += 1 if re.match('^(/[^_]?\w+/_refresh)', url): self.elastic_metric[index]['refresh']['count'] += 1 for req_time in req_times: self.elastic_metric[index]['refresh'][ 'req_times'].append(req_time) for up_time in up_times: self.elastic_metric[index]['refresh'][ 'up_times'].append(up_time) if is_error_code(line_params[4][1]): self.elastic_metric[index]['refresh'][ 'errors'] += 1 cur_tell = log_file.tell() line = log_file.readline() timestamp = int(mktime(start_time.timetuple())) self.send_data(timestamp) self.write_cur_pos_n_line(last_tell, last_line) return last_tell, last_line
def test_reader_filtered(file): rec1, rec2 = reader(file, labels=('ccc',)) assert rec1 == [['ccc', 'CCC']] assert rec2 == []
def test_reader(file): assert list(reader(file)) == _expected