def test_haproxy_log_file_cmd_requests_per_minute(self): """Check that the requests per minute command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/requests_per_minute.log', ) log_file.parse_file() requests = log_file.cmd_requests_per_minute() self.assertEqual(len(requests), 5) self.assertEqual( requests[0], (datetime(2013, 12, 11, 11, 2), 8) ) self.assertEqual( requests[1], (datetime(2013, 12, 11, 11, 3), 3) ) self.assertEqual( requests[2], (datetime(2013, 12, 11, 11, 13), 5) ) self.assertEqual( requests[3], (datetime(2013, 12, 11, 11, 52), 7) ) self.assertEqual( requests[4], (datetime(2013, 12, 11, 12, 2), 9) )
def test_haproxy_log_file_cmd_queue_peaks_no_end(self): """Check that the queue peaks command reports as expected when the last log request did not have any queue. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/queue_2.log', ) log_file.parse_file() peaks = log_file.cmd_queue_peaks() self.assertEqual(len(peaks), 3) self.assertEqual(peaks[0]['peak'], 4) self.assertEqual(peaks[0]['span'], 5) self.assertEqual(peaks[1]['peak'], 19) self.assertEqual(peaks[1]['span'], 5) self.assertEqual(peaks[2]['peak'], 49) self.assertEqual(peaks[2]['span'], 3) self.assertTrue(peaks[0]['first'] < peaks[1]['first']) self.assertTrue(peaks[1]['first'] < peaks[2]['first']) self.assertTrue(peaks[0]['last'] < peaks[1]['last']) self.assertTrue(peaks[1]['last'] < peaks[2]['last'])
def test_haproxy_log_file_cmd_average_response_time(self): """Check that the average response time returns the expected output.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/average_response.log', ) log_file.parse_file() data = log_file.cmd_average_response_time() self.assertEqual(data, 105)
def test_haproxy_log_file_parsed(self): """Check that log files are parsed.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log' ) self.assertEqual(log_file.cmd_counter(), 0) log_file.parse_file() self.assertTrue(log_file.cmd_counter() > 0)
def test_haproxy_log_file_total_lines(self): """Check that even if some lines are not valid, 'total_lines' counts all of them. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/2_ok_1_invalid.log' ) log_file.parse_file() self.assertEqual(log_file.total_lines, 3)
def main(args): if show_help(args): return # show the command list if args['list_commands']: print_commands() # no need to process further return # show the filter list if args['list_filters']: print_filters() # no need to process further return # create a HaproxyLogFile instance and parse the log file log_file = HaproxyLogFile( logfile=args['log'], ) log_file.parse_file() # apply the time frame filter if args['start'] is not None or args['delta'] is not None: start = args['start'] or '' delta = args['delta'] or '' filter_string = 'filters.filter_time_frame("{0}", "{1}")' filter_func = eval(filter_string.format(start, delta)) log_file = log_file.filter(filter_func) # apply all other filters given if args['filters'] is not None: filter_string = 'filters.filter_{0}({1})' for filter_data in args['filters']: arg = '' if filter_data[1] is not None: arg = filter_data[1] arg = "'{0}'".format(arg) filter_func = eval(filter_string.format(filter_data[0], arg)) log_file = log_file.filter(filter_func, reverse=args['negate_filter']) # run all commands command_string = 'log_file.cmd_{0}()' for command in args['commands']: string = 'command: {0}'.format(command) print(string) print('=' * len(string)) result = eval(command_string.format(command)) print(result) return log_file # return the log_file object so that tests can inspect it
def test_haproxy_log_file_cmd_connection_type(self): """Check that the connection type command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/connection.log', ) log_file.parse_file() ssl, non_ssl = log_file.cmd_connection_type() self.assertEqual(ssl, 7) self.assertEqual(non_ssl, 5)
def test_haproxy_log_file_cmd_counter_slow_requests(self): """Check that the slow requests counter command reports as expected. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() slow_requests = log_file.cmd_counter_slow_requests() self.assertEqual(slow_requests, 5)
def test_haproxy_log_file_cmd_average_waiting_time_aborted(self): """Check that the average time waiting on queues returns the expected output when there are aborted connections. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/average_waiting_aborted.log', ) log_file.parse_file() data = log_file.cmd_average_waiting_time() self.assertEqual(data, 110)
def test_haproxy_log_file_valid_and_invalid_lines(self): """Check that if some log lines can not be parsed both numbers are correctly reported. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/2_ok_1_invalid.log' ) log_file.parse_file() self.assertEqual(log_file.cmd_counter(), 2) self.assertEqual(log_file.cmd_counter_invalid(), 1)
def test_haproxy_log_file_cmd_print_empty(self): """Check that the print command prints an empty string if the log file is empty. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/empty.log', ) log_file.parse_file() data = log_file.cmd_print() self.assertEqual('', data)
def test_haproxy_log_file_cmd_average_response_time_no_wait(self): """Check that the average response time returns the expected output when there are connections that did not take any millisecond to reply. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/average_response_no_wait.log', ) log_file.parse_file() data = log_file.cmd_average_response_time() self.assertEqual(data, 74)
def test_haproxy_log_file_pickle_is_recreated(self): """Check that a pickle file is recreated if the log file is newer than the pickle file.""" filename = 'haproxy/tests/files/average_waiting_aborted.log' pickle_file = '{0}.pickle'.format(filename) # create the pickle file and get its modified time log_file = HaproxyLogFile( logfile=filename, ) log_file.parse_file() old_pickle_time = os.path.getmtime(pickle_file) # any second or nth run will not change the modified time log_file = HaproxyLogFile( logfile=filename, ) log_file.parse_file() second_old_pickle_time = os.path.getmtime(pickle_file) self.assertEqual(old_pickle_time, second_old_pickle_time) # 'update' the original log file sleep(1) # os.path.getmtime has a resolution up to seconds os.utime(filename, None) # the existing pickle file is discarded and a newer one will be created log_file = HaproxyLogFile( logfile=filename, ) log_file.parse_file() new_pickle_time = os.path.getmtime(pickle_file) self.assertTrue(new_pickle_time > old_pickle_time)
def test_haproxy_log_file_cmd_requests_per_minute_empty(self): """Check that the requests per minute command reports nothing if there are no valid lines for whichever reason. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/empty.log', ) log_file.parse_file() requests = log_file.cmd_requests_per_minute() self.assertEqual(None, requests)
def test_haproxy_log_file_cmd_average_waiting_time_no_wait(self): """Check that the average time waiting on queues returns the expected output when there are requests that did not wait at all (i.e. time_wait_queues is 0). """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/average_waiting_no_wait.log', ) log_file.parse_file() data = log_file.cmd_average_waiting_time() self.assertEqual(data, 52.5)
def test_haproxy_log_file_cmd_print(self): """Check that the print command prints the valid lines.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/2_ok_1_invalid.log', ) log_file.parse_file() data = log_file.cmd_print() self.assertNotEqual('', data) lines = data.split('\n') self.assertEqual(len(lines), 3)
def test_haproxy_log_file_cmd_status_codes(self): """Check that the status codes command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() status_codes = log_file.cmd_status_codes_counter() self.assertEqual(len(status_codes), 3) self.assertEqual(status_codes['404'], 3) self.assertEqual(status_codes['200'], 2) self.assertEqual(status_codes['300'], 4)
def test_haproxy_log_file_cmd_slow_requests(self): """Check that the slow requests command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() slow_requests = log_file.cmd_slow_requests() self.assertEqual(len(slow_requests), 5) slow_requests.sort() # sort them as the log analyzer sorts by dates self.assertEqual(slow_requests, [1293, 2936, 2942, 20095, 29408, ])
def test_haproxy_log_file_cmd_server_load(self): """Check that the server load counter command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() servers = log_file.cmd_server_load() self.assertEqual(len(servers), 3) self.assertEqual(servers['instance1'], 4) self.assertEqual(servers['instance2'], 3) self.assertEqual(servers['instance3'], 2)
def test_haproxy_log_file_cmd_http_methods(self): """Check that the http methods command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() http_methods = log_file.cmd_http_methods() self.assertEqual(len(http_methods), 3) self.assertEqual(http_methods['GET'], 4) self.assertEqual(http_methods['POST'], 2) self.assertEqual(http_methods['HEAD'], 3)
def test_haproxy_log_file_lines_sorted(self): """Check that after parsing a log file, the valid log lines are kept sorted to ease further work on them. """ log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() previous = log_file._valid_lines[0] previous_date = previous.accept_date for line in log_file._valid_lines[1:]: self.assertTrue(previous_date < line.accept_date)
def test_haproxy_log_file_cmd_ip_counter(self): """Check that the ip counter command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() ip_counter = log_file.cmd_ip_counter() self.assertEqual(len(ip_counter), 4) self.assertEqual(ip_counter['123.123.123.123'], 4) self.assertEqual(ip_counter['123.123.124.124'], 2) self.assertEqual(ip_counter['123.123.124.123'], 1) self.assertEqual(ip_counter['123.123.123.124'], 1)
def test_haproxy_log_file_pickle_exists(self): """Check that a pickle file is created after the first run.""" filename = 'haproxy/tests/files/average_waiting_aborted.log' pickle_file = '{0}.pickle'.format(filename) # pickle files does not exist self.assertFalse(os.path.exists(pickle_file)) log_file = HaproxyLogFile( logfile=filename, ) log_file.parse_file() # it does exist after parsing the file self.assertTrue(os.path.exists(pickle_file))
def test_haproxy_log_file_cmd_request_path_counter(self): """Check that the request path counter command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/small.log', ) log_file.parse_file() path_counter = log_file.cmd_request_path_counter() self.assertEqual(len(path_counter), 5) self.assertEqual(path_counter['/hello'], 3) self.assertEqual(path_counter['/world'], 2) self.assertEqual(path_counter['/free'], 2) self.assertEqual(path_counter['/fra'], 1) self.assertEqual(path_counter['/freitag'], 1)
def print_commands(): """Prints all commands available from HaproxyLogFile with their description. """ dummy_log_file = HaproxyLogFile() commands = HaproxyLogFile.commands() commands.sort() for cmd in commands: description = eval('dummy_log_file.cmd_{0}.__doc__'.format(cmd)) if description: description = re.sub(r'\n\s+', ' ', description) description = description.strip() print('{0}: {1}\n'.format(cmd, description))
def _parse_arg_commands(commands): input_commands = commands.split(',') available_commands = HaproxyLogFile.commands() for cmd in input_commands: if cmd not in available_commands: msg = 'command "{0}" is not available. Use --list-commands to ' \ 'get a list of all available commands.' raise ValueError(msg.format(cmd)) return input_commands
def test_haproxy_log_file_cmd_top_request_paths(self): """Check that the top request paths command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/top_paths.log', ) log_file.parse_file() top_paths = log_file.cmd_top_request_paths() self.assertEqual(len(top_paths), 10) self.assertEqual(top_paths[0], ('/14', 6)) self.assertEqual(top_paths[1], ('/13', 4)) # as the 3rd and 4th have the same repetitions their order is unknown self.assertEqual(top_paths[2][1], 3) self.assertEqual(top_paths[3][1], 3) self.assertEqual(top_paths[4][1], 3) self.assertTrue(top_paths[2][0] in ('/12', '/15', '/11', )) self.assertTrue(top_paths[3][0] in ('/12', '/15', '/11', )) self.assertTrue(top_paths[4][0] in ('/12', '/15', '/11', )) # the same as above for all the others other_paths = [ '/1', '/2', '/3', '/4', '/5', '/6', '/7', '/8', '/9', ] for path_info in top_paths[5:]: self.assertEqual(path_info[1], 2) self.assertTrue(path_info[0] in other_paths) # remove the other_ips to ensure all ips are there for position, current in enumerate(other_paths): if current == path_info[0]: del other_paths[position] break self.assertEqual(len(other_paths), 4)
def test_haproxy_log_file_cmd_top_ips(self): """Check that the top ips command reports as expected.""" log_file = HaproxyLogFile( logfile='haproxy/tests/files/top_ips.log', ) log_file.parse_file() top_ips = log_file.cmd_top_ips() self.assertEqual(len(top_ips), 10) self.assertEqual(top_ips[0], ('1.1.1.15', 6)) self.assertEqual(top_ips[1], ('1.1.1.11', 5)) # as the 3rd and 4th have the same repetitions their order is unknown self.assertEqual(top_ips[2][1], 4) self.assertEqual(top_ips[3][1], 4) self.assertTrue(top_ips[2][0] in ('1.1.1.10', '1.1.1.19')) self.assertTrue(top_ips[3][0] in ('1.1.1.10', '1.1.1.19')) # the same as above for all the others other_ips = [ '1.1.1.12', '1.1.1.13', '1.1.1.14', '1.1.1.16', '1.1.1.17', '1.1.1.18', ] for ip_info in top_ips[4:]: self.assertEqual(ip_info[1], 2) self.assertTrue(ip_info[0] in other_ips) # remove the other_ips to ensure all ips are there for position, current in enumerate(other_ips): if current == ip_info[0]: del other_ips[position] break self.assertEqual(other_ips, [])
def test_arg_parser_list_commands_output(self): """Test that list commands argument outputs what's expected.""" arguments = ['--list-commands', ] data = parse_arguments(self.parser.parse_args(arguments)) test_output = NamedTemporaryFile(mode='w', delete=False) with RedirectStdout(stdout=test_output): main(data) with open(test_output.name, 'r') as output_file: output_text = output_file.read() for cmd in HaproxyLogFile.commands(): self.assertIn(cmd, output_text)
def test_arg_parser_list_commands_output(self): """Test that list commands argument outputs what's expected.""" arguments = [ '--list-commands', ] data = parse_arguments(self.parser.parse_args(arguments)) test_output = NamedTemporaryFile(mode='w', delete=False) with RedirectStdout(stdout=test_output): main(data) with open(test_output.name, 'r') as output_file: output_text = output_file.read() for cmd in HaproxyLogFile.commands(): self.assertIn(cmd, output_text)
def main(args): if show_help(args): return # show the command list if args['list_commands']: print_commands() # no need to process further return # show the filter list if args['list_filters']: print_filters() # no need to process further return # create a HaproxyLogFile instance and parse the log file log_file = HaproxyLogFile(logfile=args['log'], ) log_file.parse_file() # apply the time frame filter if args['start'] is not None or args['delta'] is not None: start = args['start'] or '' delta = args['delta'] or '' filter_string = 'filters.filter_time_frame("{0}", "{1}")' filter_func = eval(filter_string.format(start, delta)) log_file = log_file.filter(filter_func) # apply all other filters given if args['filters'] is not None: filter_string = 'filters.filter_{0}({1})' for filter_data in args['filters']: arg = '' if filter_data[1] is not None: arg = filter_data[1] arg = "'{0}'".format(arg) filter_func = eval(filter_string.format(filter_data[0], arg)) log_file = log_file.filter(filter_func, reverse=args['negate_filter']) # run all commands command_string = 'log_file.cmd_{0}()' for command in args['commands']: string = 'command: {0}'.format(command) print(string) print('=' * len(string)) result = eval(command_string.format(command)) print(result) return log_file # return the log_file object so that tests can inspect it
def test_haproxy_log_file_negate_filter(self): """Check that reversing a filter output works as expected.""" filter_func = filters.filter_ssl() log_file = HaproxyLogFile( logfile='haproxy/tests/files/connection.log', ) log_file.parse_file() # total number of log lines self.assertEqual(log_file.cmd_counter(), 12) # only SSL lines only_ssl = log_file.filter(filter_func) self.assertEqual(only_ssl.cmd_counter(), 7) # non SSL lines non_ssl = log_file.filter(filter_func, reverse=True) self.assertEqual(non_ssl.cmd_counter(), 5) # we did get all lines? self.assertEqual( log_file.cmd_counter(), only_ssl.cmd_counter() + non_ssl.cmd_counter() )