def test_save_progress_success(self): self.files_to_delete.append('progressf16c93d1167446f99a26837c0fdeac6fb73869794') reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') reader._make_progress_string = lambda: 'log.log 123g456 10 19' reader._save_progress() with open('progressf16c93d1167446f99a26837c0fdeac6fb73869794', 'rb') as f: self.assertEqual(f.read(), 'log.log 123g456 10 19')
def test_seek_tail_not_enough_lines(self): self.write_log_file('2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' * 10) with open('log.log', 'rb') as f: reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), tail_length=20) reader.logfile = f reader._seek_tail() self.assertEqual(f.tell(), 0)
def test_make_progress_string_success(self): with open('log.log', 'wb') as f: f.write('Some file contents!') f.seek(10) reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') reader.logfile = f reader.logfile_id = '123g456' self.assertEqual(reader._make_progress_string(), 'log.log 123g456 10 19')
def test_seek_tail_multiple_chunks(self): self.write_log_file('2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' * 1000) with open('log.log', 'rb') as f: reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), tail_length=900) reader.CHUNK_SIZE = 1024 reader.logfile = f reader._seek_tail() self.assertEqual(f.tell(), 100 * 75)
def test_seek_first_unprocessed_position_no_progress_file(self): self.write_log_file('2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' * 20) with open('log.log', 'rb') as f: reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress', tail_length=10) reader.logfile = f reader._seek_first_unprocessed_position() self.assertEqual(f.tell(), 0) self.assertEqual(self.fake_logging.log, ['[WARN] Failed to read the progress file for "log.log".'])
def test_seek_tail_with_multiline_messages_one_chunk(self): message = '2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' + 'X' * 24 + '\n' self.write_log_file(message * 10) with open('log.log', 'rb') as f: reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), tail_length=5) reader.CHUNK_SIZE = 1024 reader.logfile = f reader._seek_tail() self.assertEqual(f.tell(), 5 * 100)
def test_seek_first_unprocessed_position(self): self.write_log_file('XXXX\n' * 100) self.write_progress_file('log.log 123g456 50 75') with open('log.log', 'rb') as f: reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') reader.logfile = f reader._seek_first_unprocessed_position() self.assertEqual(f.tell(), 50) self.assertEqual(reader.logfile_id, '123g456')
def test_close_file(self): with open('log.log', 'wb') as f: f.write('Some file contents!') reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver()) reader._open_file() f = reader.logfile reader._close_file() self.assertTrue(f.closed) self.assertEqual(reader.logfile, None)
def __enter__(self): lines = [] for i in self.seconds: lines.append(self.message_template % divmod(i, 60)) with open('log.log', 'wb') as f: f.write(''.join(lines)) self.log = open('log.log', 'rb') reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver()) reader.logfile = self.log return reader
def test_open_file_with_regular_file(self): with open('log.log', 'wb') as f: f.write('Some file contents!') reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver()) reader._open_file() try: self.assertEqual(reader.logfile.name, 'log.log') self.assertFalse(reader.logfile.closed) self.assertNotEqual(reader.logfile_id, None) self.assertEqual(reader.logfile.read(), 'Some file contents!') finally: reader.logfile.close()
def test_ensure_file_is_good_file_is_good(self): with open('log.log', 'wb') as f: f.write('Some file contents!') f.seek(10) reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver()) reader.logfile = f reader.logfile_id = get_device_and_inode_string(os.fstat(f.fileno())) reader._ensure_file_is_good() self.assertFalse(f.closed) self.assertEqual(reader.logfile, f) self.assertEqual(f.tell(), 10) self.assertEqual(self.fake_logging.log, [])
def test_open_file_with_gzip_file(self): self.files_to_delete.append('log.gz') with gzip.open('log.gz', 'wb') as f: f.write('Some file contents!') reader = LogReader(0, 'log.gz', Log4jParser(), FakeReceiver()) reader._open_file() try: self.assertEqual(reader.logfile.name, 'log.gz') self.assertFalse(reader.logfile.closed) self.assertNotEqual(reader.logfile_id, None) self.assertEqual(reader.logfile.read(), 'Some file contents!') finally: reader.logfile.close()
def test_ensure_file_is_good_file_has_been_rotated(self): with open('log.log', 'wb') as f: f.write('Some file contents!') reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver()) reader.logfile = f reader.logfile_id = 'not matching' reader._ensure_file_is_good() self.assertTrue(f.closed) self.assertNotEqual(reader.logfile, f) self.assertFalse(reader.logfile.closed) self.assertEqual(reader.logfile.readline(), 'Some file contents!') self.assertEqual(self.fake_logging.log, ['[INFO] The file log.log has been rotated.', '[INFO] Closed log.log.', '[INFO] Opened log.log.'])
def test_maybe_do_housekeeping_second_time_late_enough(self): called = [] reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') reader._ensure_file_is_good = lambda: called.append('_ensure_file_is_good') reader._save_progress = lambda: called.append('_save_progress') reader._adjust_loglevel_suppression = lambda: called.append('_adjust_loglevel_suppression') reader.last_ensure_file_is_good_call_timestamp = 23 reader.last_save_progress_call_timestamp = 23 reader.last_adjust_loglevel_suppression_call_timestamp = 23 reader._maybe_do_housekeeping(42) self.assertEqual(called, ['_ensure_file_is_good', '_save_progress', '_adjust_loglevel_suppression']) self.assertEqual(reader.last_ensure_file_is_good_call_timestamp, 42) self.assertEqual(reader.last_save_progress_call_timestamp, 42) self.assertEqual(reader.last_adjust_loglevel_suppression_call_timestamp, 42)
def test_run_with_loglevel_suppression(self): with open('log.log', 'wb') as f: f.write('2000-01-01 00:00:00,000 FlowID INFO Thread C.m(C.java:23) Info!\n') f.write('2000-01-01 00:00:01,000 FlowID ERROR Thread C.m(C.java:23) Error!\n') f.write('2000-01-01 00:00:02,000 FlowID TRACE Thread C.m(C.java:23) Trace!\n') f.write('2000-01-01 00:00:03,000 FlowID WARNING Thread C.m(C.java:23) Warning!\n') f.write('2000-01-01 00:00:04,000 FlowID FATAL Thread C.m(C.java:23) Fatal!') f.write('2000-01-01 00:00:05,000 FlowID DEBUG Thread C.m(C.java:23) Debug!') reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver()) reader.suppressed_log_level = 1 reader._maybe_do_housekeeping = lambda self: None reader.run() self.assertEqual(reader.receiver.entries[0].level, LogLevel.INFO) self.assertEqual(reader.receiver.entries[1].level, LogLevel.ERROR) self.assertEqual(reader.receiver.entries[2].level, LogLevel.WARN) self.assertEqual(reader.receiver.entries[3].level, LogLevel.FATAL) self.assertEqual(reader.receiver.entries[4], 'EOF 0') self.assertEqual(len(reader.receiver.entries), 5)
def test_ensure_file_is_good_file_does_not_exist(self): reader = LogReader(0, 'no.such.file', Log4jParser(), FakeReceiver()) reader._ensure_file_is_good() self.assertEqual(self.fake_logging.log, ['[INFO] The file no.such.file has been removed.'])
def test_make_progress_string_failure(self): reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') reader.logfile_id = '123g456' result = reader._make_progress_string() self.assertEqual(result, None) self.assertEqual(self.fake_logging.log, ['[ERROR] Failed to gather progress information for log.log.'])
def test_load_progress_with_spaces_in_filename(self): self.files_to_delete.append('progressf4a53d67a02158bcc92d7d702a8f438ad18309488') with open('progressf4a53d67a02158bcc92d7d702a8f438ad18309488', 'wb') as f: f.write('log with spaces in name.log 123g456 50 75') reader = LogReader(0, 'log with spaces in name.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') self.assertEqual(reader._load_progress(), ('log with spaces in name.log', '123g456', 50, 75))
def test_load_progress_basic(self): self.write_progress_file('log.log 123g456 50 75') reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') self.assertEqual(reader._load_progress(), ('log.log', '123g456', 50, 75))
def test_save_progress_failure(self): reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='invalid\0path') reader._make_progress_string = lambda: 'log.log 123g456 10 19' reader._save_progress() self.assertFalse(os.path.exists('progressf16c93d1167446f99a26837c0fdeac6fb73869794')) self.assertEqual(self.fake_logging.log[-1], '[ERROR] Failed to save progress for log.log.')
def test_save_progress_no_data(self): reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress') reader._make_progress_string = lambda: None reader._save_progress() self.assertFalse(os.path.exists('progressf16c93d1167446f99a26837c0fdeac6fb73869794'))
#!/usr/bin/python3 # test logreader from logreader import LogReader import os ### Example 2 ######################################################## # Now an example, reading from a file # The file we'll use is ./examples/1.data # First we need to get the path of the file we're reading from current_directory = os.path.dirname(__file__) #<-- absolute dir the script is in relative_path_to_data_file = "example/1.data" file_path = os.path.join(current_directory, relative_path_to_data_file) l = LogReader() l.readFile(file_path) # Now let's see the series of data we have for each metric for metric in l.getData(): print("%s: %s" % (metric, l.getData()[metric]))
def test_adjust_loglevel_suppression(self): fake_receiver = [1] reader = LogReader(0, 'log.log', Log4jParser(), fake_receiver) reader.START_SUPPRESSING_TRACE_ENTRIES_QUEUE_LENGTH = 2 reader.STOP_SUPPRESSING_TRACE_ENTRIES_QUEUE_LENGTH = 1 reader.START_SUPPRESSING_DEBUG_ENTRIES_QUEUE_LENGTH = 4 reader.STOP_SUPPRESSING_DEBUG_ENTRIES_QUEUE_LENGTH = 3 reader.START_SUPPRESSING_INFO_ENTRIES_QUEUE_LENGTH = 6 reader.STOP_SUPPRESSING_INFO_ENTRIES_QUEUE_LENGTH = 5 reader._adjust_loglevel_suppression() self.assertEqual(reader.suppressed_log_level, -1) fake_receiver.append(2) reader._adjust_loglevel_suppression() self.assertEqual(reader.suppressed_log_level, 0) fake_receiver.extend([3, 4]) reader._adjust_loglevel_suppression() self.assertEqual(reader.suppressed_log_level, 1) fake_receiver.remove(4) reader._adjust_loglevel_suppression() self.assertEqual(reader.suppressed_log_level, 0) fake_receiver.extend([4, 5, 6]) reader._adjust_loglevel_suppression() self.assertEqual(reader.suppressed_log_level, 2)
#!/usr/bin/python3 # test logreader from logreader import LogReader ### Example 1 ######################################################## # Let's input some log lines, just from strings # This is a nice self-contained example, further below for a file example line1 = "pan 12, tilt 15.0" line2 = "pan 13, tilt 16.0" line3 = "pan 14, tilt 20.0" line4 = "pan 16, tilt 21.0" line5 = "pan 18, tilt 20.0" l = LogReader() l.readLine(line1) l.readLine(line2) l.readLine(line3) l.readLine(line4) l.readLine(line5) # Now let's see the series of data we have for each metric for metric in l.getData(): print("%s: %s" % (metric, l.getData()[metric]))
#!/usr/bin/python3 # test logreader from logreader import LogReader ### Example 3 ######################################################## # Now an example, reading from a serial device # The serial device we'll use is "/dev/cu.usbmodem1411" # Try plugging in an Arduino that is using LogLineBuilder # and letting this read data for a bit and then unplug the Arduino # serial connection # device="/dev/cu.usbmodem1411" l = LogReader() print("Starting to listen to serial device on %s" % device) l.readSerial(device) print("Done reading") # Now let's see the series of data we have for each metric for metric in l.getData(): print("%s: %s" % (metric, l.getData()[metric]))