def test_save_progress_success(self):
self.files_to_delete.append('progressf16c93d1167446f99a26837c0fdeac6fb73869794')
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
reader._make_progress_string = lambda: 'log.log 123g456 10 19'
reader._save_progress()
with open('progressf16c93d1167446f99a26837c0fdeac6fb73869794', 'rb') as f:
self.assertEqual(f.read(), 'log.log 123g456 10 19')
def test_seek_tail_not_enough_lines(self):
self.write_log_file('2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' * 10)
with open('log.log', 'rb') as f:
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), tail_length=20)
reader.logfile = f
reader._seek_tail()
self.assertEqual(f.tell(), 0)
def test_make_progress_string_success(self):
with open('log.log', 'wb') as f:
f.write('Some file contents!')
f.seek(10)
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
reader.logfile = f
reader.logfile_id = '123g456'
self.assertEqual(reader._make_progress_string(), 'log.log 123g456 10 19')
def test_seek_tail_multiple_chunks(self):
self.write_log_file('2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' * 1000)
with open('log.log', 'rb') as f:
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), tail_length=900)
reader.CHUNK_SIZE = 1024
reader.logfile = f
reader._seek_tail()
self.assertEqual(f.tell(), 100 * 75)
def test_seek_first_unprocessed_position_no_progress_file(self):
self.write_log_file('2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' * 20)
with open('log.log', 'rb') as f:
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress', tail_length=10)
reader.logfile = f
reader._seek_first_unprocessed_position()
self.assertEqual(f.tell(), 0)
self.assertEqual(self.fake_logging.log, ['[WARN] Failed to read the progress file for "log.log".'])
def test_seek_tail_with_multiline_messages_one_chunk(self):
message = '2000-01-01 00:00:00,000 FlowID ERROR Thread C.m(C.java:23): Error! Nooooo!\n' + 'X' * 24 + '\n'
self.write_log_file(message * 10)
with open('log.log', 'rb') as f:
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), tail_length=5)
reader.CHUNK_SIZE = 1024
reader.logfile = f
reader._seek_tail()
self.assertEqual(f.tell(), 5 * 100)
def test_seek_first_unprocessed_position(self):
self.write_log_file('XXXX\n' * 100)
self.write_progress_file('log.log 123g456 50 75')
with open('log.log', 'rb') as f:
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
reader.logfile = f
reader._seek_first_unprocessed_position()
self.assertEqual(f.tell(), 50)
self.assertEqual(reader.logfile_id, '123g456')
def test_close_file(self):
with open('log.log', 'wb') as f:
f.write('Some file contents!')
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver())
reader._open_file()
f = reader.logfile
reader._close_file()
self.assertTrue(f.closed)
self.assertEqual(reader.logfile, None)
def __enter__(self):
lines = []
for i in self.seconds:
lines.append(self.message_template % divmod(i, 60))
with open('log.log', 'wb') as f:
f.write(''.join(lines))
self.log = open('log.log', 'rb')
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver())
reader.logfile = self.log
return reader
def test_open_file_with_regular_file(self):
with open('log.log', 'wb') as f:
f.write('Some file contents!')
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver())
reader._open_file()
try:
self.assertEqual(reader.logfile.name, 'log.log')
self.assertFalse(reader.logfile.closed)
self.assertNotEqual(reader.logfile_id, None)
self.assertEqual(reader.logfile.read(), 'Some file contents!')
finally:
reader.logfile.close()
def test_ensure_file_is_good_file_is_good(self):
with open('log.log', 'wb') as f:
f.write('Some file contents!')
f.seek(10)
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver())
reader.logfile = f
reader.logfile_id = get_device_and_inode_string(os.fstat(f.fileno()))
reader._ensure_file_is_good()
self.assertFalse(f.closed)
self.assertEqual(reader.logfile, f)
self.assertEqual(f.tell(), 10)
self.assertEqual(self.fake_logging.log, [])
def test_open_file_with_gzip_file(self):
self.files_to_delete.append('log.gz')
with gzip.open('log.gz', 'wb') as f:
f.write('Some file contents!')
reader = LogReader(0, 'log.gz', Log4jParser(), FakeReceiver())
reader._open_file()
try:
self.assertEqual(reader.logfile.name, 'log.gz')
self.assertFalse(reader.logfile.closed)
self.assertNotEqual(reader.logfile_id, None)
self.assertEqual(reader.logfile.read(), 'Some file contents!')
finally:
reader.logfile.close()
def test_ensure_file_is_good_file_has_been_rotated(self):
with open('log.log', 'wb') as f:
f.write('Some file contents!')
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver())
reader.logfile = f
reader.logfile_id = 'not matching'
reader._ensure_file_is_good()
self.assertTrue(f.closed)
self.assertNotEqual(reader.logfile, f)
self.assertFalse(reader.logfile.closed)
self.assertEqual(reader.logfile.readline(), 'Some file contents!')
self.assertEqual(self.fake_logging.log, ['[INFO] The file log.log has been rotated.',
'[INFO] Closed log.log.', '[INFO] Opened log.log.'])
def test_maybe_do_housekeeping_second_time_late_enough(self):
called = []
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
reader._ensure_file_is_good = lambda: called.append('_ensure_file_is_good')
reader._save_progress = lambda: called.append('_save_progress')
reader._adjust_loglevel_suppression = lambda: called.append('_adjust_loglevel_suppression')
reader.last_ensure_file_is_good_call_timestamp = 23
reader.last_save_progress_call_timestamp = 23
reader.last_adjust_loglevel_suppression_call_timestamp = 23
reader._maybe_do_housekeeping(42)
self.assertEqual(called, ['_ensure_file_is_good', '_save_progress', '_adjust_loglevel_suppression'])
self.assertEqual(reader.last_ensure_file_is_good_call_timestamp, 42)
self.assertEqual(reader.last_save_progress_call_timestamp, 42)
self.assertEqual(reader.last_adjust_loglevel_suppression_call_timestamp, 42)
def test_run_with_loglevel_suppression(self):
with open('log.log', 'wb') as f:
f.write('2000-01-01 00:00:00,000 FlowID INFO Thread C.m(C.java:23) Info!\n')
f.write('2000-01-01 00:00:01,000 FlowID ERROR Thread C.m(C.java:23) Error!\n')
f.write('2000-01-01 00:00:02,000 FlowID TRACE Thread C.m(C.java:23) Trace!\n')
f.write('2000-01-01 00:00:03,000 FlowID WARNING Thread C.m(C.java:23) Warning!\n')
f.write('2000-01-01 00:00:04,000 FlowID FATAL Thread C.m(C.java:23) Fatal!')
f.write('2000-01-01 00:00:05,000 FlowID DEBUG Thread C.m(C.java:23) Debug!')
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver())
reader.suppressed_log_level = 1
reader._maybe_do_housekeeping = lambda self: None
reader.run()
self.assertEqual(reader.receiver.entries[0].level, LogLevel.INFO)
self.assertEqual(reader.receiver.entries[1].level, LogLevel.ERROR)
self.assertEqual(reader.receiver.entries[2].level, LogLevel.WARN)
self.assertEqual(reader.receiver.entries[3].level, LogLevel.FATAL)
self.assertEqual(reader.receiver.entries[4], 'EOF 0')
self.assertEqual(len(reader.receiver.entries), 5)
def test_ensure_file_is_good_file_does_not_exist(self):
reader = LogReader(0, 'no.such.file', Log4jParser(), FakeReceiver())
reader._ensure_file_is_good()
self.assertEqual(self.fake_logging.log, ['[INFO] The file no.such.file has been removed.'])
def test_make_progress_string_failure(self):
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
reader.logfile_id = '123g456'
result = reader._make_progress_string()
self.assertEqual(result, None)
self.assertEqual(self.fake_logging.log, ['[ERROR] Failed to gather progress information for log.log.'])
def test_load_progress_with_spaces_in_filename(self):
self.files_to_delete.append('progressf4a53d67a02158bcc92d7d702a8f438ad18309488')
with open('progressf4a53d67a02158bcc92d7d702a8f438ad18309488', 'wb') as f:
f.write('log with spaces in name.log 123g456 50 75')
reader = LogReader(0, 'log with spaces in name.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
self.assertEqual(reader._load_progress(), ('log with spaces in name.log', '123g456', 50, 75))
def test_load_progress_basic(self):
self.write_progress_file('log.log 123g456 50 75')
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
self.assertEqual(reader._load_progress(), ('log.log', '123g456', 50, 75))
def test_save_progress_failure(self):
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='invalid\0path')
reader._make_progress_string = lambda: 'log.log 123g456 10 19'
reader._save_progress()
self.assertFalse(os.path.exists('progressf16c93d1167446f99a26837c0fdeac6fb73869794'))
self.assertEqual(self.fake_logging.log[-1], '[ERROR] Failed to save progress for log.log.')
def test_save_progress_no_data(self):
reader = LogReader(0, 'log.log', Log4jParser(), FakeReceiver(), progress_file_path_prefix='progress')
reader._make_progress_string = lambda: None
reader._save_progress()
self.assertFalse(os.path.exists('progressf16c93d1167446f99a26837c0fdeac6fb73869794'))
#!/usr/bin/python3
# test logreader
from logreader import LogReader
import os
### Example 2 ########################################################
# Now an example, reading from a file
# The file we'll use is ./examples/1.data
# First we need to get the path of the file we're reading from
current_directory = os.path.dirname(__file__) #<-- absolute dir the script is in
relative_path_to_data_file = "example/1.data"
file_path = os.path.join(current_directory, relative_path_to_data_file)
l = LogReader()
l.readFile(file_path)
# Now let's see the series of data we have for each metric
for metric in l.getData():
print("%s: %s" % (metric, l.getData()[metric]))
def test_adjust_loglevel_suppression(self):
fake_receiver = [1]
reader = LogReader(0, 'log.log', Log4jParser(), fake_receiver)
reader.START_SUPPRESSING_TRACE_ENTRIES_QUEUE_LENGTH = 2
reader.STOP_SUPPRESSING_TRACE_ENTRIES_QUEUE_LENGTH = 1
reader.START_SUPPRESSING_DEBUG_ENTRIES_QUEUE_LENGTH = 4
reader.STOP_SUPPRESSING_DEBUG_ENTRIES_QUEUE_LENGTH = 3
reader.START_SUPPRESSING_INFO_ENTRIES_QUEUE_LENGTH = 6
reader.STOP_SUPPRESSING_INFO_ENTRIES_QUEUE_LENGTH = 5
reader._adjust_loglevel_suppression()
self.assertEqual(reader.suppressed_log_level, -1)
fake_receiver.append(2)
reader._adjust_loglevel_suppression()
self.assertEqual(reader.suppressed_log_level, 0)
fake_receiver.extend([3, 4])
reader._adjust_loglevel_suppression()
self.assertEqual(reader.suppressed_log_level, 1)
fake_receiver.remove(4)
reader._adjust_loglevel_suppression()
self.assertEqual(reader.suppressed_log_level, 0)
fake_receiver.extend([4, 5, 6])
reader._adjust_loglevel_suppression()
self.assertEqual(reader.suppressed_log_level, 2)
#!/usr/bin/python3
# test logreader
from logreader import LogReader
### Example 1 ########################################################
# Let's input some log lines, just from strings
# This is a nice self-contained example, further below for a file example
line1 = "pan 12, tilt 15.0"
line2 = "pan 13, tilt 16.0"
line3 = "pan 14, tilt 20.0"
line4 = "pan 16, tilt 21.0"
line5 = "pan 18, tilt 20.0"
l = LogReader()
l.readLine(line1)
l.readLine(line2)
l.readLine(line3)
l.readLine(line4)
l.readLine(line5)
# Now let's see the series of data we have for each metric
for metric in l.getData():
print("%s: %s" % (metric, l.getData()[metric]))
#!/usr/bin/python3
# test logreader
from logreader import LogReader
### Example 3 ########################################################
# Now an example, reading from a serial device
# The serial device we'll use is "/dev/cu.usbmodem1411"
# Try plugging in an Arduino that is using LogLineBuilder
# and letting this read data for a bit and then unplug the Arduino
# serial connection
#
device="/dev/cu.usbmodem1411"
l = LogReader()
print("Starting to listen to serial device on %s" % device)
l.readSerial(device)
print("Done reading")
# Now let's see the series of data we have for each metric
for metric in l.getData():
print("%s: %s" % (metric, l.getData()[metric]))
