def test_log_attributes(self):
log_processor = LogFileProcessor(self.__path, file_system=self.__file_system,
log_attributes={'host': 'scalyr-1'})
log_processor.perform_processing(TestLogFileProcessor.TestAddEventsRequest(), current_time=self.__fake_time)
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.total_events(), 2)
self.assertEquals('scalyr-1', events.events[0]['attrs']['host'])
self.assertEquals('scalyr-1', events.events[1]['attrs']['host'])
def test_log_attributes(self):
log_processor = LogFileProcessor(self.__path,
file_system=self.__file_system,
log_attributes={'host': 'scalyr-1'})
log_processor.perform_processing(
TestLogFileProcessor.TestAddEventsRequest(),
current_time=self.__fake_time)
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.total_events(), 2)
self.assertEquals('scalyr-1', events.events[0]['attrs']['host'])
self.assertEquals('scalyr-1', events.events[1]['attrs']['host'])
class TestLogFileProcessor(unittest.TestCase):
def setUp(self):
self.__tempdir = tempfile.mkdtemp()
self.__file_system = FileSystem()
self.__path = os.path.join(self.__tempdir, 'text.txt')
self.__fake_time = 10
# Create the processor to test. We have it do one scan of an empty
# file so that when we next append lines to it, it will notice it.
# For now, we create one that does not have any log attributes and only
# counts the bytes of events messages as the cost.
self.write_file(self.__path, '')
self.log_processor = LogFileProcessor(self.__path, file_system=self.__file_system,
log_attributes={})
(completion_callback, buffer_full) = self.log_processor.perform_processing(
TestLogFileProcessor.TestAddEventsRequest(), current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
def test_basic_usage(self):
log_processor = self.log_processor
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
status = log_processor.generate_status()
self.assertEquals(23L, status.total_bytes_pending)
self.assertEquals(0L, status.total_bytes_copied)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
status = log_processor.generate_status()
self.assertEquals(0L, status.total_bytes_pending)
self.assertEquals(23L, status.total_bytes_copied)
# Add some more text to make sure it appears.
self.append_file(self.__path, 'Third line\n')
log_processor.scan_for_new_bytes(current_time=self.__fake_time)
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
status = log_processor.generate_status()
self.assertEquals(11L, status.total_bytes_pending)
self.assertEquals(23L, status.total_bytes_copied)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.get_message(0), 'Third line\n')
status = log_processor.generate_status()
self.assertEquals(0L, status.total_bytes_pending)
self.assertEquals(34L, status.total_bytes_copied)
def test_fail_and_retry(self):
log_processor = self.log_processor
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
self.assertFalse(completion_callback(LogFileProcessor.FAIL_AND_RETRY))
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
def test_fail_and_drop(self):
log_processor = self.log_processor
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
self.assertFalse(completion_callback(LogFileProcessor.FAIL_AND_DROP))
# Add some more text to make sure it appears.
self.append_file(self.__path, 'Third line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(1, events.total_events())
self.assertEquals(events.get_message(0), 'Third line\n')
def test_sampling_rule(self):
log_processor = self.log_processor
log_processor.add_sampler('INFO', 0)
log_processor.add_sampler('ERROR', 1)
self.append_file(self.__path, 'INFO First line\nERROR Second line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.get_message(0), 'ERROR Second line\n')
def test_redaction_rule(self):
log_processor = self.log_processor
log_processor.add_redacter('password=[^&]+', 'password=foo')
self.append_file(self.__path, 'GET /foo&password=FakePassword&start=true\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.get_message(0), 'GET /foo&password=foo&start=true\n')
def test_signals_deletion(self):
log_processor = self.log_processor
# Delete the file.
os.remove(self.__path)
# We won't signal that the file processor should be deleted until 10 mins have passed.
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(0, events.total_events())
self.__fake_time += 9 * 60
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.__fake_time += 62
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertTrue(completion_callback(LogFileProcessor.SUCCESS))
def test_log_attributes(self):
log_processor = LogFileProcessor(self.__path, file_system=self.__file_system,
log_attributes={'host': 'scalyr-1'})
log_processor.perform_processing(TestLogFileProcessor.TestAddEventsRequest(), current_time=self.__fake_time)
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.total_events(), 2)
self.assertEquals('scalyr-1', events.events[0]['attrs']['host'])
self.assertEquals('scalyr-1', events.events[1]['attrs']['host'])
def write_file(self, path, *lines):
contents = ''.join(lines)
file_handle = open(path, 'w')
file_handle.write(contents)
file_handle.close()
def append_file(self, path, *lines):
contents = ''.join(lines)
file_handle = open(path, 'a')
file_handle.write(contents)
file_handle.close()
class TestAddEventsRequest(object):
def __init__(self, limit=10):
self.events = []
self.__limit = limit
def add_event(self, event):
if len(self.events) < self.__limit:
self.events.append(event)
return True
else:
return False
def position(self):
return len(self.events)
def set_position(self, position):
self.events = self.events[0:position]
def get_message(self, index):
"""Returns the message field from an events object."""
return self.events[index]['attrs']['message']
def total_events(self):
return len(self.events)
class TestLogFileProcessor(unittest.TestCase):
def setUp(self):
self.__tempdir = tempfile.mkdtemp()
self.__file_system = FileSystem()
self.__path = os.path.join(self.__tempdir, 'text.txt')
self.__fake_time = 10
# Create the processor to test. We have it do one scan of an empty
# file so that when we next append lines to it, it will notice it.
# For now, we create one that does not have any log attributes and only
# counts the bytes of events messages as the cost.
self.write_file(self.__path, '')
self.log_processor = LogFileProcessor(self.__path,
file_system=self.__file_system,
log_attributes={})
(completion_callback,
buffer_full) = self.log_processor.perform_processing(
TestLogFileProcessor.TestAddEventsRequest(),
current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
def test_basic_usage(self):
log_processor = self.log_processor
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertEquals(len(events.threads), 1)
status = log_processor.generate_status()
self.assertEquals(23L, status.total_bytes_pending)
self.assertEquals(0L, status.total_bytes_copied)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
status = log_processor.generate_status()
self.assertEquals(0L, status.total_bytes_pending)
self.assertEquals(23L, status.total_bytes_copied)
# Add some more text to make sure it appears.
self.append_file(self.__path, 'Third line\n')
log_processor.scan_for_new_bytes(current_time=self.__fake_time)
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
status = log_processor.generate_status()
self.assertEquals(11L, status.total_bytes_pending)
self.assertEquals(23L, status.total_bytes_copied)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.get_message(0), 'Third line\n')
status = log_processor.generate_status()
self.assertEquals(0L, status.total_bytes_pending)
self.assertEquals(34L, status.total_bytes_copied)
def test_fail_and_retry(self):
log_processor = self.log_processor
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
self.assertFalse(completion_callback(LogFileProcessor.FAIL_AND_RETRY))
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
def test_fail_and_drop(self):
log_processor = self.log_processor
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
self.assertFalse(completion_callback(LogFileProcessor.FAIL_AND_DROP))
# Add some more text to make sure it appears.
self.append_file(self.__path, 'Third line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(1, events.total_events())
self.assertEquals(events.get_message(0), 'Third line\n')
def test_sampling_rule(self):
log_processor = self.log_processor
log_processor.add_sampler('INFO', 0)
log_processor.add_sampler('ERROR', 1)
self.append_file(self.__path, 'INFO First line\nERROR Second line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.get_message(0), 'ERROR Second line\n')
def test_redaction_rule(self):
log_processor = self.log_processor
log_processor.add_redacter('password=[^&]+', 'password=foo')
self.append_file(self.__path,
'GET /foo&password=FakePassword&start=true\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.get_message(0),
'GET /foo&password=foo&start=true\n')
def test_signals_deletion(self):
log_processor = self.log_processor
# Delete the file.
os.remove(self.__path)
# We won't signal that the file processor should be deleted until 10 mins have passed.
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(0, events.total_events())
self.__fake_time += 9 * 60
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.__fake_time += 62
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertTrue(completion_callback(LogFileProcessor.SUCCESS))
def test_log_attributes(self):
log_processor = LogFileProcessor(self.__path,
file_system=self.__file_system,
log_attributes={'host': 'scalyr-1'})
log_processor.perform_processing(
TestLogFileProcessor.TestAddEventsRequest(),
current_time=self.__fake_time)
self.append_file(self.__path, 'First line\nSecond line\n')
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(events.total_events(), 2)
self.assertEquals('scalyr-1', events.events[0]['attrs']['host'])
self.assertEquals('scalyr-1', events.events[1]['attrs']['host'])
def test_unique_id(self):
first_thread_id = LogFileProcessor.generate_unique_thread_id()
self.assertTrue(first_thread_id.startswith('log_'))
sequence = int(first_thread_id[4:])
self.assertTrue(sequence > 0)
self.assertEquals(first_thread_id, 'log_%d' % sequence)
self.assertEquals(LogFileProcessor.generate_unique_thread_id(),
'log_%d' % (sequence + 1))
def test_thread_id_fails_to_be_added(self):
log_processor = self.log_processor
self.append_file(self.__path, 'First line\nSecond line\n')
# Make sure if adding the thread id in fails, then unread the lines and reset everything to normal.
# We can see if it is normal by making sure the lines are read in the next successful call.
events = TestLogFileProcessor.TestAddEventsRequest(thread_limit=0)
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(0, events.total_events())
self.assertEquals(len(events.threads), 0)
# Now have a succuessful call and make sure we get the lines.
events = TestLogFileProcessor.TestAddEventsRequest()
(completion_callback, buffer_full) = log_processor.perform_processing(
events, current_time=self.__fake_time)
self.assertFalse(completion_callback(LogFileProcessor.SUCCESS))
self.assertEquals(2, events.total_events())
self.assertEquals(events.get_message(0), 'First line\n')
self.assertEquals(events.get_message(1), 'Second line\n')
self.assertEquals(len(events.threads), 1)
def write_file(self, path, *lines):
contents = ''.join(lines)
file_handle = open(path, 'wb')
file_handle.write(contents)
file_handle.close()
def append_file(self, path, *lines):
contents = ''.join(lines)
file_handle = open(path, 'ab')
file_handle.write(contents)
file_handle.close()
class TestAddEventsRequest(object):
def __init__(self, limit=10, thread_limit=10):
self.events = []
self.__limit = limit
self.__thread_limit = thread_limit
self.threads = {}
def add_event(self, event):
if len(self.events) < self.__limit:
self.events.append(event)
return True
else:
return False
def position(self):
return [len(self.events), dict(self.threads)]
def set_position(self, position):
self.events = self.events[0:position[0]]
self.threads = position[1]
def add_thread(self, thread_id, thread_name):
if self.__thread_limit == len(self.threads):
return False
self.threads[thread_id] = thread_name
return True
def get_message(self, index):
"""Returns the message field from an events object."""
return self.events[index]['attrs']['message']
def total_events(self):
return len(self.events)