def testParseDarwin(self):
"""Tests the Parse function on an Darwin-style syslog file"""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2016}
storage_writer = self._ParseFile(
['syslog_osx'],
parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 0)
self.assertEqual(storage_writer.number_of_events, 2)
def testParseRsyslogTraditional(self):
"""Tests the Parse function on a traditional rsyslog file."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2016}
storage_writer = self._ParseFile(
['syslog_rsyslog_traditional'],
parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 0)
self.assertEqual(storage_writer.number_of_events, 8)
def testParseRsyslog(self):
"""Tests the Parse function on an Ubuntu-style syslog file"""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2016}
storage_writer = self._ParseFile(
['syslog_rsyslog'],
parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_errors, 0)
self.assertEqual(storage_writer.number_of_events, 8)
def testParseRsyslog(self):
"""Tests the Parse function on a rsyslog file."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2020}
storage_writer = self._ParseFile(
['syslog_rsyslog'],
parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_events, 5)
self.assertEqual(storage_writer.number_of_extraction_warnings, 0)
self.assertEqual(storage_writer.number_of_recovery_warnings, 0)
def testEnablePlugins(self):
"""Tests the EnablePlugins function."""
parser = syslog.SyslogParser()
number_of_plugins = len(parser._plugin_classes)
parser.EnablePlugins([])
self.assertEqual(len(parser._plugins), 0)
parser.EnablePlugins(parser.ALL_PLUGINS)
self.assertEqual(len(parser._plugins), number_of_plugins)
parser.EnablePlugins(['cron'])
self.assertEqual(len(parser._plugins), 1)
def testParseChromeOS(self):
"""Tests the Parse function."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2016}
storage_writer = self._ParseFile(
['syslog_chromeos'],
parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 0)
self.assertEqual(storage_writer.number_of_events, 8)
events = list(storage_writer.GetSortedEvents())
event = events[0]
self.CheckTimestamp(event.timestamp, '2016-10-25 19:37:23.297265')
expected_message = (
'INFO [periodic_scheduler, pid: 13707] cleanup_logs: job completed'
)
self._TestGetMessageStrings(event, expected_message, expected_message)
event = events[2]
self.CheckTimestamp(event.timestamp, '2016-10-25 19:37:24.987014')
# Testing year increment.
event = events[4]
self.CheckTimestamp(event.timestamp, '2016-10-25 19:37:24.993079')
event = events[6]
expected_reporter = 'kernel'
self.assertEqual(event.reporter, expected_reporter)
event = events[7]
expected_message = (
'INFO [aprocess] [ 316.587330] cfg80211: This is a multi-line\t'
'message that screws up many syslog parsers.')
expected_short_message = (
'INFO [aprocess] [ 316.587330] cfg80211: This is a multi-line\t'
'message that sc...')
self._TestGetMessageStrings(event, expected_message,
expected_short_message)
def testParseRsyslog(self):
"""Tests the Parse function on a rsyslog file."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2020}
storage_writer = self._ParseFile(
['syslog_rsyslog'],
parser,
knowledge_base_values=knowledge_base_values)
number_of_events = storage_writer.GetNumberOfAttributeContainers(
'event')
self.assertEqual(number_of_events, 5)
number_of_warnings = storage_writer.GetNumberOfAttributeContainers(
'extraction_warning')
self.assertEqual(number_of_warnings, 0)
number_of_warnings = storage_writer.GetNumberOfAttributeContainers(
'recovery_warning')
self.assertEqual(number_of_warnings, 0)
def testParseRsyslogTraditional(self):
"""Tests the Parse function on a traditional rsyslog file."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2016}
storage_writer = self._ParseFile(
['syslog_rsyslog_traditional'], parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 0)
self.assertEqual(storage_writer.number_of_events, 8)
events = list(storage_writer.GetSortedEvents())
expected_event_values = {
'data_type': 'syslog:line',
'hostname': 'myhostname.myhost.com',
'reporter': 'Job',
'severity': None,
'timestamp': '2016-01-22 07:54:32.000000'}
self.CheckEventValues(storage_writer, events[0], expected_event_values)
def testParse(self):
"""Tests the Parse function."""
parser_object = syslog.SyslogParser()
knowledge_base_values = {u'year': 2012}
test_file = self._GetTestFilePath([u'syslog'])
event_queue_consumer = self._ParseFile(
parser_object,
test_file,
knowledge_base_values=knowledge_base_values)
event_objects = self._GetEventObjectsFromQueue(event_queue_consumer)
event_timestamp = timelib.Timestamp.CopyToIsoFormat(
event_objects[0].timestamp)
self.assertEqual(event_timestamp, u'2012-01-22T07:52:33+00:00')
self.assertEqual(event_objects[0].hostname, u'myhostname.myhost.com')
expected_string = (u'[client, pid: 30840] INFO No new content.')
self._TestGetMessageStrings(event_objects[0], expected_string,
expected_string)
event_timestamp = timelib.Timestamp.CopyToIsoFormat(
event_objects[6].timestamp)
self.assertEqual(event_timestamp, u'2012-02-29T01:15:43+00:00')
# Testing year increment.
event_timestamp = timelib.Timestamp.CopyToIsoFormat(
event_objects[8].timestamp)
self.assertEqual(event_timestamp, u'2013-03-23T23:01:18+00:00')
expected_msg = (
u'[aprocess, pid: 10100] This is a multi-line message that screws up'
u'\tmany syslog parsers.')
expected_msg_short = (
u'[aprocess, pid: 10100] This is a multi-line message that screws up'
u'\tmany syslo...')
self._TestGetMessageStrings(event_objects[11], expected_msg,
expected_msg_short)
self.assertEqual(len(event_objects), 13)
def _ParseFileWithPlugin(
self, plugin_name, path, knowledge_base_values=None):
"""Parses a syslog file with a specific plugin.
Args:
plugin_name: a string containing the name of the plugin.
path: a string containing the path to the syslog file.
knowledge_base_values: optional dictionary containing the knowledge base
values.
Returns:
An event object queue consumer object (instance of ItemQueueConsumer).
"""
event_queue = single_process.SingleProcessQueue()
event_queue_consumer = test_lib.TestItemQueueConsumer(event_queue)
parse_error_queue = single_process.SingleProcessQueue()
parser_mediator = self._GetParserMediator(
event_queue, parse_error_queue,
knowledge_base_values=knowledge_base_values)
path_spec = path_spec_factory.Factory.NewPathSpec(
definitions.TYPE_INDICATOR_OS, location=path)
file_entry = path_spec_resolver.Resolver.OpenFileEntry(path_spec)
parser_mediator.SetFileEntry(file_entry)
parser_object = syslog.SyslogParser()
parser_object.EnablePlugins([plugin_name])
file_object = file_entry.GetFileObject()
try:
parser_object.Parse(parser_mediator, file_object)
finally:
file_object.close()
return event_queue_consumer
def _ParseFileWithPlugin(self,
path_segments,
plugin_name,
knowledge_base_values=None):
"""Parses a syslog file with a specific plugin.
Args:
path_segments (list[str]): path segments inside the test data directory.
plugin_name (str): name of the plugin.
knowledge_base_values (Optional[dict]): knowledge base values.
Returns:
FakeStorageWriter: storage writer.
Raises:
SkipTest: if the path inside the test data directory does not exist and
the test should be skipped.
"""
session = sessions.Session()
storage_writer = fake_writer.FakeStorageWriter(session)
storage_writer.Open()
file_entry = self._GetTestFileEntry(path_segments)
parser_mediator = self._CreateParserMediator(
storage_writer,
file_entry=file_entry,
knowledge_base_values=knowledge_base_values)
parser = syslog.SyslogParser()
parser.EnablePlugins([plugin_name])
file_object = file_entry.GetFileObject()
try:
parser.Parse(parser_mediator, file_object)
finally:
file_object.close()
return storage_writer
def testParseRsyslogTraditional(self):
"""Tests the Parse function on a traditional rsyslog file."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2016}
storage_writer = self._ParseFile(
['syslog_rsyslog_traditional'],
parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 0)
self.assertEqual(storage_writer.number_of_events, 8)
events = list(storage_writer.GetSortedEvents())
event = events[0]
self.CheckTimestamp(event.timestamp, '2016-01-22 07:54:32.000000')
event_data = self._GetEventDataOfEvent(storage_writer, event)
self.assertEqual(event_data.data_type, 'syslog:line')
self.assertEqual(event_data.hostname, 'myhostname.myhost.com')
self.assertEqual(event_data.reporter, 'Job')
self.assertIsNone(event_data.severity)
def testParseRsyslogSysklogd(self):
"""Tests the Parse function on a syslogkd format rsyslog file."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2021}
storage_writer = self._ParseFile(
['syslog_rsyslog_SysklogdFileFormat'],
parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_events, 9)
self.assertEqual(storage_writer.number_of_extraction_warnings, 0)
self.assertEqual(storage_writer.number_of_recovery_warnings, 0)
events = list(storage_writer.GetSortedEvents())
expected_event_values = {
'date_time': '2021-03-06 04:07:28',
'data_type': 'syslog:line',
'hostname': 'hostname',
'reporter': 'log_tag',
'severity': None
}
self.CheckEventValues(storage_writer, events[0], expected_event_values)
def _ParseFileWithPlugin(self,
path_segments,
plugin_name,
knowledge_base_values=None):
"""Parses a syslog file with a specific plugin.
Args:
path_segments: a list of strings containinge the path segments inside
the test data directory.
plugin_name: a string containing the name of the plugin.
knowledge_base_values: optional dictionary containing the knowledge base
values.
Returns:
A storage writer object (instance of FakeStorageWriter).
"""
session = sessions.Session()
storage_writer = fake_storage.FakeStorageWriter(session)
storage_writer.Open()
file_entry = self._GetTestFileEntry(path_segments)
parser_mediator = self._CreateParserMediator(
storage_writer,
file_entry=file_entry,
knowledge_base_values=knowledge_base_values)
parser_object = syslog.SyslogParser()
parser_object.EnablePlugins([plugin_name])
file_object = file_entry.GetFileObject()
try:
parser_object.Parse(parser_mediator, file_object)
finally:
file_object.close()
return storage_writer
def testParse(self):
"""Tests the Parse function."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2012}
storage_writer = self._ParseFile(
['syslog'], parser, knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 1)
self.assertEqual(storage_writer.number_of_events, 16)
events = list(storage_writer.GetSortedEvents())
expected_event_values = {
'body': 'INFO No new content in ímynd.dd.',
'data_type': 'syslog:line',
'hostname': 'myhostname.myhost.com',
'pid': 30840,
'reporter': 'client',
'severity': None,
'timestamp': '2012-01-22 07:52:33.000000'}
self.CheckEventValues(storage_writer, events[0], expected_event_values)
expected_event_values = {
'data_type': 'syslog:line',
'reporter': '---',
'severity': None,
'timestamp': '2012-02-29 01:15:43.000000'}
self.CheckEventValues(storage_writer, events[6], expected_event_values)
# Testing year increment.
expected_event_values = {
'body': 'This syslog message has a fractional value for seconds.',
'data_type': 'syslog:line',
'reporter': 'somrandomexe',
'severity': None,
'timestamp': '2013-03-23 23:01:18.000000'}
self.CheckEventValues(storage_writer, events[9], expected_event_values)
expected_event_values = {
'data_type': 'syslog:line',
'reporter': '/sbin/anacron',
'severity': None}
self.CheckEventValues(storage_writer, events[11], expected_event_values)
expected_event_values = {
'body': (
'This is a multi-line message that screws up\n\tmany syslog '
'parsers.'),
'data_type': 'syslog:line',
'pid': 10100,
'reporter': 'aprocess',
'severity': None}
self.CheckEventValues(storage_writer, events[10], expected_event_values)
expected_event_values = {
'body': '[997.390602] sda2: rw=0, want=65, limit=2',
'data_type': 'syslog:line',
'hostname': None,
'reporter': 'kernel',
'severity': None}
self.CheckEventValues(storage_writer, events[14], expected_event_values)
# Testing non-leap year.
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2013}
storage_writer = self._ParseFile(
['syslog'], parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 2)
self.assertEqual(storage_writer.number_of_events, 15)
def setUp(self):
"""Sets up the needed objects used throughout the test."""
pre_obj = event.PreprocessObject()
pre_obj.year = 2012
self._parser = syslog.SyslogParser(pre_obj, None)
def setUp(self):
"""Makes preparations before running an individual test."""
self._parser = syslog.SyslogParser()
def testParseChromeOS(self):
"""Tests the Parse function."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2016}
storage_writer = self._ParseFile(
['syslog_chromeos'],
parser,
knowledge_base_values=knowledge_base_values)
number_of_events = storage_writer.GetNumberOfAttributeContainers(
'event')
self.assertEqual(number_of_events, 8)
number_of_warnings = storage_writer.GetNumberOfAttributeContainers(
'extraction_warning')
self.assertEqual(number_of_warnings, 0)
number_of_warnings = storage_writer.GetNumberOfAttributeContainers(
'recovery_warning')
self.assertEqual(number_of_warnings, 0)
events = list(storage_writer.GetSortedEvents())
# Note that syslog_chromeos contains -07:00 as time zone offset.
expected_event_values = {
'body': 'cleanup_logs: job completed',
'date_time': '2016-10-25 12:37:23.297265',
'data_type': 'syslog:line',
'reporter': 'periodic_scheduler',
'pid': 13707,
'severity': 'INFO',
'timestamp': '2016-10-25 19:37:23.297265'
}
self.CheckEventValues(storage_writer, events[0], expected_event_values)
expected_event_values = {
'date_time': '2016-10-25 12:37:24.987014',
'data_type': 'syslog:line',
'reporter': 'kernel',
'severity': 'DEBUG'
}
self.CheckEventValues(storage_writer, events[2], expected_event_values)
# Testing year increment.
expected_event_values = {
'date_time': '2016-10-25 12:37:24.993079',
'data_type': 'syslog:line',
'reporter': 'kernel',
'severity': 'DEBUG'
}
self.CheckEventValues(storage_writer, events[4], expected_event_values)
expected_event_values = {
'date_time': '2016-10-25 12:37:25.007963',
'data_type': 'syslog:line',
'reporter': 'kernel',
'severity': 'ERR'
}
self.CheckEventValues(storage_writer, events[6], expected_event_values)
expected_event_values = {
'body':
('[ 316.587330] cfg80211: This is a multi-line\n\tmessage that '
'screws up many syslog parsers.'),
'date_time':
'2016-10-25 12:37:25.014015',
'data_type':
'syslog:line',
'reporter':
'aprocess',
'severity':
'INFO'
}
self.CheckEventValues(storage_writer, events[7], expected_event_values)
def testParse(self):
"""Tests the Parse function."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2012}
storage_writer = self._ParseFile(
['syslog'], parser, knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_events, 16)
events = list(storage_writer.GetSortedEvents())
event = events[1]
self.CheckTimestamp(event.timestamp, '2012-01-22 07:52:33.000000')
self.assertEqual(event.hostname, 'myhostname.myhost.com')
expected_message = (
'[client, pid: 30840] INFO No new content in ímynd.dd.')
self._TestGetMessageStrings(event, expected_message, expected_message)
event = events[6]
self.CheckTimestamp(event.timestamp, '2012-02-29 01:15:43.000000')
# Testing year increment.
event = events[8]
self.CheckTimestamp(event.timestamp, '2013-03-23 23:01:18.000000')
event = events[11]
expected_reporter = '/sbin/anacron'
self.assertEqual(event.reporter, expected_reporter)
event = events[10]
expected_message = (
'[aprocess, pid: 10100] This is a multi-line message that screws up'
'\tmany syslog parsers.')
expected_short_message = (
'[aprocess, pid: 10100] This is a multi-line message that screws up'
'\tmany syslo...')
self._TestGetMessageStrings(event, expected_message,
expected_short_message)
event = events[14]
self.assertEqual(event.reporter, 'kernel')
self.assertIsNone(event.hostname)
expected_message = (
'[kernel] [997.390602] sda2: rw=0, want=65, limit=2')
expected_short_message = (
'[kernel] [997.390602] sda2: rw=0, want=65, limit=2')
self._TestGetMessageStrings(event, expected_message,
expected_short_message)
# Testing non-leap year.
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2013}
storage_writer = self._ParseFile(
['syslog'], parser, knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_events, 15)
def testParse(self):
"""Tests the Parse function."""
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2012}
storage_writer = self._ParseFile(
['syslog'], parser, knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 1)
self.assertEqual(storage_writer.number_of_events, 16)
events = list(storage_writer.GetSortedEvents())
event = events[0]
self.CheckTimestamp(event.timestamp, '2012-01-22 07:52:33.000000')
event_data = self._GetEventDataOfEvent(storage_writer, event)
self.assertEqual(event_data.data_type, 'syslog:line')
self.assertEqual(event_data.hostname, 'myhostname.myhost.com')
self.assertEqual(event_data.reporter, 'client')
self.assertIsNone(event_data.severity)
expected_message = (
'[client, pid: 30840] INFO No new content in ímynd.dd.')
self._TestGetMessageStrings(
event_data, expected_message, expected_message)
event = events[6]
self.CheckTimestamp(event.timestamp, '2012-02-29 01:15:43.000000')
event_data = self._GetEventDataOfEvent(storage_writer, event)
self.assertEqual(event_data.reporter, '---')
self.assertIsNone(event_data.severity)
# Testing year increment.
event = events[9]
self.CheckTimestamp(event.timestamp, '2013-03-23 23:01:18.000000')
event_data = self._GetEventDataOfEvent(storage_writer, event)
self.assertEqual(
event_data.body,
'This syslog message has a fractional value for seconds.')
self.assertEqual(event_data.reporter, 'somrandomexe')
self.assertIsNone(event_data.severity)
event = events[11]
event_data = self._GetEventDataOfEvent(storage_writer, event)
self.assertEqual(event_data.reporter, '/sbin/anacron')
self.assertIsNone(event_data.severity)
event = events[10]
event_data = self._GetEventDataOfEvent(storage_writer, event)
self.assertEqual(event_data.reporter, 'aprocess')
self.assertIsNone(event_data.severity)
expected_message = (
'[aprocess, pid: 10100] This is a multi-line message that screws up'
'\tmany syslog parsers.')
expected_short_message = (
'[aprocess, pid: 10100] This is a multi-line message that screws up'
'\tmany syslo...')
self._TestGetMessageStrings(
event_data, expected_message, expected_short_message)
event = events[14]
event_data = self._GetEventDataOfEvent(storage_writer, event)
self.assertIsNone(event_data.hostname)
self.assertEqual(event_data.reporter, 'kernel')
self.assertIsNone(event_data.severity)
expected_message = (
'[kernel] [997.390602] sda2: rw=0, want=65, limit=2')
expected_short_message = (
'[kernel] [997.390602] sda2: rw=0, want=65, limit=2')
self._TestGetMessageStrings(
event_data, expected_message, expected_short_message)
# Testing non-leap year.
parser = syslog.SyslogParser()
knowledge_base_values = {'year': 2013}
storage_writer = self._ParseFile(
['syslog'], parser,
knowledge_base_values=knowledge_base_values)
self.assertEqual(storage_writer.number_of_warnings, 2)
self.assertEqual(storage_writer.number_of_events, 15)
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self._parser = syslog.SyslogParser()
