def testParseDarwin(self): """Tests the Parse function on an Darwin-style syslog file""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2016} storage_writer = self._ParseFile( ['syslog_osx'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 0) self.assertEqual(storage_writer.number_of_events, 2)
def testParseRsyslogTraditional(self): """Tests the Parse function on a traditional rsyslog file.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2016} storage_writer = self._ParseFile( ['syslog_rsyslog_traditional'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 0) self.assertEqual(storage_writer.number_of_events, 8)
def testParseRsyslog(self): """Tests the Parse function on an Ubuntu-style syslog file""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2016} storage_writer = self._ParseFile( ['syslog_rsyslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_errors, 0) self.assertEqual(storage_writer.number_of_events, 8)
def testParseRsyslog(self): """Tests the Parse function on a rsyslog file.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2020} storage_writer = self._ParseFile( ['syslog_rsyslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_events, 5) self.assertEqual(storage_writer.number_of_extraction_warnings, 0) self.assertEqual(storage_writer.number_of_recovery_warnings, 0)
def testEnablePlugins(self): """Tests the EnablePlugins function.""" parser = syslog.SyslogParser() number_of_plugins = len(parser._plugin_classes) parser.EnablePlugins([]) self.assertEqual(len(parser._plugins), 0) parser.EnablePlugins(parser.ALL_PLUGINS) self.assertEqual(len(parser._plugins), number_of_plugins) parser.EnablePlugins(['cron']) self.assertEqual(len(parser._plugins), 1)
def testParseChromeOS(self): """Tests the Parse function.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2016} storage_writer = self._ParseFile( ['syslog_chromeos'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 0) self.assertEqual(storage_writer.number_of_events, 8) events = list(storage_writer.GetSortedEvents()) event = events[0] self.CheckTimestamp(event.timestamp, '2016-10-25 19:37:23.297265') expected_message = ( 'INFO [periodic_scheduler, pid: 13707] cleanup_logs: job completed' ) self._TestGetMessageStrings(event, expected_message, expected_message) event = events[2] self.CheckTimestamp(event.timestamp, '2016-10-25 19:37:24.987014') # Testing year increment. event = events[4] self.CheckTimestamp(event.timestamp, '2016-10-25 19:37:24.993079') event = events[6] expected_reporter = 'kernel' self.assertEqual(event.reporter, expected_reporter) event = events[7] expected_message = ( 'INFO [aprocess] [ 316.587330] cfg80211: This is a multi-line\t' 'message that screws up many syslog parsers.') expected_short_message = ( 'INFO [aprocess] [ 316.587330] cfg80211: This is a multi-line\t' 'message that sc...') self._TestGetMessageStrings(event, expected_message, expected_short_message)
def testParseRsyslog(self): """Tests the Parse function on a rsyslog file.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2020} storage_writer = self._ParseFile( ['syslog_rsyslog'], parser, knowledge_base_values=knowledge_base_values) number_of_events = storage_writer.GetNumberOfAttributeContainers( 'event') self.assertEqual(number_of_events, 5) number_of_warnings = storage_writer.GetNumberOfAttributeContainers( 'extraction_warning') self.assertEqual(number_of_warnings, 0) number_of_warnings = storage_writer.GetNumberOfAttributeContainers( 'recovery_warning') self.assertEqual(number_of_warnings, 0)
def testParseRsyslogTraditional(self): """Tests the Parse function on a traditional rsyslog file.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2016} storage_writer = self._ParseFile( ['syslog_rsyslog_traditional'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 0) self.assertEqual(storage_writer.number_of_events, 8) events = list(storage_writer.GetSortedEvents()) expected_event_values = { 'data_type': 'syslog:line', 'hostname': 'myhostname.myhost.com', 'reporter': 'Job', 'severity': None, 'timestamp': '2016-01-22 07:54:32.000000'} self.CheckEventValues(storage_writer, events[0], expected_event_values)
def testParse(self): """Tests the Parse function.""" parser_object = syslog.SyslogParser() knowledge_base_values = {u'year': 2012} test_file = self._GetTestFilePath([u'syslog']) event_queue_consumer = self._ParseFile( parser_object, test_file, knowledge_base_values=knowledge_base_values) event_objects = self._GetEventObjectsFromQueue(event_queue_consumer) event_timestamp = timelib.Timestamp.CopyToIsoFormat( event_objects[0].timestamp) self.assertEqual(event_timestamp, u'2012-01-22T07:52:33+00:00') self.assertEqual(event_objects[0].hostname, u'myhostname.myhost.com') expected_string = (u'[client, pid: 30840] INFO No new content.') self._TestGetMessageStrings(event_objects[0], expected_string, expected_string) event_timestamp = timelib.Timestamp.CopyToIsoFormat( event_objects[6].timestamp) self.assertEqual(event_timestamp, u'2012-02-29T01:15:43+00:00') # Testing year increment. event_timestamp = timelib.Timestamp.CopyToIsoFormat( event_objects[8].timestamp) self.assertEqual(event_timestamp, u'2013-03-23T23:01:18+00:00') expected_msg = ( u'[aprocess, pid: 10100] This is a multi-line message that screws up' u'\tmany syslog parsers.') expected_msg_short = ( u'[aprocess, pid: 10100] This is a multi-line message that screws up' u'\tmany syslo...') self._TestGetMessageStrings(event_objects[11], expected_msg, expected_msg_short) self.assertEqual(len(event_objects), 13)
def _ParseFileWithPlugin( self, plugin_name, path, knowledge_base_values=None): """Parses a syslog file with a specific plugin. Args: plugin_name: a string containing the name of the plugin. path: a string containing the path to the syslog file. knowledge_base_values: optional dictionary containing the knowledge base values. Returns: An event object queue consumer object (instance of ItemQueueConsumer). """ event_queue = single_process.SingleProcessQueue() event_queue_consumer = test_lib.TestItemQueueConsumer(event_queue) parse_error_queue = single_process.SingleProcessQueue() parser_mediator = self._GetParserMediator( event_queue, parse_error_queue, knowledge_base_values=knowledge_base_values) path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_OS, location=path) file_entry = path_spec_resolver.Resolver.OpenFileEntry(path_spec) parser_mediator.SetFileEntry(file_entry) parser_object = syslog.SyslogParser() parser_object.EnablePlugins([plugin_name]) file_object = file_entry.GetFileObject() try: parser_object.Parse(parser_mediator, file_object) finally: file_object.close() return event_queue_consumer
def _ParseFileWithPlugin(self, path_segments, plugin_name, knowledge_base_values=None): """Parses a syslog file with a specific plugin. Args: path_segments (list[str]): path segments inside the test data directory. plugin_name (str): name of the plugin. knowledge_base_values (Optional[dict]): knowledge base values. Returns: FakeStorageWriter: storage writer. Raises: SkipTest: if the path inside the test data directory does not exist and the test should be skipped. """ session = sessions.Session() storage_writer = fake_writer.FakeStorageWriter(session) storage_writer.Open() file_entry = self._GetTestFileEntry(path_segments) parser_mediator = self._CreateParserMediator( storage_writer, file_entry=file_entry, knowledge_base_values=knowledge_base_values) parser = syslog.SyslogParser() parser.EnablePlugins([plugin_name]) file_object = file_entry.GetFileObject() try: parser.Parse(parser_mediator, file_object) finally: file_object.close() return storage_writer
def testParseRsyslogTraditional(self): """Tests the Parse function on a traditional rsyslog file.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2016} storage_writer = self._ParseFile( ['syslog_rsyslog_traditional'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 0) self.assertEqual(storage_writer.number_of_events, 8) events = list(storage_writer.GetSortedEvents()) event = events[0] self.CheckTimestamp(event.timestamp, '2016-01-22 07:54:32.000000') event_data = self._GetEventDataOfEvent(storage_writer, event) self.assertEqual(event_data.data_type, 'syslog:line') self.assertEqual(event_data.hostname, 'myhostname.myhost.com') self.assertEqual(event_data.reporter, 'Job') self.assertIsNone(event_data.severity)
def testParseRsyslogSysklogd(self): """Tests the Parse function on a syslogkd format rsyslog file.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2021} storage_writer = self._ParseFile( ['syslog_rsyslog_SysklogdFileFormat'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_events, 9) self.assertEqual(storage_writer.number_of_extraction_warnings, 0) self.assertEqual(storage_writer.number_of_recovery_warnings, 0) events = list(storage_writer.GetSortedEvents()) expected_event_values = { 'date_time': '2021-03-06 04:07:28', 'data_type': 'syslog:line', 'hostname': 'hostname', 'reporter': 'log_tag', 'severity': None } self.CheckEventValues(storage_writer, events[0], expected_event_values)
def _ParseFileWithPlugin(self, path_segments, plugin_name, knowledge_base_values=None): """Parses a syslog file with a specific plugin. Args: path_segments: a list of strings containinge the path segments inside the test data directory. plugin_name: a string containing the name of the plugin. knowledge_base_values: optional dictionary containing the knowledge base values. Returns: A storage writer object (instance of FakeStorageWriter). """ session = sessions.Session() storage_writer = fake_storage.FakeStorageWriter(session) storage_writer.Open() file_entry = self._GetTestFileEntry(path_segments) parser_mediator = self._CreateParserMediator( storage_writer, file_entry=file_entry, knowledge_base_values=knowledge_base_values) parser_object = syslog.SyslogParser() parser_object.EnablePlugins([plugin_name]) file_object = file_entry.GetFileObject() try: parser_object.Parse(parser_mediator, file_object) finally: file_object.close() return storage_writer
def testParse(self): """Tests the Parse function.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2012} storage_writer = self._ParseFile( ['syslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 1) self.assertEqual(storage_writer.number_of_events, 16) events = list(storage_writer.GetSortedEvents()) expected_event_values = { 'body': 'INFO No new content in ímynd.dd.', 'data_type': 'syslog:line', 'hostname': 'myhostname.myhost.com', 'pid': 30840, 'reporter': 'client', 'severity': None, 'timestamp': '2012-01-22 07:52:33.000000'} self.CheckEventValues(storage_writer, events[0], expected_event_values) expected_event_values = { 'data_type': 'syslog:line', 'reporter': '---', 'severity': None, 'timestamp': '2012-02-29 01:15:43.000000'} self.CheckEventValues(storage_writer, events[6], expected_event_values) # Testing year increment. expected_event_values = { 'body': 'This syslog message has a fractional value for seconds.', 'data_type': 'syslog:line', 'reporter': 'somrandomexe', 'severity': None, 'timestamp': '2013-03-23 23:01:18.000000'} self.CheckEventValues(storage_writer, events[9], expected_event_values) expected_event_values = { 'data_type': 'syslog:line', 'reporter': '/sbin/anacron', 'severity': None} self.CheckEventValues(storage_writer, events[11], expected_event_values) expected_event_values = { 'body': ( 'This is a multi-line message that screws up\n\tmany syslog ' 'parsers.'), 'data_type': 'syslog:line', 'pid': 10100, 'reporter': 'aprocess', 'severity': None} self.CheckEventValues(storage_writer, events[10], expected_event_values) expected_event_values = { 'body': '[997.390602] sda2: rw=0, want=65, limit=2', 'data_type': 'syslog:line', 'hostname': None, 'reporter': 'kernel', 'severity': None} self.CheckEventValues(storage_writer, events[14], expected_event_values) # Testing non-leap year. parser = syslog.SyslogParser() knowledge_base_values = {'year': 2013} storage_writer = self._ParseFile( ['syslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 2) self.assertEqual(storage_writer.number_of_events, 15)
def setUp(self): """Sets up the needed objects used throughout the test.""" pre_obj = event.PreprocessObject() pre_obj.year = 2012 self._parser = syslog.SyslogParser(pre_obj, None)
def setUp(self): """Makes preparations before running an individual test.""" self._parser = syslog.SyslogParser()
def testParseChromeOS(self): """Tests the Parse function.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2016} storage_writer = self._ParseFile( ['syslog_chromeos'], parser, knowledge_base_values=knowledge_base_values) number_of_events = storage_writer.GetNumberOfAttributeContainers( 'event') self.assertEqual(number_of_events, 8) number_of_warnings = storage_writer.GetNumberOfAttributeContainers( 'extraction_warning') self.assertEqual(number_of_warnings, 0) number_of_warnings = storage_writer.GetNumberOfAttributeContainers( 'recovery_warning') self.assertEqual(number_of_warnings, 0) events = list(storage_writer.GetSortedEvents()) # Note that syslog_chromeos contains -07:00 as time zone offset. expected_event_values = { 'body': 'cleanup_logs: job completed', 'date_time': '2016-10-25 12:37:23.297265', 'data_type': 'syslog:line', 'reporter': 'periodic_scheduler', 'pid': 13707, 'severity': 'INFO', 'timestamp': '2016-10-25 19:37:23.297265' } self.CheckEventValues(storage_writer, events[0], expected_event_values) expected_event_values = { 'date_time': '2016-10-25 12:37:24.987014', 'data_type': 'syslog:line', 'reporter': 'kernel', 'severity': 'DEBUG' } self.CheckEventValues(storage_writer, events[2], expected_event_values) # Testing year increment. expected_event_values = { 'date_time': '2016-10-25 12:37:24.993079', 'data_type': 'syslog:line', 'reporter': 'kernel', 'severity': 'DEBUG' } self.CheckEventValues(storage_writer, events[4], expected_event_values) expected_event_values = { 'date_time': '2016-10-25 12:37:25.007963', 'data_type': 'syslog:line', 'reporter': 'kernel', 'severity': 'ERR' } self.CheckEventValues(storage_writer, events[6], expected_event_values) expected_event_values = { 'body': ('[ 316.587330] cfg80211: This is a multi-line\n\tmessage that ' 'screws up many syslog parsers.'), 'date_time': '2016-10-25 12:37:25.014015', 'data_type': 'syslog:line', 'reporter': 'aprocess', 'severity': 'INFO' } self.CheckEventValues(storage_writer, events[7], expected_event_values)
def testParse(self): """Tests the Parse function.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2012} storage_writer = self._ParseFile( ['syslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_events, 16) events = list(storage_writer.GetSortedEvents()) event = events[1] self.CheckTimestamp(event.timestamp, '2012-01-22 07:52:33.000000') self.assertEqual(event.hostname, 'myhostname.myhost.com') expected_message = ( '[client, pid: 30840] INFO No new content in ímynd.dd.') self._TestGetMessageStrings(event, expected_message, expected_message) event = events[6] self.CheckTimestamp(event.timestamp, '2012-02-29 01:15:43.000000') # Testing year increment. event = events[8] self.CheckTimestamp(event.timestamp, '2013-03-23 23:01:18.000000') event = events[11] expected_reporter = '/sbin/anacron' self.assertEqual(event.reporter, expected_reporter) event = events[10] expected_message = ( '[aprocess, pid: 10100] This is a multi-line message that screws up' '\tmany syslog parsers.') expected_short_message = ( '[aprocess, pid: 10100] This is a multi-line message that screws up' '\tmany syslo...') self._TestGetMessageStrings(event, expected_message, expected_short_message) event = events[14] self.assertEqual(event.reporter, 'kernel') self.assertIsNone(event.hostname) expected_message = ( '[kernel] [997.390602] sda2: rw=0, want=65, limit=2') expected_short_message = ( '[kernel] [997.390602] sda2: rw=0, want=65, limit=2') self._TestGetMessageStrings(event, expected_message, expected_short_message) # Testing non-leap year. parser = syslog.SyslogParser() knowledge_base_values = {'year': 2013} storage_writer = self._ParseFile( ['syslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_events, 15)
def testParse(self): """Tests the Parse function.""" parser = syslog.SyslogParser() knowledge_base_values = {'year': 2012} storage_writer = self._ParseFile( ['syslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 1) self.assertEqual(storage_writer.number_of_events, 16) events = list(storage_writer.GetSortedEvents()) event = events[0] self.CheckTimestamp(event.timestamp, '2012-01-22 07:52:33.000000') event_data = self._GetEventDataOfEvent(storage_writer, event) self.assertEqual(event_data.data_type, 'syslog:line') self.assertEqual(event_data.hostname, 'myhostname.myhost.com') self.assertEqual(event_data.reporter, 'client') self.assertIsNone(event_data.severity) expected_message = ( '[client, pid: 30840] INFO No new content in ímynd.dd.') self._TestGetMessageStrings( event_data, expected_message, expected_message) event = events[6] self.CheckTimestamp(event.timestamp, '2012-02-29 01:15:43.000000') event_data = self._GetEventDataOfEvent(storage_writer, event) self.assertEqual(event_data.reporter, '---') self.assertIsNone(event_data.severity) # Testing year increment. event = events[9] self.CheckTimestamp(event.timestamp, '2013-03-23 23:01:18.000000') event_data = self._GetEventDataOfEvent(storage_writer, event) self.assertEqual( event_data.body, 'This syslog message has a fractional value for seconds.') self.assertEqual(event_data.reporter, 'somrandomexe') self.assertIsNone(event_data.severity) event = events[11] event_data = self._GetEventDataOfEvent(storage_writer, event) self.assertEqual(event_data.reporter, '/sbin/anacron') self.assertIsNone(event_data.severity) event = events[10] event_data = self._GetEventDataOfEvent(storage_writer, event) self.assertEqual(event_data.reporter, 'aprocess') self.assertIsNone(event_data.severity) expected_message = ( '[aprocess, pid: 10100] This is a multi-line message that screws up' '\tmany syslog parsers.') expected_short_message = ( '[aprocess, pid: 10100] This is a multi-line message that screws up' '\tmany syslo...') self._TestGetMessageStrings( event_data, expected_message, expected_short_message) event = events[14] event_data = self._GetEventDataOfEvent(storage_writer, event) self.assertIsNone(event_data.hostname) self.assertEqual(event_data.reporter, 'kernel') self.assertIsNone(event_data.severity) expected_message = ( '[kernel] [997.390602] sda2: rw=0, want=65, limit=2') expected_short_message = ( '[kernel] [997.390602] sda2: rw=0, want=65, limit=2') self._TestGetMessageStrings( event_data, expected_message, expected_short_message) # Testing non-leap year. parser = syslog.SyslogParser() knowledge_base_values = {'year': 2013} storage_writer = self._ParseFile( ['syslog'], parser, knowledge_base_values=knowledge_base_values) self.assertEqual(storage_writer.number_of_warnings, 2) self.assertEqual(storage_writer.number_of_events, 15)
def setUp(self): """Sets up the needed objects used throughout the test.""" self._parser = syslog.SyslogParser()