def testMergeFromStorage(self): """Tests the MergeFromStorage function.""" session = sessions.Session() storage_writer = fake_storage.FakeStorageWriter(session) storage_writer.Open() test_file = self._GetTestFilePath([u'psort_test.json.plaso']) storage_reader = zip_file.ZIPStorageFileReader(test_file) storage_writer.MergeFromStorage(storage_reader) test_file = self._GetTestFilePath([u'pinfo_test.json.plaso']) storage_reader = zip_file.ZIPStorageFileReader(test_file) storage_writer.MergeFromStorage(storage_reader) storage_writer.Close()
def testOutput(self): """Tests the Output function.""" test_filename = os.path.join(u'test_data', u'psort_test.json.plaso') with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, u'pstorage.plaso') storage_file = storage_zip_file.StorageFile(test_filename, read_only=True) with storage_zip_file.ZIPStorageFileReader( storage_file) as storage_reader: output_mediator = self._CreateOutputMediator( storage_file=storage_file) output_module = pstorage.PlasoStorageOutputModule( output_mediator) output_module.SetFilePath(temp_file) with event_buffer.EventBuffer( output_module, check_dedups=False) as output_buffer: for event_object in storage_reader.GetEvents(): output_buffer.Append(event_object) original_zip_file = storage_zip_file.StorageFile(test_filename, read_only=True) pstorage_zip_file = storage_zip_file.StorageFile(temp_file, read_only=True) original_list = [] pstorage_list = [] event_object_original = original_zip_file.GetSortedEntry() event_object_pstorage = pstorage_zip_file.GetSortedEntry() while event_object_original: original_equality_string = event_object_original.EqualityString( ) pstorage_equality_string = event_object_pstorage.EqualityString( ) # Remove the UUID for comparision. original_equality_string, _, _ = original_equality_string.rpartition( u'|') pstorage_equality_string, _, _ = pstorage_equality_string.rpartition( u'|') original_list.append(original_equality_string) pstorage_list.append(pstorage_equality_string) event_object_original = original_zip_file.GetSortedEntry() event_object_pstorage = pstorage_zip_file.GetSortedEntry() self.assertFalse(event_object_pstorage) for original_str, dump_str in zip(sorted(original_list), sorted(pstorage_list)): self.assertEqual(original_str, dump_str)
def CreateStorageReader(self, storage_file_path): """Creates a storage reader. Args: storage_file_path (str): path of the storage file. Returns: StorageReader: storage reader. """ return storage_zip_file.ZIPStorageFileReader(storage_file_path)
def CreateStorageReaderForFile(cls, path): """Creates a storage reader based on the file. Args: path (str): path to the storage file. Returns: StorageReader: a storage reader or None if the storage file cannot be opened or the storage format is not supported. """ if storage_sqlite_file.SQLiteStorageFile.CheckSupportedFormat(path): return storage_sqlite_file.SQLiteStorageFileReader(path) elif storage_zip_file.ZIPStorageFile.CheckSupportedFormat(path): return storage_zip_file.ZIPStorageFileReader(path)
def export_storage_file(self, storage_file): """Extracts and exports plaso event data and sources into the graph.""" with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: knowledge_base = KnowledgeBase() storage_reader.ReadPreprocessingInformation(knowledge_base) # TODO: Export knowledge base. for session in storage_reader._storage_file.GetSessions(): self.export_session(session) for source in storage_reader.GetEventSources(): self.export_event_source(source) for event in storage_reader.GetEvents(): self.export_event(event)
def testInternalExportEvents(self): """Tests the _ExportEvents function.""" knowledge_base_object = knowledge_base.KnowledgeBase() output_writer = cli_test_lib.TestOutputWriter() formatter_mediator = formatters_mediator.FormatterMediator() output_mediator_object = output_mediator.OutputMediator( knowledge_base_object, formatter_mediator) output_module = TestOutputModule(output_mediator_object) output_module.SetOutputWriter(output_writer) test_engine = psort.PsortMultiProcessEngine() formatters_manager.FormattersManager.RegisterFormatter( TestEventFormatter) with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, u'storage.plaso') self._CreateTestStorageFile(temp_file) storage_reader = storage_zip_file.ZIPStorageFileReader(temp_file) storage_reader.ReadPreprocessingInformation(knowledge_base_object) event_buffer = TestEventBuffer(output_module, check_dedups=False) test_engine._ExportEvents(storage_reader, event_buffer) event_buffer.Flush() formatters_manager.FormattersManager.DeregisterFormatter( TestEventFormatter) lines = [] output = output_writer.ReadOutput() for line in output.split(b'\n'): lines.append(line) self.assertEqual(len(lines), 8) self.assertTrue(b'My text goes along: My text dude. lines' in lines[2]) self.assertTrue(b'LOG/' in lines[2]) self.assertTrue(b'None in Particular' in lines[2]) self.assertEqual(lines[0], ( b'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,' b'version,filename,inode,notes,format,extra'))
def testReadEntries(self): """Ensure returned EventObjects from the storage are within time bounds.""" storage_file_path = self._GetTestFilePath([u'psort_test.proto.plaso']) storage_file = storage_zip_file.StorageFile(storage_file_path, read_only=True) time_range = storage_time_range.TimeRange(self._start_timestamp, self._end_timestamp) timestamp_list = [] with storage_zip_file.ZIPStorageFileReader( storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=time_range): timestamp_list.append(event_object.timestamp) self.assertEqual(len(timestamp_list), 15) self.assertEqual(timestamp_list[0], self._start_timestamp) self.assertEqual(timestamp_list[-1], self._end_timestamp)
def testOutput(self): """Tests the Output function.""" with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, u'pstorage.plaso') # Copy events to pstorage dump. storage_file = storage_zip_file.StorageFile(self._test_filename, read_only=True) with storage_zip_file.ZIPStorageFileReader( storage_file) as storage_reader: output_mediator = self._CreateOutputMediator( storage_file=storage_file) output_module = pstorage.PlasoStorageOutputModule( output_mediator) output_module.SetFilePath(temp_file) with event_buffer.EventBuffer( output_module, check_dedups=False) as output_buffer: for event_object in storage_reader.GetEvents(): output_buffer.Append(event_object) # Make sure original and dump have the same events. original = storage_zip_file.StorageFile(self._test_filename, read_only=True) dump = storage_zip_file.StorageFile(temp_file, read_only=True) event_object_original = original.GetSortedEntry() event_object_dump = dump.GetSortedEntry() original_list = [] dump_list = [] while event_object_original: original_list.append(event_object_original.EqualityString()) dump_list.append(event_object_dump.EqualityString()) event_object_original = original.GetSortedEntry() event_object_dump = dump.GetSortedEntry() self.assertFalse(event_object_dump) for original_str, dump_str in zip(sorted(original_list), sorted(dump_list)): self.assertEqual(original_str, dump_str)
def testExportEvents(self): """Tests the ExportEvents function.""" storage_file_path = self._GetTestFilePath([u'psort_test.json.plaso']) knowledge_base_object = knowledge_base.KnowledgeBase() output_writer = cli_test_lib.TestOutputWriter() formatter_mediator = formatters_mediator.FormatterMediator() formatter_mediator.SetPreferredLanguageIdentifier(u'en-US') output_mediator_object = output_mediator.OutputMediator( knowledge_base_object, formatter_mediator) output_module = dynamic.DynamicOutputModule(output_mediator_object) output_module.SetOutputWriter(output_writer) storage_reader = storage_zip_file.ZIPStorageFileReader(storage_file_path) test_engine = psort.PsortMultiProcessEngine() counter = test_engine.ExportEvents( knowledge_base_object, storage_reader, output_module) # TODO: refactor preprocessing object. self.assertEqual(counter[u'Stored Events'], 0) lines = [] output = output_writer.ReadOutput() for line in output.split(b'\n'): lines.append(line) self.assertEqual(len(lines), 24) expected_line = ( u'2016-10-16T15:13:43+00:00,' u'mtime,' u'FILE,' u'OS mtime,' u'OS:/tmp/test/test_data/syslog Type: file,' u'filestat,' u'OS:/tmp/test/test_data/syslog,-') self.assertEquals(lines[14], expected_line)
def testExportEvents(self): """Tests the ExportEvents function.""" storage_file_path = self._GetTestFilePath(['psort_test.json.plaso']) knowledge_base_object = knowledge_base.KnowledgeBase() output_writer = cli_test_lib.TestOutputWriter() formatter_mediator = formatters_mediator.FormatterMediator() formatter_mediator.SetPreferredLanguageIdentifier('en-US') output_mediator_object = output_mediator.OutputMediator( knowledge_base_object, formatter_mediator) output_module = dynamic.DynamicOutputModule(output_mediator_object) output_module.SetOutputWriter(output_writer) storage_reader = storage_zip_file.ZIPStorageFileReader( storage_file_path) test_engine = psort.PsortMultiProcessEngine() counter = test_engine.ExportEvents(knowledge_base_object, storage_reader, output_module) self.assertEqual(counter['Stored Events'], 0) lines = [] output = output_writer.ReadOutput() for line in output.split(b'\n'): lines.append(line) self.assertEqual(len(lines), 22) expected_line = ('2014-11-18T01:15:43+00:00,' 'Content Modification Time,' 'LOG,' 'Log File,' '[---] last message repeated 5 times ---,' 'syslog,' 'OS:/tmp/test/test_data/syslog,' 'repeated') self.assertEqual(lines[14], expected_line)
def testInternalExportEvents(self): """Tests the _ExportEvents function.""" knowledge_base_object = knowledge_base.KnowledgeBase() output_writer = cli_test_lib.TestOutputWriter() formatter_mediator = formatters_mediator.FormatterMediator() output_mediator_object = output_mediator.OutputMediator( knowledge_base_object, formatter_mediator) output_module = TestOutputModule(output_mediator_object) output_module.SetOutputWriter(output_writer) test_engine = psort.PsortMultiProcessEngine() formatters_manager.FormattersManager.RegisterFormatter( TestEventFormatter) with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') self._CreateTestStorageFile(temp_file) storage_reader = storage_zip_file.ZIPStorageFileReader(temp_file) storage_reader.ReadPreprocessingInformation(knowledge_base_object) test_engine._ExportEvents(storage_reader, output_module, deduplicate_events=False) formatters_manager.FormattersManager.DeregisterFormatter( TestEventFormatter) lines = [] output = output_writer.ReadOutput() for line in output.split(b'\n'): lines.append(line) self.assertEqual(len(output_module.events), 17) self.assertEqual(len(output_module.macb_groups), 3)
def testGetEvents(self): """Tests the GetEvents function.""" test_file = self._GetTestFilePath([u'psort_test.json.plaso']) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents(): timestamps.append(event_object.timestamp) expected_timestamps = [ 1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000, 1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000, 1456708543000000, 1456708543000000, 1462105168000000, 1462105168000000, 1462105168000000, 1462105168000000, 1462105169000000, 1462105170000000, 1482083672000000, 1482083672000000, 1490310078000000, 1490310078000000, 1490310078000123, 1490310078000123, 1514742872000000, 1514742872000000, 1542503720000000, 1542503720000000, 1542503743000000, 1542503743000000] self.assertEqual(len(timestamps), 32) self.assertEqual(sorted(timestamps), expected_timestamps) # Test lower bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'), timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59')) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1462105168000000, 1462105168000000, 1462105168000000, 1462105168000000, 1462105169000000, 1462105170000000, 1482083672000000, 1482083672000000, 1490310078000000, 1490310078000000, 1490310078000123, 1490310078000123, 1514742872000000, 1514742872000000, 1542503720000000, 1542503720000000, 1542503743000000, 1542503743000000] self.assertEqual(sorted(timestamps), expected_timestamps) # Test upper bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'), timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49')) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000, 1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000, 1456708543000000, 1456708543000000] self.assertEqual(sorted(timestamps), expected_timestamps)
def ProcessStorage(self, output_module, storage_file, storage_file_path, analysis_plugins, event_queue_producers, command_line_arguments=None, deduplicate_events=True, preferred_encoding=u'utf-8', time_slice=None, use_time_slicer=False): """Processes a plaso storage file. Args: output_module: an output module (instance of OutputModule). storage_file: the storage file object (instance of StorageFile). storage_file_path: string containing the path of the storage file. analysis_plugins: list of analysis plugin objects (instance of AnalysisPlugin). event_queue_producers: list of event queue producer objects (instance of ItemQueueProducer). command_line_arguments: optional string of the command line arguments or None if not set. deduplicate_events: optional boolean value to indicate if the event objects should be deduplicated. preferred_encoding: optional preferred encoding. time_slice: optional time slice object (instance of TimeSlice). use_time_slicer: optional boolean value to indicate the 'time slicer' should be used. The 'time slicer' will provide a context of events around an event of interest. Returns: A counter (an instance of collections.Counter) that tracks the number of events extracted from storage, and the analysis plugin results. Raises: RuntimeError: if a non-recoverable situation is encountered. """ if time_slice: if time_slice.event_timestamp is not None: time_slice = storage_time_range.TimeRange( time_slice.start_timestamp, time_slice.end_timestamp) elif use_time_slicer: self._filter_buffer = bufferlib.CircularBuffer( time_slice.duration) # TODO: allow for single processing. # TODO: add upper queue limit. analysis_queue_port = None if self._use_zeromq: analysis_report_incoming_queue = zeromq_queue.ZeroMQPullBindQueue( delay_open=False, port=None, linger_seconds=5) analysis_queue_port = analysis_report_incoming_queue.port else: analysis_report_incoming_queue = multi_process.MultiProcessingQueue( timeout=5) pre_obj = self._GetLastGoodPreprocess(storage_file) if pre_obj is None: pre_obj = event.PreprocessObject() if analysis_plugins: self._StartAnalysisPlugins( storage_file_path, analysis_plugins, pre_obj, analysis_queue_port=analysis_queue_port, analysis_report_incoming_queue=analysis_report_incoming_queue, command_line_arguments=command_line_arguments) # Assign the preprocessing object to the storage. # This is normally done in the construction of the storage object, # however we cannot do that here since the preprocessing object is # stored inside the storage file, so we need to open it first to # be able to read it in, before we make changes to it. Thus we need # to access this protected member of the class. # pylint: disable=protected-access storage_file._pre_obj = pre_obj else: event_queue_producers = [] output_buffer = output_event_buffer.EventBuffer( output_module, deduplicate_events) with output_buffer: storage_reader = storage_zip_file.ZIPStorageFileReader( storage_file) counter = self.ProcessEventsFromStorage( storage_reader, output_buffer, analysis_queues=event_queue_producers, filter_buffer=self._filter_buffer, my_filter=self._filter_object, time_slice=time_slice) for information in storage_file.GetStorageInformation(): if hasattr(information, u'counter'): counter[u'Stored Events'] += information.counter[u'total'] # Get all reports and tags from analysis plugins. self._ProcessAnalysisPlugins(analysis_plugins, analysis_report_incoming_queue, storage_file, counter, pre_obj, preferred_encoding=preferred_encoding) if self._filter_object and not counter[u'Limited By']: counter[u'Filter By Date'] = (counter[u'Stored Events'] - counter[u'Events Included'] - counter[u'Events Filtered Out']) return counter
def AnalyzeEvents(self): """Analyzes events from a plaso storage file and generate a report. Raises: BadConfigOption: when a configuration parameter fails validation. RuntimeError: if a non-recoverable situation is encountered. """ session = engine.BaseEngine.CreateSession( command_line_arguments=self._command_line_arguments, preferred_encoding=self.preferred_encoding) storage_reader = storage_zip_file.ZIPStorageFileReader( self._storage_file_path) self._number_of_analysis_reports = ( storage_reader.GetNumberOfAnalysisReports()) storage_reader.Close() counter = collections.Counter() if self._output_format != u'null': self._status_view.SetMode(self._status_view_mode) self._status_view.SetStorageFileInformation( self._storage_file_path) status_update_callback = ( self._status_view.GetAnalysisStatusUpdateCallback()) storage_reader = storage_zip_file.ZIPStorageFileReader( self._storage_file_path) # TODO: add single processing support. analysis_engine = psort.PsortMultiProcessEngine( use_zeromq=self._use_zeromq) # TODO: pass configuration object. events_counter = analysis_engine.ExportEvents( self._knowledge_base, storage_reader, self._output_module, deduplicate_events=self._deduplicate_events, status_update_callback=status_update_callback, time_slice=self._time_slice, use_time_slicer=self._use_time_slicer) counter += events_counter for item, value in iter(session.analysis_reports_counter.items()): counter[item] = value if self._quiet_mode: return self._output_writer.Write(u'Processing completed.\n') table_view = views.ViewsFactory.GetTableView(self._views_format_type, title=u'Counter') for element, count in counter.most_common(): if not element: element = u'N/A' table_view.AddRow([element, count]) table_view.Write(self._output_writer) storage_reader = storage_zip_file.ZIPStorageFileReader( self._storage_file_path) self._PrintAnalysisReportsDetails(storage_reader, self._number_of_analysis_reports) self._output_writer.Write(u'Storage file is {0:s}\n'.format( self._storage_file_path))
def testOutput(self): """Testing if psort can output data.""" formatters_manager.FormattersManager.RegisterFormatter( PsortTestEventFormatter) events = [ PsortTestEvent(5134324321), PsortTestEvent(2134324321), PsortTestEvent(9134324321), PsortTestEvent(15134324321), PsortTestEvent(5134324322), PsortTestEvent(5134024321) ] output_writer = cli_test_lib.TestOutputWriter() with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, u'plaso.db') storage_file = storage_zip_file.StorageFile(temp_file) storage_file.AddEventObjects(events) storage_file.Close() storage_file = storage_zip_file.StorageFile(temp_file, read_only=True) with storage_zip_file.ZIPStorageFileReader( storage_file) as storage_reader: output_mediator_object = output_mediator.OutputMediator( self._formatter_mediator) output_mediator_object.SetStorageFile(storage_file) output_module = TestOutputModule(output_mediator_object) output_module.SetOutputWriter(output_writer) event_buffer = TestEventBuffer(output_module, check_dedups=False, store=storage_file) self._front_end.ProcessEventsFromStorage( storage_reader, event_buffer) event_buffer.Flush() lines = [] output = output_writer.ReadOutput() for line in output.split(b'\n'): if line == b'.': continue if line: lines.append(line) # One more line than events (header row). self.assertEqual(len(lines), 7) self.assertTrue(b'My text goes along: My text dude. lines' in lines[2]) self.assertTrue(b'LOG/' in lines[2]) self.assertTrue(b'None in Particular' in lines[2]) self.assertEqual(lines[0], ( b'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,' b'version,filename,inode,notes,format,extra')) formatters_manager.FormattersManager.DeregisterFormatter( PsortTestEventFormatter)
def testGetEvents(self): """Tests the GetEvents function.""" test_file = self._GetTestFilePath([u'psort_test.proto.plaso']) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents(): timestamps.append(event_object.timestamp) expected_timestamps = [ 1343166324000000, 1344270407000000, 1390377153000000, 1390377153000000, 1390377181000000, 1390377241000000, 1390377241000000, 1390377272000000, 1392438730000000, 1418925272000000, 1427151678000000, 1427151678000123, 1451584472000000, 1479431720000000, 1479431743000000 ] self.assertEqual(sorted(timestamps), expected_timestamps) # Test lower bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00'), timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59')) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1418925272000000, 1427151678000000, 1427151678000123, 1451584472000000, 1479431720000000, 1479431743000000 ] self.assertEqual(sorted(timestamps), expected_timestamps) # Test upper bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'), timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00')) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1343166324000000, 1344270407000000, 1390377153000000, 1390377153000000, 1390377181000000, 1390377241000000, 1390377241000000, 1390377272000000, 1392438730000000 ] self.assertEqual(sorted(timestamps), expected_timestamps)