def testOpenClose(self): """Tests the Open and Close functions.""" with shared_test_lib.TempDirectory() as temp_directory: test_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = sqlite_writer.SQLiteStorageFileWriter() storage_writer.Open(path=test_path) storage_writer.Close() storage_writer.Open(path=test_path) storage_writer.Close() storage_writer = sqlite_writer.SQLiteStorageFileWriter( storage_type=definitions.STORAGE_TYPE_TASK) storage_writer.Open(path=test_path) storage_writer.Close() storage_writer.Open(path=test_path) with self.assertRaises(IOError): storage_writer.Open(path=test_path) storage_writer.Close() with self.assertRaises(IOError): storage_writer.Close()
def testProcessSources(self): """Tests the PreprocessSources and ProcessSources function.""" registry = artifacts_registry.ArtifactDefinitionsRegistry() reader = artifacts_reader.YamlArtifactsReader() path = shared_test_lib.GetTestFilePath(['artifacts']) registry.ReadFromDirectory(reader, path) test_engine = task_engine.TaskMultiProcessEngine( maximum_number_of_tasks=100) source_path = self._GetTestFilePath(['ímynd.dd']) os_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path) source_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_TSK, location='/', parent=os_path_spec) test_engine.PreprocessSources(registry, [source_path_spec]) session = sessions.Session() configuration = configurations.ProcessingConfiguration() configuration.parser_filter_expression = 'filestat' with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') storage_writer = sqlite_writer.SQLiteStorageFileWriter( session, temp_file) test_engine.ProcessSources(session.identifier, [source_path_spec], storage_writer, configuration)
def testMergeAttributeContainers(self): """Tests the MergeAttributeContainers function.""" session = sessions.Session() client = self._GetRedisClient() with shared_test_lib.TempDirectory() as temp_directory: task = self._CreateTaskStore(session, redis_client=client) session_storage_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = writer.SQLiteStorageFileWriter( session, session_storage_path) test_reader = merge_reader.RedisMergeReader(storage_writer, task, redis_client=client) ready_tasks, _ = redis_store.RedisStore.ScanForProcessedTasks( session_identifier=session.identifier, redis_client=client) self.assertEqual(1, len(ready_tasks)) storage_writer.Open() result = test_reader.MergeAttributeContainers() self.assertTrue(result) storage_writer.Close()
def testMergeAttributeContainersWithDeserializationError(self): """Tests MergeAttributeContainers with a deserialization error.""" session = sessions.Session() with shared_test_lib.TempDirectory() as temp_directory: task_storage_path = os.path.join(temp_directory, 'task.sqlite') self._CreateTaskStorageFile( task_storage_path, self._TEST_EVENTS_WITH_DESERIALIZATION_ERROR) session_storage_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = sqlite_writer.SQLiteStorageFileWriter( session_storage_path) task_storage_reader = sqlite_reader.SQLiteStorageFileReader( task_storage_path) test_reader = merge_reader.StorageMergeReader( session, storage_writer, task_storage_reader) storage_writer.Open(path=task_storage_path) result = test_reader.MergeAttributeContainers() self.assertTrue(result) storage_writer.Close()
def _CreateTaskStorageFile(self, session, path, event_values_list): """Creates a task storage file for testing. Args: session (Session): session the task storage is part of. path (str): path to the task storage file that should be merged. event_values_list (list[dict[str, str]]): list of event values. """ task = tasks.Task(session_identifier=session.identifier) storage_file = writer.SQLiteStorageFileWriter( session, path, storage_type=definitions.STORAGE_TYPE_TASK, task=task) storage_file.Open() for event, event_data in containers_test_lib.CreateEventsFromValues( event_values_list): storage_file.AddEventData(event_data) event.SetEventDataIdentifier(event_data.GetIdentifier()) storage_file.AddEvent(event) storage_file.Close()
def testAddAttributeContainer(self): """Tests the AddAttributeContainer function.""" event_data_stream = events.EventDataStream() with shared_test_lib.TempDirectory() as temp_directory: test_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = sqlite_writer.SQLiteStorageFileWriter() storage_writer.Open(path=test_path) try: number_of_containers = storage_writer.GetNumberOfAttributeContainers( event_data_stream.CONTAINER_TYPE) self.assertEqual(number_of_containers, 0) storage_writer.AddAttributeContainer(event_data_stream) number_of_containers = storage_writer.GetNumberOfAttributeContainers( event_data_stream.CONTAINER_TYPE) self.assertEqual(number_of_containers, 1) finally: storage_writer.Close() with self.assertRaises(IOError): storage_writer.AddAttributeContainer(event_data_stream)
def testAddOrUpdateEventTag(self): """Tests the AddOrUpdateEventTag function.""" with shared_test_lib.TempDirectory() as temp_directory: test_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = sqlite_writer.SQLiteStorageFileWriter() storage_writer.Open(path=test_path) try: test_events = self._AddTestEvents(storage_writer) event_tag = events.EventTag() event_identifier = test_events[1].GetIdentifier() event_tag.SetEventIdentifier(event_identifier) event_tag.AddLabel('Label1') number_of_containers = storage_writer.GetNumberOfAttributeContainers( event_tag.CONTAINER_TYPE) self.assertEqual(number_of_containers, 0) storage_writer.AddOrUpdateEventTag(event_tag) number_of_containers = storage_writer.GetNumberOfAttributeContainers( event_tag.CONTAINER_TYPE) self.assertEqual(number_of_containers, 1) event_tag = events.EventTag() event_identifier = test_events[2].GetIdentifier() event_tag.SetEventIdentifier(event_identifier) event_tag.AddLabel('Label2') storage_writer.AddOrUpdateEventTag(event_tag) number_of_containers = storage_writer.GetNumberOfAttributeContainers( event_tag.CONTAINER_TYPE) self.assertEqual(number_of_containers, 2) event_tag = events.EventTag() event_identifier = test_events[1].GetIdentifier() event_tag.SetEventIdentifier(event_identifier) event_tag.AddLabel('AnotherLabel1') storage_writer.AddOrUpdateEventTag(event_tag) number_of_containers = storage_writer.GetNumberOfAttributeContainers( event_tag.CONTAINER_TYPE) self.assertEqual(number_of_containers, 2) event_tags = list( storage_writer.GetAttributeContainers( event_tag.CONTAINER_TYPE)) self.assertEqual(event_tags[0].labels, ['Label1', 'AnotherLabel1']) self.assertEqual(event_tags[1].labels, ['Label2']) finally: storage_writer.Close()
def CreateStorageWriterForFile(cls, session, path): """Creates a storage writer based on the file. Args: session (Session): session the storage changes are part of. path (str): path to the storage file. Returns: StorageWriter: a storage writer or None if the storage file cannot be opened or the storage format is not supported. """ if sqlite_file.SQLiteStorageFile.CheckSupportedFormat(path): return sqlite_writer.SQLiteStorageFileWriter(session, path)
def CreateStorageWriter(cls, storage_format, session, path): """Creates a storage writer. Args: session (Session): session the storage changes are part of. path (str): path to the storage file. storage_format (str): storage format. Returns: StorageWriter: a storage writer or None if the storage file cannot be opened or the storage format is not supported. """ if storage_format == definitions.STORAGE_FORMAT_SQLITE: return sqlite_writer.SQLiteStorageFileWriter(session, path)
def testGetSortedEvents(self): """Tests the GetSortedEvents function.""" with shared_test_lib.TempDirectory() as temp_directory: test_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = sqlite_writer.SQLiteStorageFileWriter() storage_writer.Open(path=test_path) try: self._AddTestEvents(storage_writer) test_events = list(storage_writer.GetSortedEvents()) self.assertEqual(len(test_events), 4) finally: storage_writer.Close()
def CreateStorageWriter(cls, storage_format): """Creates a storage writer. Args: storage_format (str): storage format. Returns: StorageWriter: a storage writer or None if the storage file cannot be opened or the storage format is not supported. """ if storage_format == definitions.STORAGE_FORMAT_SQLITE: return sqlite_writer.SQLiteStorageFileWriter() if storage_format == definitions.STORAGE_FORMAT_REDIS: return redis_writer.RedisStorageWriter() return None
def testReadStorageMetadata(self): """Tests the _ReadStorageMetadata function.""" session = sessions.Session() with shared_test_lib.TempDirectory() as temp_directory: task_storage_path = os.path.join(temp_directory, 'task.sqlite') self._CreateTaskStorageFile(session, task_storage_path) session_storage_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = writer.SQLiteStorageFileWriter( session, session_storage_path) test_reader = merge_reader.SQLiteStorageMergeReader( storage_writer, task_storage_path) test_reader._Open() test_reader._ReadStorageMetadata() test_reader._Close()
def _CreateTaskStorageFile(self, session, path): """Creates a task storage file for testing. Args: session (Session): session the task storage is part of. path (str): path to the task storage file that should be merged. """ task = tasks.Task(session_identifier=session.identifier) test_events = self._CreateTestEvents() storage_file = writer.SQLiteStorageFileWriter( session, path, storage_type=definitions.STORAGE_TYPE_TASK, task=task) storage_file.Open() for event in test_events: storage_file.AddEvent(event) storage_file.Close()
def testMergeAttributeContainers(self): """Tests the MergeAttributeContainers function.""" session = sessions.Session() with shared_test_lib.TempDirectory() as temp_directory: task_storage_path = os.path.join(temp_directory, 'task.sqlite') self._CreateTaskStorageFile(session, task_storage_path) session_storage_path = os.path.join(temp_directory, 'plaso.sqlite') storage_writer = writer.SQLiteStorageFileWriter( session, session_storage_path) test_reader = merge_reader.SQLiteStorageMergeReader( storage_writer, task_storage_path) storage_writer.Open() result = test_reader.MergeAttributeContainers() self.assertTrue(result) storage_writer.Close()
def _CreateTaskStorageFile(self, path, event_values_list): """Creates a task storage file for testing. Args: path (str): path to the task storage file that should be merged. event_values_list (list[dict[str, str]]): list of event values. """ storage_writer = sqlite_writer.SQLiteStorageFileWriter( storage_type=definitions.STORAGE_TYPE_TASK) storage_writer.Open(path=path) for event, event_data, event_data_stream in ( containers_test_lib.CreateEventsFromValues(event_values_list)): storage_writer.AddAttributeContainer(event_data_stream) event_data.SetEventDataStreamIdentifier( event_data_stream.GetIdentifier()) storage_writer.AddAttributeContainer(event_data) event.SetEventDataIdentifier(event_data.GetIdentifier()) storage_writer.AddAttributeContainer(event) storage_writer.Close()
def testProcessSources(self): """Tests the PreprocessSources and ProcessSources function.""" artifacts_path = shared_test_lib.GetTestFilePath(['artifacts']) self._SkipIfPathNotExists(artifacts_path) test_engine = extraction_engine.ExtractionMultiProcessEngine( maximum_number_of_tasks=100) test_file_path = self._GetTestFilePath(['ímynd.dd']) self._SkipIfPathNotExists(test_file_path) os_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_OS, location=test_file_path) source_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_TSK, location='/', parent=os_path_spec) source_configuration = artifacts.SourceConfigurationArtifact( path_spec=source_path_spec) session = sessions.Session() configuration = configurations.ProcessingConfiguration() configuration.parser_filter_expression = 'filestat' configuration.task_storage_format = definitions.STORAGE_FORMAT_SQLITE with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') storage_writer = sqlite_writer.SQLiteStorageFileWriter() storage_writer.Open(path=temp_file) try: test_engine.PreprocessSources(artifacts_path, None, [source_path_spec], session, storage_writer) processing_status = test_engine.ProcessSources( [source_configuration], storage_writer, session.identifier, configuration, storage_file_path=temp_directory) number_of_events = storage_writer.GetNumberOfAttributeContainers( 'event') number_of_extraction_warnings = ( storage_writer.GetNumberOfAttributeContainers( 'extraction_warning')) number_of_recovery_warnings = ( storage_writer.GetNumberOfAttributeContainers( 'recovery_warning')) parsers_counter = collections.Counter({ parser_count.name: parser_count.number_of_events for parser_count in storage_writer.GetAttributeContainers( 'parser_count') }) finally: storage_writer.Close() self.assertFalse(processing_status.aborted) self.assertEqual(number_of_events, 15) self.assertEqual(number_of_extraction_warnings, 0) self.assertEqual(number_of_recovery_warnings, 0) expected_parsers_counter = collections.Counter({ 'filestat': 15, 'total': 15 }) self.assertEqual(parsers_counter, expected_parsers_counter)