def testOpenClose(self):
"""Tests the Open and Close functions."""
with shared_test_lib.TempDirectory() as temp_directory:
test_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = sqlite_writer.SQLiteStorageFileWriter()
storage_writer.Open(path=test_path)
storage_writer.Close()
storage_writer.Open(path=test_path)
storage_writer.Close()
storage_writer = sqlite_writer.SQLiteStorageFileWriter(
storage_type=definitions.STORAGE_TYPE_TASK)
storage_writer.Open(path=test_path)
storage_writer.Close()
storage_writer.Open(path=test_path)
with self.assertRaises(IOError):
storage_writer.Open(path=test_path)
storage_writer.Close()
with self.assertRaises(IOError):
storage_writer.Close()
def testProcessSources(self):
"""Tests the PreprocessSources and ProcessSources function."""
registry = artifacts_registry.ArtifactDefinitionsRegistry()
reader = artifacts_reader.YamlArtifactsReader()
path = shared_test_lib.GetTestFilePath(['artifacts'])
registry.ReadFromDirectory(reader, path)
test_engine = task_engine.TaskMultiProcessEngine(
maximum_number_of_tasks=100)
source_path = self._GetTestFilePath(['ímynd.dd'])
os_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
source_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location='/',
parent=os_path_spec)
test_engine.PreprocessSources(registry, [source_path_spec])
session = sessions.Session()
configuration = configurations.ProcessingConfiguration()
configuration.parser_filter_expression = 'filestat'
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, 'storage.plaso')
storage_writer = sqlite_writer.SQLiteStorageFileWriter(
session, temp_file)
test_engine.ProcessSources(session.identifier, [source_path_spec],
storage_writer, configuration)
def testMergeAttributeContainers(self):
"""Tests the MergeAttributeContainers function."""
session = sessions.Session()
client = self._GetRedisClient()
with shared_test_lib.TempDirectory() as temp_directory:
task = self._CreateTaskStore(session, redis_client=client)
session_storage_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = writer.SQLiteStorageFileWriter(
session, session_storage_path)
test_reader = merge_reader.RedisMergeReader(storage_writer,
task,
redis_client=client)
ready_tasks, _ = redis_store.RedisStore.ScanForProcessedTasks(
session_identifier=session.identifier, redis_client=client)
self.assertEqual(1, len(ready_tasks))
storage_writer.Open()
result = test_reader.MergeAttributeContainers()
self.assertTrue(result)
storage_writer.Close()
def testMergeAttributeContainersWithDeserializationError(self):
"""Tests MergeAttributeContainers with a deserialization error."""
session = sessions.Session()
with shared_test_lib.TempDirectory() as temp_directory:
task_storage_path = os.path.join(temp_directory, 'task.sqlite')
self._CreateTaskStorageFile(
task_storage_path,
self._TEST_EVENTS_WITH_DESERIALIZATION_ERROR)
session_storage_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = sqlite_writer.SQLiteStorageFileWriter(
session_storage_path)
task_storage_reader = sqlite_reader.SQLiteStorageFileReader(
task_storage_path)
test_reader = merge_reader.StorageMergeReader(
session, storage_writer, task_storage_reader)
storage_writer.Open(path=task_storage_path)
result = test_reader.MergeAttributeContainers()
self.assertTrue(result)
storage_writer.Close()
def _CreateTaskStorageFile(self, session, path, event_values_list):
"""Creates a task storage file for testing.
Args:
session (Session): session the task storage is part of.
path (str): path to the task storage file that should be merged.
event_values_list (list[dict[str, str]]): list of event values.
"""
task = tasks.Task(session_identifier=session.identifier)
storage_file = writer.SQLiteStorageFileWriter(
session,
path,
storage_type=definitions.STORAGE_TYPE_TASK,
task=task)
storage_file.Open()
for event, event_data in containers_test_lib.CreateEventsFromValues(
event_values_list):
storage_file.AddEventData(event_data)
event.SetEventDataIdentifier(event_data.GetIdentifier())
storage_file.AddEvent(event)
storage_file.Close()
def testAddAttributeContainer(self):
"""Tests the AddAttributeContainer function."""
event_data_stream = events.EventDataStream()
with shared_test_lib.TempDirectory() as temp_directory:
test_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = sqlite_writer.SQLiteStorageFileWriter()
storage_writer.Open(path=test_path)
try:
number_of_containers = storage_writer.GetNumberOfAttributeContainers(
event_data_stream.CONTAINER_TYPE)
self.assertEqual(number_of_containers, 0)
storage_writer.AddAttributeContainer(event_data_stream)
number_of_containers = storage_writer.GetNumberOfAttributeContainers(
event_data_stream.CONTAINER_TYPE)
self.assertEqual(number_of_containers, 1)
finally:
storage_writer.Close()
with self.assertRaises(IOError):
storage_writer.AddAttributeContainer(event_data_stream)
def testAddOrUpdateEventTag(self):
"""Tests the AddOrUpdateEventTag function."""
with shared_test_lib.TempDirectory() as temp_directory:
test_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = sqlite_writer.SQLiteStorageFileWriter()
storage_writer.Open(path=test_path)
try:
test_events = self._AddTestEvents(storage_writer)
event_tag = events.EventTag()
event_identifier = test_events[1].GetIdentifier()
event_tag.SetEventIdentifier(event_identifier)
event_tag.AddLabel('Label1')
number_of_containers = storage_writer.GetNumberOfAttributeContainers(
event_tag.CONTAINER_TYPE)
self.assertEqual(number_of_containers, 0)
storage_writer.AddOrUpdateEventTag(event_tag)
number_of_containers = storage_writer.GetNumberOfAttributeContainers(
event_tag.CONTAINER_TYPE)
self.assertEqual(number_of_containers, 1)
event_tag = events.EventTag()
event_identifier = test_events[2].GetIdentifier()
event_tag.SetEventIdentifier(event_identifier)
event_tag.AddLabel('Label2')
storage_writer.AddOrUpdateEventTag(event_tag)
number_of_containers = storage_writer.GetNumberOfAttributeContainers(
event_tag.CONTAINER_TYPE)
self.assertEqual(number_of_containers, 2)
event_tag = events.EventTag()
event_identifier = test_events[1].GetIdentifier()
event_tag.SetEventIdentifier(event_identifier)
event_tag.AddLabel('AnotherLabel1')
storage_writer.AddOrUpdateEventTag(event_tag)
number_of_containers = storage_writer.GetNumberOfAttributeContainers(
event_tag.CONTAINER_TYPE)
self.assertEqual(number_of_containers, 2)
event_tags = list(
storage_writer.GetAttributeContainers(
event_tag.CONTAINER_TYPE))
self.assertEqual(event_tags[0].labels,
['Label1', 'AnotherLabel1'])
self.assertEqual(event_tags[1].labels, ['Label2'])
finally:
storage_writer.Close()
def CreateStorageWriterForFile(cls, session, path):
"""Creates a storage writer based on the file.
Args:
session (Session): session the storage changes are part of.
path (str): path to the storage file.
Returns:
StorageWriter: a storage writer or None if the storage file cannot be
opened or the storage format is not supported.
"""
if sqlite_file.SQLiteStorageFile.CheckSupportedFormat(path):
return sqlite_writer.SQLiteStorageFileWriter(session, path)
def CreateStorageWriter(cls, storage_format, session, path):
"""Creates a storage writer.
Args:
session (Session): session the storage changes are part of.
path (str): path to the storage file.
storage_format (str): storage format.
Returns:
StorageWriter: a storage writer or None if the storage file cannot be
opened or the storage format is not supported.
"""
if storage_format == definitions.STORAGE_FORMAT_SQLITE:
return sqlite_writer.SQLiteStorageFileWriter(session, path)
def testGetSortedEvents(self):
"""Tests the GetSortedEvents function."""
with shared_test_lib.TempDirectory() as temp_directory:
test_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = sqlite_writer.SQLiteStorageFileWriter()
storage_writer.Open(path=test_path)
try:
self._AddTestEvents(storage_writer)
test_events = list(storage_writer.GetSortedEvents())
self.assertEqual(len(test_events), 4)
finally:
storage_writer.Close()
def CreateStorageWriter(cls, storage_format):
"""Creates a storage writer.
Args:
storage_format (str): storage format.
Returns:
StorageWriter: a storage writer or None if the storage file cannot be
opened or the storage format is not supported.
"""
if storage_format == definitions.STORAGE_FORMAT_SQLITE:
return sqlite_writer.SQLiteStorageFileWriter()
if storage_format == definitions.STORAGE_FORMAT_REDIS:
return redis_writer.RedisStorageWriter()
return None
def testReadStorageMetadata(self):
"""Tests the _ReadStorageMetadata function."""
session = sessions.Session()
with shared_test_lib.TempDirectory() as temp_directory:
task_storage_path = os.path.join(temp_directory, 'task.sqlite')
self._CreateTaskStorageFile(session, task_storage_path)
session_storage_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = writer.SQLiteStorageFileWriter(
session, session_storage_path)
test_reader = merge_reader.SQLiteStorageMergeReader(
storage_writer, task_storage_path)
test_reader._Open()
test_reader._ReadStorageMetadata()
test_reader._Close()
def _CreateTaskStorageFile(self, session, path):
"""Creates a task storage file for testing.
Args:
session (Session): session the task storage is part of.
path (str): path to the task storage file that should be merged.
"""
task = tasks.Task(session_identifier=session.identifier)
test_events = self._CreateTestEvents()
storage_file = writer.SQLiteStorageFileWriter(
session, path, storage_type=definitions.STORAGE_TYPE_TASK, task=task)
storage_file.Open()
for event in test_events:
storage_file.AddEvent(event)
storage_file.Close()
def testMergeAttributeContainers(self):
"""Tests the MergeAttributeContainers function."""
session = sessions.Session()
with shared_test_lib.TempDirectory() as temp_directory:
task_storage_path = os.path.join(temp_directory, 'task.sqlite')
self._CreateTaskStorageFile(session, task_storage_path)
session_storage_path = os.path.join(temp_directory, 'plaso.sqlite')
storage_writer = writer.SQLiteStorageFileWriter(
session, session_storage_path)
test_reader = merge_reader.SQLiteStorageMergeReader(
storage_writer, task_storage_path)
storage_writer.Open()
result = test_reader.MergeAttributeContainers()
self.assertTrue(result)
storage_writer.Close()
def _CreateTaskStorageFile(self, path, event_values_list):
"""Creates a task storage file for testing.
Args:
path (str): path to the task storage file that should be merged.
event_values_list (list[dict[str, str]]): list of event values.
"""
storage_writer = sqlite_writer.SQLiteStorageFileWriter(
storage_type=definitions.STORAGE_TYPE_TASK)
storage_writer.Open(path=path)
for event, event_data, event_data_stream in (
containers_test_lib.CreateEventsFromValues(event_values_list)):
storage_writer.AddAttributeContainer(event_data_stream)
event_data.SetEventDataStreamIdentifier(
event_data_stream.GetIdentifier())
storage_writer.AddAttributeContainer(event_data)
event.SetEventDataIdentifier(event_data.GetIdentifier())
storage_writer.AddAttributeContainer(event)
storage_writer.Close()
def testProcessSources(self):
"""Tests the PreprocessSources and ProcessSources function."""
artifacts_path = shared_test_lib.GetTestFilePath(['artifacts'])
self._SkipIfPathNotExists(artifacts_path)
test_engine = extraction_engine.ExtractionMultiProcessEngine(
maximum_number_of_tasks=100)
test_file_path = self._GetTestFilePath(['ímynd.dd'])
self._SkipIfPathNotExists(test_file_path)
os_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=test_file_path)
source_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location='/',
parent=os_path_spec)
source_configuration = artifacts.SourceConfigurationArtifact(
path_spec=source_path_spec)
session = sessions.Session()
configuration = configurations.ProcessingConfiguration()
configuration.parser_filter_expression = 'filestat'
configuration.task_storage_format = definitions.STORAGE_FORMAT_SQLITE
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, 'storage.plaso')
storage_writer = sqlite_writer.SQLiteStorageFileWriter()
storage_writer.Open(path=temp_file)
try:
test_engine.PreprocessSources(artifacts_path, None,
[source_path_spec], session,
storage_writer)
processing_status = test_engine.ProcessSources(
[source_configuration],
storage_writer,
session.identifier,
configuration,
storage_file_path=temp_directory)
number_of_events = storage_writer.GetNumberOfAttributeContainers(
'event')
number_of_extraction_warnings = (
storage_writer.GetNumberOfAttributeContainers(
'extraction_warning'))
number_of_recovery_warnings = (
storage_writer.GetNumberOfAttributeContainers(
'recovery_warning'))
parsers_counter = collections.Counter({
parser_count.name: parser_count.number_of_events
for parser_count in storage_writer.GetAttributeContainers(
'parser_count')
})
finally:
storage_writer.Close()
self.assertFalse(processing_status.aborted)
self.assertEqual(number_of_events, 15)
self.assertEqual(number_of_extraction_warnings, 0)
self.assertEqual(number_of_recovery_warnings, 0)
expected_parsers_counter = collections.Counter({
'filestat': 15,
'total': 15
})
self.assertEqual(parsers_counter, expected_parsers_counter)
