def testSetEnableProfiling(self):
"""Tests the SetEnableProfiling function."""
self._test_engine = engine.BaseEngine(None, None, None)
self._test_engine.SetEnableProfiling(True,
profiling_sample_rate=5000,
profiling_type=u'all')
def testSupportsMemoryProfiling(self):
"""Tests the SupportsMemoryProfiling function."""
test_engine = engine.BaseEngine(None, None, None)
expected_result = hpy is not None
result = test_engine.SupportsMemoryProfiling()
self.assertEqual(result, expected_result)
def testGetSourceFileSystem(self):
"""Tests the GetSourceFileSystem function."""
test_engine = engine.BaseEngine()
test_file_path = self._GetTestFilePath(['ímynd.dd'])
self._SkipIfPathNotExists(test_file_path)
os_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=test_file_path)
source_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location='/',
parent=os_path_spec)
resolver_context = context.Context()
test_file_system, test_mount_point = test_engine.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
self.assertIsNotNone(test_file_system)
self.assertIsInstance(test_file_system, dfvfs_file_system.FileSystem)
self.assertIsNotNone(test_mount_point)
self.assertIsInstance(test_mount_point, path_spec.PathSpec)
with self.assertRaises(RuntimeError):
test_engine.GetSourceFileSystem(None)
def _ExtractWithFilter(self,
source_path_specs,
destination_path,
output_writer,
artifact_filters,
filter_file,
artifact_definitions_path,
custom_artifacts_path,
skip_duplicates=True):
"""Extracts files using a filter expression.
This method runs the file extraction process on the image and
potentially on every VSS if that is wanted.
Args:
source_path_specs (list[dfvfs.PathSpec]): path specifications to extract.
destination_path (str): path where the extracted files should be stored.
output_writer (CLIOutputWriter): output writer.
artifact_definitions_path (str): path to artifact definitions file.
custom_artifacts_path (str): path to custom artifact definitions file.
artifact_filters (list[str]): names of artifact definitions that are
used for filtering file system and Windows Registry key paths.
filter_file (str): path of the file that contains the filter file path
filters.
skip_duplicates (Optional[bool]): True if files with duplicate content
should be skipped.
"""
extraction_engine = engine.BaseEngine()
# If the source is a directory or a storage media image
# run pre-processing.
if self._source_type in self._SOURCE_TYPES_TO_PREPROCESS:
self._PreprocessSources(extraction_engine)
for source_path_spec in source_path_specs:
file_system, mount_point = self._GetSourceFileSystem(
source_path_spec, resolver_context=self._resolver_context)
display_name = path_helper.PathHelper.GetDisplayNameForPathSpec(
source_path_spec)
output_writer.Write(
'Extracting file entries from: {0:s}\n'.format(display_name))
filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs(
artifact_definitions_path, custom_artifacts_path,
extraction_engine.knowledge_base, artifact_filters,
filter_file)
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
for path_spec in searcher.Find(find_specs=filter_find_specs):
self._ExtractFileEntry(path_spec,
destination_path,
output_writer,
skip_duplicates=skip_duplicates)
file_system.Close()
def testStartStopProfiling(self):
"""Tests the _StartProfiling and _StopProfiling functions."""
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.profiling.directory = temp_directory
configuration.profiling.profilers = set([
'memory', 'parsers', 'processing', 'serializers', 'storage',
'task_queue'])
test_engine = engine.BaseEngine()
test_engine._StartProfiling(None)
test_engine._StartProfiling(configuration.profiling)
test_engine._StopProfiling()
def ParseFile(file_entry):
"""Parse a file given a file entry or path and return a list of results.
Args:
file_entry: Either a file entry object (instance of dfvfs.FileEntry)
or a string containing a path (absolute or relative) to a
local file.
Returns:
A list of event object (instance of EventObject) that were extracted from
the file (or an empty list if no events were extracted).
"""
if not file_entry:
return
if isinstance(file_entry, basestring):
file_entry = OpenOSFile(file_entry)
# Set up the engine.
# TODO: refactor and add queue limit.
collection_queue = single_process.SingleProcessQueue()
storage_queue = single_process.SingleProcessQueue()
parse_error_queue = single_process.SingleProcessQueue()
engine_object = engine.BaseEngine(collection_queue, storage_queue,
parse_error_queue)
# Create a worker.
worker_object = engine_object.CreateExtractionWorker(0)
# TODO: add support for parser_filter_string.
worker_object.InitializeParserObjects()
worker_object.ParseFileEntry(file_entry)
collection_queue.SignalEndOfInput()
engine_object.SignalEndOfInputStorageQueue()
results = []
while True:
try:
item = storage_queue.PopItem()
except errors.QueueEmpty:
break
if isinstance(item, queue.QueueEndOfInput):
break
results.append(item)
return results
def testDetermineOperatingSystem(self):
"""Tests the _DetermineOperatingSystem function."""
test_engine = engine.BaseEngine()
source_path = self._GetTestFilePath(['ímynd.dd'])
os_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
source_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK, location='/',
parent=os_path_spec)
file_system, mount_point = test_engine.GetSourceFileSystem(source_path_spec)
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
operating_system = test_engine._DetermineOperatingSystem(searcher)
self.assertEqual(
operating_system, definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN)
def testGetSourceFileSystem(self):
"""Tests the GetSourceFileSystem function."""
test_engine = engine.BaseEngine(None, None, None)
source_path = os.path.join(self._TEST_DATA_PATH, u'ímynd.dd')
os_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
source_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK, location=u'/',
parent=os_path_spec)
resolver_context = context.Context()
test_file_system, test_mount_point = test_engine.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
self.assertIsNotNone(test_file_system)
self.assertIsInstance(test_file_system, file_system.FileSystem)
self.assertIsNotNone(test_mount_point)
self.assertIsInstance(test_mount_point, path_spec.PathSpec)
test_file_system.Close()
def testSetEnableDebugOutput(self):
"""Tests the SetDebugMode function."""
test_engine = engine.BaseEngine(None, None, None)
test_engine.SetEnableDebugOutput(True)
def testCreateSession(self):
"""Tests the CreateSession function."""
test_engine = engine.BaseEngine()
session = test_engine.CreateSession()
self.assertIsNotNone(session)
def _Extract(self,
source_path_specs,
destination_path,
output_writer,
artifact_filters,
filter_file,
artifact_definitions_path,
custom_artifacts_path,
skip_duplicates=True):
"""Extracts files.
This method runs the file extraction process on the image and
potentially on every VSS if that is wanted.
Args:
source_path_specs (list[dfvfs.PathSpec]): path specifications to extract.
destination_path (str): path where the extracted files should be stored.
output_writer (CLIOutputWriter): output writer.
artifact_definitions_path (str): path to artifact definitions file.
custom_artifacts_path (str): path to custom artifact definitions file.
artifact_filters (list[str]): names of artifact definitions that are
used for filtering file system and Windows Registry key paths.
filter_file (str): path of the file that contains the filter file path
filters.
skip_duplicates (Optional[bool]): True if files with duplicate content
should be skipped.
Raises:
BadConfigOption: if an invalid collection filter was specified.
"""
extraction_engine = engine.BaseEngine()
# If the source is a directory or a storage media image
# run pre-processing.
if self._source_type in self._SOURCE_TYPES_TO_PREPROCESS:
self._PreprocessSources(extraction_engine)
try:
extraction_engine.BuildCollectionFilters(
artifact_definitions_path, custom_artifacts_path,
extraction_engine.knowledge_base, artifact_filters,
filter_file)
except errors.InvalidFilter as exception:
raise errors.BadConfigOption(
'Unable to build collection filters with error: {0!s}'.format(
exception))
filters_helper = extraction_engine.collection_filters_helper
excluded_find_specs = None
included_find_specs = None
if filters_helper:
excluded_find_specs = filters_helper.excluded_file_system_find_specs
included_find_specs = filters_helper.included_file_system_find_specs
output_writer.Write('Extracting file entries.\n')
path_spec_generator = self._path_spec_extractor.ExtractPathSpecs(
source_path_specs,
find_specs=included_find_specs,
resolver_context=self._resolver_context)
for path_spec in path_spec_generator:
file_entry = path_spec_resolver.Resolver.OpenFileEntry(
path_spec, resolver_context=self._resolver_context)
if not file_entry:
logger.warning(
'Unable to open file entry for path spec: {0:s}'.format(
path_spec.comparable))
continue
skip_file_entry = False
for find_spec in excluded_find_specs or []:
skip_file_entry = find_spec.CompareLocation(file_entry)
if skip_file_entry:
break
if skip_file_entry:
logger.info(
'Skipped: {0:s} because of exclusion filter.'.format(
file_entry.path_spec.location))
continue
self._ExtractFileEntry(file_entry,
destination_path,
skip_duplicates=skip_duplicates)
