def PreprocessSources(
self, artifact_definitions_path, custom_artifacts_path,
source_path_specs, session, storage_writer, resolver_context=None):
"""Preprocesses the sources.
Args:
artifact_definitions_path (str): path to artifact definitions directory
or file.
custom_artifacts_path (str): path to custom artifact definitions
directory or file.
source_path_specs (list[dfvfs.PathSpec]): path specifications of
the sources to process.
session (Session): session the preprocessing is part of.
storage_writer (StorageWriter): storage writer.
resolver_context (Optional[dfvfs.Context]): resolver context.
"""
artifacts_registry_object = self._BuildArtifactsRegistry(
artifact_definitions_path, custom_artifacts_path)
mediator = preprocess_mediator.PreprocessMediator(
session, storage_writer, self.knowledge_base)
detected_operating_systems = []
for source_path_spec in source_path_specs:
try:
file_system, mount_point = self.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
except (RuntimeError, dfvfs_errors.BackEndError) as exception:
logger.error(exception)
continue
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
try:
operating_system = self._DetermineOperatingSystem(searcher)
except (ValueError, dfvfs_errors.PathSpecError) as exception:
logger.error(exception)
continue
if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN:
preprocess_manager.PreprocessPluginsManager.RunPlugins(
artifacts_registry_object, file_system, mount_point, mediator)
detected_operating_systems.append(operating_system)
if detected_operating_systems:
logger.info('Preprocessing detected operating systems: {0:s}'.format(
', '.join(detected_operating_systems)))
self.knowledge_base.SetValue(
'operating_system', detected_operating_systems[0])
def _ZeroMQResponder(self, source_queue):
"""Listens for requests and replies to clients.
Args:
source_queue (Queue.queue): queue to use to pull items from.
Raises:
RuntimeError: if closed or terminate event is missing.
"""
if not self._closed_event or not self._terminate_event:
raise RuntimeError('Missing closed or terminate event.')
logger.debug('{0:s} responder thread started'.format(self.name))
item = None
while not self._terminate_event.is_set():
if not item:
try:
if self._closed_event.is_set():
item = source_queue.get_nowait()
else:
item = source_queue.get(True,
self._buffer_timeout_seconds)
except Queue.Empty:
if self._closed_event.is_set():
break
continue
try:
# We need to receive a request before we can reply with the item.
self._ReceiveItemOnActivity(self._zmq_socket)
except errors.QueueEmpty:
if self._closed_event.is_set() and self._queue.empty():
break
continue
sent_successfully = self._SendItem(self._zmq_socket, item)
item = None
if not sent_successfully:
logger.error('Queue {0:s} unable to send item.'.format(
self.name))
break
logger.info('Queue {0:s} responder exiting.'.format(self.name))
self._zmq_socket.close(self._linger_seconds)
def PreprocessSources(self,
artifacts_registry_object,
source_path_specs,
resolver_context=None):
"""Preprocesses the sources.
Args:
artifacts_registry_object (artifacts.ArtifactDefinitionsRegistry):
artifact definitions registry.
source_path_specs (list[dfvfs.PathSpec]): path specifications of
the sources to process.
resolver_context (Optional[dfvfs.Context]): resolver context.
"""
detected_operating_systems = []
for source_path_spec in source_path_specs:
try:
file_system, mount_point = self.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
except (RuntimeError, dfvfs_errors.BackEndError) as exception:
logger.error(exception)
continue
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
try:
operating_system = self._DetermineOperatingSystem(searcher)
except (ValueError, dfvfs_errors.PathSpecError) as exception:
logger.error(exception)
continue
if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN:
preprocess_manager.PreprocessPluginsManager.RunPlugins(
artifacts_registry_object, file_system, mount_point,
self.knowledge_base)
detected_operating_systems.append(operating_system)
if detected_operating_systems:
logger.info(
'Preprocessing detected operating systems: {0:s}'.format(
', '.join(detected_operating_systems)))
self.knowledge_base.SetValue('operating_system',
detected_operating_systems[0])
def ProcessPathSpec(self, mediator, path_spec, excluded_find_specs=None):
"""Processes a path specification.
Args:
mediator (ParserMediator): mediates the interactions between
parsers and other components, such as storage and abort signals.
path_spec (dfvfs.PathSpec): path specification.
excluded_find_specs (Optional[list[dfvfs.FindSpec]]): find specifications
that are excluded from processing.
"""
self.last_activity_timestamp = time.time()
self.processing_status = definitions.STATUS_INDICATOR_RUNNING
file_entry = path_spec_resolver.Resolver.OpenFileEntry(
path_spec, resolver_context=mediator.resolver_context)
if file_entry is None:
display_name = mediator.GetDisplayNameForPathSpec(path_spec)
logger.warning(
'Unable to open file entry with path spec: {0:s}'.format(
display_name))
self.processing_status = definitions.STATUS_INDICATOR_IDLE
return
for find_spec in excluded_find_specs or []:
if find_spec.CompareLocation(file_entry):
logger.info(
'Skipped: {0:s} because of exclusion filter.'.format(
file_entry.path_spec.location))
self.processing_status = definitions.STATUS_INDICATOR_IDLE
return
mediator.SetFileEntry(file_entry)
try:
if file_entry.IsDirectory():
self._ProcessDirectory(mediator, file_entry)
self._ProcessFileEntry(mediator, file_entry)
finally:
mediator.ResetFileEntry()
self.last_activity_timestamp = time.time()
self.processing_status = definitions.STATUS_INDICATOR_IDLE
def ProcessPathSpec(self, mediator, path_spec, excluded_find_specs=None):
"""Processes a path specification.
Args:
mediator (ParserMediator): mediates the interactions between
parsers and other components, such as storage and abort signals.
path_spec (dfvfs.PathSpec): path specification.
excluded_find_specs (Optional[list[dfvfs.FindSpec]]): find specifications
that are excluded from processing.
"""
self.last_activity_timestamp = time.time()
self.processing_status = definitions.STATUS_INDICATOR_RUNNING
file_entry = path_spec_resolver.Resolver.OpenFileEntry(
path_spec, resolver_context=mediator.resolver_context)
if file_entry is None:
display_name = mediator.GetDisplayNameForPathSpec(path_spec)
logger.warning(
'Unable to open file entry with path spec: {0:s}'.format(
display_name))
self.processing_status = definitions.STATUS_INDICATOR_IDLE
return
for find_spec in excluded_find_specs or []:
if find_spec.Matches(file_entry) == (True, True):
logger.info('Skipped: {0:s} because of exclusion filter.'.format(
file_entry.path_spec.location))
return
mediator.SetFileEntry(file_entry)
try:
if file_entry.IsDirectory():
self._ProcessDirectory(mediator, file_entry)
self._ProcessFileEntry(mediator, file_entry)
finally:
mediator.ResetFileEntry()
self.last_activity_timestamp = time.time()
self.processing_status = definitions.STATUS_INDICATOR_IDLE
def PreprocessSources(
self, artifacts_registry_object, source_path_specs,
resolver_context=None):
"""Preprocesses the sources.
Args:
artifacts_registry_object (artifacts.ArtifactDefinitionsRegistry):
artifact definitions registry.
source_path_specs (list[dfvfs.PathSpec]): path specifications of
the sources to process.
resolver_context (Optional[dfvfs.Context]): resolver context.
"""
detected_operating_systems = []
for source_path_spec in source_path_specs:
try:
file_system, mount_point = self.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
except (RuntimeError, dfvfs_errors.BackEndError) as exception:
logger.error(exception)
continue
try:
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
operating_system = self._DetermineOperatingSystem(searcher)
if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN:
preprocess_manager.PreprocessPluginsManager.RunPlugins(
artifacts_registry_object, file_system, mount_point,
self.knowledge_base)
detected_operating_systems.append(operating_system)
finally:
file_system.Close()
if detected_operating_systems:
logger.info('Preprocessing detected operating systems: {0:s}'.format(
', '.join(detected_operating_systems)))
self.knowledge_base.SetValue(
'operating_system', detected_operating_systems[0])
