def PreprocessSources( self, artifact_definitions_path, custom_artifacts_path, source_path_specs, session, storage_writer, resolver_context=None): """Preprocesses the sources. Args: artifact_definitions_path (str): path to artifact definitions directory or file. custom_artifacts_path (str): path to custom artifact definitions directory or file. source_path_specs (list[dfvfs.PathSpec]): path specifications of the sources to process. session (Session): session the preprocessing is part of. storage_writer (StorageWriter): storage writer. resolver_context (Optional[dfvfs.Context]): resolver context. """ artifacts_registry_object = self._BuildArtifactsRegistry( artifact_definitions_path, custom_artifacts_path) mediator = preprocess_mediator.PreprocessMediator( session, storage_writer, self.knowledge_base) detected_operating_systems = [] for source_path_spec in source_path_specs: try: file_system, mount_point = self.GetSourceFileSystem( source_path_spec, resolver_context=resolver_context) except (RuntimeError, dfvfs_errors.BackEndError) as exception: logger.error(exception) continue searcher = file_system_searcher.FileSystemSearcher( file_system, mount_point) try: operating_system = self._DetermineOperatingSystem(searcher) except (ValueError, dfvfs_errors.PathSpecError) as exception: logger.error(exception) continue if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN: preprocess_manager.PreprocessPluginsManager.RunPlugins( artifacts_registry_object, file_system, mount_point, mediator) detected_operating_systems.append(operating_system) if detected_operating_systems: logger.info('Preprocessing detected operating systems: {0:s}'.format( ', '.join(detected_operating_systems))) self.knowledge_base.SetValue( 'operating_system', detected_operating_systems[0])
def _ZeroMQResponder(self, source_queue): """Listens for requests and replies to clients. Args: source_queue (Queue.queue): queue to use to pull items from. Raises: RuntimeError: if closed or terminate event is missing. """ if not self._closed_event or not self._terminate_event: raise RuntimeError('Missing closed or terminate event.') logger.debug('{0:s} responder thread started'.format(self.name)) item = None while not self._terminate_event.is_set(): if not item: try: if self._closed_event.is_set(): item = source_queue.get_nowait() else: item = source_queue.get(True, self._buffer_timeout_seconds) except Queue.Empty: if self._closed_event.is_set(): break continue try: # We need to receive a request before we can reply with the item. self._ReceiveItemOnActivity(self._zmq_socket) except errors.QueueEmpty: if self._closed_event.is_set() and self._queue.empty(): break continue sent_successfully = self._SendItem(self._zmq_socket, item) item = None if not sent_successfully: logger.error('Queue {0:s} unable to send item.'.format( self.name)) break logger.info('Queue {0:s} responder exiting.'.format(self.name)) self._zmq_socket.close(self._linger_seconds)
def PreprocessSources(self, artifacts_registry_object, source_path_specs, resolver_context=None): """Preprocesses the sources. Args: artifacts_registry_object (artifacts.ArtifactDefinitionsRegistry): artifact definitions registry. source_path_specs (list[dfvfs.PathSpec]): path specifications of the sources to process. resolver_context (Optional[dfvfs.Context]): resolver context. """ detected_operating_systems = [] for source_path_spec in source_path_specs: try: file_system, mount_point = self.GetSourceFileSystem( source_path_spec, resolver_context=resolver_context) except (RuntimeError, dfvfs_errors.BackEndError) as exception: logger.error(exception) continue searcher = file_system_searcher.FileSystemSearcher( file_system, mount_point) try: operating_system = self._DetermineOperatingSystem(searcher) except (ValueError, dfvfs_errors.PathSpecError) as exception: logger.error(exception) continue if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN: preprocess_manager.PreprocessPluginsManager.RunPlugins( artifacts_registry_object, file_system, mount_point, self.knowledge_base) detected_operating_systems.append(operating_system) if detected_operating_systems: logger.info( 'Preprocessing detected operating systems: {0:s}'.format( ', '.join(detected_operating_systems))) self.knowledge_base.SetValue('operating_system', detected_operating_systems[0])
def ProcessPathSpec(self, mediator, path_spec, excluded_find_specs=None): """Processes a path specification. Args: mediator (ParserMediator): mediates the interactions between parsers and other components, such as storage and abort signals. path_spec (dfvfs.PathSpec): path specification. excluded_find_specs (Optional[list[dfvfs.FindSpec]]): find specifications that are excluded from processing. """ self.last_activity_timestamp = time.time() self.processing_status = definitions.STATUS_INDICATOR_RUNNING file_entry = path_spec_resolver.Resolver.OpenFileEntry( path_spec, resolver_context=mediator.resolver_context) if file_entry is None: display_name = mediator.GetDisplayNameForPathSpec(path_spec) logger.warning( 'Unable to open file entry with path spec: {0:s}'.format( display_name)) self.processing_status = definitions.STATUS_INDICATOR_IDLE return for find_spec in excluded_find_specs or []: if find_spec.CompareLocation(file_entry): logger.info( 'Skipped: {0:s} because of exclusion filter.'.format( file_entry.path_spec.location)) self.processing_status = definitions.STATUS_INDICATOR_IDLE return mediator.SetFileEntry(file_entry) try: if file_entry.IsDirectory(): self._ProcessDirectory(mediator, file_entry) self._ProcessFileEntry(mediator, file_entry) finally: mediator.ResetFileEntry() self.last_activity_timestamp = time.time() self.processing_status = definitions.STATUS_INDICATOR_IDLE
def ProcessPathSpec(self, mediator, path_spec, excluded_find_specs=None): """Processes a path specification. Args: mediator (ParserMediator): mediates the interactions between parsers and other components, such as storage and abort signals. path_spec (dfvfs.PathSpec): path specification. excluded_find_specs (Optional[list[dfvfs.FindSpec]]): find specifications that are excluded from processing. """ self.last_activity_timestamp = time.time() self.processing_status = definitions.STATUS_INDICATOR_RUNNING file_entry = path_spec_resolver.Resolver.OpenFileEntry( path_spec, resolver_context=mediator.resolver_context) if file_entry is None: display_name = mediator.GetDisplayNameForPathSpec(path_spec) logger.warning( 'Unable to open file entry with path spec: {0:s}'.format( display_name)) self.processing_status = definitions.STATUS_INDICATOR_IDLE return for find_spec in excluded_find_specs or []: if find_spec.Matches(file_entry) == (True, True): logger.info('Skipped: {0:s} because of exclusion filter.'.format( file_entry.path_spec.location)) return mediator.SetFileEntry(file_entry) try: if file_entry.IsDirectory(): self._ProcessDirectory(mediator, file_entry) self._ProcessFileEntry(mediator, file_entry) finally: mediator.ResetFileEntry() self.last_activity_timestamp = time.time() self.processing_status = definitions.STATUS_INDICATOR_IDLE
def PreprocessSources( self, artifacts_registry_object, source_path_specs, resolver_context=None): """Preprocesses the sources. Args: artifacts_registry_object (artifacts.ArtifactDefinitionsRegistry): artifact definitions registry. source_path_specs (list[dfvfs.PathSpec]): path specifications of the sources to process. resolver_context (Optional[dfvfs.Context]): resolver context. """ detected_operating_systems = [] for source_path_spec in source_path_specs: try: file_system, mount_point = self.GetSourceFileSystem( source_path_spec, resolver_context=resolver_context) except (RuntimeError, dfvfs_errors.BackEndError) as exception: logger.error(exception) continue try: searcher = file_system_searcher.FileSystemSearcher( file_system, mount_point) operating_system = self._DetermineOperatingSystem(searcher) if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN: preprocess_manager.PreprocessPluginsManager.RunPlugins( artifacts_registry_object, file_system, mount_point, self.knowledge_base) detected_operating_systems.append(operating_system) finally: file_system.Close() if detected_operating_systems: logger.info('Preprocessing detected operating systems: {0:s}'.format( ', '.join(detected_operating_systems))) self.knowledge_base.SetValue( 'operating_system', detected_operating_systems[0])