def _SendItem(self, zmq_socket, item, block=True):
"""Attempts to send an item to a ZeroMQ socket.
Args:
zmq_socket (zmq.Socket): used to the send the item.
item (object): sent on the queue. Will be pickled prior to sending.
Returns:
bool: whether the item was sent successfully.
"""
try:
logger.debug('{0:s} sending item'.format(self.name))
if block:
zmq_socket.send_pyobj(item)
else:
zmq_socket.send_pyobj(item, zmq.DONTWAIT)
logger.debug('{0:s} sent item'.format(self.name))
return True
except zmq.error.Again:
logger.debug('{0:s} could not send an item'.format(self.name))
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logger.error('ZMQ syscall interrupted in {0:s}.'.format(
self.name))
return False
def _ReceiveItemOnActivity(self, zmq_socket):
"""Attempts to receive an item from a ZeroMQ socket.
Args:
zmq_socket (zmq.Socket): used to the receive the item.
Returns:
object: item from the socket.
Raises:
QueueEmpty: if no item could be received within the timeout.
zmq.error.ZMQError: if an error occurs in ZeroMQ
"""
events = zmq_socket.poll(self._ZMQ_SOCKET_RECEIVE_TIMEOUT_MILLISECONDS)
if events:
try:
received_object = self._zmq_socket.recv_pyobj()
return received_object
except zmq.error.Again:
logger.error('{0:s}. Failed to receive item in time.'.format(
self.name))
raise
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logger.error(
'ZMQ syscall interrupted in {0:s}. Queue aborting.'.
format(self.name))
raise
raise errors.QueueEmpty
def _ProcessPathSpec(self, extraction_worker, parser_mediator, path_spec):
"""Processes a path specification.
Args:
extraction_worker (worker.ExtractionWorker): extraction worker.
parser_mediator (ParserMediator): parser mediator.
path_spec (dfvfs.PathSpec): path specification.
"""
self._current_display_name = parser_mediator.GetDisplayNameForPathSpec(
path_spec)
self._CacheFileSystem(path_spec)
excluded_find_specs = None
if self.collection_filters_helper:
excluded_find_specs = (
self.collection_filters_helper.excluded_file_system_find_specs)
try:
extraction_worker.ProcessPathSpec(
parser_mediator,
path_spec,
excluded_find_specs=excluded_find_specs)
except KeyboardInterrupt:
self._abort = True
self._processing_status.aborted = True
if self._status_update_callback:
self._status_update_callback(self._processing_status)
# We cannot recover from a CacheFullError and abort processing when
# it is raised.
except dfvfs_errors.CacheFullError:
# TODO: signal engine of failure.
self._abort = True
logger.error(
('ABORT: detected cache full error while processing '
'path spec: {0:s}').format(self._current_display_name))
# All exceptions need to be caught here to prevent the worker
# from being killed by an uncaught exception.
except Exception as exception: # pylint: disable=broad-except
parser_mediator.ProduceExtractionWarning(
('unable to process path specification with error: '
'{0!s}').format(exception),
path_spec=path_spec)
if getattr(self._processing_configuration, 'debug_output', False):
self._StopStatusUpdateThread()
logger.warning(
'Unhandled exception while processing path spec: {0:s}.'.
format(self._current_display_name))
logger.exception(exception)
pdb.post_mortem()
self._StartStatusUpdateThread()
def BuildFindSpecsFromFileArtifact(self, source_path, path_separator,
environment_variables, user_accounts):
"""Builds find specifications from a file source type.
Args:
source_path (str): file system path defined by the source.
path_separator (str): file system path segment separator.
environment_variables (list[str]): environment variable attributes used to
dynamically populate environment variables in key.
user_accounts (list[str]): identified user accounts stored in the
knowledge base.
Returns:
list[dfvfs.FindSpec]: find specifications for the file source type.
"""
find_specs = []
for glob_path in path_helper.PathHelper.ExpandRecursiveGlobs(
source_path, path_separator):
for path in path_helper.PathHelper.ExpandUsersHomeDirectoryPath(
glob_path, user_accounts):
if '%' in path:
path = path_helper.PathHelper.ExpandWindowsPath(
path, environment_variables)
if not path.startswith(path_separator):
logger.warning((
'The path filter must be defined as an absolute path: '
'"{0:s}"').format(path))
continue
# Convert the path filters into a list of path segments and
# strip the root path segment.
path_segments = path.split(path_separator)
# Remove initial root entry
path_segments.pop(0)
if not path_segments[-1]:
logger.warning(
'Empty last path segment in path filter: "{0:s}"'.
format(path))
path_segments.pop(-1)
try:
find_spec = file_system_searcher.FindSpec(
location_glob=path_segments, case_sensitive=False)
except ValueError as exception:
logger.error((
'Unable to build find specification for path: "{0:s}" with '
'error: {1!s}').format(path, exception))
continue
find_specs.append(find_spec)
return find_specs
def _ExtractPathSpecsFromFileSystem(self,
path_spec,
find_specs=None,
recurse_file_system=True,
resolver_context=None):
"""Extracts path specification from a file system within a specific source.
Args:
path_spec (dfvfs.PathSpec): path specification of the root of
the file system.
find_specs (Optional[list[dfvfs.FindSpec]]): find specifications
used in path specification extraction.
recurse_file_system (Optional[bool]): True if extraction should
recurse into a file system.
resolver_context (Optional[dfvfs.Context]): resolver context.
Yields:
dfvfs.PathSpec: path specification of a file entry found in
the file system.
"""
file_system = None
try:
file_system = path_spec_resolver.Resolver.OpenFileSystem(
path_spec, resolver_context=resolver_context)
except (dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
dfvfs_errors.PathSpecError) as exception:
logger.error('Unable to open file system with error: {0!s}'.format(
exception))
if file_system:
try:
if find_specs:
searcher = file_system_searcher.FileSystemSearcher(
file_system, path_spec)
for extracted_path_spec in searcher.Find(
find_specs=find_specs):
yield extracted_path_spec
elif recurse_file_system:
file_entry = file_system.GetFileEntryByPathSpec(path_spec)
if file_entry:
for extracted_path_spec in self._ExtractPathSpecsFromDirectory(
file_entry):
yield extracted_path_spec
else:
yield path_spec
except (dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
dfvfs_errors.PathSpecError) as exception:
logger.warning('{0!s}'.format(exception))
finally:
file_system.Close()
def _BuildFindSpecsFromFileSourcePath(self, source_path, path_separator,
environment_variables,
user_accounts):
"""Builds find specifications from a file source type.
Args:
source_path (str): file system path defined by the source.
path_separator (str): file system path segment separator.
environment_variables (list[str]): environment variable attributes used to
dynamically populate environment variables in key.
user_accounts (list[str]): identified user accounts stored in the
knowledge base.
Returns:
list[dfvfs.FindSpec]: find specifications for the file source type.
"""
find_specs = []
for path_glob in path_helper.PathHelper.ExpandGlobStars(
source_path, path_separator):
logger.debug(
'building find spec from path glob: {0:s}'.format(path_glob))
for path in path_helper.PathHelper.ExpandUsersVariablePath(
path_glob, path_separator, user_accounts):
logger.debug(
'building find spec from path: {0:s}'.format(path))
if '%' in path:
path = path_helper.PathHelper.ExpandWindowsPath(
path, environment_variables)
logger.debug(
'building find spec from expanded path: {0:s}'.format(
path))
if not path.startswith(path_separator):
logger.warning((
'The path filter must be defined as an absolute path: '
'"{0:s}"').format(path))
continue
try:
find_spec = file_system_searcher.FindSpec(
case_sensitive=False,
location_glob=path,
location_separator=path_separator)
except ValueError as exception:
logger.error((
'Unable to build find specification for path: "{0:s}" with '
'error: {1!s}').format(path, exception))
continue
find_specs.append(find_spec)
return find_specs
def BuildFindSpecsFromFileArtifact(
self, source_path, path_separator, environment_variables, user_accounts):
"""Builds find specifications from a file source type.
Args:
source_path (str): file system path defined by the source.
path_separator (str): file system path segment separator.
environment_variables (list[str]): environment variable attributes used to
dynamically populate environment variables in key.
user_accounts (list[str]): identified user accounts stored in the
knowledge base.
Returns:
list[dfvfs.FindSpec]: find specifications for the file source type.
"""
find_specs = []
for glob_path in path_helper.PathHelper.ExpandRecursiveGlobs(
source_path, path_separator):
for path in path_helper.PathHelper.ExpandUsersHomeDirectoryPath(
glob_path, path_separator, user_accounts):
if '%' in path:
path = path_helper.PathHelper.ExpandWindowsPath(
path, environment_variables)
if not path.startswith(path_separator):
logger.warning((
'The path filter must be defined as an absolute path: '
'"{0:s}"').format(path))
continue
# Convert the path filters into a list of path segments and
# strip the root path segment.
path_segments = path.split(path_separator)
# Remove initial root entry
path_segments.pop(0)
if not path_segments[-1]:
logger.warning(
'Empty last path segment in path filter: "{0:s}"'.format(path))
path_segments.pop(-1)
try:
find_spec = file_system_searcher.FindSpec(
location_glob=path_segments, case_sensitive=False)
except ValueError as exception:
logger.error((
'Unable to build find specification for path: "{0:s}" with '
'error: {1!s}').format(path, exception))
continue
find_specs.append(find_spec)
return find_specs
def _ExtractPathSpecsFromFileSystem(
self, path_spec, find_specs=None, recurse_file_system=True,
resolver_context=None):
"""Extracts path specification from a file system within a specific source.
Args:
path_spec (dfvfs.PathSpec): path specification of the root of
the file system.
find_specs (Optional[list[dfvfs.FindSpec]]): find specifications.
recurse_file_system (Optional[bool]): True if extraction should
recurse into a file system.
resolver_context (Optional[dfvfs.Context]): resolver context.
Yields:
dfvfs.PathSpec: path specification of a file entry found in
the file system.
"""
try:
file_system = path_spec_resolver.Resolver.OpenFileSystem(
path_spec, resolver_context=resolver_context)
except (
dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
dfvfs_errors.PathSpecError) as exception:
logger.error(
'Unable to open file system with error: {0!s}'.format(exception))
return
try:
if find_specs:
searcher = file_system_searcher.FileSystemSearcher(
file_system, path_spec)
for extracted_path_spec in searcher.Find(find_specs=find_specs):
yield extracted_path_spec
elif recurse_file_system:
file_entry = file_system.GetFileEntryByPathSpec(path_spec)
if file_entry:
for extracted_path_spec in self._ExtractPathSpecsFromDirectory(
file_entry):
yield extracted_path_spec
else:
yield path_spec
except (
dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
dfvfs_errors.PathSpecError) as exception:
logger.warning('{0!s}'.format(exception))
finally:
file_system.Close()
def PreprocessSources(
self, artifact_definitions_path, custom_artifacts_path,
source_path_specs, session, storage_writer, resolver_context=None):
"""Preprocesses the sources.
Args:
artifact_definitions_path (str): path to artifact definitions directory
or file.
custom_artifacts_path (str): path to custom artifact definitions
directory or file.
source_path_specs (list[dfvfs.PathSpec]): path specifications of
the sources to process.
session (Session): session the preprocessing is part of.
storage_writer (StorageWriter): storage writer.
resolver_context (Optional[dfvfs.Context]): resolver context.
"""
artifacts_registry_object = self._BuildArtifactsRegistry(
artifact_definitions_path, custom_artifacts_path)
mediator = preprocess_mediator.PreprocessMediator(
session, storage_writer, self.knowledge_base)
detected_operating_systems = []
for source_path_spec in source_path_specs:
try:
file_system, mount_point = self.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
except (RuntimeError, dfvfs_errors.BackEndError) as exception:
logger.error(exception)
continue
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
try:
operating_system = self._DetermineOperatingSystem(searcher)
except (ValueError, dfvfs_errors.PathSpecError) as exception:
logger.error(exception)
continue
if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN:
preprocess_manager.PreprocessPluginsManager.RunPlugins(
artifacts_registry_object, file_system, mount_point, mediator)
detected_operating_systems.append(operating_system)
if detected_operating_systems:
logger.info('Preprocessing detected operating systems: {0:s}'.format(
', '.join(detected_operating_systems)))
self.knowledge_base.SetValue(
'operating_system', detected_operating_systems[0])
def _ProcessPathSpec(self, extraction_worker, parser_mediator, path_spec):
"""Processes a path specification.
Args:
extraction_worker (worker.ExtractionWorker): extraction worker.
parser_mediator (ParserMediator): parser mediator.
path_spec (dfvfs.PathSpec): path specification.
"""
self._current_display_name = parser_mediator.GetDisplayNameForPathSpec(
path_spec)
excluded_find_specs = None
if self.collection_filters_helper:
excluded_find_specs = (
self.collection_filters_helper.excluded_file_system_find_specs)
try:
extraction_worker.ProcessPathSpec(
parser_mediator, path_spec, excluded_find_specs=excluded_find_specs)
except KeyboardInterrupt:
self._abort = True
self._processing_status.aborted = True
if self._status_update_callback:
self._status_update_callback(self._processing_status)
# We cannot recover from a CacheFullError and abort processing when
# it is raised.
except dfvfs_errors.CacheFullError:
# TODO: signal engine of failure.
self._abort = True
logger.error((
'ABORT: detected cache full error while processing '
'path spec: {0:s}').format(self._current_display_name))
# All exceptions need to be caught here to prevent the worker
# from being killed by an uncaught exception.
except Exception as exception: # pylint: disable=broad-except
parser_mediator.ProduceExtractionWarning((
'unable to process path specification with error: '
'{0!s}').format(exception), path_spec=path_spec)
if getattr(self._processing_configuration, 'debug_output', False):
logger.warning(
'Unhandled exception while processing path spec: {0:s}.'.format(
self._current_display_name))
logger.exception(exception)
pdb.post_mortem()
def _ExtractPathSpecs(self,
path_spec,
find_specs=None,
recurse_file_system=True,
resolver_context=None):
"""Extracts path specification from a specific source.
Args:
path_spec (dfvfs.PathSpec): path specification.
find_specs (Optional[list[dfvfs.FindSpec]]): find specifications
used in path specification extraction.
recurse_file_system (Optional[bool]): True if extraction should
recurse into a file system.
resolver_context (Optional[dfvfs.Context]): resolver context.
Yields:
dfvfs.PathSpec: path specification of a file entry found in the source.
"""
try:
file_entry = path_spec_resolver.Resolver.OpenFileEntry(
path_spec, resolver_context=resolver_context)
except (dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
dfvfs_errors.PathSpecError) as exception:
logger.error('Unable to open file entry with error: {0!s}'.format(
exception))
return
if not file_entry:
logger.warning('Unable to open: {0:s}'.format(
path_spec.comparable))
return
if (not file_entry.IsDirectory() and not file_entry.IsFile()
and not file_entry.IsDevice()):
logger.warning(
('Source path specification not a device, file or directory.\n'
'{0:s}').format(path_spec.comparable))
return
if file_entry.IsFile():
yield path_spec
else:
for extracted_path_spec in self._ExtractPathSpecsFromFileSystem(
path_spec,
find_specs=find_specs,
recurse_file_system=recurse_file_system,
resolver_context=resolver_context):
yield extracted_path_spec
def _ZeroMQResponder(self, source_queue):
"""Listens for requests and replies to clients.
Args:
source_queue (Queue.queue): queue to use to pull items from.
Raises:
RuntimeError: if closed or terminate event is missing.
"""
if not self._closed_event or not self._terminate_event:
raise RuntimeError('Missing closed or terminate event.')
logger.debug('{0:s} responder thread started'.format(self.name))
item = None
while not self._terminate_event.is_set():
if not item:
try:
if self._closed_event.is_set():
item = source_queue.get_nowait()
else:
item = source_queue.get(True,
self._buffer_timeout_seconds)
except Queue.Empty:
if self._closed_event.is_set():
break
continue
try:
# We need to receive a request before we can reply with the item.
self._ReceiveItemOnActivity(self._zmq_socket)
except errors.QueueEmpty:
if self._closed_event.is_set() and self._queue.empty():
break
continue
sent_successfully = self._SendItem(self._zmq_socket, item)
item = None
if not sent_successfully:
logger.error('Queue {0:s} unable to send item.'.format(
self.name))
break
logger.info('Queue {0:s} responder exiting.'.format(self.name))
self._zmq_socket.close(self._linger_seconds)
def _ExtractPathSpecs(
self, path_spec, find_specs=None, recurse_file_system=True,
resolver_context=None):
"""Extracts path specification from a specific source.
Args:
path_spec (dfvfs.PathSpec): path specification.
find_specs (Optional[list[dfvfs.FindSpec]]): find specifications
used in path specification extraction.
recurse_file_system (Optional[bool]): True if extraction should
recurse into a file system.
resolver_context (Optional[dfvfs.Context]): resolver context.
Yields:
dfvfs.PathSpec: path specification of a file entry found in the source.
"""
try:
file_entry = path_spec_resolver.Resolver.OpenFileEntry(
path_spec, resolver_context=resolver_context)
except (
dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
dfvfs_errors.PathSpecError) as exception:
logger.error(
'Unable to open file entry with error: {0!s}'.format(exception))
return
if not file_entry:
logger.warning('Unable to open: {0:s}'.format(path_spec.comparable))
return
if (not file_entry.IsDirectory() and not file_entry.IsFile() and
not file_entry.IsDevice()):
logger.warning((
'Source path specification not a device, file or directory.\n'
'{0:s}').format(path_spec.comparable))
return
if file_entry.IsFile():
yield path_spec
else:
for extracted_path_spec in self._ExtractPathSpecsFromFileSystem(
path_spec, find_specs=find_specs,
recurse_file_system=recurse_file_system,
resolver_context=resolver_context):
yield extracted_path_spec
def Close(self, abort=False):
"""Closes the queue.
Args:
abort (Optional[bool]): whether the Close is the result of an abort
condition. If True, queue contents may be lost.
Raises:
QueueAlreadyClosed: If the queue is not started, or has already been
closed.
RuntimeError: if closed or terminate event is missing.
"""
if not self._closed_event or not self._terminate_event:
raise RuntimeError('Missing closed or terminate event.')
if not abort and self._closed_event.is_set():
raise errors.QueueAlreadyClosed()
self._closed_event.set()
if abort:
if not self._closed_event.is_set():
logger.warning(
'{0:s} queue aborting. Contents may be lost.'.format(
self.name))
# We can't determine whether a there might be an operation being performed
# on the socket in a separate method or thread, so we'll signal that any
# such operation should cease.
self._terminate_event.set()
self._linger_seconds = 0
if self._zmq_thread:
logger.debug('[{0:s}] Waiting for thread to exit.'.format(
self.name))
self._zmq_thread.join(timeout=self.timeout_seconds)
if self._zmq_thread.isAlive():
logger.error((
'{0:s} ZMQ responder thread did not exit within timeout'
).format(self.name))
else:
logger.debug(
'{0:s} queue closing, will linger for up to {1:d} seconds'.
format(self.name, self._linger_seconds))
def PreprocessSources(self,
artifacts_registry_object,
source_path_specs,
resolver_context=None):
"""Preprocesses the sources.
Args:
artifacts_registry_object (artifacts.ArtifactDefinitionsRegistry):
artifact definitions registry.
source_path_specs (list[dfvfs.PathSpec]): path specifications of
the sources to process.
resolver_context (Optional[dfvfs.Context]): resolver context.
"""
detected_operating_systems = []
for source_path_spec in source_path_specs:
try:
file_system, mount_point = self.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
except (RuntimeError, dfvfs_errors.BackEndError) as exception:
logger.error(exception)
continue
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
try:
operating_system = self._DetermineOperatingSystem(searcher)
except (ValueError, dfvfs_errors.PathSpecError) as exception:
logger.error(exception)
continue
if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN:
preprocess_manager.PreprocessPluginsManager.RunPlugins(
artifacts_registry_object, file_system, mount_point,
self.knowledge_base)
detected_operating_systems.append(operating_system)
if detected_operating_systems:
logger.info(
'Preprocessing detected operating systems: {0:s}'.format(
', '.join(detected_operating_systems)))
self.knowledge_base.SetValue('operating_system',
detected_operating_systems[0])
def PushItem(self, item, block=True):
"""Push an item on to the queue.
If no ZeroMQ socket has been created, one will be created the first time
this method is called.
Args:
item (object): item to push on the queue.
block (Optional[bool]): whether the push should be performed in blocking
or non-block mode.
Raises:
KeyboardInterrupt: if the process is sent a KeyboardInterrupt while
pushing an item.
QueueFull: if it was not possible to push the item to the queue
within the timeout.
RuntimeError: if terminate event is missing.
zmq.error.ZMQError: if a ZeroMQ specific error occurs.
"""
if not self._zmq_socket:
self._CreateZMQSocket()
if not self._terminate_event:
raise RuntimeError('Missing terminate event.')
logger.debug('Push on {0:s} queue, port {1:d}'.format(
self.name, self.port))
last_retry_timestamp = time.time() + self.timeout_seconds
while not self._terminate_event.is_set():
try:
send_successful = self._SendItem(self._zmq_socket, item, block)
if send_successful:
break
if time.time() > last_retry_timestamp:
logger.error('{0:s} unable to push item, raising.'.format(
self.name))
raise errors.QueueFull
except KeyboardInterrupt:
self.Close(abort=True)
raise
def PreprocessSources(
self, artifacts_registry_object, source_path_specs,
resolver_context=None):
"""Preprocesses the sources.
Args:
artifacts_registry_object (artifacts.ArtifactDefinitionsRegistry):
artifact definitions registry.
source_path_specs (list[dfvfs.PathSpec]): path specifications of
the sources to process.
resolver_context (Optional[dfvfs.Context]): resolver context.
"""
detected_operating_systems = []
for source_path_spec in source_path_specs:
try:
file_system, mount_point = self.GetSourceFileSystem(
source_path_spec, resolver_context=resolver_context)
except (RuntimeError, dfvfs_errors.BackEndError) as exception:
logger.error(exception)
continue
try:
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
operating_system = self._DetermineOperatingSystem(searcher)
if operating_system != definitions.OPERATING_SYSTEM_FAMILY_UNKNOWN:
preprocess_manager.PreprocessPluginsManager.RunPlugins(
artifacts_registry_object, file_system, mount_point,
self.knowledge_base)
detected_operating_systems.append(operating_system)
finally:
file_system.Close()
if detected_operating_systems:
logger.info('Preprocessing detected operating systems: {0:s}'.format(
', '.join(detected_operating_systems)))
self.knowledge_base.SetValue(
'operating_system', detected_operating_systems[0])
def _BuildFindSpecsFromRegistrySourceKey(self, key_path):
"""Build find specifications from a Windows Registry source type.
Args:
key_path (str): Windows Registry key path defined by the source.
Returns:
list[dfwinreg.FindSpec]: find specifications for the Windows Registry
source type.
"""
find_specs = []
for key_path_glob in path_helper.PathHelper.ExpandRecursiveGlobs(
key_path, '\\'):
if '%%' in key_path_glob:
logger.error('Unable to expand key path: "{0:s}"'.format(key_path_glob))
continue
find_spec = registry_searcher.FindSpec(key_path_glob=key_path_glob)
find_specs.append(find_spec)
return find_specs
def BuildFindSpecsFromRegistryArtifact(self, source_key_path):
"""Build find specifications from a Windows Registry source type.
Args:
source_key_path (str): Windows Registry key path defined by the source.
Returns:
list[dfwinreg.FindSpec]: find specifications for the Windows Registry
source type.
"""
find_specs = []
for key_path in path_helper.PathHelper.ExpandRecursiveGlobs(
source_key_path, '\\'):
if '%%' in key_path:
logger.error('Unable to expand key path: "{0:s}"'.format(key_path))
continue
find_spec = registry_searcher.FindSpec(key_path_glob=key_path)
find_specs.append(find_spec)
return find_specs
def BuildFindSpecs(self, environment_variables=None):
"""Build find specification from a filter file.
Args:
environment_variables (Optional[list[EnvironmentVariableArtifact]]):
environment variables.
Returns:
list[dfvfs.FindSpec]: find specification.
"""
path_attributes = {}
if environment_variables:
for environment_variable in environment_variables:
attribute_name = environment_variable.name.lower()
attribute_value = environment_variable.value
if not isinstance(attribute_value, py2to3.STRING_TYPES):
continue
# Remove the drive letter.
if len(attribute_value) > 2 and attribute_value[1] == ':':
_, _, attribute_value = attribute_value.rpartition(':')
if attribute_value.startswith('\\'):
attribute_value = attribute_value.replace('\\', '/')
path_attributes[attribute_name] = attribute_value
find_specs = []
with open(self._path, 'r') as file_object:
for line in file_object:
line = line.strip()
if line.startswith('#'):
continue
if path_attributes:
try:
line = line.format(**path_attributes)
except KeyError as exception:
logger.error(
('Unable to expand path filter: {0:s} with error: '
'{1:s}').format(line, exception))
continue
if not line.startswith('/'):
logger.warning((
'The path filter must be defined as an absolute path: '
'{0:s}').format(line))
continue
# Convert the path filters into a list of path segments and strip
# the root path segment.
path_segments = line.split('/')
path_segments.pop(0)
if not path_segments[-1]:
logger.warning(
'Empty last path segment in path filter: {0:s}'.format(
line))
continue
find_spec = file_system_searcher.FindSpec(
location_regex=path_segments, case_sensitive=False)
find_specs.append(find_spec)
return find_specs
def BuildFindSpecs(self, environment_variables=None):
"""Build find specification from a filter file.
Args:
environment_variables (Optional[list[EnvironmentVariableArtifact]]):
environment variables.
Returns:
list[dfvfs.FindSpec]: find specification.
"""
path_attributes = {}
if environment_variables:
for environment_variable in environment_variables:
attribute_name = environment_variable.name.lower()
attribute_value = environment_variable.value
if not isinstance(attribute_value, py2to3.STRING_TYPES):
continue
# Remove the drive letter.
if len(attribute_value) > 2 and attribute_value[1] == ':':
_, _, attribute_value = attribute_value.rpartition(':')
if attribute_value.startswith('\\'):
attribute_value = attribute_value.replace('\\', '/')
path_attributes[attribute_name] = attribute_value
find_specs = []
with open(self._path, 'r') as file_object:
for line in file_object:
line = line.strip()
if line.startswith('#'):
continue
if path_attributes:
try:
line = line.format(**path_attributes)
except KeyError as exception:
logger.error((
'Unable to expand path filter: {0:s} with error: '
'{1!s}').format(line, exception))
continue
if not line.startswith('/'):
logger.warning((
'The path filter must be defined as an absolute path: '
'{0:s}').format(line))
continue
# Convert the path filters into a list of path segments and strip
# the root path segment.
path_segments = line.split('/')
path_segments.pop(0)
if not path_segments[-1]:
logger.warning(
'Empty last path segment in path filter: {0:s}'.format(line))
continue
find_spec = file_system_searcher.FindSpec(
location_regex=path_segments, case_sensitive=False)
find_specs.append(find_spec)
return find_specs
