def GetCommandLineArguments(self):
"""Retrieves the command line arguments.
Returns:
str: command line arguments.
"""
command_line_arguments = sys.argv
if not command_line_arguments:
return ''
if isinstance(command_line_arguments[0], py2to3.BYTES_TYPE):
encoding = sys.stdin.encoding
# Note that sys.stdin.encoding can be None.
if not encoding:
encoding = self.preferred_encoding
try:
command_line_arguments = [
argument.decode(encoding) for argument in command_line_arguments]
except UnicodeDecodeError:
logger.error(
'Unable to properly read command line input due to encoding '
'error. Replacing non Basic Latin (C0) characters with "?" or '
'"\\ufffd".')
command_line_arguments = [
argument.decode(encoding, errors='replace')
for argument in command_line_arguments]
return ' '.join(command_line_arguments)
def Read(self):
"""Reads a string from the input.
Returns:
str: input.
"""
encoded_string = self._file_object.readline()
if isinstance(encoded_string, py2to3.UNICODE_TYPE):
return encoded_string
try:
string = codecs.decode(encoded_string, self._encoding, self._errors)
except UnicodeDecodeError:
if self._errors == 'strict':
logger.error(
'Unable to properly read input due to encoding error. '
'Switching to error tolerant encoding which can result in '
'non Basic Latin (C0) characters to be replaced with "?" or '
'"\\ufffd".')
self._errors = 'replace'
string = codecs.decode(encoded_string, self._encoding, self._errors)
return string
def ListTimeZones(self):
"""Lists the timezones."""
max_length = 0
for timezone_name in pytz.all_timezones:
if len(timezone_name) > max_length:
max_length = len(timezone_name)
utc_date_time = datetime.datetime.utcnow()
table_view = views.ViewsFactory.GetTableView(
self._views_format_type, column_names=['Timezone', 'UTC Offset'],
title='Zones')
for timezone_name in pytz.all_timezones:
try:
local_timezone = pytz.timezone(timezone_name)
except AssertionError as exception:
logger.error((
'Unable to determine information about timezone: {0:s} with '
'error: {1!s}').format(timezone_name, exception))
continue
local_date_string = '{0!s}'.format(
local_timezone.localize(utc_date_time))
if '+' in local_date_string:
_, _, diff = local_date_string.rpartition('+')
diff_string = '+{0:s}'.format(diff)
else:
_, _, diff = local_date_string.rpartition('-')
diff_string = '-{0:s}'.format(diff)
table_view.AddRow([timezone_name, diff_string])
table_view.Write(self._output_writer)
def PrintStorageInformation(self):
"""Prints the storage information."""
storage_file = storage_factory.StorageFactory.CreateStorageFileForFile(
self._storage_file_path)
if not storage_file:
logger.error(
'Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return
try:
storage_file.Open(path=self._storage_file_path, read_only=True)
except IOError as exception:
logger.error(
'Unable to open storage file: {0:s} with error: {1!s}'.format(
self._storage_file_path, exception))
return
try:
if self._output_format == 'json':
self._PrintStorageInformationAsJSON(storage_file)
elif self._output_format == 'text':
self._PrintStorageInformationAsText(storage_file)
finally:
storage_file.Close()
def PrintStorageInformation(self):
"""Prints the storage information."""
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
if not storage_reader:
logger.error(
'Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return
try:
if self._output_format == 'json':
self._PrintStorageInformationAsJSON(storage_reader)
elif self._output_format == 'text':
self._PrintStorageInformationAsText(storage_reader)
finally:
storage_reader.Close()
def CompareStores(self):
"""Compares the contents of two stores.
Returns:
bool: True if the content of the stores is identical.
"""
storage_file = storage_factory.StorageFactory.CreateStorageFileForFile(
self._storage_file_path)
if not storage_file:
logger.error(
'Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return False
try:
storage_file.Open(path=self._storage_file_path, read_only=True)
except IOError as exception:
logger.error(
'Unable to open storage file: {0:s} with error: {1!s}'.format(
self._storage_file_path, exception))
return False
compare_storage_file = (
storage_factory.StorageFactory.CreateStorageFileForFile(
self._compare_storage_file_path))
if not compare_storage_file:
logger.error(
'Format of storage file: {0:s} not supported'.format(
self._compare_storage_file_path))
return False
try:
compare_storage_file.Open(
path=self._compare_storage_file_path, read_only=True)
except IOError as exception:
logger.error(
'Unable to open storage file: {0:s} with error: {1!s}'.format(
self._compare_storage_file_path, exception))
storage_file.Close()
return False
try:
result = self._CompareStores(storage_file, compare_storage_file)
finally:
compare_storage_file.Close()
storage_file.Close()
if result:
self._output_writer.Write('Storage files are identical.\n')
else:
self._output_writer.Write('Storage files are different.\n')
return result
def _Preprocess(self, file_system, mount_point):
"""Preprocesses the image.
Args:
file_system (dfvfs.FileSystem): file system to be preprocessed.
mount_point (dfvfs.PathSpec): mount point path specification that refers
to the base location of the file system.
"""
logger.debug('Starting preprocessing.')
try:
preprocess_manager.PreprocessPluginsManager.RunPlugins(
self._artifacts_registry, file_system, mount_point,
self._knowledge_base)
except IOError as exception:
logger.error('Unable to preprocess with error: {0!s}'.format(exception))
logger.debug('Preprocessing done.')
def _PreprocessSources(self, extraction_engine):
"""Preprocesses the sources.
Args:
extraction_engine (BaseEngine): extraction engine to preprocess
the sources.
"""
logger.debug('Starting preprocessing.')
try:
artifacts_registry = engine.BaseEngine.BuildArtifactsRegistry(
self._artifact_definitions_path, self._custom_artifacts_path)
extraction_engine.PreprocessSources(
artifacts_registry, self._source_path_specs,
resolver_context=self._resolver_context)
except IOError as exception:
logger.error('Unable to preprocess with error: {0!s}'.format(exception))
logger.debug('Preprocessing done.')
def Write(self, string):
"""Writes a string to the output.
Args:
string (str): output.
"""
try:
# Note that encode() will first convert string into a Unicode string
# if necessary.
encoded_string = codecs.encode(string, self._encoding, self._errors)
except UnicodeEncodeError:
if self._errors == 'strict':
logger.error(
'Unable to properly write output due to encoding error. '
'Switching to error tolerant encoding which can result in '
'non Basic Latin (C0) characters to be replaced with "?" or '
'"\\ufffd".')
self._errors = 'replace'
encoded_string = codecs.encode(string, self._encoding, self._errors)
self._file_object.write(encoded_string)
def CompareStores(self):
"""Compares the contents of two stores.
Returns:
bool: True if the content of the stores is identical.
"""
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
if not storage_reader:
logger.error(
'Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return False
compare_storage_reader = (
storage_factory.StorageFactory.CreateStorageReaderForFile(
self._compare_storage_file_path))
if not compare_storage_reader:
logger.error(
'Format of storage file: {0:s} not supported'.format(
self._compare_storage_file_path))
return False
try:
result = self._CompareStores(storage_reader, compare_storage_reader)
finally:
compare_storage_reader.Close()
storage_reader.Close()
if result:
self._output_writer.Write('Storage files are identical.\n')
else:
self._output_writer.Write('Storage files are different.\n')
return result
def _EncodeString(self, string):
"""Encodes a string in the preferred encoding.
Returns:
bytes: encoded string.
"""
try:
# Note that encode() will first convert string into a Unicode string
# if necessary.
encoded_string = string.encode(
self.preferred_encoding, errors=self._encode_errors)
except UnicodeEncodeError:
if self._encode_errors == 'strict':
logger.error(
'Unable to properly write output due to encoding error. '
'Switching to error tolerant encoding which can result in '
'non Basic Latin (C0) characters to be replaced with "?" or '
'"\\ufffd".')
self._encode_errors = 'replace'
encoded_string = string.encode(
self.preferred_encoding, errors=self._encode_errors)
return encoded_string
def _ReadSpecificationFile(self, path):
"""Reads the format specification file.
Args:
path (str): path of the format specification file.
Returns:
FormatSpecificationStore: format specification store.
"""
specification_store = specification.FormatSpecificationStore()
with io.open(
path, 'rt', encoding=self._SPECIFICATION_FILE_ENCODING) as file_object:
for line in file_object.readlines():
line = line.strip()
if not line or line.startswith('#'):
continue
try:
identifier, offset, pattern = line.split()
except ValueError:
logger.error('[skipping] invalid line: {0:s}'.format(line))
continue
try:
offset = int(offset, 10)
except ValueError:
logger.error('[skipping] invalid offset in line: {0:s}'.format(line))
continue
try:
# TODO: find another way to do this that doesn't use an undocumented
# API.
pattern = codecs.escape_decode(pattern)[0]
# ValueError is raised e.g. when the patterns contains "\xg1".
except ValueError:
logger.error(
'[skipping] invalid pattern in line: {0:s}'.format(line))
continue
format_specification = specification.FormatSpecification(identifier)
format_specification.AddNewSignature(pattern, offset=offset)
specification_store.AddSpecification(format_specification)
return specification_store
def ProcessStorage(self):
"""Processes a plaso storage file.
Raises:
BadConfigOption: when a configuration parameter fails validation.
RuntimeError: if a non-recoverable situation is encountered.
"""
self._CheckStorageFile(self._storage_file_path)
self._status_view.SetMode(self._status_view_mode)
self._status_view.SetStorageFileInformation(self._storage_file_path)
status_update_callback = (
self._status_view.GetAnalysisStatusUpdateCallback())
session = engine.BaseEngine.CreateSession(
command_line_arguments=self._command_line_arguments,
preferred_encoding=self.preferred_encoding)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
if not storage_reader:
logger.error('Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return
self._number_of_analysis_reports = (
storage_reader.GetNumberOfAnalysisReports())
storage_reader.Close()
configuration = configurations.ProcessingConfiguration()
configuration.data_location = self._data_location
configuration.profiling.directory = self._profiling_directory
configuration.profiling.sample_rate = self._profiling_sample_rate
configuration.profiling.profilers = self._profilers
analysis_counter = None
if self._analysis_plugins:
storage_writer = (
storage_factory.StorageFactory.CreateStorageWriterForFile(
session, self._storage_file_path))
# TODO: add single processing support.
analysis_engine = psort.PsortMultiProcessEngine(
use_zeromq=self._use_zeromq)
analysis_engine.AnalyzeEvents(
self._knowledge_base, storage_writer, self._data_location,
self._analysis_plugins, configuration,
event_filter=self._event_filter,
event_filter_expression=self._event_filter_expression,
status_update_callback=status_update_callback,
worker_memory_limit=self._worker_memory_limit)
analysis_counter = collections.Counter()
for item, value in iter(session.analysis_reports_counter.items()):
analysis_counter[item] = value
events_counter = None
if self._output_format != 'null':
storage_reader = (
storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path))
# TODO: add single processing support.
analysis_engine = psort.PsortMultiProcessEngine(
use_zeromq=self._use_zeromq)
events_counter = analysis_engine.ExportEvents(
self._knowledge_base, storage_reader, self._output_module,
configuration, deduplicate_events=self._deduplicate_events,
event_filter=self._event_filter,
status_update_callback=status_update_callback,
time_slice=self._time_slice, use_time_slicer=self._use_time_slicer)
if self._quiet_mode:
return
self._output_writer.Write('Processing completed.\n')
if analysis_counter:
table_view = views.ViewsFactory.GetTableView(
self._views_format_type, title='Analysis reports generated')
for element, count in analysis_counter.most_common():
if element != 'total':
table_view.AddRow([element, count])
table_view.AddRow(['Total', analysis_counter['total']])
table_view.Write(self._output_writer)
if events_counter:
table_view = views.ViewsFactory.GetTableView(
self._views_format_type, title='Export results')
for element, count in events_counter.most_common():
table_view.AddRow([element, count])
table_view.Write(self._output_writer)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
self._PrintAnalysisReportsDetails(storage_reader)
def AnalyzeEvents(self):
"""Analyzes events from a plaso storage file and generate a report.
Raises:
BadConfigOption: when a configuration parameter fails validation.
RuntimeError: if a non-recoverable situation is encountered.
"""
session = engine.BaseEngine.CreateSession(
command_line_arguments=self._command_line_arguments,
preferred_encoding=self.preferred_encoding)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
if not storage_reader:
logger.error('Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return
self._number_of_analysis_reports = (
storage_reader.GetNumberOfAnalysisReports())
storage_reader.Close()
configuration = self._CreateProcessingConfiguration(
self._knowledge_base)
counter = collections.Counter()
if self._output_format != 'null':
self._status_view.SetMode(self._status_view_mode)
self._status_view.SetStorageFileInformation(self._storage_file_path)
status_update_callback = (
self._status_view.GetAnalysisStatusUpdateCallback())
storage_reader = (
storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path))
# TODO: add single processing support.
analysis_engine = psort.PsortMultiProcessEngine(
use_zeromq=self._use_zeromq)
events_counter = analysis_engine.ExportEvents(
self._knowledge_base, storage_reader, self._output_module,
configuration, deduplicate_events=self._deduplicate_events,
status_update_callback=status_update_callback,
time_slice=self._time_slice, use_time_slicer=self._use_time_slicer)
counter += events_counter
for item, value in iter(session.analysis_reports_counter.items()):
counter[item] = value
if self._quiet_mode:
return
self._output_writer.Write('Processing completed.\n')
table_view = views.ViewsFactory.GetTableView(
self._views_format_type, title='Counter')
for element, count in counter.most_common():
if not element:
element = 'N/A'
table_view.AddRow([element, count])
table_view.Write(self._output_writer)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
self._PrintAnalysisReportsDetails(
storage_reader, self._number_of_analysis_reports)
self._output_writer.Write('Storage file is {0:s}\n'.format(
self._storage_file_path))
def AnalyzeEvents(self):
"""Analyzes events from a plaso storage file and generate a report.
Raises:
BadConfigOption: when a configuration parameter fails validation.
RuntimeError: if a non-recoverable situation is encountered.
"""
session = engine.BaseEngine.CreateSession(
command_line_arguments=self._command_line_arguments,
preferred_encoding=self.preferred_encoding)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
if not storage_reader:
logger.error('Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return
self._number_of_analysis_reports = (
storage_reader.GetNumberOfAnalysisReports())
storage_reader.Close()
configuration = self._CreateProcessingConfiguration(
self._knowledge_base)
counter = collections.Counter()
if self._output_format != 'null':
self._status_view.SetMode(self._status_view_mode)
self._status_view.SetStorageFileInformation(
self._storage_file_path)
status_update_callback = (
self._status_view.GetAnalysisStatusUpdateCallback())
storage_reader = (
storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path))
# TODO: add single processing support.
analysis_engine = psort.PsortMultiProcessEngine()
analysis_engine.ExportEvents(
self._knowledge_base,
storage_reader,
self._output_module,
configuration,
deduplicate_events=self._deduplicate_events,
status_update_callback=status_update_callback,
time_slice=self._time_slice,
use_time_slicer=self._use_time_slicer)
for item, value in iter(session.analysis_reports_counter.items()):
counter[item] = value
if self._quiet_mode:
return
self._output_writer.Write('Processing completed.\n')
table_view = views.ViewsFactory.GetTableView(self._views_format_type,
title='Counter')
for element, count in counter.most_common():
if not element:
element = 'N/A'
table_view.AddRow([element, count])
table_view.Write(self._output_writer)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
self._PrintAnalysisReportsDetails(storage_reader,
self._number_of_analysis_reports)
self._output_writer.Write('Storage file is {0:s}\n'.format(
self._storage_file_path))
def _ExtractDataStream(self,
file_entry,
data_stream_name,
destination_path,
skip_duplicates=True):
"""Extracts a data stream.
Args:
file_entry (dfvfs.FileEntry): file entry containing the data stream.
data_stream_name (str): name of the data stream.
destination_path (str): path where the extracted files should be stored.
skip_duplicates (Optional[bool]): True if files with duplicate content
should be skipped.
"""
if not data_stream_name and not file_entry.IsFile():
return
display_name = path_helper.PathHelper.GetDisplayNameForPathSpec(
file_entry.path_spec)
try:
digest = self._CalculateDigestHash(file_entry, data_stream_name)
except (IOError, dfvfs_errors.BackEndError) as exception:
logger.error(
('[skipping] unable to read content of file entry: {0:s} '
'with error: {1!s}').format(display_name, exception))
return
if not digest:
logger.error(
'[skipping] unable to read content of file entry: {0:s}'.
format(display_name))
return
target_directory, target_filename = self._CreateSanitizedDestination(
file_entry, file_entry.path_spec, data_stream_name,
destination_path)
# If does not exist, append path separator to have consistant behaviour.
if not destination_path.endswith(os.path.sep):
destination_path = destination_path + os.path.sep
target_path = os.path.join(target_directory, target_filename)
if target_path.startswith(destination_path):
path = target_path[len(destination_path):]
self._paths_by_hash[digest].append(path)
if skip_duplicates:
duplicate_display_name = self._digests.get(digest, None)
if duplicate_display_name:
logger.warning((
'[skipping] file entry: {0:s} is a duplicate of: {1:s} with '
'digest: {2:s}').format(display_name,
duplicate_display_name, digest))
return
self._digests[digest] = display_name
if not os.path.isdir(target_directory):
os.makedirs(target_directory)
if os.path.exists(target_path):
logger.warning(
('[skipping] unable to export contents of file entry: {0:s} '
'because exported file: {1:s} already exists.').format(
display_name, target_path))
return
try:
self._WriteFileEntry(file_entry, data_stream_name, target_path)
except (IOError, dfvfs_errors.BackEndError) as exception:
logger.error(
('[skipping] unable to export contents of file entry: {0:s} '
'with error: {1!s}').format(display_name, exception))
try:
os.remove(target_path)
except (IOError, OSError):
pass
def ProcessStorage(self):
"""Processes a plaso storage file.
Raises:
BadConfigOption: when a configuration parameter fails validation.
RuntimeError: if a non-recoverable situation is encountered.
"""
self._CheckStorageFile(self._storage_file_path)
self._status_view.SetMode(self._status_view_mode)
self._status_view.SetStorageFileInformation(self._storage_file_path)
status_update_callback = (
self._status_view.GetAnalysisStatusUpdateCallback())
session = engine.BaseEngine.CreateSession(
command_line_arguments=self._command_line_arguments,
preferred_encoding=self.preferred_encoding)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
if not storage_reader:
logger.error('Format of storage file: {0:s} not supported'.format(
self._storage_file_path))
return
self._number_of_analysis_reports = (
storage_reader.GetNumberOfAnalysisReports())
storage_reader.Close()
configuration = configurations.ProcessingConfiguration()
configuration.data_location = self._data_location
configuration.profiling.directory = self._profiling_directory
configuration.profiling.sample_rate = self._profiling_sample_rate
configuration.profiling.profilers = self._profilers
analysis_counter = None
if self._analysis_plugins:
storage_writer = (
storage_factory.StorageFactory.CreateStorageWriterForFile(
session, self._storage_file_path))
if not storage_writer:
logger.error(
'Format of storage file: {0:s} not supported for writing'.format(
self._storage_file_path))
return
# TODO: add single processing support.
analysis_engine = psort.PsortMultiProcessEngine()
analysis_engine.AnalyzeEvents(
self._knowledge_base, storage_writer, self._data_location,
self._analysis_plugins, configuration,
event_filter=self._event_filter,
event_filter_expression=self._event_filter_expression,
status_update_callback=status_update_callback,
worker_memory_limit=self._worker_memory_limit)
analysis_counter = collections.Counter()
for item, value in iter(session.analysis_reports_counter.items()):
analysis_counter[item] = value
if self._output_format != 'null':
storage_reader = (
storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path))
# TODO: add single processing support.
analysis_engine = psort.PsortMultiProcessEngine()
analysis_engine.ExportEvents(
self._knowledge_base, storage_reader, self._output_module,
configuration, deduplicate_events=self._deduplicate_events,
event_filter=self._event_filter,
status_update_callback=status_update_callback,
time_slice=self._time_slice, use_time_slicer=self._use_time_slicer)
if self._quiet_mode:
return
self._output_writer.Write('Processing completed.\n')
if analysis_counter:
table_view = views.ViewsFactory.GetTableView(
self._views_format_type, title='Analysis reports generated')
for element, count in analysis_counter.most_common():
if element != 'total':
table_view.AddRow([element, count])
table_view.AddRow(['Total', analysis_counter['total']])
table_view.Write(self._output_writer)
storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile(
self._storage_file_path)
self._PrintAnalysisReportsDetails(storage_reader)