def GetCommandLineArguments(self): """Retrieves the command line arguments. Returns: str: command line arguments. """ command_line_arguments = sys.argv if not command_line_arguments: return '' if isinstance(command_line_arguments[0], py2to3.BYTES_TYPE): encoding = sys.stdin.encoding # Note that sys.stdin.encoding can be None. if not encoding: encoding = self.preferred_encoding try: command_line_arguments = [ argument.decode(encoding) for argument in command_line_arguments] except UnicodeDecodeError: logger.error( 'Unable to properly read command line input due to encoding ' 'error. Replacing non Basic Latin (C0) characters with "?" or ' '"\\ufffd".') command_line_arguments = [ argument.decode(encoding, errors='replace') for argument in command_line_arguments] return ' '.join(command_line_arguments)
def Read(self): """Reads a string from the input. Returns: str: input. """ encoded_string = self._file_object.readline() if isinstance(encoded_string, py2to3.UNICODE_TYPE): return encoded_string try: string = codecs.decode(encoded_string, self._encoding, self._errors) except UnicodeDecodeError: if self._errors == 'strict': logger.error( 'Unable to properly read input due to encoding error. ' 'Switching to error tolerant encoding which can result in ' 'non Basic Latin (C0) characters to be replaced with "?" or ' '"\\ufffd".') self._errors = 'replace' string = codecs.decode(encoded_string, self._encoding, self._errors) return string
def ListTimeZones(self): """Lists the timezones.""" max_length = 0 for timezone_name in pytz.all_timezones: if len(timezone_name) > max_length: max_length = len(timezone_name) utc_date_time = datetime.datetime.utcnow() table_view = views.ViewsFactory.GetTableView( self._views_format_type, column_names=['Timezone', 'UTC Offset'], title='Zones') for timezone_name in pytz.all_timezones: try: local_timezone = pytz.timezone(timezone_name) except AssertionError as exception: logger.error(( 'Unable to determine information about timezone: {0:s} with ' 'error: {1!s}').format(timezone_name, exception)) continue local_date_string = '{0!s}'.format( local_timezone.localize(utc_date_time)) if '+' in local_date_string: _, _, diff = local_date_string.rpartition('+') diff_string = '+{0:s}'.format(diff) else: _, _, diff = local_date_string.rpartition('-') diff_string = '-{0:s}'.format(diff) table_view.AddRow([timezone_name, diff_string]) table_view.Write(self._output_writer)
def PrintStorageInformation(self): """Prints the storage information.""" storage_file = storage_factory.StorageFactory.CreateStorageFileForFile( self._storage_file_path) if not storage_file: logger.error( 'Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return try: storage_file.Open(path=self._storage_file_path, read_only=True) except IOError as exception: logger.error( 'Unable to open storage file: {0:s} with error: {1!s}'.format( self._storage_file_path, exception)) return try: if self._output_format == 'json': self._PrintStorageInformationAsJSON(storage_file) elif self._output_format == 'text': self._PrintStorageInformationAsText(storage_file) finally: storage_file.Close()
def PrintStorageInformation(self): """Prints the storage information.""" storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) if not storage_reader: logger.error( 'Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return try: if self._output_format == 'json': self._PrintStorageInformationAsJSON(storage_reader) elif self._output_format == 'text': self._PrintStorageInformationAsText(storage_reader) finally: storage_reader.Close()
def CompareStores(self): """Compares the contents of two stores. Returns: bool: True if the content of the stores is identical. """ storage_file = storage_factory.StorageFactory.CreateStorageFileForFile( self._storage_file_path) if not storage_file: logger.error( 'Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return False try: storage_file.Open(path=self._storage_file_path, read_only=True) except IOError as exception: logger.error( 'Unable to open storage file: {0:s} with error: {1!s}'.format( self._storage_file_path, exception)) return False compare_storage_file = ( storage_factory.StorageFactory.CreateStorageFileForFile( self._compare_storage_file_path)) if not compare_storage_file: logger.error( 'Format of storage file: {0:s} not supported'.format( self._compare_storage_file_path)) return False try: compare_storage_file.Open( path=self._compare_storage_file_path, read_only=True) except IOError as exception: logger.error( 'Unable to open storage file: {0:s} with error: {1!s}'.format( self._compare_storage_file_path, exception)) storage_file.Close() return False try: result = self._CompareStores(storage_file, compare_storage_file) finally: compare_storage_file.Close() storage_file.Close() if result: self._output_writer.Write('Storage files are identical.\n') else: self._output_writer.Write('Storage files are different.\n') return result
def _Preprocess(self, file_system, mount_point): """Preprocesses the image. Args: file_system (dfvfs.FileSystem): file system to be preprocessed. mount_point (dfvfs.PathSpec): mount point path specification that refers to the base location of the file system. """ logger.debug('Starting preprocessing.') try: preprocess_manager.PreprocessPluginsManager.RunPlugins( self._artifacts_registry, file_system, mount_point, self._knowledge_base) except IOError as exception: logger.error('Unable to preprocess with error: {0!s}'.format(exception)) logger.debug('Preprocessing done.')
def _PreprocessSources(self, extraction_engine): """Preprocesses the sources. Args: extraction_engine (BaseEngine): extraction engine to preprocess the sources. """ logger.debug('Starting preprocessing.') try: artifacts_registry = engine.BaseEngine.BuildArtifactsRegistry( self._artifact_definitions_path, self._custom_artifacts_path) extraction_engine.PreprocessSources( artifacts_registry, self._source_path_specs, resolver_context=self._resolver_context) except IOError as exception: logger.error('Unable to preprocess with error: {0!s}'.format(exception)) logger.debug('Preprocessing done.')
def Write(self, string): """Writes a string to the output. Args: string (str): output. """ try: # Note that encode() will first convert string into a Unicode string # if necessary. encoded_string = codecs.encode(string, self._encoding, self._errors) except UnicodeEncodeError: if self._errors == 'strict': logger.error( 'Unable to properly write output due to encoding error. ' 'Switching to error tolerant encoding which can result in ' 'non Basic Latin (C0) characters to be replaced with "?" or ' '"\\ufffd".') self._errors = 'replace' encoded_string = codecs.encode(string, self._encoding, self._errors) self._file_object.write(encoded_string)
def CompareStores(self): """Compares the contents of two stores. Returns: bool: True if the content of the stores is identical. """ storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) if not storage_reader: logger.error( 'Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return False compare_storage_reader = ( storage_factory.StorageFactory.CreateStorageReaderForFile( self._compare_storage_file_path)) if not compare_storage_reader: logger.error( 'Format of storage file: {0:s} not supported'.format( self._compare_storage_file_path)) return False try: result = self._CompareStores(storage_reader, compare_storage_reader) finally: compare_storage_reader.Close() storage_reader.Close() if result: self._output_writer.Write('Storage files are identical.\n') else: self._output_writer.Write('Storage files are different.\n') return result
def _EncodeString(self, string): """Encodes a string in the preferred encoding. Returns: bytes: encoded string. """ try: # Note that encode() will first convert string into a Unicode string # if necessary. encoded_string = string.encode( self.preferred_encoding, errors=self._encode_errors) except UnicodeEncodeError: if self._encode_errors == 'strict': logger.error( 'Unable to properly write output due to encoding error. ' 'Switching to error tolerant encoding which can result in ' 'non Basic Latin (C0) characters to be replaced with "?" or ' '"\\ufffd".') self._encode_errors = 'replace' encoded_string = string.encode( self.preferred_encoding, errors=self._encode_errors) return encoded_string
def _ReadSpecificationFile(self, path): """Reads the format specification file. Args: path (str): path of the format specification file. Returns: FormatSpecificationStore: format specification store. """ specification_store = specification.FormatSpecificationStore() with io.open( path, 'rt', encoding=self._SPECIFICATION_FILE_ENCODING) as file_object: for line in file_object.readlines(): line = line.strip() if not line or line.startswith('#'): continue try: identifier, offset, pattern = line.split() except ValueError: logger.error('[skipping] invalid line: {0:s}'.format(line)) continue try: offset = int(offset, 10) except ValueError: logger.error('[skipping] invalid offset in line: {0:s}'.format(line)) continue try: # TODO: find another way to do this that doesn't use an undocumented # API. pattern = codecs.escape_decode(pattern)[0] # ValueError is raised e.g. when the patterns contains "\xg1". except ValueError: logger.error( '[skipping] invalid pattern in line: {0:s}'.format(line)) continue format_specification = specification.FormatSpecification(identifier) format_specification.AddNewSignature(pattern, offset=offset) specification_store.AddSpecification(format_specification) return specification_store
def ProcessStorage(self): """Processes a plaso storage file. Raises: BadConfigOption: when a configuration parameter fails validation. RuntimeError: if a non-recoverable situation is encountered. """ self._CheckStorageFile(self._storage_file_path) self._status_view.SetMode(self._status_view_mode) self._status_view.SetStorageFileInformation(self._storage_file_path) status_update_callback = ( self._status_view.GetAnalysisStatusUpdateCallback()) session = engine.BaseEngine.CreateSession( command_line_arguments=self._command_line_arguments, preferred_encoding=self.preferred_encoding) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) if not storage_reader: logger.error('Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return self._number_of_analysis_reports = ( storage_reader.GetNumberOfAnalysisReports()) storage_reader.Close() configuration = configurations.ProcessingConfiguration() configuration.data_location = self._data_location configuration.profiling.directory = self._profiling_directory configuration.profiling.sample_rate = self._profiling_sample_rate configuration.profiling.profilers = self._profilers analysis_counter = None if self._analysis_plugins: storage_writer = ( storage_factory.StorageFactory.CreateStorageWriterForFile( session, self._storage_file_path)) # TODO: add single processing support. analysis_engine = psort.PsortMultiProcessEngine( use_zeromq=self._use_zeromq) analysis_engine.AnalyzeEvents( self._knowledge_base, storage_writer, self._data_location, self._analysis_plugins, configuration, event_filter=self._event_filter, event_filter_expression=self._event_filter_expression, status_update_callback=status_update_callback, worker_memory_limit=self._worker_memory_limit) analysis_counter = collections.Counter() for item, value in iter(session.analysis_reports_counter.items()): analysis_counter[item] = value events_counter = None if self._output_format != 'null': storage_reader = ( storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path)) # TODO: add single processing support. analysis_engine = psort.PsortMultiProcessEngine( use_zeromq=self._use_zeromq) events_counter = analysis_engine.ExportEvents( self._knowledge_base, storage_reader, self._output_module, configuration, deduplicate_events=self._deduplicate_events, event_filter=self._event_filter, status_update_callback=status_update_callback, time_slice=self._time_slice, use_time_slicer=self._use_time_slicer) if self._quiet_mode: return self._output_writer.Write('Processing completed.\n') if analysis_counter: table_view = views.ViewsFactory.GetTableView( self._views_format_type, title='Analysis reports generated') for element, count in analysis_counter.most_common(): if element != 'total': table_view.AddRow([element, count]) table_view.AddRow(['Total', analysis_counter['total']]) table_view.Write(self._output_writer) if events_counter: table_view = views.ViewsFactory.GetTableView( self._views_format_type, title='Export results') for element, count in events_counter.most_common(): table_view.AddRow([element, count]) table_view.Write(self._output_writer) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) self._PrintAnalysisReportsDetails(storage_reader)
def AnalyzeEvents(self): """Analyzes events from a plaso storage file and generate a report. Raises: BadConfigOption: when a configuration parameter fails validation. RuntimeError: if a non-recoverable situation is encountered. """ session = engine.BaseEngine.CreateSession( command_line_arguments=self._command_line_arguments, preferred_encoding=self.preferred_encoding) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) if not storage_reader: logger.error('Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return self._number_of_analysis_reports = ( storage_reader.GetNumberOfAnalysisReports()) storage_reader.Close() configuration = self._CreateProcessingConfiguration( self._knowledge_base) counter = collections.Counter() if self._output_format != 'null': self._status_view.SetMode(self._status_view_mode) self._status_view.SetStorageFileInformation(self._storage_file_path) status_update_callback = ( self._status_view.GetAnalysisStatusUpdateCallback()) storage_reader = ( storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path)) # TODO: add single processing support. analysis_engine = psort.PsortMultiProcessEngine( use_zeromq=self._use_zeromq) events_counter = analysis_engine.ExportEvents( self._knowledge_base, storage_reader, self._output_module, configuration, deduplicate_events=self._deduplicate_events, status_update_callback=status_update_callback, time_slice=self._time_slice, use_time_slicer=self._use_time_slicer) counter += events_counter for item, value in iter(session.analysis_reports_counter.items()): counter[item] = value if self._quiet_mode: return self._output_writer.Write('Processing completed.\n') table_view = views.ViewsFactory.GetTableView( self._views_format_type, title='Counter') for element, count in counter.most_common(): if not element: element = 'N/A' table_view.AddRow([element, count]) table_view.Write(self._output_writer) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) self._PrintAnalysisReportsDetails( storage_reader, self._number_of_analysis_reports) self._output_writer.Write('Storage file is {0:s}\n'.format( self._storage_file_path))
def AnalyzeEvents(self): """Analyzes events from a plaso storage file and generate a report. Raises: BadConfigOption: when a configuration parameter fails validation. RuntimeError: if a non-recoverable situation is encountered. """ session = engine.BaseEngine.CreateSession( command_line_arguments=self._command_line_arguments, preferred_encoding=self.preferred_encoding) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) if not storage_reader: logger.error('Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return self._number_of_analysis_reports = ( storage_reader.GetNumberOfAnalysisReports()) storage_reader.Close() configuration = self._CreateProcessingConfiguration( self._knowledge_base) counter = collections.Counter() if self._output_format != 'null': self._status_view.SetMode(self._status_view_mode) self._status_view.SetStorageFileInformation( self._storage_file_path) status_update_callback = ( self._status_view.GetAnalysisStatusUpdateCallback()) storage_reader = ( storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path)) # TODO: add single processing support. analysis_engine = psort.PsortMultiProcessEngine() analysis_engine.ExportEvents( self._knowledge_base, storage_reader, self._output_module, configuration, deduplicate_events=self._deduplicate_events, status_update_callback=status_update_callback, time_slice=self._time_slice, use_time_slicer=self._use_time_slicer) for item, value in iter(session.analysis_reports_counter.items()): counter[item] = value if self._quiet_mode: return self._output_writer.Write('Processing completed.\n') table_view = views.ViewsFactory.GetTableView(self._views_format_type, title='Counter') for element, count in counter.most_common(): if not element: element = 'N/A' table_view.AddRow([element, count]) table_view.Write(self._output_writer) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) self._PrintAnalysisReportsDetails(storage_reader, self._number_of_analysis_reports) self._output_writer.Write('Storage file is {0:s}\n'.format( self._storage_file_path))
def _ExtractDataStream(self, file_entry, data_stream_name, destination_path, skip_duplicates=True): """Extracts a data stream. Args: file_entry (dfvfs.FileEntry): file entry containing the data stream. data_stream_name (str): name of the data stream. destination_path (str): path where the extracted files should be stored. skip_duplicates (Optional[bool]): True if files with duplicate content should be skipped. """ if not data_stream_name and not file_entry.IsFile(): return display_name = path_helper.PathHelper.GetDisplayNameForPathSpec( file_entry.path_spec) try: digest = self._CalculateDigestHash(file_entry, data_stream_name) except (IOError, dfvfs_errors.BackEndError) as exception: logger.error( ('[skipping] unable to read content of file entry: {0:s} ' 'with error: {1!s}').format(display_name, exception)) return if not digest: logger.error( '[skipping] unable to read content of file entry: {0:s}'. format(display_name)) return target_directory, target_filename = self._CreateSanitizedDestination( file_entry, file_entry.path_spec, data_stream_name, destination_path) # If does not exist, append path separator to have consistant behaviour. if not destination_path.endswith(os.path.sep): destination_path = destination_path + os.path.sep target_path = os.path.join(target_directory, target_filename) if target_path.startswith(destination_path): path = target_path[len(destination_path):] self._paths_by_hash[digest].append(path) if skip_duplicates: duplicate_display_name = self._digests.get(digest, None) if duplicate_display_name: logger.warning(( '[skipping] file entry: {0:s} is a duplicate of: {1:s} with ' 'digest: {2:s}').format(display_name, duplicate_display_name, digest)) return self._digests[digest] = display_name if not os.path.isdir(target_directory): os.makedirs(target_directory) if os.path.exists(target_path): logger.warning( ('[skipping] unable to export contents of file entry: {0:s} ' 'because exported file: {1:s} already exists.').format( display_name, target_path)) return try: self._WriteFileEntry(file_entry, data_stream_name, target_path) except (IOError, dfvfs_errors.BackEndError) as exception: logger.error( ('[skipping] unable to export contents of file entry: {0:s} ' 'with error: {1!s}').format(display_name, exception)) try: os.remove(target_path) except (IOError, OSError): pass
def ProcessStorage(self): """Processes a plaso storage file. Raises: BadConfigOption: when a configuration parameter fails validation. RuntimeError: if a non-recoverable situation is encountered. """ self._CheckStorageFile(self._storage_file_path) self._status_view.SetMode(self._status_view_mode) self._status_view.SetStorageFileInformation(self._storage_file_path) status_update_callback = ( self._status_view.GetAnalysisStatusUpdateCallback()) session = engine.BaseEngine.CreateSession( command_line_arguments=self._command_line_arguments, preferred_encoding=self.preferred_encoding) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) if not storage_reader: logger.error('Format of storage file: {0:s} not supported'.format( self._storage_file_path)) return self._number_of_analysis_reports = ( storage_reader.GetNumberOfAnalysisReports()) storage_reader.Close() configuration = configurations.ProcessingConfiguration() configuration.data_location = self._data_location configuration.profiling.directory = self._profiling_directory configuration.profiling.sample_rate = self._profiling_sample_rate configuration.profiling.profilers = self._profilers analysis_counter = None if self._analysis_plugins: storage_writer = ( storage_factory.StorageFactory.CreateStorageWriterForFile( session, self._storage_file_path)) if not storage_writer: logger.error( 'Format of storage file: {0:s} not supported for writing'.format( self._storage_file_path)) return # TODO: add single processing support. analysis_engine = psort.PsortMultiProcessEngine() analysis_engine.AnalyzeEvents( self._knowledge_base, storage_writer, self._data_location, self._analysis_plugins, configuration, event_filter=self._event_filter, event_filter_expression=self._event_filter_expression, status_update_callback=status_update_callback, worker_memory_limit=self._worker_memory_limit) analysis_counter = collections.Counter() for item, value in iter(session.analysis_reports_counter.items()): analysis_counter[item] = value if self._output_format != 'null': storage_reader = ( storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path)) # TODO: add single processing support. analysis_engine = psort.PsortMultiProcessEngine() analysis_engine.ExportEvents( self._knowledge_base, storage_reader, self._output_module, configuration, deduplicate_events=self._deduplicate_events, event_filter=self._event_filter, status_update_callback=status_update_callback, time_slice=self._time_slice, use_time_slicer=self._use_time_slicer) if self._quiet_mode: return self._output_writer.Write('Processing completed.\n') if analysis_counter: table_view = views.ViewsFactory.GetTableView( self._views_format_type, title='Analysis reports generated') for element, count in analysis_counter.most_common(): if element != 'total': table_view.AddRow([element, count]) table_view.AddRow(['Total', analysis_counter['total']]) table_view.Write(self._output_writer) storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( self._storage_file_path) self._PrintAnalysisReportsDetails(storage_reader)