def ParseOptions(cls, options, configuration_object): """Parses and validates options. Args: options (argparse.Namespace): parser options. configuration_object (CLITool): object to be configured by the argument helper. Raises: BadConfigObject: when the configuration object is of the wrong type. BadConfigOption: when the location of the data files cannot be determined. """ if not isinstance(configuration_object, tools.CLITool): raise errors.BadConfigObject( 'Configuration object is not an instance of CLITool') data_location = cls._ParseStringOption(options, 'data_location') if not data_location: # Determine the source root path, which is 3 directories up from # the location of the script. data_location = os.path.dirname(cls._PATH) data_location = os.path.dirname(data_location) data_location = os.path.dirname(data_location) data_location = os.path.dirname(data_location) # There are multiple options to run a tool e.g. running from source or # from an egg file. data_location_egg = os.path.join(data_location, 'share', 'plaso') data_location_source = os.path.join(data_location, 'data') data_location = None if os.path.exists(data_location_egg) and os.path.isfile(os.path.join( data_location_egg, 'plaso-data.README')): data_location = data_location_egg elif os.path.exists(data_location_source) and os.path.isfile(os.path.join( data_location_source, 'plaso-data.README')): data_location = data_location_source if not data_location or not os.path.exists(data_location): data_location = os.path.join(sys.prefix, 'share', 'plaso') if not os.path.exists(data_location): data_location = os.path.join(sys.prefix, 'local', 'share', 'plaso') if sys.prefix != '/usr': if not os.path.exists(data_location): data_location = os.path.join('/usr', 'share', 'plaso') if not os.path.exists(data_location): data_location = os.path.join('/usr', 'local', 'share', 'plaso') if not os.path.exists(data_location) or not os.path.isfile(os.path.join( data_location, 'plaso-data.README')): data_location = None if not data_location: raise errors.BadConfigOption( 'Unable to determine location of data files.') logger.info('Determined data location: {0:s}'.format(data_location)) setattr(configuration_object, '_data_location', data_location)
def _CreateProcessingConfiguration(self, knowledge_base): """Creates a processing configuration. Args: knowledge_base (KnowledgeBase): contains information from the source data needed for parsing. Returns: ProcessingConfiguration: processing configuration. Raises: BadConfigOption: if more than 1 parser and parser plugins preset was found for the detected operating system. """ # TODO: pass preferred_encoding. configuration = configurations.ProcessingConfiguration() configuration.artifact_filters = self._artifact_filters configuration.credentials = self._credential_configurations configuration.debug_output = self._debug_mode configuration.event_extraction.text_prepend = self._text_prepend configuration.extraction.hasher_file_size_limit = ( self._hasher_file_size_limit) configuration.extraction.hasher_names_string = self._hasher_names_string configuration.extraction.process_archives = self._process_archives configuration.extraction.process_compressed_streams = ( self._process_compressed_streams) configuration.extraction.yara_rules_string = self._yara_rules_string configuration.filter_file = self._filter_file configuration.input_source.mount_path = self._mount_path configuration.log_filename = self._log_file configuration.parser_filter_expression = self._parser_filter_expression configuration.preferred_year = self._preferred_year configuration.profiling.directory = self._profiling_directory configuration.profiling.sample_rate = self._profiling_sample_rate configuration.profiling.profilers = self._profilers configuration.temporary_directory = self._temporary_directory if not configuration.parser_filter_expression: operating_system = knowledge_base.GetValue('operating_system') operating_system_product = knowledge_base.GetValue( 'operating_system_product') operating_system_version = knowledge_base.GetValue( 'operating_system_version') preset_definitions = ( parsers_manager.ParsersManager.GetPresetsForOperatingSystem( operating_system, operating_system_product, operating_system_version)) if preset_definitions: preset_names = [ preset_definition.name for preset_definition in preset_definitions ] filter_expression = ','.join(preset_names) logger.info('Parser filter expression set to: {0:s}'.format( filter_expression)) configuration.parser_filter_expression = filter_expression return configuration
def _CreateProcessingConfiguration(self, knowledge_base): """Creates a processing configuration. Args: knowledge_base (KnowledgeBase): contains information from the source data needed for parsing. Returns: ProcessingConfiguration: processing configuration. Raises: BadConfigOption: if presets in the parser filter expression could not be expanded or if an invalid parser or plugin name is specified. """ parser_filter_expression = self._parser_filter_expression if not parser_filter_expression: operating_system_family = knowledge_base.GetValue('operating_system') operating_system_product = knowledge_base.GetValue( 'operating_system_product') operating_system_version = knowledge_base.GetValue( 'operating_system_version') operating_system_artifact = artifacts.OperatingSystemArtifact( family=operating_system_family, product=operating_system_product, version=operating_system_version) preset_definitions = self._presets_manager.GetPresetsByOperatingSystem( operating_system_artifact) if preset_definitions: preset_names = [ preset_definition.name for preset_definition in preset_definitions] filter_expression = ','.join(preset_names) logger.info('Parser filter expression set to: {0:s}'.format( filter_expression)) parser_filter_expression = filter_expression parser_filter_helper = parser_filter.ParserFilterExpressionHelper() try: parser_filter_expression = parser_filter_helper.ExpandPresets( self._presets_manager, parser_filter_expression) except RuntimeError as exception: raise errors.BadConfigOption(( 'Unable to expand presets in parser filter expression with ' 'error: {0!s}').format(exception)) _, invalid_parser_elements = ( parsers_manager.ParsersManager.CheckFilterExpression( parser_filter_expression)) if invalid_parser_elements: invalid_parser_names_string = ','.join(invalid_parser_elements) raise errors.BadConfigOption( 'Unknown parser or plugin names in element(s): "{0:s}" of ' 'parser filter expression: {1:s}'.format( invalid_parser_names_string, parser_filter_expression)) # TODO: pass preferred_encoding. configuration = configurations.ProcessingConfiguration() configuration.artifact_filters = self._artifact_filters configuration.credentials = self._credential_configurations configuration.debug_output = self._debug_mode configuration.event_extraction.text_prepend = self._text_prepend configuration.extraction.hasher_file_size_limit = ( self._hasher_file_size_limit) configuration.extraction.hasher_names_string = self._hasher_names_string configuration.extraction.process_archives = self._process_archives configuration.extraction.process_compressed_streams = ( self._process_compressed_streams) configuration.extraction.yara_rules_string = self._yara_rules_string configuration.filter_file = self._filter_file configuration.input_source.mount_path = self._mount_path configuration.log_filename = self._log_file configuration.parser_filter_expression = parser_filter_expression configuration.preferred_year = self._preferred_year configuration.profiling.directory = self._profiling_directory configuration.profiling.sample_rate = self._profiling_sample_rate configuration.profiling.profilers = self._profilers configuration.task_storage_format = self._task_storage_format configuration.temporary_directory = self._temporary_directory return configuration
def ParseOptions(cls, options, configuration_object): """Parses and validates options. Args: options (argparse.Namespace): parser options. configuration_object (CLITool): object to be configured by the argument helper. Raises: BadConfigObject: when the configuration object is of the wrong type. BadConfigOption: if the required artifact definitions are not defined. """ if not isinstance(configuration_object, tools.CLITool): raise errors.BadConfigObject( 'Configuration object is not an instance of CLITool') artifacts_path = getattr(options, 'artifact_definitions_path', None) if ((not artifacts_path or not os.path.exists(artifacts_path)) and configuration_object.data_location): artifacts_path = os.path.dirname( configuration_object.data_location) artifacts_path = os.path.join(artifacts_path, 'artifacts') if not os.path.exists( artifacts_path) and 'VIRTUAL_ENV' in os.environ: artifacts_path = os.path.join(os.environ['VIRTUAL_ENV'], 'share', 'artifacts') if not os.path.exists(artifacts_path): artifacts_path = os.path.join(sys.prefix, 'share', 'artifacts') if not os.path.exists(artifacts_path): artifacts_path = os.path.join(sys.prefix, 'local', 'share', 'artifacts') if sys.prefix != '/usr': if not os.path.exists(artifacts_path): artifacts_path = os.path.join('/usr', 'share', 'artifacts') if not os.path.exists(artifacts_path): artifacts_path = os.path.join('/usr', 'local', 'share', 'artifacts') if not os.path.exists(artifacts_path): artifacts_path = None if not artifacts_path or not os.path.exists(artifacts_path): raise errors.BadConfigOption( 'Unable to determine path to artifact definitions.') custom_artifacts_path = getattr(options, 'custom_artifact_definitions_path', None) if custom_artifacts_path and not os.path.exists(custom_artifacts_path): raise errors.BadConfigOption( ('Unable to determine path to custom artifact definitions: ' '{0:s}.').format(custom_artifacts_path)) if custom_artifacts_path: logger.info('Custom artifact definitions path: {0:s}'.format( custom_artifacts_path)) registry = artifacts_registry.ArtifactDefinitionsRegistry() reader = artifacts_reader.YamlArtifactsReader() logger.info('Determined artifact definitions path: {0:s}'.format( artifacts_path)) try: if os.path.isdir(artifacts_path): registry.ReadFromDirectory(reader, artifacts_path) else: registry.ReadFromFile(reader, artifacts_path) except (KeyError, artifacts_errors.FormatError) as exception: raise errors.BadConfigOption( ('Unable to read artifact definitions from: {0:s} with error: ' '{1!s}').format(artifacts_path, exception)) if custom_artifacts_path: try: if os.path.isdir(custom_artifacts_path): registry.ReadFromDirectory(reader, custom_artifacts_path) else: registry.ReadFromFile(reader, custom_artifacts_path) except (KeyError, artifacts_errors.FormatError) as exception: raise errors.BadConfigOption(( 'Unable to read custorm artifact definitions from: {0:s} with ' 'error: {1!s}').format(custom_artifacts_path, exception)) for name in preprocessors_manager.PreprocessPluginsManager.GetNames(): if not registry.GetDefinitionByName(name): raise errors.BadConfigOption( 'Missing required artifact definition: {0:s}'.format(name)) setattr(configuration_object, '_artifact_definitions_path', artifacts_path) setattr(configuration_object, '_custom_artifacts_path', custom_artifacts_path)
def _Extract(self, source_path_specs, destination_path, output_writer, artifact_filters, filter_file, artifact_definitions_path, custom_artifacts_path, skip_duplicates=True): """Extracts files. This method runs the file extraction process on the image and potentially on every VSS if that is wanted. Args: source_path_specs (list[dfvfs.PathSpec]): path specifications to extract. destination_path (str): path where the extracted files should be stored. output_writer (CLIOutputWriter): output writer. artifact_definitions_path (str): path to artifact definitions file. custom_artifacts_path (str): path to custom artifact definitions file. artifact_filters (list[str]): names of artifact definitions that are used for filtering file system and Windows Registry key paths. filter_file (str): path of the file that contains the filter file path filters. skip_duplicates (Optional[bool]): True if files with duplicate content should be skipped. Raises: BadConfigOption: if an invalid collection filter was specified. """ extraction_engine = engine.BaseEngine() # If the source is a directory or a storage media image # run pre-processing. if self._source_type in self._SOURCE_TYPES_TO_PREPROCESS: self._PreprocessSources(extraction_engine) try: extraction_engine.BuildCollectionFilters( artifact_definitions_path, custom_artifacts_path, extraction_engine.knowledge_base, artifact_filters, filter_file) except errors.InvalidFilter as exception: raise errors.BadConfigOption( 'Unable to build collection filters with error: {0!s}'.format( exception)) filters_helper = extraction_engine.collection_filters_helper excluded_find_specs = None included_find_specs = None if filters_helper: excluded_find_specs = filters_helper.excluded_file_system_find_specs included_find_specs = filters_helper.included_file_system_find_specs output_writer.Write('Extracting file entries.\n') path_spec_generator = self._path_spec_extractor.ExtractPathSpecs( source_path_specs, find_specs=included_find_specs, resolver_context=self._resolver_context) for path_spec in path_spec_generator: file_entry = path_spec_resolver.Resolver.OpenFileEntry( path_spec, resolver_context=self._resolver_context) if not file_entry: logger.warning( 'Unable to open file entry for path spec: {0:s}'.format( path_spec.comparable)) continue skip_file_entry = False for find_spec in excluded_find_specs or []: skip_file_entry = find_spec.CompareLocation(file_entry) if skip_file_entry: break if skip_file_entry: logger.info( 'Skipped: {0:s} because of exclusion filter.'.format( file_entry.path_spec.location)) continue self._ExtractFileEntry(file_entry, destination_path, skip_duplicates=skip_duplicates)