def _GetParsers(cls, parser_filter_expression=None): """Retrieves the registered parsers and plugins. Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. A parser filter expression is a comma separated value string that denotes which parsers and plugins should be used. See filters/parser_filter.py for details of the expression syntax. This function does not support presets, and requires a parser filter expression where presets have been expanded. Yields: tuple: containing: * str: name of the parser: * type: parser class (subclass of BaseParser). """ parser_filter_helper = parser_filter.ParserFilterExpressionHelper() excludes, includes = parser_filter_helper.SplitExpression( parser_filter_expression) for parser_name, parser_class in cls._parser_classes.items(): # If there are no includes all parsers are included by default. if not includes and parser_name in excludes: continue if includes and parser_name not in includes: continue yield parser_name, parser_class
def testGetParserAndPluginsList(self): """Tests the _GetParserAndPluginsList function.""" test_helper = parser_filter.ParserFilterExpressionHelper() expression = test_helper._GetParserAndPluginsList( {'excluded': set(['*', 'plugin1'])}) self.assertEqual(expression, ['excluded', 'excluded/plugin1'])
def _GetExpandedParserFilterExpression(self, knowledge_base): """Determines the expanded parser filter expression. Args: knowledge_base (KnowledgeBase): contains information from the source data needed for parsing. Returns: str: expanded parser filter expression. Raises: BadConfigOption: if presets in the parser filter expression could not be expanded or if an invalid parser or plugin name is specified. """ parser_filter_expression = self._parser_filter_expression if not parser_filter_expression: operating_system_family = knowledge_base.GetValue('operating_system') operating_system_product = knowledge_base.GetValue( 'operating_system_product') operating_system_version = knowledge_base.GetValue( 'operating_system_version') operating_system_artifact = artifacts.OperatingSystemArtifact( family=operating_system_family, product=operating_system_product, version=operating_system_version) preset_definitions = self._presets_manager.GetPresetsByOperatingSystem( operating_system_artifact) if preset_definitions: self._parser_filter_expression = ','.join([ preset_definition.name for preset_definition in preset_definitions]) logger.debug('Parser filter expression set to preset: {0:s}'.format( self._parser_filter_expression)) parser_filter_helper = parser_filter.ParserFilterExpressionHelper() try: parser_filter_expression = parser_filter_helper.ExpandPresets( self._presets_manager, self._parser_filter_expression) logger.debug('Parser filter expression set to: {0:s}'.format( parser_filter_expression or 'N/A')) except RuntimeError as exception: raise errors.BadConfigOption(( 'Unable to expand presets in parser filter expression with ' 'error: {0!s}').format(exception)) parser_elements, invalid_parser_elements = ( parsers_manager.ParsersManager.CheckFilterExpression( parser_filter_expression)) if invalid_parser_elements: invalid_parser_names_string = ','.join(invalid_parser_elements) raise errors.BadConfigOption( 'Unknown parser or plugin names in element(s): "{0:s}" of ' 'parser filter expression: {1:s}'.format( invalid_parser_names_string, parser_filter_expression)) return ','.join(sorted(parser_elements))
def testExpandPresets(self): """Tests the ExpandPresets function.""" presets_file = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(presets_file) presets_manager = parsers_presets.ParserPresetsManager() presets_manager.ReadFromFile(presets_file) test_helper = parser_filter.ParserFilterExpressionHelper() expected_parser_filter_expression = ','.join( sorted([ '!utmp', 'bencode', 'binary_cookies', 'chrome_cache', 'chrome_preferences', 'czip/oxml', 'esedb', 'esedb/msie_webcache', 'filestat', 'firefox_cache', 'gdrive_synclog', 'java_idx', 'lnk', 'mcafee_protection', 'msiecf', 'olecf', 'opera_global', 'opera_typed_history', 'pe', 'plist/safari_history', 'prefetch', 'sccm', 'skydrive_log', 'skydrive_log_old', 'sqlite/chrome_27_history', 'sqlite/chrome_8_history', 'sqlite/chrome_autofill', 'sqlite/chrome_cookies', 'sqlite/chrome_extension_activity', 'sqlite/firefox_cookies', 'sqlite/firefox_downloads', 'sqlite/firefox_history', 'sqlite/google_drive', 'sqlite/skype', 'symantec_scanlog', 'usnjrnl', 'winfirewall', 'winjob', 'winreg' ])) parser_filter_expression = test_helper.ExpandPresets( presets_manager, 'win_gen,!utmp') self.assertEqual(parser_filter_expression, expected_parser_filter_expression) parser_filter_expression = test_helper.ExpandPresets( presets_manager, 'olecf,!utmp') self.assertEqual(parser_filter_expression, '!utmp,olecf')
def testParsersAndPresets(self): """Tests that all parsers/plugins in the default presets are valid.""" presets_file_path = self._GetDataFilePath(['presets.yaml']) preset_manager = presets.ParserPresetsManager() preset_manager.ReadFromFile(presets_file_path) filter_helper = parser_filter.ParserFilterExpressionHelper() for name in preset_manager.GetNames(): expanded_preset = filter_helper.ExpandPresets(preset_manager, name) _, invalid_parser_elements = ( parsers_manager.ParsersManager.CheckFilterExpression(expanded_preset)) self.assertFalse( invalid_parser_elements, msg='Invalid parser/plugin name(s)')
def _GetParsers(cls, parser_filter_expression=None): """Retrieves the registered parsers and plugins. Retrieves a dictionary of all registered parsers and associated plugins from a parser filter string. The filter string can contain direct names of parsers or plugins. The filter string can also negate selection if prepended with an exclamation point, such as: "foo,!foo/bar" would include parser foo but not include plugin bar. A list of specific included and excluded plugins is also passed to each parser's class. The three types of entries in the filter string: * name of a parser: this would be the exact name of a single parser to include (or exclude), such as foo; * name of a plugin: if a plugin name is included the parent parser will be included in the list of registered parsers; Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. The parser filter expression is a comma separated value string that denotes a list of parser names to include and/or exclude. Each entry can have the value of: * A name of a single parser (case insensitive), such as msiecf. * A glob name for a single parser, such as '*msie*' (case insensitive). Yields: tuple: containing: * str: name of the parser: * type: parser class (subclass of BaseParser). """ parser_filter_helper = parser_filter.ParserFilterExpressionHelper() excludes, includes = parser_filter_helper.SplitExpression( parser_filter_expression) for parser_name, parser_class in iter(cls._parser_classes.items()): # If there are no includes all parsers are included by default. if not includes and parser_name in excludes: continue if includes and parser_name not in includes: continue yield parser_name, parser_class
def testJoinExpression(self): """Tests the _JoinExpression function.""" test_helper = parser_filter.ParserFilterExpressionHelper() expression = test_helper._JoinExpression({'excluded': set(['*'])}, {}) self.assertEqual(expression, '!excluded') expression = test_helper._JoinExpression( {'excluded': set(['plugin1'])}, {'excluded': set(['*'])}) self.assertEqual(expression, '!excluded/plugin1,excluded') expression = test_helper._JoinExpression( {'excluded': set(['plugin1', 'plugin2'])}, {'excluded': set(['*'])}) self.assertEqual(expression, '!excluded/plugin1,!excluded/plugin2,excluded') expression = test_helper._JoinExpression({}, {'included': set(['*'])}) self.assertEqual(expression, 'included') expression = test_helper._JoinExpression( {}, {'included': set(['*', 'plugin1'])}) self.assertEqual(expression, 'included,included/plugin1') expression = test_helper._JoinExpression( {}, {'included': set(['plugin1', 'plugin2'])}) self.assertEqual(expression, 'included/plugin1,included/plugin2') expression = test_helper._JoinExpression( {'excluded': set(['plugin1'])}, { 'excluded': set(['*']), 'included': set(['plugin2']) }) self.assertEqual(expression, '!excluded/plugin1,excluded,included/plugin2') with self.assertRaises(RuntimeError): test_helper._JoinExpression({'excluded': set(['plugin1'])}, {'excluded': set(['plugin1'])}) with self.assertRaises(RuntimeError): test_helper._JoinExpression({'excluded': set(['plugin1'])}, {})
def GetParserObjects(cls, parser_filter_expression=None): """Retrieves the parser objects. Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. The parser filter expression is a comma separated value string that denotes a list of parser names to include and/or exclude. Each entry can have the value of: * A name of a single parser (case insensitive), such as msiecf. * A glob name for a single parser, such as '*msie*' (case insensitive). Returns: dict[str, BaseParser]: parsers per name. """ parser_filter_helper = parser_filter.ParserFilterExpressionHelper() excludes, includes = parser_filter_helper.SplitExpression( parser_filter_expression) parser_objects = {} for parser_name, parser_class in iter(cls._parser_classes.items()): # If there are no includes all parsers are included by default. if not includes and parser_name in excludes: continue if includes and parser_name not in includes: continue parser_object = parser_class() if parser_class.SupportsPlugins(): plugin_includes = None if parser_name in includes: plugin_includes = includes[parser_name] parser_object.EnablePlugins(plugin_includes) parser_objects[parser_name] = parser_object return parser_objects
def GetParserObjects(cls, parser_filter_expression=None): """Retrieves the parser objects. Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. A parser filter expression is a comma separated value string that denotes which parsers and plugins should be used. See filters/parser_filter.py for details of the expression syntax. This function does not support presets, and requires a parser filter expression where presets have been expanded. Returns: dict[str, BaseParser]: parsers per name. """ parser_filter_helper = parser_filter.ParserFilterExpressionHelper() excludes, includes = parser_filter_helper.SplitExpression( parser_filter_expression) parser_objects = {} for parser_name, parser_class in cls._parser_classes.items(): # If there are no includes all parsers are included by default. if not includes and parser_name in excludes: continue if includes and parser_name not in includes: continue parser_object = parser_class() if parser_class.SupportsPlugins(): plugin_includes = None if parser_name in includes: plugin_includes = includes[parser_name] parser_object.EnablePlugins(plugin_includes) parser_objects[parser_name] = parser_object return parser_objects
def testExpandPreset(self): """Tests the _ExpandPreset function.""" presets_file = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(presets_file) presets_manager = parsers_presets.ParserPresetsManager() presets_manager.ReadFromFile(presets_file) test_helper = parser_filter.ParserFilterExpressionHelper() parsers_and_plugins = {'win_gen': set(['*'])} test_helper._ExpandPreset(presets_manager, 'win_gen', parsers_and_plugins) expected_parsers_and_plugins = { 'bencode': set(['*']), 'czip': set(['oxml']), 'filestat': set(['*']), 'gdrive_synclog': set(['*']), 'java_idx': set(['*']), 'lnk': set(['*']), 'mcafee_protection': set(['*']), 'olecf': set(['*']), 'pe': set(['*']), 'prefetch': set(['*']), 'sccm': set(['*']), 'skydrive_log': set(['*']), 'skydrive_log_old': set(['*']), 'sqlite': set(['google_drive', 'skype']), 'symantec_scanlog': set(['*']), 'usnjrnl': set(['*']), 'webhist': set(['*']), 'winfirewall': set(['*']), 'winjob': set(['*']), 'winreg': set(['*']) } self.assertEqual(parsers_and_plugins, expected_parsers_and_plugins)
def testSplitExpression(self): """Tests the SplitExpression function.""" test_helper = parser_filter.ParserFilterExpressionHelper() excludes, includes = test_helper.SplitExpression('!excluded') self.assertEqual(excludes, {'excluded': set(['*'])}) self.assertEqual(includes, {}) excludes, includes = test_helper.SplitExpression( '!excluded,!excluded/plugin1,') self.assertEqual(excludes, {'excluded': set(['*', 'plugin1'])}) self.assertEqual(includes, {}) excludes, includes = test_helper.SplitExpression( '!excluded/plugin1,!excluded/plugin2,') self.assertEqual(excludes, {'excluded': set(['plugin1', 'plugin2'])}) self.assertEqual(includes, {}) excludes, includes = test_helper.SplitExpression('included') self.assertEqual(excludes, {}) self.assertEqual(includes, {'included': set(['*'])}) excludes, includes = test_helper.SplitExpression( 'included,included/plugin1') self.assertEqual(excludes, {}) self.assertEqual(includes, {'included': set(['*', 'plugin1'])}) excludes, includes = test_helper.SplitExpression( 'included/plugin1,included/plugin2') self.assertEqual(excludes, {}) self.assertEqual(includes, {'included': set(['plugin1', 'plugin2'])}) excludes, includes = test_helper.SplitExpression( '!excluded/plugin1,included/plugin2') self.assertEqual(excludes, {'excluded': set(['plugin1'])}) self.assertEqual(includes, {'included': set(['plugin2'])})
def _CreateProcessingConfiguration(self, knowledge_base): """Creates a processing configuration. Args: knowledge_base (KnowledgeBase): contains information from the source data needed for parsing. Returns: ProcessingConfiguration: processing configuration. Raises: BadConfigOption: if presets in the parser filter expression could not be expanded or if an invalid parser or plugin name is specified. """ parser_filter_expression = self._parser_filter_expression if not parser_filter_expression: operating_system_family = knowledge_base.GetValue('operating_system') operating_system_product = knowledge_base.GetValue( 'operating_system_product') operating_system_version = knowledge_base.GetValue( 'operating_system_version') operating_system_artifact = artifacts.OperatingSystemArtifact( family=operating_system_family, product=operating_system_product, version=operating_system_version) preset_definitions = self._presets_manager.GetPresetsByOperatingSystem( operating_system_artifact) if preset_definitions: preset_names = [ preset_definition.name for preset_definition in preset_definitions] filter_expression = ','.join(preset_names) logger.info('Parser filter expression set to: {0:s}'.format( filter_expression)) parser_filter_expression = filter_expression parser_filter_helper = parser_filter.ParserFilterExpressionHelper() try: parser_filter_expression = parser_filter_helper.ExpandPresets( self._presets_manager, parser_filter_expression) except RuntimeError as exception: raise errors.BadConfigOption(( 'Unable to expand presets in parser filter expression with ' 'error: {0!s}').format(exception)) _, invalid_parser_elements = ( parsers_manager.ParsersManager.CheckFilterExpression( parser_filter_expression)) if invalid_parser_elements: invalid_parser_names_string = ','.join(invalid_parser_elements) raise errors.BadConfigOption( 'Unknown parser or plugin names in element(s): "{0:s}" of ' 'parser filter expression: {1:s}'.format( invalid_parser_names_string, parser_filter_expression)) # TODO: pass preferred_encoding. configuration = configurations.ProcessingConfiguration() configuration.artifact_filters = self._artifact_filters configuration.credentials = self._credential_configurations configuration.debug_output = self._debug_mode configuration.event_extraction.text_prepend = self._text_prepend configuration.extraction.hasher_file_size_limit = ( self._hasher_file_size_limit) configuration.extraction.hasher_names_string = self._hasher_names_string configuration.extraction.process_archives = self._process_archives configuration.extraction.process_compressed_streams = ( self._process_compressed_streams) configuration.extraction.yara_rules_string = self._yara_rules_string configuration.filter_file = self._filter_file configuration.input_source.mount_path = self._mount_path configuration.log_filename = self._log_file configuration.parser_filter_expression = parser_filter_expression configuration.preferred_year = self._preferred_year configuration.profiling.directory = self._profiling_directory configuration.profiling.sample_rate = self._profiling_sample_rate configuration.profiling.profilers = self._profilers configuration.task_storage_format = self._task_storage_format configuration.temporary_directory = self._temporary_directory return configuration