def _GetParsers(cls, parser_filter_expression=None):
"""Retrieves the registered parsers and plugins.
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
A parser filter expression is a comma separated value string that
denotes which parsers and plugins should be used. See
filters/parser_filter.py for details of the expression syntax.
This function does not support presets, and requires a parser
filter expression where presets have been expanded.
Yields:
tuple: containing:
* str: name of the parser:
* type: parser class (subclass of BaseParser).
"""
parser_filter_helper = parser_filter.ParserFilterExpressionHelper()
excludes, includes = parser_filter_helper.SplitExpression(
parser_filter_expression)
for parser_name, parser_class in cls._parser_classes.items():
# If there are no includes all parsers are included by default.
if not includes and parser_name in excludes:
continue
if includes and parser_name not in includes:
continue
yield parser_name, parser_class
def testGetParserAndPluginsList(self):
"""Tests the _GetParserAndPluginsList function."""
test_helper = parser_filter.ParserFilterExpressionHelper()
expression = test_helper._GetParserAndPluginsList(
{'excluded': set(['*', 'plugin1'])})
self.assertEqual(expression, ['excluded', 'excluded/plugin1'])
def _GetExpandedParserFilterExpression(self, knowledge_base):
"""Determines the expanded parser filter expression.
Args:
knowledge_base (KnowledgeBase): contains information from the source
data needed for parsing.
Returns:
str: expanded parser filter expression.
Raises:
BadConfigOption: if presets in the parser filter expression could not
be expanded or if an invalid parser or plugin name is specified.
"""
parser_filter_expression = self._parser_filter_expression
if not parser_filter_expression:
operating_system_family = knowledge_base.GetValue('operating_system')
operating_system_product = knowledge_base.GetValue(
'operating_system_product')
operating_system_version = knowledge_base.GetValue(
'operating_system_version')
operating_system_artifact = artifacts.OperatingSystemArtifact(
family=operating_system_family, product=operating_system_product,
version=operating_system_version)
preset_definitions = self._presets_manager.GetPresetsByOperatingSystem(
operating_system_artifact)
if preset_definitions:
self._parser_filter_expression = ','.join([
preset_definition.name
for preset_definition in preset_definitions])
logger.debug('Parser filter expression set to preset: {0:s}'.format(
self._parser_filter_expression))
parser_filter_helper = parser_filter.ParserFilterExpressionHelper()
try:
parser_filter_expression = parser_filter_helper.ExpandPresets(
self._presets_manager, self._parser_filter_expression)
logger.debug('Parser filter expression set to: {0:s}'.format(
parser_filter_expression or 'N/A'))
except RuntimeError as exception:
raise errors.BadConfigOption((
'Unable to expand presets in parser filter expression with '
'error: {0!s}').format(exception))
parser_elements, invalid_parser_elements = (
parsers_manager.ParsersManager.CheckFilterExpression(
parser_filter_expression))
if invalid_parser_elements:
invalid_parser_names_string = ','.join(invalid_parser_elements)
raise errors.BadConfigOption(
'Unknown parser or plugin names in element(s): "{0:s}" of '
'parser filter expression: {1:s}'.format(
invalid_parser_names_string, parser_filter_expression))
return ','.join(sorted(parser_elements))
def testExpandPresets(self):
"""Tests the ExpandPresets function."""
presets_file = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(presets_file)
presets_manager = parsers_presets.ParserPresetsManager()
presets_manager.ReadFromFile(presets_file)
test_helper = parser_filter.ParserFilterExpressionHelper()
expected_parser_filter_expression = ','.join(
sorted([
'!utmp', 'bencode', 'binary_cookies', 'chrome_cache',
'chrome_preferences', 'czip/oxml', 'esedb',
'esedb/msie_webcache', 'filestat', 'firefox_cache',
'gdrive_synclog', 'java_idx', 'lnk', 'mcafee_protection',
'msiecf', 'olecf', 'opera_global', 'opera_typed_history', 'pe',
'plist/safari_history', 'prefetch', 'sccm', 'skydrive_log',
'skydrive_log_old', 'sqlite/chrome_27_history',
'sqlite/chrome_8_history', 'sqlite/chrome_autofill',
'sqlite/chrome_cookies', 'sqlite/chrome_extension_activity',
'sqlite/firefox_cookies', 'sqlite/firefox_downloads',
'sqlite/firefox_history', 'sqlite/google_drive',
'sqlite/skype', 'symantec_scanlog', 'usnjrnl', 'winfirewall',
'winjob', 'winreg'
]))
parser_filter_expression = test_helper.ExpandPresets(
presets_manager, 'win_gen,!utmp')
self.assertEqual(parser_filter_expression,
expected_parser_filter_expression)
parser_filter_expression = test_helper.ExpandPresets(
presets_manager, 'olecf,!utmp')
self.assertEqual(parser_filter_expression, '!utmp,olecf')
def testParsersAndPresets(self):
"""Tests that all parsers/plugins in the default presets are valid."""
presets_file_path = self._GetDataFilePath(['presets.yaml'])
preset_manager = presets.ParserPresetsManager()
preset_manager.ReadFromFile(presets_file_path)
filter_helper = parser_filter.ParserFilterExpressionHelper()
for name in preset_manager.GetNames():
expanded_preset = filter_helper.ExpandPresets(preset_manager, name)
_, invalid_parser_elements = (
parsers_manager.ParsersManager.CheckFilterExpression(expanded_preset))
self.assertFalse(
invalid_parser_elements, msg='Invalid parser/plugin name(s)')
def _GetParsers(cls, parser_filter_expression=None):
"""Retrieves the registered parsers and plugins.
Retrieves a dictionary of all registered parsers and associated plugins
from a parser filter string. The filter string can contain direct names of
parsers or plugins. The filter string can also negate selection if prepended
with an exclamation point, such as: "foo,!foo/bar" would include parser foo
but not include plugin bar. A list of specific included and excluded
plugins is also passed to each parser's class.
The three types of entries in the filter string:
* name of a parser: this would be the exact name of a single parser to
include (or exclude), such as foo;
* name of a plugin: if a plugin name is included the parent parser will be
included in the list of registered parsers;
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
The parser filter expression is a comma separated value string that
denotes a list of parser names to include and/or exclude. Each entry
can have the value of:
* A name of a single parser (case insensitive), such as msiecf.
* A glob name for a single parser, such as '*msie*' (case
insensitive).
Yields:
tuple: containing:
* str: name of the parser:
* type: parser class (subclass of BaseParser).
"""
parser_filter_helper = parser_filter.ParserFilterExpressionHelper()
excludes, includes = parser_filter_helper.SplitExpression(
parser_filter_expression)
for parser_name, parser_class in iter(cls._parser_classes.items()):
# If there are no includes all parsers are included by default.
if not includes and parser_name in excludes:
continue
if includes and parser_name not in includes:
continue
yield parser_name, parser_class
def testJoinExpression(self):
"""Tests the _JoinExpression function."""
test_helper = parser_filter.ParserFilterExpressionHelper()
expression = test_helper._JoinExpression({'excluded': set(['*'])}, {})
self.assertEqual(expression, '!excluded')
expression = test_helper._JoinExpression(
{'excluded': set(['plugin1'])}, {'excluded': set(['*'])})
self.assertEqual(expression, '!excluded/plugin1,excluded')
expression = test_helper._JoinExpression(
{'excluded': set(['plugin1', 'plugin2'])},
{'excluded': set(['*'])})
self.assertEqual(expression,
'!excluded/plugin1,!excluded/plugin2,excluded')
expression = test_helper._JoinExpression({}, {'included': set(['*'])})
self.assertEqual(expression, 'included')
expression = test_helper._JoinExpression(
{}, {'included': set(['*', 'plugin1'])})
self.assertEqual(expression, 'included,included/plugin1')
expression = test_helper._JoinExpression(
{}, {'included': set(['plugin1', 'plugin2'])})
self.assertEqual(expression, 'included/plugin1,included/plugin2')
expression = test_helper._JoinExpression(
{'excluded': set(['plugin1'])}, {
'excluded': set(['*']),
'included': set(['plugin2'])
})
self.assertEqual(expression,
'!excluded/plugin1,excluded,included/plugin2')
with self.assertRaises(RuntimeError):
test_helper._JoinExpression({'excluded': set(['plugin1'])},
{'excluded': set(['plugin1'])})
with self.assertRaises(RuntimeError):
test_helper._JoinExpression({'excluded': set(['plugin1'])}, {})
def GetParserObjects(cls, parser_filter_expression=None):
"""Retrieves the parser objects.
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
The parser filter expression is a comma separated value string that
denotes a list of parser names to include and/or exclude. Each entry
can have the value of:
* A name of a single parser (case insensitive), such as msiecf.
* A glob name for a single parser, such as '*msie*' (case
insensitive).
Returns:
dict[str, BaseParser]: parsers per name.
"""
parser_filter_helper = parser_filter.ParserFilterExpressionHelper()
excludes, includes = parser_filter_helper.SplitExpression(
parser_filter_expression)
parser_objects = {}
for parser_name, parser_class in iter(cls._parser_classes.items()):
# If there are no includes all parsers are included by default.
if not includes and parser_name in excludes:
continue
if includes and parser_name not in includes:
continue
parser_object = parser_class()
if parser_class.SupportsPlugins():
plugin_includes = None
if parser_name in includes:
plugin_includes = includes[parser_name]
parser_object.EnablePlugins(plugin_includes)
parser_objects[parser_name] = parser_object
return parser_objects
def GetParserObjects(cls, parser_filter_expression=None):
"""Retrieves the parser objects.
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
A parser filter expression is a comma separated value string that
denotes which parsers and plugins should be used. See
filters/parser_filter.py for details of the expression syntax.
This function does not support presets, and requires a parser
filter expression where presets have been expanded.
Returns:
dict[str, BaseParser]: parsers per name.
"""
parser_filter_helper = parser_filter.ParserFilterExpressionHelper()
excludes, includes = parser_filter_helper.SplitExpression(
parser_filter_expression)
parser_objects = {}
for parser_name, parser_class in cls._parser_classes.items():
# If there are no includes all parsers are included by default.
if not includes and parser_name in excludes:
continue
if includes and parser_name not in includes:
continue
parser_object = parser_class()
if parser_class.SupportsPlugins():
plugin_includes = None
if parser_name in includes:
plugin_includes = includes[parser_name]
parser_object.EnablePlugins(plugin_includes)
parser_objects[parser_name] = parser_object
return parser_objects
def testExpandPreset(self):
"""Tests the _ExpandPreset function."""
presets_file = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(presets_file)
presets_manager = parsers_presets.ParserPresetsManager()
presets_manager.ReadFromFile(presets_file)
test_helper = parser_filter.ParserFilterExpressionHelper()
parsers_and_plugins = {'win_gen': set(['*'])}
test_helper._ExpandPreset(presets_manager, 'win_gen',
parsers_and_plugins)
expected_parsers_and_plugins = {
'bencode': set(['*']),
'czip': set(['oxml']),
'filestat': set(['*']),
'gdrive_synclog': set(['*']),
'java_idx': set(['*']),
'lnk': set(['*']),
'mcafee_protection': set(['*']),
'olecf': set(['*']),
'pe': set(['*']),
'prefetch': set(['*']),
'sccm': set(['*']),
'skydrive_log': set(['*']),
'skydrive_log_old': set(['*']),
'sqlite': set(['google_drive', 'skype']),
'symantec_scanlog': set(['*']),
'usnjrnl': set(['*']),
'webhist': set(['*']),
'winfirewall': set(['*']),
'winjob': set(['*']),
'winreg': set(['*'])
}
self.assertEqual(parsers_and_plugins, expected_parsers_and_plugins)
def testSplitExpression(self):
"""Tests the SplitExpression function."""
test_helper = parser_filter.ParserFilterExpressionHelper()
excludes, includes = test_helper.SplitExpression('!excluded')
self.assertEqual(excludes, {'excluded': set(['*'])})
self.assertEqual(includes, {})
excludes, includes = test_helper.SplitExpression(
'!excluded,!excluded/plugin1,')
self.assertEqual(excludes, {'excluded': set(['*', 'plugin1'])})
self.assertEqual(includes, {})
excludes, includes = test_helper.SplitExpression(
'!excluded/plugin1,!excluded/plugin2,')
self.assertEqual(excludes, {'excluded': set(['plugin1', 'plugin2'])})
self.assertEqual(includes, {})
excludes, includes = test_helper.SplitExpression('included')
self.assertEqual(excludes, {})
self.assertEqual(includes, {'included': set(['*'])})
excludes, includes = test_helper.SplitExpression(
'included,included/plugin1')
self.assertEqual(excludes, {})
self.assertEqual(includes, {'included': set(['*', 'plugin1'])})
excludes, includes = test_helper.SplitExpression(
'included/plugin1,included/plugin2')
self.assertEqual(excludes, {})
self.assertEqual(includes, {'included': set(['plugin1', 'plugin2'])})
excludes, includes = test_helper.SplitExpression(
'!excluded/plugin1,included/plugin2')
self.assertEqual(excludes, {'excluded': set(['plugin1'])})
self.assertEqual(includes, {'included': set(['plugin2'])})
def _CreateProcessingConfiguration(self, knowledge_base):
"""Creates a processing configuration.
Args:
knowledge_base (KnowledgeBase): contains information from the source
data needed for parsing.
Returns:
ProcessingConfiguration: processing configuration.
Raises:
BadConfigOption: if presets in the parser filter expression could not
be expanded or if an invalid parser or plugin name is specified.
"""
parser_filter_expression = self._parser_filter_expression
if not parser_filter_expression:
operating_system_family = knowledge_base.GetValue('operating_system')
operating_system_product = knowledge_base.GetValue(
'operating_system_product')
operating_system_version = knowledge_base.GetValue(
'operating_system_version')
operating_system_artifact = artifacts.OperatingSystemArtifact(
family=operating_system_family, product=operating_system_product,
version=operating_system_version)
preset_definitions = self._presets_manager.GetPresetsByOperatingSystem(
operating_system_artifact)
if preset_definitions:
preset_names = [
preset_definition.name for preset_definition in preset_definitions]
filter_expression = ','.join(preset_names)
logger.info('Parser filter expression set to: {0:s}'.format(
filter_expression))
parser_filter_expression = filter_expression
parser_filter_helper = parser_filter.ParserFilterExpressionHelper()
try:
parser_filter_expression = parser_filter_helper.ExpandPresets(
self._presets_manager, parser_filter_expression)
except RuntimeError as exception:
raise errors.BadConfigOption((
'Unable to expand presets in parser filter expression with '
'error: {0!s}').format(exception))
_, invalid_parser_elements = (
parsers_manager.ParsersManager.CheckFilterExpression(
parser_filter_expression))
if invalid_parser_elements:
invalid_parser_names_string = ','.join(invalid_parser_elements)
raise errors.BadConfigOption(
'Unknown parser or plugin names in element(s): "{0:s}" of '
'parser filter expression: {1:s}'.format(
invalid_parser_names_string, parser_filter_expression))
# TODO: pass preferred_encoding.
configuration = configurations.ProcessingConfiguration()
configuration.artifact_filters = self._artifact_filters
configuration.credentials = self._credential_configurations
configuration.debug_output = self._debug_mode
configuration.event_extraction.text_prepend = self._text_prepend
configuration.extraction.hasher_file_size_limit = (
self._hasher_file_size_limit)
configuration.extraction.hasher_names_string = self._hasher_names_string
configuration.extraction.process_archives = self._process_archives
configuration.extraction.process_compressed_streams = (
self._process_compressed_streams)
configuration.extraction.yara_rules_string = self._yara_rules_string
configuration.filter_file = self._filter_file
configuration.input_source.mount_path = self._mount_path
configuration.log_filename = self._log_file
configuration.parser_filter_expression = parser_filter_expression
configuration.preferred_year = self._preferred_year
configuration.profiling.directory = self._profiling_directory
configuration.profiling.sample_rate = self._profiling_sample_rate
configuration.profiling.profilers = self._profilers
configuration.task_storage_format = self._task_storage_format
configuration.temporary_directory = self._temporary_directory
return configuration
