def testGetPresetsByOperatingSystem(self):
"""Tests the GetPresetsByOperatingSystem function."""
test_manager = presets.ParserPresetsManager()
test_path = self._GetTestFilePath(['presets.yaml'])
test_manager.ReadFromFile(test_path)
operating_system = artifacts.OperatingSystemArtifact(family='MacOS')
test_presets = test_manager.GetPresetsByOperatingSystem(
operating_system)
self.assertEqual(len(test_presets), 1)
self.assertEqual(test_presets[0].name, 'macos')
expected_parsers = [
'asl_log', 'bash_history', 'bencode', 'bsm_log', 'cups_ipp',
'czip/oxml', 'filestat', 'fseventsd', 'gdrive_synclog', 'java_idx',
'mac_appfirewall_log', 'mac_keychain', 'mac_securityd', 'macwifi',
'olecf', 'plist', 'sqlite/appusage', 'sqlite/google_drive',
'sqlite/imessage', 'sqlite/ls_quarantine',
'sqlite/mac_document_versions', 'sqlite/mackeeper_cache',
'sqlite/skype', 'syslog', 'utmpx', 'webhist',
'zsh_extended_history'
]
self.assertEqual(test_presets[0].parsers, expected_parsers)
operating_system = artifacts.OperatingSystemArtifact(family='bogus')
test_presets = test_manager.GetPresetsByOperatingSystem(
operating_system)
self.assertEqual(len(test_presets), 0)
def testGetPresetByName(self):
"""Tests the GetPresetByName function."""
test_manager = presets.ParserPresetsManager()
test_file_path = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(test_file_path)
test_manager.ReadFromFile(test_file_path)
test_preset = test_manager.GetPresetByName('linux')
self.assertIsNotNone(test_preset)
self.assertEqual(test_preset.name, 'linux')
expected_parsers = [
'bash_history', 'bencode', 'czip/oxml', 'dockerjson', 'dpkg',
'filestat', 'gdrive_synclog', 'java_idx', 'olecf', 'pls_recall',
'popularity_contest', 'selinux', 'sqlite/google_drive',
'sqlite/skype', 'sqlite/zeitgeist', 'syslog', 'systemd_journal',
'utmp', 'webhist', 'xchatlog', 'xchatscrollback',
'zsh_extended_history'
]
self.assertEqual(test_preset.parsers, expected_parsers)
test_preset = test_manager.GetPresetByName('bogus')
self.assertIsNone(test_preset)
def __init__(self, input_reader=None, output_writer=None):
"""Initializes an CLI tool.
Args:
input_reader (Optional[InputReader]): input reader, where None indicates
that the stdin input reader should be used.
output_writer (Optional[OutputWriter]): output writer, where None
indicates that the stdout output writer should be used.
"""
super(ExtractionTool, self).__init__(
input_reader=input_reader, output_writer=output_writer)
self._artifacts_registry = None
self._buffer_size = 0
self._mount_path = None
self._operating_system = None
self._parser_filter_expression = None
self._preferred_year = None
self._presets_file = None
self._presets_manager = parsers_presets.ParserPresetsManager()
self._process_archives = False
self._process_compressed_streams = True
self._process_memory_limit = None
self._queue_size = self._DEFAULT_QUEUE_SIZE
self._resolver_context = dfvfs_context.Context()
self._single_process_mode = False
self._storage_file_path = None
self._storage_format = definitions.STORAGE_FORMAT_SQLITE
self._task_storage_format = definitions.STORAGE_FORMAT_SQLITE
self._temporary_directory = None
self._text_prepend = None
self._yara_rules_string = None
def testExpandPresets(self):
"""Tests the ExpandPresets function."""
presets_file = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(presets_file)
presets_manager = parsers_presets.ParserPresetsManager()
presets_manager.ReadFromFile(presets_file)
test_helper = parser_filter.ParserFilterExpressionHelper()
expected_parser_filter_expression = ','.join(
sorted([
'!utmp', 'bencode', 'binary_cookies', 'chrome_cache',
'chrome_preferences', 'czip/oxml', 'esedb',
'esedb/msie_webcache', 'filestat', 'firefox_cache',
'gdrive_synclog', 'java_idx', 'lnk', 'mcafee_protection',
'msiecf', 'olecf', 'opera_global', 'opera_typed_history', 'pe',
'plist/safari_history', 'prefetch', 'sccm', 'skydrive_log',
'skydrive_log_old', 'sqlite/chrome_27_history',
'sqlite/chrome_8_history', 'sqlite/chrome_autofill',
'sqlite/chrome_cookies', 'sqlite/chrome_extension_activity',
'sqlite/firefox_cookies', 'sqlite/firefox_downloads',
'sqlite/firefox_history', 'sqlite/google_drive',
'sqlite/skype', 'symantec_scanlog', 'usnjrnl', 'winfirewall',
'winjob', 'winreg'
]))
parser_filter_expression = test_helper.ExpandPresets(
presets_manager, 'win_gen,!utmp')
self.assertEqual(parser_filter_expression,
expected_parser_filter_expression)
parser_filter_expression = test_helper.ExpandPresets(
presets_manager, 'olecf,!utmp')
self.assertEqual(parser_filter_expression, '!utmp,olecf')
def testGetPresets(self):
"""Tests the GetPresets function."""
test_manager = presets.ParserPresetsManager()
test_path = self._GetTestFilePath(['presets.yaml'])
test_manager.ReadFromFile(test_path)
test_presets = list(test_manager.GetPresets())
self.assertEqual(len(test_presets), 7)
def testGetParsersByPreset(self):
"""Tests the GetParsersByPreset function."""
test_file_path = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(test_file_path)
test_manager = presets.ParserPresetsManager()
test_manager.ReadFromFile(test_file_path)
parser_names = test_manager.GetParsersByPreset('linux')
self.assertEqual(parser_names, self._LINUX_PARSERS)
with self.assertRaises(KeyError):
test_manager.GetParsersByPreset('bogus')
def testGetPresetsInformation(self):
"""Tests the GetPresetsInformation function."""
test_file_path = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(test_file_path)
test_manager = presets.ParserPresetsManager()
test_manager.ReadFromFile(test_file_path)
parser_presets_information = test_manager.GetPresetsInformation()
self.assertGreaterEqual(len(parser_presets_information), 1)
available_parser_names = [name for name, _ in parser_presets_information]
self.assertIn('linux', available_parser_names)
def testGetNames(self):
"""Tests the GetNames function."""
test_manager = presets.ParserPresetsManager()
test_path = self._GetTestFilePath(['presets.yaml'])
test_manager.ReadFromFile(test_path)
test_names = list(test_manager.GetNames())
self.assertEqual(len(test_names), 7)
expected_names = sorted([
'android', 'linux', 'macos', 'webhist', 'win7', 'win_gen', 'winxp'
])
self.assertEqual(test_names, expected_names)
def testParsersAndPresets(self):
"""Tests that all parsers/plugins in the default presets are valid."""
presets_file_path = self._GetDataFilePath(['presets.yaml'])
preset_manager = presets.ParserPresetsManager()
preset_manager.ReadFromFile(presets_file_path)
filter_helper = parser_filter.ParserFilterExpressionHelper()
for name in preset_manager.GetNames():
expanded_preset = filter_helper.ExpandPresets(preset_manager, name)
_, invalid_parser_elements = (
parsers_manager.ParsersManager.CheckFilterExpression(expanded_preset))
self.assertFalse(
invalid_parser_elements, msg='Invalid parser/plugin name(s)')
def testGetPresetByName(self):
"""Tests the GetPresetByName function."""
test_file_path = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(test_file_path)
test_manager = presets.ParserPresetsManager()
test_manager.ReadFromFile(test_file_path)
test_preset = test_manager.GetPresetByName('linux')
self.assertIsNotNone(test_preset)
self.assertEqual(test_preset.name, 'linux')
self.assertEqual(test_preset.parsers, self._LINUX_PARSERS)
test_preset = test_manager.GetPresetByName('bogus')
self.assertIsNone(test_preset)
def __init__(self, input_reader=None, output_writer=None):
"""Initializes an CLI tool.
Args:
input_reader (Optional[InputReader]): input reader, where None indicates
that the stdin input reader should be used.
output_writer (Optional[OutputWriter]): output writer, where None
indicates that the stdout output writer should be used.
"""
super(ExtractionTool, self).__init__(input_reader=input_reader,
output_writer=output_writer)
self._artifacts_registry = None
self._buffer_size = 0
self._command_line_arguments = None
self._enable_sigsegv_handler = False
self._expanded_parser_filter_expression = None
self._extract_winevt_resources = True
self._number_of_extraction_workers = 0
self._parser_filter_expression = None
self._preferred_codepage = None
self._preferred_language = None
self._preferred_time_zone = None
self._preferred_year = None
self._presets_file = None
self._presets_manager = parsers_presets.ParserPresetsManager()
self._process_archives = False
self._process_compressed_streams = True
self._process_memory_limit = None
self._queue_size = self._DEFAULT_QUEUE_SIZE
self._resolver_context = dfvfs_context.Context()
self._single_process_mode = False
self._status_view_mode = status_view.StatusView.MODE_WINDOW
self._status_view = status_view.StatusView(self._output_writer,
self.NAME)
self._storage_file_path = None
self._storage_format = definitions.STORAGE_FORMAT_SQLITE
self._task_storage_format = definitions.STORAGE_FORMAT_SQLITE
self._temporary_directory = None
self._text_prepend = None
self._worker_memory_limit = None
self._worker_timeout = None
self._yara_rules_string = None
self.list_language_tags = False
self.list_time_zones = False
def testGetPresetsByOperatingSystem(self):
"""Tests the GetPresetsByOperatingSystem function."""
test_file_path = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(test_file_path)
test_manager = presets.ParserPresetsManager()
test_manager.ReadFromFile(test_file_path)
operating_system = artifacts.OperatingSystemArtifact(family='MacOS')
test_presets = test_manager.GetPresetsByOperatingSystem(operating_system)
self.assertEqual(len(test_presets), 1)
self.assertEqual(test_presets[0].name, 'macos')
self.assertEqual(test_presets[0].parsers, self._MACOS_PARSERS)
operating_system = artifacts.OperatingSystemArtifact(family='bogus')
test_presets = test_manager.GetPresetsByOperatingSystem(operating_system)
self.assertEqual(len(test_presets), 0)
def testExpandPreset(self):
"""Tests the _ExpandPreset function."""
presets_file = self._GetTestFilePath(['presets.yaml'])
self._SkipIfPathNotExists(presets_file)
presets_manager = parsers_presets.ParserPresetsManager()
presets_manager.ReadFromFile(presets_file)
test_helper = parser_filter.ParserFilterExpressionHelper()
parsers_and_plugins = {'win_gen': set(['*'])}
test_helper._ExpandPreset(presets_manager, 'win_gen',
parsers_and_plugins)
expected_parsers_and_plugins = {
'bencode': set(['*']),
'czip': set(['oxml']),
'filestat': set(['*']),
'gdrive_synclog': set(['*']),
'java_idx': set(['*']),
'lnk': set(['*']),
'mcafee_protection': set(['*']),
'olecf': set(['*']),
'pe': set(['*']),
'prefetch': set(['*']),
'sccm': set(['*']),
'skydrive_log': set(['*']),
'skydrive_log_old': set(['*']),
'sqlite': set(['google_drive', 'skype']),
'symantec_scanlog': set(['*']),
'usnjrnl': set(['*']),
'webhist': set(['*']),
'winfirewall': set(['*']),
'winjob': set(['*']),
'winreg': set(['*'])
}
self.assertEqual(parsers_and_plugins, expected_parsers_and_plugins)
class ParsersManager(object):
"""The parsers and plugins manager."""
_parser_classes = {}
_presets = presets.ParserPresetsManager()
@classmethod
def _GetParserFilters(cls, parser_filter_expression):
"""Retrieves the parsers and plugins to include and exclude.
Takes a comma separated string and splits it up into two dictionaries,
of parsers and plugins to include and to exclude from selection. If a
particular filter is prepended with an exclamation point it will be
added to the exclude section, otherwise in the include.
Args:
parser_filter_expression (str): parser filter expression, where None
represents all parsers and plugins.
Returns:
tuple: containing:
* dict[str, BaseParser]: included parsers and plugins by name.
* dict[str, BaseParser]: excluded parsers and plugins by name.
"""
if not parser_filter_expression:
return {}, {}
includes = {}
excludes = {}
preset_names = cls._presets.GetNames()
for parser_filter in parser_filter_expression.split(','):
parser_filter = parser_filter.strip()
if not parser_filter:
continue
if parser_filter.startswith('!'):
parser_filter = parser_filter[1:]
active_dict = excludes
else:
active_dict = includes
parser_filter = parser_filter.lower()
if parser_filter in preset_names:
for parser_in_category in cls._GetParsersFromPresetCategory(
parser_filter):
parser, _, plugin = parser_in_category.partition('/')
active_dict.setdefault(parser, [])
if plugin:
active_dict[parser].append(plugin)
else:
parser, _, plugin = parser_filter.partition('/')
active_dict.setdefault(parser, [])
if plugin:
active_dict[parser].append(plugin)
cls._ReduceParserFilters(includes, excludes)
return includes, excludes
@classmethod
def _GetParsersFromPresetCategory(cls, category):
"""Retrieves the parser names of specific preset category.
Args:
category (str): parser preset categories.
Returns:
list[str]: parser names in alphabetical order.
"""
preset_definition = cls._presets.GetPresetByName(category)
if preset_definition is None:
return []
preset_names = cls._presets.GetNames()
parser_names = set()
for element_name in preset_definition.parsers:
if element_name in preset_names:
category_parser_names = cls._GetParsersFromPresetCategory(element_name)
parser_names.update(category_parser_names)
else:
parser_names.add(element_name)
return sorted(parser_names)
@classmethod
def _ReduceParserFilters(cls, includes, excludes):
"""Reduces the parsers and plugins to include and exclude.
If an intersection is found, the parser or plugin is removed from
the inclusion set. If a parser is not in inclusion set there is no need
to have it in the exclusion set.
Args:
includes (dict[str, BaseParser]): included parsers and plugins by name.
excludes (dict[str, BaseParser]): excluded parsers and plugins by name.
"""
if not includes or not excludes:
return
for parser_name in set(includes).intersection(excludes):
# Check parser and plugin list for exact equivalence.
if includes[parser_name] == excludes[parser_name]:
logger.warning(
'Parser {0:s} was in both the inclusion and exclusion lists. '
'Ignoring included parser.'.format(parser_name))
includes.pop(parser_name)
continue
# Remove plugins that defined are in both inclusion and exclusion lists.
plugin_includes = includes[parser_name]
plugin_excludes = excludes[parser_name]
intersection = set(plugin_includes).intersection(plugin_excludes)
if not intersection:
continue
logger.warning(
'Parser {0:s} plugins: {1:s} in both the inclusion and exclusion '
'lists. Ignoring included plugins.'.format(
parser_name, ', '.join(intersection)))
plugins_list = list(set(plugin_includes).difference(intersection))
includes[parser_name] = plugins_list
# Remove excluded parsers that do not run.
parsers_to_pop = []
for parser_name in excludes:
if parser_name in includes:
continue
logger.warning(
'The excluded parser: {0:s} is not associated with the included '
'parsers: {1:s}. Ignoring excluded parser.'.format(
parser_name, ', '.join(includes.keys())))
parsers_to_pop.append(parser_name)
for parser_name in parsers_to_pop:
excludes.pop(parser_name)
@classmethod
def CreateSignatureScanner(cls, specification_store):
"""Creates a signature scanner for format specifications with signatures.
Args:
specification_store (FormatSpecificationStore): format specifications
with signatures.
Returns:
pysigscan.scanner: signature scanner.
"""
scanner_object = pysigscan.scanner()
for format_specification in specification_store.specifications:
for signature in format_specification.signatures:
pattern_offset = signature.offset
if pattern_offset is None:
signature_flags = pysigscan.signature_flags.NO_OFFSET
elif pattern_offset < 0:
pattern_offset *= -1
signature_flags = pysigscan.signature_flags.RELATIVE_FROM_END
else:
signature_flags = pysigscan.signature_flags.RELATIVE_FROM_START
scanner_object.add_signature(
signature.identifier, pattern_offset, signature.pattern,
signature_flags)
return scanner_object
@classmethod
def DeregisterParser(cls, parser_class):
"""Deregisters a parser class.
The parser classes are identified based on their lower case name.
Args:
parser_class (type): parser class (subclass of BaseParser).
Raises:
KeyError: if parser class is not set for the corresponding name.
"""
parser_name = parser_class.NAME.lower()
if parser_name not in cls._parser_classes:
raise KeyError('Parser class not set for name: {0:s}.'.format(
parser_class.NAME))
del cls._parser_classes[parser_name]
@classmethod
def GetFormatsWithSignatures(cls, parser_filter_expression=None):
"""Retrieves the format specifications that have signatures.
This method will create a specification store for parsers that define
a format specification with signatures and a list of parser names for
those that do not.
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
Returns:
tuple: containing:
* FormatSpecificationStore: format specifications with signatures.
* list[str]: names of parsers that do not have format specifications with
signatures, or have signatures but also need to be applied 'brute
force'.
"""
specification_store = specification.FormatSpecificationStore()
remainder_list = []
for parser_name, parser_class in cls.GetParsers(
parser_filter_expression=parser_filter_expression):
format_specification = parser_class.GetFormatSpecification()
if format_specification and format_specification.signatures:
specification_store.AddSpecification(format_specification)
# The plist parser is a special case, where it both defines a signature
# and also needs to be applied 'brute-force' to non-matching files,
# as the signature matches binary plists, but not XML or JSON plists.
if parser_name == 'plist':
remainder_list.append(parser_name)
else:
remainder_list.append(parser_name)
return specification_store, remainder_list
@classmethod
def GetNamesOfParsersWithPlugins(cls):
"""Retrieves the names of all parsers with plugins.
Returns:
list[str]: names of all parsers with plugins.
"""
parser_names = []
for parser_name, parser_class in cls.GetParsers():
if parser_class.SupportsPlugins():
parser_names.append(parser_name)
return sorted(parser_names)
@classmethod
def GetParserAndPluginNames(cls, parser_filter_expression=None):
"""Retrieves the parser and parser plugin names.
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
Returns:
list[str]: parser and parser plugin names.
"""
parser_and_plugin_names = []
for parser_name, parser_class in cls.GetParsers(
parser_filter_expression=parser_filter_expression):
parser_and_plugin_names.append(parser_name)
if parser_class.SupportsPlugins():
for plugin_name, _ in parser_class.GetPlugins():
parser_and_plugin_names.append(
'{0:s}/{1:s}'.format(parser_name, plugin_name))
return parser_and_plugin_names
@classmethod
def GetParserPluginsInformation(cls, parser_filter_expression=None):
"""Retrieves the parser plugins information.
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
Returns:
list[tuple[str, str]]: pairs of parser plugin names and descriptions.
"""
parser_plugins_information = []
for _, parser_class in cls.GetParsers(
parser_filter_expression=parser_filter_expression):
if parser_class.SupportsPlugins():
for plugin_name, plugin_class in parser_class.GetPlugins():
description = getattr(plugin_class, 'DESCRIPTION', '')
parser_plugins_information.append((plugin_name, description))
return parser_plugins_information
# Note this method is used by l2tpreg.
@classmethod
def GetParserObjectByName(cls, parser_name):
"""Retrieves a specific parser object by its name.
Args:
parser_name (str): name of the parser.
Returns:
BaseParser: parser object or None.
"""
parser_class = cls._parser_classes.get(parser_name, None)
if parser_class:
return parser_class()
return None
@classmethod
def GetParserObjects(cls, parser_filter_expression=None):
"""Retrieves the parser objects.
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
Returns:
dict[str, BaseParser]: parsers per name.
"""
includes, excludes = cls._GetParserFilters(parser_filter_expression)
parser_objects = {}
for parser_name, parser_class in iter(cls._parser_classes.items()):
# If there are no includes all parsers are included by default.
if not includes and parser_name in excludes:
continue
if includes and parser_name not in includes:
continue
parser_object = parser_class()
if parser_class.SupportsPlugins():
plugin_includes = None
if parser_name in includes:
plugin_includes = includes[parser_name]
parser_object.EnablePlugins(plugin_includes)
parser_objects[parser_name] = parser_object
return parser_objects
@classmethod
def GetParsers(cls, parser_filter_expression=None):
"""Retrieves the registered parsers and plugins.
Retrieves a dictionary of all registered parsers and associated plugins
from a parser filter string. The filter string can contain direct names of
parsers, presets or plugins. The filter string can also negate selection
if prepended with an exclamation point, e.g.: "foo,!foo/bar" would include
parser foo but not include plugin bar. A list of specific included and
excluded plugins is also passed to each parser's class.
The three types of entries in the filter string:
* name of a parser: this would be the exact name of a single parser to
include (or exclude), e.g. foo;
* name of a preset, e.g. win7: the presets are defined in
plaso/parsers/presets.py;
* name of a plugin: if a plugin name is included the parent parser will be
included in the list of registered parsers;
Args:
parser_filter_expression (Optional[str]): parser filter expression,
where None represents all parsers and plugins.
Yields:
tuple: containing:
* str: name of the parser:
* type: parser class (subclass of BaseParser).
"""
includes, excludes = cls._GetParserFilters(parser_filter_expression)
for parser_name, parser_class in iter(cls._parser_classes.items()):
# If there are no includes all parsers are included by default.
if not includes and parser_name in excludes:
continue
if includes and parser_name not in includes:
continue
yield parser_name, parser_class
@classmethod
def GetParsersInformation(cls):
"""Retrieves the parsers information.
Returns:
list[tuple[str, str]]: parser names and descriptions.
"""
parsers_information = []
for _, parser_class in cls.GetParsers():
description = getattr(parser_class, 'DESCRIPTION', '')
parsers_information.append((parser_class.NAME, description))
return parsers_information
@classmethod
def GetPresetsForOperatingSystem(
cls, operating_system, operating_system_product,
operating_system_version):
"""Determines the presets for a specific operating system.
Args:
operating_system (str): operating system for example "Windows". This
should be one of the values in definitions.OPERATING_SYSTEM_FAMILIES.
operating_system_product (str): operating system product for
example "Windows XP" as determined by preprocessing.
operating_system_version (str): operating system version for
example "5.1" as determined by preprocessing.
Returns:
list[PresetDefinition]: preset definitions, where an empty list
represents all parsers and parser plugins (no preset).
"""
operating_system = artifacts.OperatingSystemArtifact(
family=operating_system, product=operating_system_product,
version=operating_system_version)
return cls._presets.GetPresetsByOperatingSystem(operating_system)
@classmethod
def GetPresets(cls):
"""Retrieves the preset definitions.
Returns:
generator[PresetDefinition]: preset definition generator in alphabetical
order by name.
"""
return cls._presets.GetPresets()
@classmethod
def ReadPresetsFromFile(cls, path):
"""Reads parser and parser plugin presets from a file.
Args:
path (str): path of file that contains the the parser and parser plugin
presets configuration.
"""
cls._presets.ReadFromFile(path)
@classmethod
def RegisterParser(cls, parser_class):
"""Registers a parser class.
The parser classes are identified based on their lower case name.
Args:
parser_class (type): parser class (subclass of BaseParser).
Raises:
KeyError: if parser class is already set for the corresponding name.
"""
parser_name = parser_class.NAME.lower()
if parser_name in cls._parser_classes:
raise KeyError('Parser class already set for name: {0:s}.'.format(
parser_class.NAME))
cls._parser_classes[parser_name] = parser_class
@classmethod
def RegisterParsers(cls, parser_classes):
"""Registers parser classes.
The parser classes are identified based on their lower case name.
Args:
parser_classes (list[type]): parsers classes (subclasses of BaseParser).
Raises:
KeyError: if parser class is already set for the corresponding name.
"""
for parser_class in parser_classes:
cls.RegisterParser(parser_class)
