def testGetPresetsByOperatingSystem(self): """Tests the GetPresetsByOperatingSystem function.""" test_manager = presets.ParserPresetsManager() test_path = self._GetTestFilePath(['presets.yaml']) test_manager.ReadFromFile(test_path) operating_system = artifacts.OperatingSystemArtifact(family='MacOS') test_presets = test_manager.GetPresetsByOperatingSystem( operating_system) self.assertEqual(len(test_presets), 1) self.assertEqual(test_presets[0].name, 'macos') expected_parsers = [ 'asl_log', 'bash_history', 'bencode', 'bsm_log', 'cups_ipp', 'czip/oxml', 'filestat', 'fseventsd', 'gdrive_synclog', 'java_idx', 'mac_appfirewall_log', 'mac_keychain', 'mac_securityd', 'macwifi', 'olecf', 'plist', 'sqlite/appusage', 'sqlite/google_drive', 'sqlite/imessage', 'sqlite/ls_quarantine', 'sqlite/mac_document_versions', 'sqlite/mackeeper_cache', 'sqlite/skype', 'syslog', 'utmpx', 'webhist', 'zsh_extended_history' ] self.assertEqual(test_presets[0].parsers, expected_parsers) operating_system = artifacts.OperatingSystemArtifact(family='bogus') test_presets = test_manager.GetPresetsByOperatingSystem( operating_system) self.assertEqual(len(test_presets), 0)
def testGetPresetByName(self): """Tests the GetPresetByName function.""" test_manager = presets.ParserPresetsManager() test_file_path = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(test_file_path) test_manager.ReadFromFile(test_file_path) test_preset = test_manager.GetPresetByName('linux') self.assertIsNotNone(test_preset) self.assertEqual(test_preset.name, 'linux') expected_parsers = [ 'bash_history', 'bencode', 'czip/oxml', 'dockerjson', 'dpkg', 'filestat', 'gdrive_synclog', 'java_idx', 'olecf', 'pls_recall', 'popularity_contest', 'selinux', 'sqlite/google_drive', 'sqlite/skype', 'sqlite/zeitgeist', 'syslog', 'systemd_journal', 'utmp', 'webhist', 'xchatlog', 'xchatscrollback', 'zsh_extended_history' ] self.assertEqual(test_preset.parsers, expected_parsers) test_preset = test_manager.GetPresetByName('bogus') self.assertIsNone(test_preset)
def __init__(self, input_reader=None, output_writer=None): """Initializes an CLI tool. Args: input_reader (Optional[InputReader]): input reader, where None indicates that the stdin input reader should be used. output_writer (Optional[OutputWriter]): output writer, where None indicates that the stdout output writer should be used. """ super(ExtractionTool, self).__init__( input_reader=input_reader, output_writer=output_writer) self._artifacts_registry = None self._buffer_size = 0 self._mount_path = None self._operating_system = None self._parser_filter_expression = None self._preferred_year = None self._presets_file = None self._presets_manager = parsers_presets.ParserPresetsManager() self._process_archives = False self._process_compressed_streams = True self._process_memory_limit = None self._queue_size = self._DEFAULT_QUEUE_SIZE self._resolver_context = dfvfs_context.Context() self._single_process_mode = False self._storage_file_path = None self._storage_format = definitions.STORAGE_FORMAT_SQLITE self._task_storage_format = definitions.STORAGE_FORMAT_SQLITE self._temporary_directory = None self._text_prepend = None self._yara_rules_string = None
def testExpandPresets(self): """Tests the ExpandPresets function.""" presets_file = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(presets_file) presets_manager = parsers_presets.ParserPresetsManager() presets_manager.ReadFromFile(presets_file) test_helper = parser_filter.ParserFilterExpressionHelper() expected_parser_filter_expression = ','.join( sorted([ '!utmp', 'bencode', 'binary_cookies', 'chrome_cache', 'chrome_preferences', 'czip/oxml', 'esedb', 'esedb/msie_webcache', 'filestat', 'firefox_cache', 'gdrive_synclog', 'java_idx', 'lnk', 'mcafee_protection', 'msiecf', 'olecf', 'opera_global', 'opera_typed_history', 'pe', 'plist/safari_history', 'prefetch', 'sccm', 'skydrive_log', 'skydrive_log_old', 'sqlite/chrome_27_history', 'sqlite/chrome_8_history', 'sqlite/chrome_autofill', 'sqlite/chrome_cookies', 'sqlite/chrome_extension_activity', 'sqlite/firefox_cookies', 'sqlite/firefox_downloads', 'sqlite/firefox_history', 'sqlite/google_drive', 'sqlite/skype', 'symantec_scanlog', 'usnjrnl', 'winfirewall', 'winjob', 'winreg' ])) parser_filter_expression = test_helper.ExpandPresets( presets_manager, 'win_gen,!utmp') self.assertEqual(parser_filter_expression, expected_parser_filter_expression) parser_filter_expression = test_helper.ExpandPresets( presets_manager, 'olecf,!utmp') self.assertEqual(parser_filter_expression, '!utmp,olecf')
def testGetPresets(self): """Tests the GetPresets function.""" test_manager = presets.ParserPresetsManager() test_path = self._GetTestFilePath(['presets.yaml']) test_manager.ReadFromFile(test_path) test_presets = list(test_manager.GetPresets()) self.assertEqual(len(test_presets), 7)
def testGetParsersByPreset(self): """Tests the GetParsersByPreset function.""" test_file_path = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(test_file_path) test_manager = presets.ParserPresetsManager() test_manager.ReadFromFile(test_file_path) parser_names = test_manager.GetParsersByPreset('linux') self.assertEqual(parser_names, self._LINUX_PARSERS) with self.assertRaises(KeyError): test_manager.GetParsersByPreset('bogus')
def testGetPresetsInformation(self): """Tests the GetPresetsInformation function.""" test_file_path = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(test_file_path) test_manager = presets.ParserPresetsManager() test_manager.ReadFromFile(test_file_path) parser_presets_information = test_manager.GetPresetsInformation() self.assertGreaterEqual(len(parser_presets_information), 1) available_parser_names = [name for name, _ in parser_presets_information] self.assertIn('linux', available_parser_names)
def testGetNames(self): """Tests the GetNames function.""" test_manager = presets.ParserPresetsManager() test_path = self._GetTestFilePath(['presets.yaml']) test_manager.ReadFromFile(test_path) test_names = list(test_manager.GetNames()) self.assertEqual(len(test_names), 7) expected_names = sorted([ 'android', 'linux', 'macos', 'webhist', 'win7', 'win_gen', 'winxp' ]) self.assertEqual(test_names, expected_names)
def testParsersAndPresets(self): """Tests that all parsers/plugins in the default presets are valid.""" presets_file_path = self._GetDataFilePath(['presets.yaml']) preset_manager = presets.ParserPresetsManager() preset_manager.ReadFromFile(presets_file_path) filter_helper = parser_filter.ParserFilterExpressionHelper() for name in preset_manager.GetNames(): expanded_preset = filter_helper.ExpandPresets(preset_manager, name) _, invalid_parser_elements = ( parsers_manager.ParsersManager.CheckFilterExpression(expanded_preset)) self.assertFalse( invalid_parser_elements, msg='Invalid parser/plugin name(s)')
def testGetPresetByName(self): """Tests the GetPresetByName function.""" test_file_path = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(test_file_path) test_manager = presets.ParserPresetsManager() test_manager.ReadFromFile(test_file_path) test_preset = test_manager.GetPresetByName('linux') self.assertIsNotNone(test_preset) self.assertEqual(test_preset.name, 'linux') self.assertEqual(test_preset.parsers, self._LINUX_PARSERS) test_preset = test_manager.GetPresetByName('bogus') self.assertIsNone(test_preset)
def __init__(self, input_reader=None, output_writer=None): """Initializes an CLI tool. Args: input_reader (Optional[InputReader]): input reader, where None indicates that the stdin input reader should be used. output_writer (Optional[OutputWriter]): output writer, where None indicates that the stdout output writer should be used. """ super(ExtractionTool, self).__init__(input_reader=input_reader, output_writer=output_writer) self._artifacts_registry = None self._buffer_size = 0 self._command_line_arguments = None self._enable_sigsegv_handler = False self._expanded_parser_filter_expression = None self._extract_winevt_resources = True self._number_of_extraction_workers = 0 self._parser_filter_expression = None self._preferred_codepage = None self._preferred_language = None self._preferred_time_zone = None self._preferred_year = None self._presets_file = None self._presets_manager = parsers_presets.ParserPresetsManager() self._process_archives = False self._process_compressed_streams = True self._process_memory_limit = None self._queue_size = self._DEFAULT_QUEUE_SIZE self._resolver_context = dfvfs_context.Context() self._single_process_mode = False self._status_view_mode = status_view.StatusView.MODE_WINDOW self._status_view = status_view.StatusView(self._output_writer, self.NAME) self._storage_file_path = None self._storage_format = definitions.STORAGE_FORMAT_SQLITE self._task_storage_format = definitions.STORAGE_FORMAT_SQLITE self._temporary_directory = None self._text_prepend = None self._worker_memory_limit = None self._worker_timeout = None self._yara_rules_string = None self.list_language_tags = False self.list_time_zones = False
def testGetPresetsByOperatingSystem(self): """Tests the GetPresetsByOperatingSystem function.""" test_file_path = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(test_file_path) test_manager = presets.ParserPresetsManager() test_manager.ReadFromFile(test_file_path) operating_system = artifacts.OperatingSystemArtifact(family='MacOS') test_presets = test_manager.GetPresetsByOperatingSystem(operating_system) self.assertEqual(len(test_presets), 1) self.assertEqual(test_presets[0].name, 'macos') self.assertEqual(test_presets[0].parsers, self._MACOS_PARSERS) operating_system = artifacts.OperatingSystemArtifact(family='bogus') test_presets = test_manager.GetPresetsByOperatingSystem(operating_system) self.assertEqual(len(test_presets), 0)
def testExpandPreset(self): """Tests the _ExpandPreset function.""" presets_file = self._GetTestFilePath(['presets.yaml']) self._SkipIfPathNotExists(presets_file) presets_manager = parsers_presets.ParserPresetsManager() presets_manager.ReadFromFile(presets_file) test_helper = parser_filter.ParserFilterExpressionHelper() parsers_and_plugins = {'win_gen': set(['*'])} test_helper._ExpandPreset(presets_manager, 'win_gen', parsers_and_plugins) expected_parsers_and_plugins = { 'bencode': set(['*']), 'czip': set(['oxml']), 'filestat': set(['*']), 'gdrive_synclog': set(['*']), 'java_idx': set(['*']), 'lnk': set(['*']), 'mcafee_protection': set(['*']), 'olecf': set(['*']), 'pe': set(['*']), 'prefetch': set(['*']), 'sccm': set(['*']), 'skydrive_log': set(['*']), 'skydrive_log_old': set(['*']), 'sqlite': set(['google_drive', 'skype']), 'symantec_scanlog': set(['*']), 'usnjrnl': set(['*']), 'webhist': set(['*']), 'winfirewall': set(['*']), 'winjob': set(['*']), 'winreg': set(['*']) } self.assertEqual(parsers_and_plugins, expected_parsers_and_plugins)
class ParsersManager(object): """The parsers and plugins manager.""" _parser_classes = {} _presets = presets.ParserPresetsManager() @classmethod def _GetParserFilters(cls, parser_filter_expression): """Retrieves the parsers and plugins to include and exclude. Takes a comma separated string and splits it up into two dictionaries, of parsers and plugins to include and to exclude from selection. If a particular filter is prepended with an exclamation point it will be added to the exclude section, otherwise in the include. Args: parser_filter_expression (str): parser filter expression, where None represents all parsers and plugins. Returns: tuple: containing: * dict[str, BaseParser]: included parsers and plugins by name. * dict[str, BaseParser]: excluded parsers and plugins by name. """ if not parser_filter_expression: return {}, {} includes = {} excludes = {} preset_names = cls._presets.GetNames() for parser_filter in parser_filter_expression.split(','): parser_filter = parser_filter.strip() if not parser_filter: continue if parser_filter.startswith('!'): parser_filter = parser_filter[1:] active_dict = excludes else: active_dict = includes parser_filter = parser_filter.lower() if parser_filter in preset_names: for parser_in_category in cls._GetParsersFromPresetCategory( parser_filter): parser, _, plugin = parser_in_category.partition('/') active_dict.setdefault(parser, []) if plugin: active_dict[parser].append(plugin) else: parser, _, plugin = parser_filter.partition('/') active_dict.setdefault(parser, []) if plugin: active_dict[parser].append(plugin) cls._ReduceParserFilters(includes, excludes) return includes, excludes @classmethod def _GetParsersFromPresetCategory(cls, category): """Retrieves the parser names of specific preset category. Args: category (str): parser preset categories. Returns: list[str]: parser names in alphabetical order. """ preset_definition = cls._presets.GetPresetByName(category) if preset_definition is None: return [] preset_names = cls._presets.GetNames() parser_names = set() for element_name in preset_definition.parsers: if element_name in preset_names: category_parser_names = cls._GetParsersFromPresetCategory(element_name) parser_names.update(category_parser_names) else: parser_names.add(element_name) return sorted(parser_names) @classmethod def _ReduceParserFilters(cls, includes, excludes): """Reduces the parsers and plugins to include and exclude. If an intersection is found, the parser or plugin is removed from the inclusion set. If a parser is not in inclusion set there is no need to have it in the exclusion set. Args: includes (dict[str, BaseParser]): included parsers and plugins by name. excludes (dict[str, BaseParser]): excluded parsers and plugins by name. """ if not includes or not excludes: return for parser_name in set(includes).intersection(excludes): # Check parser and plugin list for exact equivalence. if includes[parser_name] == excludes[parser_name]: logger.warning( 'Parser {0:s} was in both the inclusion and exclusion lists. ' 'Ignoring included parser.'.format(parser_name)) includes.pop(parser_name) continue # Remove plugins that defined are in both inclusion and exclusion lists. plugin_includes = includes[parser_name] plugin_excludes = excludes[parser_name] intersection = set(plugin_includes).intersection(plugin_excludes) if not intersection: continue logger.warning( 'Parser {0:s} plugins: {1:s} in both the inclusion and exclusion ' 'lists. Ignoring included plugins.'.format( parser_name, ', '.join(intersection))) plugins_list = list(set(plugin_includes).difference(intersection)) includes[parser_name] = plugins_list # Remove excluded parsers that do not run. parsers_to_pop = [] for parser_name in excludes: if parser_name in includes: continue logger.warning( 'The excluded parser: {0:s} is not associated with the included ' 'parsers: {1:s}. Ignoring excluded parser.'.format( parser_name, ', '.join(includes.keys()))) parsers_to_pop.append(parser_name) for parser_name in parsers_to_pop: excludes.pop(parser_name) @classmethod def CreateSignatureScanner(cls, specification_store): """Creates a signature scanner for format specifications with signatures. Args: specification_store (FormatSpecificationStore): format specifications with signatures. Returns: pysigscan.scanner: signature scanner. """ scanner_object = pysigscan.scanner() for format_specification in specification_store.specifications: for signature in format_specification.signatures: pattern_offset = signature.offset if pattern_offset is None: signature_flags = pysigscan.signature_flags.NO_OFFSET elif pattern_offset < 0: pattern_offset *= -1 signature_flags = pysigscan.signature_flags.RELATIVE_FROM_END else: signature_flags = pysigscan.signature_flags.RELATIVE_FROM_START scanner_object.add_signature( signature.identifier, pattern_offset, signature.pattern, signature_flags) return scanner_object @classmethod def DeregisterParser(cls, parser_class): """Deregisters a parser class. The parser classes are identified based on their lower case name. Args: parser_class (type): parser class (subclass of BaseParser). Raises: KeyError: if parser class is not set for the corresponding name. """ parser_name = parser_class.NAME.lower() if parser_name not in cls._parser_classes: raise KeyError('Parser class not set for name: {0:s}.'.format( parser_class.NAME)) del cls._parser_classes[parser_name] @classmethod def GetFormatsWithSignatures(cls, parser_filter_expression=None): """Retrieves the format specifications that have signatures. This method will create a specification store for parsers that define a format specification with signatures and a list of parser names for those that do not. Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. Returns: tuple: containing: * FormatSpecificationStore: format specifications with signatures. * list[str]: names of parsers that do not have format specifications with signatures, or have signatures but also need to be applied 'brute force'. """ specification_store = specification.FormatSpecificationStore() remainder_list = [] for parser_name, parser_class in cls.GetParsers( parser_filter_expression=parser_filter_expression): format_specification = parser_class.GetFormatSpecification() if format_specification and format_specification.signatures: specification_store.AddSpecification(format_specification) # The plist parser is a special case, where it both defines a signature # and also needs to be applied 'brute-force' to non-matching files, # as the signature matches binary plists, but not XML or JSON plists. if parser_name == 'plist': remainder_list.append(parser_name) else: remainder_list.append(parser_name) return specification_store, remainder_list @classmethod def GetNamesOfParsersWithPlugins(cls): """Retrieves the names of all parsers with plugins. Returns: list[str]: names of all parsers with plugins. """ parser_names = [] for parser_name, parser_class in cls.GetParsers(): if parser_class.SupportsPlugins(): parser_names.append(parser_name) return sorted(parser_names) @classmethod def GetParserAndPluginNames(cls, parser_filter_expression=None): """Retrieves the parser and parser plugin names. Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. Returns: list[str]: parser and parser plugin names. """ parser_and_plugin_names = [] for parser_name, parser_class in cls.GetParsers( parser_filter_expression=parser_filter_expression): parser_and_plugin_names.append(parser_name) if parser_class.SupportsPlugins(): for plugin_name, _ in parser_class.GetPlugins(): parser_and_plugin_names.append( '{0:s}/{1:s}'.format(parser_name, plugin_name)) return parser_and_plugin_names @classmethod def GetParserPluginsInformation(cls, parser_filter_expression=None): """Retrieves the parser plugins information. Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. Returns: list[tuple[str, str]]: pairs of parser plugin names and descriptions. """ parser_plugins_information = [] for _, parser_class in cls.GetParsers( parser_filter_expression=parser_filter_expression): if parser_class.SupportsPlugins(): for plugin_name, plugin_class in parser_class.GetPlugins(): description = getattr(plugin_class, 'DESCRIPTION', '') parser_plugins_information.append((plugin_name, description)) return parser_plugins_information # Note this method is used by l2tpreg. @classmethod def GetParserObjectByName(cls, parser_name): """Retrieves a specific parser object by its name. Args: parser_name (str): name of the parser. Returns: BaseParser: parser object or None. """ parser_class = cls._parser_classes.get(parser_name, None) if parser_class: return parser_class() return None @classmethod def GetParserObjects(cls, parser_filter_expression=None): """Retrieves the parser objects. Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. Returns: dict[str, BaseParser]: parsers per name. """ includes, excludes = cls._GetParserFilters(parser_filter_expression) parser_objects = {} for parser_name, parser_class in iter(cls._parser_classes.items()): # If there are no includes all parsers are included by default. if not includes and parser_name in excludes: continue if includes and parser_name not in includes: continue parser_object = parser_class() if parser_class.SupportsPlugins(): plugin_includes = None if parser_name in includes: plugin_includes = includes[parser_name] parser_object.EnablePlugins(plugin_includes) parser_objects[parser_name] = parser_object return parser_objects @classmethod def GetParsers(cls, parser_filter_expression=None): """Retrieves the registered parsers and plugins. Retrieves a dictionary of all registered parsers and associated plugins from a parser filter string. The filter string can contain direct names of parsers, presets or plugins. The filter string can also negate selection if prepended with an exclamation point, e.g.: "foo,!foo/bar" would include parser foo but not include plugin bar. A list of specific included and excluded plugins is also passed to each parser's class. The three types of entries in the filter string: * name of a parser: this would be the exact name of a single parser to include (or exclude), e.g. foo; * name of a preset, e.g. win7: the presets are defined in plaso/parsers/presets.py; * name of a plugin: if a plugin name is included the parent parser will be included in the list of registered parsers; Args: parser_filter_expression (Optional[str]): parser filter expression, where None represents all parsers and plugins. Yields: tuple: containing: * str: name of the parser: * type: parser class (subclass of BaseParser). """ includes, excludes = cls._GetParserFilters(parser_filter_expression) for parser_name, parser_class in iter(cls._parser_classes.items()): # If there are no includes all parsers are included by default. if not includes and parser_name in excludes: continue if includes and parser_name not in includes: continue yield parser_name, parser_class @classmethod def GetParsersInformation(cls): """Retrieves the parsers information. Returns: list[tuple[str, str]]: parser names and descriptions. """ parsers_information = [] for _, parser_class in cls.GetParsers(): description = getattr(parser_class, 'DESCRIPTION', '') parsers_information.append((parser_class.NAME, description)) return parsers_information @classmethod def GetPresetsForOperatingSystem( cls, operating_system, operating_system_product, operating_system_version): """Determines the presets for a specific operating system. Args: operating_system (str): operating system for example "Windows". This should be one of the values in definitions.OPERATING_SYSTEM_FAMILIES. operating_system_product (str): operating system product for example "Windows XP" as determined by preprocessing. operating_system_version (str): operating system version for example "5.1" as determined by preprocessing. Returns: list[PresetDefinition]: preset definitions, where an empty list represents all parsers and parser plugins (no preset). """ operating_system = artifacts.OperatingSystemArtifact( family=operating_system, product=operating_system_product, version=operating_system_version) return cls._presets.GetPresetsByOperatingSystem(operating_system) @classmethod def GetPresets(cls): """Retrieves the preset definitions. Returns: generator[PresetDefinition]: preset definition generator in alphabetical order by name. """ return cls._presets.GetPresets() @classmethod def ReadPresetsFromFile(cls, path): """Reads parser and parser plugin presets from a file. Args: path (str): path of file that contains the the parser and parser plugin presets configuration. """ cls._presets.ReadFromFile(path) @classmethod def RegisterParser(cls, parser_class): """Registers a parser class. The parser classes are identified based on their lower case name. Args: parser_class (type): parser class (subclass of BaseParser). Raises: KeyError: if parser class is already set for the corresponding name. """ parser_name = parser_class.NAME.lower() if parser_name in cls._parser_classes: raise KeyError('Parser class already set for name: {0:s}.'.format( parser_class.NAME)) cls._parser_classes[parser_name] = parser_class @classmethod def RegisterParsers(cls, parser_classes): """Registers parser classes. The parser classes are identified based on their lower case name. Args: parser_classes (list[type]): parsers classes (subclasses of BaseParser). Raises: KeyError: if parser class is already set for the corresponding name. """ for parser_class in parser_classes: cls.RegisterParser(parser_class)