def testValidation(self): glob_expression = rdf_paths.GlobExpression( "/home/%%users.username%%/**/.mozilla/") glob_expression.Validate() glob_expression = rdf_paths.GlobExpression("/home/**/**") self.assertRaises(ValueError, glob_expression.Validate)
def testGlobExpressionSplitsIntoExplainableComponents(self): kb = rdf_client.KnowledgeBase(users=[ rdf_client.User(homedir="/home/foo"), rdf_client.User(homedir="/home/bar"), rdf_client.User(homedir="/home/baz"), ]) # Test for preservation of **/ because it behaves different to **. ge = rdf_paths.GlobExpression("/foo/**/{bar,baz}/bar?/.*baz") components = ge.ExplainComponents(2, kb) self.assertEqual( [c.glob_expression for c in components], ["/foo/", "**/", "{bar,baz}", "/bar", "?", "/.", "*", "baz"]) ge = rdf_paths.GlobExpression("/foo/**bar") components = ge.ExplainComponents(2, kb) self.assertEqual([c.glob_expression for c in components], ["/foo/", "**", "bar"]) ge = rdf_paths.GlobExpression("/foo/**10bar") components = ge.ExplainComponents(2, kb) self.assertEqual([c.glob_expression for c in components], ["/foo/", "**10", "bar"]) ge = rdf_paths.GlobExpression("/{foo,bar,baz}") components = ge.ExplainComponents(2, kb) self.assertEqual(components[1].examples, ["foo", "bar"]) ge = rdf_paths.GlobExpression("%%users.homedir%%/foo") components = ge.ExplainComponents(2, kb) self.assertEqual([c.glob_expression for c in components], ["%%users.homedir%%", "/foo"]) self.assertEqual(components[0].examples, ["/home/foo", "/home/bar"])
def _BuildPredicate(self, client_id, token=None): if self.path_globs_whitelist is None: return None client_obj = aff4.FACTORY.Open(client_id.ToClientURN(), aff4_type=aff4_grr.VFSGRRClient, token=token) blacklist_regexes = [] for expression in self.path_globs_blacklist: for pattern in expression.Interpolate(client=client_obj): blacklist_regexes.append( rdf_paths.GlobExpression(pattern).AsRegEx()) whitelist_regexes = [] for expression in self.path_globs_whitelist: for pattern in expression.Interpolate(client=client_obj): whitelist_regexes.append( rdf_paths.GlobExpression(pattern).AsRegEx()) def Predicate(fd): pathspec = fd.Get(fd.Schema.PATHSPEC) path = pathspec.CollapsePath() return (not any(r.Match(path) for r in blacklist_regexes) and any(r.Match(path) for r in whitelist_regexes)) return Predicate
def _BuildPredicate(self, client_id, token=None): if self.path_globs_whitelist is None: return None kb = data_store_utils.GetClientKnowledgeBase(client_id) blacklist_regexes = [] for expression in self.path_globs_blacklist: for pattern in expression.Interpolate(knowledge_base=kb): blacklist_regexes.append( rdf_paths.GlobExpression(pattern).AsRegEx()) whitelist_regexes = [] for expression in self.path_globs_whitelist: for pattern in expression.Interpolate(knowledge_base=kb): whitelist_regexes.append( rdf_paths.GlobExpression(pattern).AsRegEx()) def Predicate(client_path): # Enforce leading / since Regexes require it. path = "/" + client_path.Path().lstrip("/") return (not any(r.Match(path) for r in blacklist_regexes) and any(r.Match(path) for r in whitelist_regexes)) return Predicate
def FetchBinaries(self, responses): """Parses the Rekall response and initiates FileFinder flows.""" if not responses.success: self.Log("Error fetching VAD data: %s", responses.status) return self.Log("Found %d binaries", len(responses)) if self.args.filename_regex: binaries = [] for response in responses: if self.args.filename_regex.Match(response.CollapsePath()): binaries.append(response) self.Log("Applied filename regex. Have %d files after filtering.", len(binaries)) else: binaries = responses if self.args.fetch_binaries: self.CallFlow(file_finder.FileFinder.__name__, next_state="HandleDownloadedFiles", paths=[ rdf_paths.GlobExpression(b.CollapsePath()) for b in binaries ], pathtype=rdf_paths.PathSpec.PathType.OS, action=rdf_file_finder.FileFinderAction.Download()) else: for b in binaries: self.SendReply(b)
def testRegExIsCorrectForGlobWithoutStars(self): glob_expression = rdf_paths.GlobExpression("/foo/bar/blah.txt") regex = glob_expression.AsRegEx() self.assertTrue(regex.Match("/foo/bar/blah.txt")) self.assertFalse(regex.Match("/foo/bar/blah2.txt")) self.assertFalse(regex.Match("/some/foo/bar/blah2.txt")) self.assertFalse(regex.Match("/some/foo/bar/blah2.txt/other"))
def Handle(self, args, token=None): ge = rdf_paths.GlobExpression(args.glob_expression) ge.Validate() kb = data_store_utils.GetClientKnowledgeBase(str(args.client_id)) components = ge.ExplainComponents(args.example_count, kb) return ApiExplainGlobExpressionResult(components=components)
def testIgnoresFileNotMatchingPathGlobsExclusionList(self): handler = flow_plugin.ApiGetFlowFilesArchiveHandler( include_only_path_globs=[rdf_paths.GlobExpression("/**/*/test.plist")], exclude_path_globs=[rdf_paths.GlobExpression("**/*.plist")]) result = handler.Handle( flow_plugin.ApiGetFlowFilesArchiveArgs( client_id=self.client_id, flow_id=self.flow_id, archive_format="ZIP"), context=self.context) manifest = self._GetZipManifest(result) self.assertEqual(manifest["archived_files"], 0) self.assertEqual(manifest["failed_files"], 0) self.assertEqual(manifest["processed_files"], 1) self.assertEqual(manifest["ignored_files"], 1) self.assertEqual( manifest["ignored_files_list"], ["aff4:/%s/fs/os%s/test.plist" % (self.client_id, self.base_path)])
def testRegExIsCorrectForGlobWithSingleStar(self): glob_expression = rdf_paths.GlobExpression("/foo/bar/*.txt") regex = glob_expression.AsRegEx() self.assertTrue(regex.Match("/foo/bar/blah.txt")) self.assertFalse(regex.Match("/foo/bar/blah.plist")) self.assertFalse(regex.Match("/foo/bar/blah/blah.txt")) self.assertFalse(regex.Match("/foo/blah1/blah2/bar/blah.txt"))
def testIgnoresFileNotMatchingPathGlobsBlacklist(self): handler = flow_plugin.ApiGetFlowFilesArchiveHandler( path_globs_whitelist=[rdf_paths.GlobExpression("/**/*/test.plist")], path_globs_blacklist=[rdf_paths.GlobExpression("**/*.plist")]) result = handler.Handle( flow_plugin.ApiGetFlowFilesArchiveArgs( client_id=self.client_id, flow_id=self.flow_urn.Basename(), archive_format="ZIP"), token=self.token) manifest = self._GetZipManifest(result) self.assertEqual(manifest["archived_files"], 0) self.assertEqual(manifest["failed_files"], 0) self.assertEqual(manifest["processed_files"], 1) self.assertEqual(manifest["ignored_files"], 1) self.assertEqual(manifest["ignored_files_list"], [ utils.SmartUnicode( self.client_id.Add("fs/os").Add(self.base_path).Add("test.plist")) ])
def testRegExIsCorrectForGlobWithGrouping(self): glob_expression = rdf_paths.GlobExpression("/foo/{bar,other}/*{.txt,.exe}") regex = glob_expression.AsRegEx() self.assertTrue(regex.Match("/foo/bar/blah.txt")) self.assertTrue(regex.Match("/foo/other/blah2.txt")) self.assertTrue(regex.Match("/foo/bar/blah.exe")) self.assertTrue(regex.Match("/foo/other/blah2.exe")) self.assertFalse(regex.Match("/foo/other2/blah.txt")) self.assertFalse(regex.Match("/foo/bar/blah.com"))
def testRegExIsCaseInsensitive(self): glob_expression = rdf_paths.GlobExpression("/foo/**/bar?/*{.txt,.exe}") regex = glob_expression.AsRegEx() self.assertTrue(regex.Match("/foo/bar1/blah.txt")) self.assertTrue(regex.Match("/foO/bAr1/blah.txt")) self.assertTrue(regex.Match("/foo/bar1/blah.TXT")) self.assertFalse(regex.Match("/foo/bar2/blah.com")) self.assertFalse(regex.Match("/foO/bAr2/blah.com")) self.assertFalse(regex.Match("/foo/bar2/blah.COM"))
def testRegExIsCorrectForGlobWithTwoStars(self): glob_expression = rdf_paths.GlobExpression("/foo/**/bar.txt") regex = glob_expression.AsRegEx() self.assertTrue(regex.Match("/foo/bar.txt")) self.assertTrue(regex.Match("/foo/blah/bar.txt")) self.assertTrue(regex.Match("/foo/blah1/blah2/bar.txt")) self.assertFalse(regex.Match("/foo/bar.plist")) self.assertFalse(regex.Match("/foo/blah/bar.txt/res")) self.assertFalse(regex.Match("/foo/blah1/blah2/bar.txt2"))
def testGroupingInterpolation(self): glob_expression = rdf_paths.GlobExpression() interpolated = glob_expression.InterpolateGrouping("/home/*.{sh,deb}") self.assertCountEqual(interpolated, [u"/home/*.deb", u"/home/*.sh"]) interpolated = glob_expression.InterpolateGrouping("/home/*.{sh, deb}") self.assertCountEqual(interpolated, [u"/home/*. deb", u"/home/*.sh"]) interpolated = glob_expression.InterpolateGrouping( "HKEY_CLASSES_ROOT/CLSID/{16d12736-7a9e-4765-bec6-f301d679caaa}") self.assertCountEqual(interpolated, [ u"HKEY_CLASSES_ROOT/CLSID/{16d12736-7a9e-4765-bec6-f301d679caaa}" ])
def testRegExIsCorrectForComplexGlob(self): glob_expression = rdf_paths.GlobExpression("/foo/**/bar?/*{.txt,.exe}") regex = glob_expression.AsRegEx() self.assertTrue(regex.Match("/foo/bar1/blah.txt")) self.assertTrue(regex.Match("/foo/bar2/blah.exe")) self.assertTrue(regex.Match("/foo/c1/c2/c3/bar1/blah.txt")) self.assertTrue(regex.Match("/foo/c1/c2/c3/bar2/blah.exe")) self.assertFalse(regex.Match("/foo/bar/blah.txt")) self.assertFalse(regex.Match("/foo/bar2/blah.com")) self.assertFalse(regex.Match("/foo/c1/c2/c3/bar1/blah.txt/res.txt")) self.assertFalse(regex.Match("/foo/c1/c2/c3/bar2/blah.exe/res.exe"))
def _BuildPredicate(self, client_id, context=None): if self.include_only_path_globs is None: return None kb = data_store_utils.GetClientKnowledgeBase(client_id) exclude_regexes = [] for expression in self.exclude_path_globs: for pattern in expression.Interpolate(knowledge_base=kb): exclude_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx()) include_only_regexes = [] for expression in self.include_only_path_globs: for pattern in expression.Interpolate(knowledge_base=kb): include_only_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx()) def Predicate(client_path): # Enforce leading / since Regexes require it. path = "/" + client_path.Path().lstrip("/") return (not any(r.Match(path) for r in exclude_regexes) and any(r.Match(path) for r in include_only_regexes)) return Predicate
def _BuildPredicate(self, client_id, token=None): if self.path_globs_whitelist is None: return None client_obj = aff4.FACTORY.Open( client_id.ToClientURN(), aff4_type=aff4_grr.VFSGRRClient, token=token) blacklist_regexes = [] for expression in self.path_globs_blacklist: for pattern in expression.Interpolate(client=client_obj): blacklist_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx()) whitelist_regexes = [] for expression in self.path_globs_whitelist: for pattern in expression.Interpolate(client=client_obj): whitelist_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx()) def Predicate(client_path): # Enforce leading / since Regexes require it. path = "/" + client_path.Path().lstrip("/") return (not any(r.Match(path) for r in blacklist_regexes) and any(r.Match(path) for r in whitelist_regexes)) return Predicate
def testArchivesFileMatchingPathGlobsWhitelist(self): handler = flow_plugin.ApiGetFlowFilesArchiveHandler( path_globs_blacklist=[], path_globs_whitelist=[rdf_paths.GlobExpression("/**/*/test.plist")]) result = handler.Handle( flow_plugin.ApiGetFlowFilesArchiveArgs( client_id=self.client_id, flow_id=self.flow_urn.Basename(), archive_format="ZIP"), token=self.token) manifest = self._GetZipManifest(result) self.assertEqual(manifest["archived_files"], 1) self.assertEqual(manifest["failed_files"], 0) self.assertEqual(manifest["processed_files"], 1) self.assertEqual(manifest["ignored_files"], 0)
def testArchivesFileMatchingPathGlobsInclusionList(self): handler = flow_plugin.ApiGetFlowFilesArchiveHandler( exclude_path_globs=[], include_only_path_globs=[ rdf_paths.GlobExpression("/**/*/test.plist") ]) result = handler.Handle(flow_plugin.ApiGetFlowFilesArchiveArgs( client_id=self.client_id, flow_id=self.flow_id, archive_format="ZIP"), token=self.token) manifest = self._GetZipManifest(result) self.assertEqual(manifest["archived_files"], 1) self.assertEqual(manifest["failed_files"], 0) self.assertEqual(manifest["processed_files"], 1) self.assertEqual(manifest["ignored_files"], 0)
def testIgnoresFileNotMatchingPathGlobsWhitelist(self): handler = flow_plugin.ApiGetFlowFilesArchiveHandler( path_globs_blacklist=[], path_globs_whitelist=[rdf_paths.GlobExpression("/**/foo.bar")]) result = handler.Handle(flow_plugin.ApiGetFlowFilesArchiveArgs( client_id=self.client_id, flow_id=self.flow_id, archive_format="ZIP"), token=self.token) manifest = self._GetZipManifest(result) self.assertEqual(manifest["archived_files"], 0) self.assertEqual(manifest["failed_files"], 0) self.assertEqual(manifest["processed_files"], 1) self.assertEqual(manifest["ignored_files"], 1) self.assertEqual( manifest["ignored_files_list"], ["aff4:/%s/fs/os%s/test.plist" % (self.client_id, self.base_path)])
def testExplainComponentsReturnsEmptyExamplesOnKbError(self): kb = rdf_client.KnowledgeBase(users=[rdf_client.User()]) ge = rdf_paths.GlobExpression("%%users.appdir%%/foo") components = ge.ExplainComponents(2, kb) self.assertEqual(components[0].examples, [])
def testRegExIsCorrectForGlobWithQuestion(self): glob_expression = rdf_paths.GlobExpression("/foo/bar/???.txt") regex = glob_expression.AsRegEx() self.assertTrue(regex.Match("/foo/bar/bla.txt")) self.assertFalse(regex.Match("/foo/bar/blah.txt"))