def testValidation(self):
glob_expression = rdf_paths.GlobExpression(
"/home/%%users.username%%/**/.mozilla/")
glob_expression.Validate()
glob_expression = rdf_paths.GlobExpression("/home/**/**")
self.assertRaises(ValueError, glob_expression.Validate)
def testGlobExpressionSplitsIntoExplainableComponents(self):
kb = rdf_client.KnowledgeBase(users=[
rdf_client.User(homedir="/home/foo"),
rdf_client.User(homedir="/home/bar"),
rdf_client.User(homedir="/home/baz"),
])
# Test for preservation of **/ because it behaves different to **.
ge = rdf_paths.GlobExpression("/foo/**/{bar,baz}/bar?/.*baz")
components = ge.ExplainComponents(2, kb)
self.assertEqual(
[c.glob_expression for c in components],
["/foo/", "**/", "{bar,baz}", "/bar", "?", "/.", "*", "baz"])
ge = rdf_paths.GlobExpression("/foo/**bar")
components = ge.ExplainComponents(2, kb)
self.assertEqual([c.glob_expression for c in components],
["/foo/", "**", "bar"])
ge = rdf_paths.GlobExpression("/foo/**10bar")
components = ge.ExplainComponents(2, kb)
self.assertEqual([c.glob_expression for c in components],
["/foo/", "**10", "bar"])
ge = rdf_paths.GlobExpression("/{foo,bar,baz}")
components = ge.ExplainComponents(2, kb)
self.assertEqual(components[1].examples, ["foo", "bar"])
ge = rdf_paths.GlobExpression("%%users.homedir%%/foo")
components = ge.ExplainComponents(2, kb)
self.assertEqual([c.glob_expression for c in components],
["%%users.homedir%%", "/foo"])
self.assertEqual(components[0].examples, ["/home/foo", "/home/bar"])
def _BuildPredicate(self, client_id, token=None):
if self.path_globs_whitelist is None:
return None
client_obj = aff4.FACTORY.Open(client_id.ToClientURN(),
aff4_type=aff4_grr.VFSGRRClient,
token=token)
blacklist_regexes = []
for expression in self.path_globs_blacklist:
for pattern in expression.Interpolate(client=client_obj):
blacklist_regexes.append(
rdf_paths.GlobExpression(pattern).AsRegEx())
whitelist_regexes = []
for expression in self.path_globs_whitelist:
for pattern in expression.Interpolate(client=client_obj):
whitelist_regexes.append(
rdf_paths.GlobExpression(pattern).AsRegEx())
def Predicate(fd):
pathspec = fd.Get(fd.Schema.PATHSPEC)
path = pathspec.CollapsePath()
return (not any(r.Match(path) for r in blacklist_regexes)
and any(r.Match(path) for r in whitelist_regexes))
return Predicate
def _BuildPredicate(self, client_id, token=None):
if self.path_globs_whitelist is None:
return None
kb = data_store_utils.GetClientKnowledgeBase(client_id)
blacklist_regexes = []
for expression in self.path_globs_blacklist:
for pattern in expression.Interpolate(knowledge_base=kb):
blacklist_regexes.append(
rdf_paths.GlobExpression(pattern).AsRegEx())
whitelist_regexes = []
for expression in self.path_globs_whitelist:
for pattern in expression.Interpolate(knowledge_base=kb):
whitelist_regexes.append(
rdf_paths.GlobExpression(pattern).AsRegEx())
def Predicate(client_path):
# Enforce leading / since Regexes require it.
path = "/" + client_path.Path().lstrip("/")
return (not any(r.Match(path) for r in blacklist_regexes)
and any(r.Match(path) for r in whitelist_regexes))
return Predicate
def FetchBinaries(self, responses):
"""Parses the Rekall response and initiates FileFinder flows."""
if not responses.success:
self.Log("Error fetching VAD data: %s", responses.status)
return
self.Log("Found %d binaries", len(responses))
if self.args.filename_regex:
binaries = []
for response in responses:
if self.args.filename_regex.Match(response.CollapsePath()):
binaries.append(response)
self.Log("Applied filename regex. Have %d files after filtering.",
len(binaries))
else:
binaries = responses
if self.args.fetch_binaries:
self.CallFlow(file_finder.FileFinder.__name__,
next_state="HandleDownloadedFiles",
paths=[
rdf_paths.GlobExpression(b.CollapsePath())
for b in binaries
],
pathtype=rdf_paths.PathSpec.PathType.OS,
action=rdf_file_finder.FileFinderAction.Download())
else:
for b in binaries:
self.SendReply(b)
def testRegExIsCorrectForGlobWithoutStars(self):
glob_expression = rdf_paths.GlobExpression("/foo/bar/blah.txt")
regex = glob_expression.AsRegEx()
self.assertTrue(regex.Match("/foo/bar/blah.txt"))
self.assertFalse(regex.Match("/foo/bar/blah2.txt"))
self.assertFalse(regex.Match("/some/foo/bar/blah2.txt"))
self.assertFalse(regex.Match("/some/foo/bar/blah2.txt/other"))
def Handle(self, args, token=None):
ge = rdf_paths.GlobExpression(args.glob_expression)
ge.Validate()
kb = data_store_utils.GetClientKnowledgeBase(str(args.client_id))
components = ge.ExplainComponents(args.example_count, kb)
return ApiExplainGlobExpressionResult(components=components)
def testIgnoresFileNotMatchingPathGlobsExclusionList(self):
handler = flow_plugin.ApiGetFlowFilesArchiveHandler(
include_only_path_globs=[rdf_paths.GlobExpression("/**/*/test.plist")],
exclude_path_globs=[rdf_paths.GlobExpression("**/*.plist")])
result = handler.Handle(
flow_plugin.ApiGetFlowFilesArchiveArgs(
client_id=self.client_id,
flow_id=self.flow_id,
archive_format="ZIP"),
context=self.context)
manifest = self._GetZipManifest(result)
self.assertEqual(manifest["archived_files"], 0)
self.assertEqual(manifest["failed_files"], 0)
self.assertEqual(manifest["processed_files"], 1)
self.assertEqual(manifest["ignored_files"], 1)
self.assertEqual(
manifest["ignored_files_list"],
["aff4:/%s/fs/os%s/test.plist" % (self.client_id, self.base_path)])
def testRegExIsCorrectForGlobWithSingleStar(self):
glob_expression = rdf_paths.GlobExpression("/foo/bar/*.txt")
regex = glob_expression.AsRegEx()
self.assertTrue(regex.Match("/foo/bar/blah.txt"))
self.assertFalse(regex.Match("/foo/bar/blah.plist"))
self.assertFalse(regex.Match("/foo/bar/blah/blah.txt"))
self.assertFalse(regex.Match("/foo/blah1/blah2/bar/blah.txt"))
def testIgnoresFileNotMatchingPathGlobsBlacklist(self):
handler = flow_plugin.ApiGetFlowFilesArchiveHandler(
path_globs_whitelist=[rdf_paths.GlobExpression("/**/*/test.plist")],
path_globs_blacklist=[rdf_paths.GlobExpression("**/*.plist")])
result = handler.Handle(
flow_plugin.ApiGetFlowFilesArchiveArgs(
client_id=self.client_id,
flow_id=self.flow_urn.Basename(),
archive_format="ZIP"),
token=self.token)
manifest = self._GetZipManifest(result)
self.assertEqual(manifest["archived_files"], 0)
self.assertEqual(manifest["failed_files"], 0)
self.assertEqual(manifest["processed_files"], 1)
self.assertEqual(manifest["ignored_files"], 1)
self.assertEqual(manifest["ignored_files_list"], [
utils.SmartUnicode(
self.client_id.Add("fs/os").Add(self.base_path).Add("test.plist"))
])
def testRegExIsCorrectForGlobWithGrouping(self):
glob_expression = rdf_paths.GlobExpression("/foo/{bar,other}/*{.txt,.exe}")
regex = glob_expression.AsRegEx()
self.assertTrue(regex.Match("/foo/bar/blah.txt"))
self.assertTrue(regex.Match("/foo/other/blah2.txt"))
self.assertTrue(regex.Match("/foo/bar/blah.exe"))
self.assertTrue(regex.Match("/foo/other/blah2.exe"))
self.assertFalse(regex.Match("/foo/other2/blah.txt"))
self.assertFalse(regex.Match("/foo/bar/blah.com"))
def testRegExIsCaseInsensitive(self):
glob_expression = rdf_paths.GlobExpression("/foo/**/bar?/*{.txt,.exe}")
regex = glob_expression.AsRegEx()
self.assertTrue(regex.Match("/foo/bar1/blah.txt"))
self.assertTrue(regex.Match("/foO/bAr1/blah.txt"))
self.assertTrue(regex.Match("/foo/bar1/blah.TXT"))
self.assertFalse(regex.Match("/foo/bar2/blah.com"))
self.assertFalse(regex.Match("/foO/bAr2/blah.com"))
self.assertFalse(regex.Match("/foo/bar2/blah.COM"))
def testRegExIsCorrectForGlobWithTwoStars(self):
glob_expression = rdf_paths.GlobExpression("/foo/**/bar.txt")
regex = glob_expression.AsRegEx()
self.assertTrue(regex.Match("/foo/bar.txt"))
self.assertTrue(regex.Match("/foo/blah/bar.txt"))
self.assertTrue(regex.Match("/foo/blah1/blah2/bar.txt"))
self.assertFalse(regex.Match("/foo/bar.plist"))
self.assertFalse(regex.Match("/foo/blah/bar.txt/res"))
self.assertFalse(regex.Match("/foo/blah1/blah2/bar.txt2"))
def testGroupingInterpolation(self):
glob_expression = rdf_paths.GlobExpression()
interpolated = glob_expression.InterpolateGrouping("/home/*.{sh,deb}")
self.assertCountEqual(interpolated, [u"/home/*.deb", u"/home/*.sh"])
interpolated = glob_expression.InterpolateGrouping("/home/*.{sh, deb}")
self.assertCountEqual(interpolated, [u"/home/*. deb", u"/home/*.sh"])
interpolated = glob_expression.InterpolateGrouping(
"HKEY_CLASSES_ROOT/CLSID/{16d12736-7a9e-4765-bec6-f301d679caaa}")
self.assertCountEqual(interpolated, [
u"HKEY_CLASSES_ROOT/CLSID/{16d12736-7a9e-4765-bec6-f301d679caaa}"
])
def testRegExIsCorrectForComplexGlob(self):
glob_expression = rdf_paths.GlobExpression("/foo/**/bar?/*{.txt,.exe}")
regex = glob_expression.AsRegEx()
self.assertTrue(regex.Match("/foo/bar1/blah.txt"))
self.assertTrue(regex.Match("/foo/bar2/blah.exe"))
self.assertTrue(regex.Match("/foo/c1/c2/c3/bar1/blah.txt"))
self.assertTrue(regex.Match("/foo/c1/c2/c3/bar2/blah.exe"))
self.assertFalse(regex.Match("/foo/bar/blah.txt"))
self.assertFalse(regex.Match("/foo/bar2/blah.com"))
self.assertFalse(regex.Match("/foo/c1/c2/c3/bar1/blah.txt/res.txt"))
self.assertFalse(regex.Match("/foo/c1/c2/c3/bar2/blah.exe/res.exe"))
def _BuildPredicate(self, client_id, context=None):
if self.include_only_path_globs is None:
return None
kb = data_store_utils.GetClientKnowledgeBase(client_id)
exclude_regexes = []
for expression in self.exclude_path_globs:
for pattern in expression.Interpolate(knowledge_base=kb):
exclude_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx())
include_only_regexes = []
for expression in self.include_only_path_globs:
for pattern in expression.Interpolate(knowledge_base=kb):
include_only_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx())
def Predicate(client_path):
# Enforce leading / since Regexes require it.
path = "/" + client_path.Path().lstrip("/")
return (not any(r.Match(path) for r in exclude_regexes) and
any(r.Match(path) for r in include_only_regexes))
return Predicate
def _BuildPredicate(self, client_id, token=None):
if self.path_globs_whitelist is None:
return None
client_obj = aff4.FACTORY.Open(
client_id.ToClientURN(), aff4_type=aff4_grr.VFSGRRClient, token=token)
blacklist_regexes = []
for expression in self.path_globs_blacklist:
for pattern in expression.Interpolate(client=client_obj):
blacklist_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx())
whitelist_regexes = []
for expression in self.path_globs_whitelist:
for pattern in expression.Interpolate(client=client_obj):
whitelist_regexes.append(rdf_paths.GlobExpression(pattern).AsRegEx())
def Predicate(client_path):
# Enforce leading / since Regexes require it.
path = "/" + client_path.Path().lstrip("/")
return (not any(r.Match(path) for r in blacklist_regexes) and
any(r.Match(path) for r in whitelist_regexes))
return Predicate
def testArchivesFileMatchingPathGlobsWhitelist(self):
handler = flow_plugin.ApiGetFlowFilesArchiveHandler(
path_globs_blacklist=[],
path_globs_whitelist=[rdf_paths.GlobExpression("/**/*/test.plist")])
result = handler.Handle(
flow_plugin.ApiGetFlowFilesArchiveArgs(
client_id=self.client_id,
flow_id=self.flow_urn.Basename(),
archive_format="ZIP"),
token=self.token)
manifest = self._GetZipManifest(result)
self.assertEqual(manifest["archived_files"], 1)
self.assertEqual(manifest["failed_files"], 0)
self.assertEqual(manifest["processed_files"], 1)
self.assertEqual(manifest["ignored_files"], 0)
def testArchivesFileMatchingPathGlobsInclusionList(self):
handler = flow_plugin.ApiGetFlowFilesArchiveHandler(
exclude_path_globs=[],
include_only_path_globs=[
rdf_paths.GlobExpression("/**/*/test.plist")
])
result = handler.Handle(flow_plugin.ApiGetFlowFilesArchiveArgs(
client_id=self.client_id,
flow_id=self.flow_id,
archive_format="ZIP"),
token=self.token)
manifest = self._GetZipManifest(result)
self.assertEqual(manifest["archived_files"], 1)
self.assertEqual(manifest["failed_files"], 0)
self.assertEqual(manifest["processed_files"], 1)
self.assertEqual(manifest["ignored_files"], 0)
def testIgnoresFileNotMatchingPathGlobsWhitelist(self):
handler = flow_plugin.ApiGetFlowFilesArchiveHandler(
path_globs_blacklist=[],
path_globs_whitelist=[rdf_paths.GlobExpression("/**/foo.bar")])
result = handler.Handle(flow_plugin.ApiGetFlowFilesArchiveArgs(
client_id=self.client_id,
flow_id=self.flow_id,
archive_format="ZIP"),
token=self.token)
manifest = self._GetZipManifest(result)
self.assertEqual(manifest["archived_files"], 0)
self.assertEqual(manifest["failed_files"], 0)
self.assertEqual(manifest["processed_files"], 1)
self.assertEqual(manifest["ignored_files"], 1)
self.assertEqual(
manifest["ignored_files_list"],
["aff4:/%s/fs/os%s/test.plist" % (self.client_id, self.base_path)])
def testExplainComponentsReturnsEmptyExamplesOnKbError(self):
kb = rdf_client.KnowledgeBase(users=[rdf_client.User()])
ge = rdf_paths.GlobExpression("%%users.appdir%%/foo")
components = ge.ExplainComponents(2, kb)
self.assertEqual(components[0].examples, [])
def testRegExIsCorrectForGlobWithQuestion(self):
glob_expression = rdf_paths.GlobExpression("/foo/bar/???.txt")
regex = glob_expression.AsRegEx()
self.assertTrue(regex.Match("/foo/bar/bla.txt"))
self.assertFalse(regex.Match("/foo/bar/blah.txt"))