def _RunGlob(self, paths):
self.flow_replies = []
client_mock = action_mocks.GlobClientMock()
with utils.Stubber(flow.GRRFlow, "SendReply", self._MockSendReply):
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token):
pass
def _RunRegistryFinder(self, paths=None):
client_mock = action_mocks.GlobClientMock()
client_id = self.SetupClients(1)[0]
for s in test_lib.TestFlowHelper("RegistryFinder",
client_mock,
client_id=client_id,
keys_paths=paths,
conditions=[],
token=self.token):
session_id = s
return list(
flow.GRRFlow.ResultCollectionForFID(session_id, token=self.token))
def testGlobGrouping(self):
"""Test that glob expands directories."""
pattern = "test_data/{ntfs_img.dd,*.log,*.raw}"
client_mock = action_mocks.GlobClientMock()
path = os.path.join(os.path.dirname(self.base_path), pattern)
# Run the flow.
for _ in test_lib.TestFlowHelper("Glob",
client_mock,
client_id=self.client_id,
paths=[path],
token=self.token):
pass
def testGlobRoundtrips(self):
"""Tests that glob doesn't use too many client round trips."""
for pattern, num_find, num_stat, duplicated_ok in [
("test_data/test_artifact.json", 0, 1, False),
("test_data/test_*", 1, 0, False),
("test_*/test_artifact.json", 1, 1, False),
("test_*/test_*", 2, 0, False),
("test_*/test_{artifact,artifacts}.json", 1, 2, True),
("test_data/test_{artifact,artifacts}.json", 0, 2, False),
("test_data/{ntfs_img.dd,*.log,*.raw}", 1, 1, False),
("test_data/{*.log,*.raw}", 1, 0, False),
("test_data/a/**/helloc.txt", 1, None, False),
("test_data/a/**/hello{c,d}.txt", 1, None, True),
("test_data/a/**/hello*.txt", 4, None, False),
("test_data/a/**.txt", 1, None, False),
("test_data/a/**5*.txt", 1, None, False),
("test_data/a/**{.json,.txt}", 1, 0, False),
]:
path = os.path.join(os.path.dirname(self.base_path), pattern)
client_mock = action_mocks.GlobClientMock()
# Run the flow.
for _ in test_lib.TestFlowHelper("Glob",
client_mock,
client_id=self.client_id,
paths=[path],
token=self.token):
pass
if num_find is not None:
self.assertEqual(client_mock.action_counts.get("Find", 0),
num_find)
if num_stat is not None:
self.assertEqual(client_mock.action_counts.get("StatFile", 0),
num_stat)
if not duplicated_ok:
# Check for duplicate client calls. There might be duplicates that are
# very cheap when we look for a wildcard (* or **) first and later in
# the pattern for a group of files ({}).
for method in "StatFile", "Find":
stat_args = client_mock.recorded_args.get(method, [])
stat_paths = [c.pathspec.CollapsePath() for c in stat_args]
self.assertListEqual(sorted(stat_paths),
sorted(set(stat_paths)))
def testGlobWithInvalidStarStar(self):
client_mock = action_mocks.GlobClientMock()
# This glob is invalid since it uses 2 ** expressions..
paths = [
os.path.join(self.base_path, "test_img.dd", "**", "**", "foo")
]
# Make sure the flow raises.
self.assertRaises(
ValueError, list,
test_lib.TestFlowHelper("Glob",
client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token))
def testGlobWildcardsAndTSK(self):
client_mock = action_mocks.GlobClientMock()
# This glob should find this file in test data: glob_test/a/b/foo.
path = os.path.join(self.base_path, "test_IMG.dd", "glob_test", "a",
"b", "FOO*")
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=[path],
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token):
pass
output_path = self.client_id.Add("fs/tsk").Add(
os.path.join(self.base_path, "test_img.dd", "glob_test", "a", "b"))
fd = aff4.FACTORY.Open(output_path, token=self.token)
children = list(fd.ListChildren())
self.assertEqual(len(children), 1)
self.assertEqual(children[0].Basename(), "foo")
aff4.FACTORY.Delete(self.client_id.Add("fs/tsk").Add(
os.path.join(self.base_path)),
token=self.token)
# Specifying a wildcard for the image will not open it.
path = os.path.join(self.base_path, "*.dd", "glob_test", "a", "b",
"FOO*")
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=[path],
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token):
pass
fd = aff4.FACTORY.Open(output_path, token=self.token)
children = list(fd.ListChildren())
self.assertEqual(len(children), 0)
def testGlobWithStarStarRootPath(self):
"""Test ** expressions with root_path."""
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(rdf_client.User(username="******"))
kb.MergeOrAddUser(rdf_client.User(username="******"))
client.Set(kb)
client.Close()
client_mock = action_mocks.GlobClientMock()
# Glob for foo at a depth of 4.
path = os.path.join("foo**4")
root_path = rdf_paths.PathSpec(path=os.path.join(
self.base_path, "test_img.dd"),
pathtype=rdf_paths.PathSpec.PathType.OS)
root_path.Append(path="/", pathtype=rdf_paths.PathSpec.PathType.TSK)
# Run the flow.
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=[path],
root_path=root_path,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token):
pass
output_path = self.client_id.Add("fs/tsk").Add(
self.base_path.replace("\\", "/")).Add("test_img.dd/glob_test/a/b")
children = []
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
# We should find some files.
self.assertEqual(children, ["foo"])
def _RunRegistryFinder(self, paths=None):
client_mock = action_mocks.GlobClientMock()
client_id = self.SetupClients(1)[0]
for s in test_lib.TestFlowHelper("RegistryFinder",
client_mock,
client_id=client_id,
keys_paths=paths,
conditions=[],
token=self.token):
session_id = s
try:
return list(
aff4.FACTORY.Open(
session_id.Add(flow_runner.RESULTS_SUFFIX),
aff4_type=sequential_collection.GeneralIndexedCollection,
token=self.token))
except aff4.InstantiationError:
return []
def testGlobDirectory(self):
"""Test that glob expands directories."""
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(
rdf_client.User(username="******", appdata="test_data/index.dat"))
kb.MergeOrAddUser(
rdf_client.User(username="******", appdata="test_data/History"))
# This is a record which means something to the interpolation system. We
# should not process this especially.
kb.MergeOrAddUser(rdf_client.User(username="******",
appdata="%%PATH%%"))
client.Set(kb)
client.Close()
client_mock = action_mocks.GlobClientMock()
# This glob selects all files which start with the username on this system.
path = os.path.join(os.path.dirname(self.base_path),
"%%users.appdata%%")
# Run the flow.
for _ in test_lib.TestFlowHelper("Glob",
client_mock,
client_id=self.client_id,
paths=[path],
token=self.token):
pass
path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat")
aff4.FACTORY.Open(path, aff4_type=aff4_grr.VFSFile, token=self.token)
path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat")
aff4.FACTORY.Open(path, aff4_type=aff4_grr.VFSFile, token=self.token)
def testIllegalGlobAsync(self):
# When running the flow asynchronously, we will not receive any errors from
# the Start method, but the flow should still fail.
paths = ["Test/%%Weird_illegal_attribute%%"]
client_mock = action_mocks.GlobClientMock()
# Run the flow.
session_id = None
# This should not raise here since the flow is run asynchronously.
for session_id in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
check_flow_errors=False,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token,
sync=False):
pass
fd = aff4.FACTORY.Open(session_id, token=self.token)
self.assertIn("KnowledgeBaseInterpolationError", fd.context.backtrace)
self.assertEqual("ERROR", str(fd.context.state))
def testGlob(self):
"""Test that glob works properly."""
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(rdf_client.User(username="******"))
kb.MergeOrAddUser(rdf_client.User(username="******"))
client.Set(kb)
client.Close()
client_mock = action_mocks.GlobClientMock()
# This glob selects all files which start with the username on this system.
paths = [
os.path.join(self.base_path, "%%Users.username%%*"),
os.path.join(self.base_path, "VFSFixture/var/*/wtmp")
]
# Set iterator really low to force iteration.
with utils.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2):
for _ in test_lib.TestFlowHelper(
"Glob",
client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token,
sync=False,
check_flow_errors=False):
pass
output_path = self.client_id.Add("fs/os").Add(
self.base_path.replace("\\", "/"))
children = []
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
filename = child.Basename()
if filename != "VFSFixture":
children.append(filename)
expected = [
filename for filename in os.listdir(self.base_path)
if filename.startswith("test") or filename.startswith("syslog")
]
self.assertTrue([x for x in expected if x.startswith("test")],
"Need a file starting with 'test'"
" in test_data for this test!")
self.assertTrue([x for x in expected if x.startswith("syslog")],
"Need a file starting with 'syslog'"
" in test_data for this test!")
self.assertItemsEqual(expected, children)
children = []
fd = aff4.FACTORY.Open(output_path.Add("VFSFixture/var/log"),
token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
self.assertItemsEqual(children, ["wtmp"])
