def testGlobWithStarStarRootPath(self):
"""Test ** expressions with root_path."""
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(rdf_client.User(username="******"))
kb.MergeOrAddUser(rdf_client.User(username="******"))
client.Set(kb)
client.Close()
client_mock = action_mocks.GlobClientMock()
# Glob for foo at a depth of 4.
path = os.path.join("foo**4")
root_path = rdf_paths.PathSpec(path=os.path.join(
self.base_path, "test_img.dd"),
pathtype=rdf_paths.PathSpec.PathType.OS)
root_path.Append(path="/", pathtype=rdf_paths.PathSpec.PathType.TSK)
# Run the flow.
for _ in flow_test_lib.TestFlowHelper(
filesystem.Glob.__name__,
client_mock,
client_id=self.client_id,
paths=[path],
root_path=root_path,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token):
pass
output_path = self.client_id.Add("fs/tsk").Add(
self.base_path.replace("\\", "/")).Add("test_img.dd/glob_test/a/b")
children = []
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
# We should find some files.
self.assertEqual(children, ["foo"])
def testGlobWildcardsAndTSK(self):
client_mock = action_mocks.GlobClientMock()
# This glob should find this file in test data: glob_test/a/b/foo.
path = os.path.join(self.base_path, "test_IMG.dd", "glob_test", "a", "b",
"FOO*")
flow_test_lib.TestFlowHelper(
filesystem.Glob.__name__,
client_mock,
client_id=self.client_id,
paths=[path],
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token)
output_path = self.client_id.Add("fs/tsk").Add(
os.path.join(self.base_path, "test_img.dd", "glob_test", "a", "b"))
fd = aff4.FACTORY.Open(output_path, token=self.token)
children = list(fd.ListChildren())
self.assertEqual(len(children), 1)
self.assertEqual(children[0].Basename(), "foo")
aff4.FACTORY.Delete(
self.client_id.Add("fs/tsk").Add(os.path.join(self.base_path)),
token=self.token)
# Specifying a wildcard for the image will not open it.
path = os.path.join(self.base_path, "*.dd", "glob_test", "a", "b", "FOO*")
flow_test_lib.TestFlowHelper(
filesystem.Glob.__name__,
client_mock,
client_id=self.client_id,
paths=[path],
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token)
fd = aff4.FACTORY.Open(output_path, token=self.token)
children = list(fd.ListChildren())
self.assertEqual(len(children), 0)
def testGlobWithStarStarRootPath(self):
"""Test ** expressions with root_path."""
users = [
rdf_client.User(username="******"),
rdf_client.User(username="******")
]
self.client_id = self.SetupClient(0, users=users)
client_mock = action_mocks.GlobClientMock()
# Glob for foo at a depth of 4.
path = os.path.join("foo**4")
root_path = rdf_paths.PathSpec(
path=os.path.join(self.base_path, "test_img.dd"),
pathtype=rdf_paths.PathSpec.PathType.OS)
root_path.Append(path="/", pathtype=rdf_paths.PathSpec.PathType.TSK)
# Run the flow.
flow_test_lib.TestFlowHelper(
compatibility.GetName(filesystem.Glob),
client_mock,
client_id=self.client_id,
paths=[path],
root_path=root_path,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token)
if data_store.AFF4Enabled():
children = []
output_path = self.client_id.Add("fs/tsk").Add(
self.base_path).Add("test_img.dd/glob_test/a/b")
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
# We should find some files.
self.assertEqual(children, ["foo"])
else:
children = self._ListTestChildPathInfos(
["test_img.dd", "glob_test", "a", "b"])
self.assertLen(children, 1)
self.assertEqual(children[0].components[-1], "foo")
def testGlob(self):
"""Test that glob works properly."""
users = [
rdf_client.User(username="******"),
rdf_client.User(username="******")
]
client_id = self.SetupClient(0, users=users)
client_mock = action_mocks.GlobClientMock()
# This glob selects all files which start with the username on this system.
paths = [
os.path.join(self.base_path, "%%Users.username%%*"),
os.path.join(self.base_path, "VFSFixture/var/*/wtmp")
]
flow_test_lib.TestFlowHelper(compatibility.GetName(filesystem.Glob),
client_mock,
client_id=client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token,
check_flow_errors=False)
expected_files = [
filename for filename in os.listdir(self.base_path)
if filename.startswith("test") or filename.startswith("syslog")
]
expected_files.append("VFSFixture")
children = self._ListTestChildPathInfos(
[], path_type=rdf_objects.PathInfo.PathType.OS)
found_files = [child.components[-1] for child in children]
self.assertCountEqual(expected_files, found_files)
children = self._ListTestChildPathInfos(
["VFSFixture", "var", "log"],
path_type=rdf_objects.PathInfo.PathType.OS)
self.assertLen(children, 1)
self.assertEqual(children[0].components[-1], "wtmp")
def testGlobDirectory(self):
"""Test that glob expands directories."""
self.client_id = self.SetupClient(0)
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(
rdf_client.User(username="******", appdata="test_data/index.dat"))
kb.MergeOrAddUser(
rdf_client.User(username="******", appdata="test_data/History"))
# This is a record which means something to the interpolation system. We
# should not process this especially.
kb.MergeOrAddUser(rdf_client.User(username="******", appdata="%%PATH%%"))
client.Set(kb)
client.Close()
client_mock = action_mocks.GlobClientMock()
# This glob selects all files which start with the username on this system.
path = os.path.join(os.path.dirname(self.base_path), "%%users.appdata%%")
# Run the flow.
for _ in flow_test_lib.TestFlowHelper(
filesystem.Glob.__name__,
client_mock,
client_id=self.client_id,
paths=[path],
token=self.token):
pass
path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat")
aff4.FACTORY.Open(path, aff4_type=aff4_grr.VFSFile, token=self.token)
path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat")
aff4.FACTORY.Open(path, aff4_type=aff4_grr.VFSFile, token=self.token)
def testGlobWildcardOnImage(self):
client_mock = action_mocks.GlobClientMock()
# Specifying a wildcard for the image will not open it.
path = os.path.join(self.base_path, "*.dd", "glob_test", "a", "b", "FOO*")
flow_test_lib.TestFlowHelper(
compatibility.GetName(filesystem.Glob),
client_mock,
client_id=self.client_id,
paths=[path],
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token)
if data_store.AFF4Enabled():
output_path = self.client_id.Add("fs/tsk").Add(
os.path.join(self.base_path, "test_img.dd", "glob_test", "a", "b"))
fd = aff4.FACTORY.Open(output_path, token=self.token)
children = list(fd.ListChildren())
self.assertEmpty(children)
else:
children = self._ListTestChildPathInfos(
["test_img.dd", "glob_test", "a", "b"])
self.assertEmpty(children)
def testGlobWithWildcardsInsideTSKFileCaseInsensitive(self):
client_mock = action_mocks.GlobClientMock()
# This glob should find this file in test data: glob_test/a/b/foo.
path = os.path.join("*", "a", "b", "FOO*")
root_path = rdf_paths.PathSpec(path=os.path.join(
self.base_path, "test_IMG.dd"),
pathtype=rdf_paths.PathSpec.PathType.OS)
root_path.Append(path="/", pathtype=rdf_paths.PathSpec.PathType.TSK)
# Run the flow.
flow_test_lib.TestFlowHelper(compatibility.GetName(filesystem.Glob),
client_mock,
client_id=self.client_id,
paths=[path],
root_path=root_path,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token)
children = self._ListTestChildPathInfos(
["test_img.dd", "glob_test", "a", "b"])
self.assertLen(children, 1)
self.assertEqual(children[0].components[-1], "foo")
def testIllegalGlobAsync(self):
# When running the flow asynchronously, we will not receive any errors from
# the Start method, but the flow should still fail.
paths = ["Test/%%Weird_illegal_attribute%%"]
client_mock = action_mocks.GlobClientMock()
# Run the flow.
session_id = None
# This should not raise here since the flow is run asynchronously.
session_id = flow_test_lib.TestFlowHelper(
filesystem.Glob.__name__,
client_mock,
client_id=self.client_id,
check_flow_errors=False,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token,
sync=False)
fd = aff4.FACTORY.Open(session_id, token=self.token)
self.assertIn("KnowledgeBaseInterpolationError", fd.context.backtrace)
self.assertEqual("ERROR", str(fd.context.state))
def testGlob(self):
"""Test that glob works properly."""
client_id = self.SetupClient(0)
# Add some usernames we can interpolate later.
client = aff4.FACTORY.Open(client_id, mode="rw", token=self.token)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
kb.MergeOrAddUser(rdf_client.User(username="******"))
kb.MergeOrAddUser(rdf_client.User(username="******"))
client.Set(kb)
client.Close()
client_mock = action_mocks.GlobClientMock()
# This glob selects all files which start with the username on this system.
paths = [
os.path.join(self.base_path, "%%Users.username%%*"),
os.path.join(self.base_path, "VFSFixture/var/*/wtmp")
]
# Set iterator really low to force iteration.
with utils.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2):
for _ in flow_test_lib.TestFlowHelper(
filesystem.Glob.__name__,
client_mock,
client_id=client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
token=self.token,
sync=False,
check_flow_errors=False):
pass
output_path = client_id.Add("fs/os").Add(self.base_path.replace("\\", "/"))
children = []
fd = aff4.FACTORY.Open(output_path, token=self.token)
for child in fd.ListChildren():
filename = child.Basename()
if filename != "VFSFixture":
children.append(filename)
expected = [
filename for filename in os.listdir(self.base_path)
if filename.startswith("test") or filename.startswith("syslog")
]
self.assertTrue([x for x in expected if x.startswith("test")],
"Need a file starting with 'test'"
" in test_data for this test!")
self.assertTrue([x for x in expected if x.startswith("syslog")],
"Need a file starting with 'syslog'"
" in test_data for this test!")
self.assertItemsEqual(expected, children)
children = []
fd = aff4.FACTORY.Open(
output_path.Add("VFSFixture/var/log"), token=self.token)
for child in fd.ListChildren():
children.append(child.Basename())
self.assertItemsEqual(children, ["wtmp"])
def testOldClientSnapshotFallbackIfInterpolationFails(self):
rel_db = data_store.REL_DB
client_id = "C.0123456789abcdef"
rel_db.WriteClientMetadata(client_id, first_seen=rdfvalue.RDFDatetime.Now())
# Write some fake snapshot history.
kb_0 = rdf_client.KnowledgeBase(os="Linux")
kb_0.users = [
rdf_client.User(username="******"),
rdf_client.User(username="******"),
]
snapshot_0 = rdf_objects.ClientSnapshot(
client_id=client_id, knowledge_base=kb_0)
rel_db.WriteClientSnapshot(snapshot_0)
kb_1 = rdf_client.KnowledgeBase(os="Linux")
snapshot_1 = rdf_objects.ClientSnapshot(
client_id=client_id, knowledge_base=kb_1)
rel_db.WriteClientSnapshot(snapshot_1)
kb_2 = rdf_client.KnowledgeBase(os="Linux")
snapshot_2 = rdf_objects.ClientSnapshot(
client_id=client_id, knowledge_base=kb_2)
rel_db.WriteClientSnapshot(snapshot_2)
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
filesystem_test_lib.CreateFile(
os.path.join(dirpath, "user1", "quux", "thud"))
filesystem_test_lib.CreateFile(
os.path.join(dirpath, "user2", "quux", "norf"))
# Write a fake artifact.
path = os.path.join(dirpath, "%%users.username%%", "quux", "*")
art = rdf_artifacts.Artifact(
name="Quux",
doc="Lorem ipsum.",
sources=[
rdf_artifacts.ArtifactSource(
type=rdf_artifacts.ArtifactSource.SourceType.DIRECTORY,
attributes={
"paths": [path],
}),
])
rel_db.WriteArtifact(art)
artifact_registry.REGISTRY.ReloadDatastoreArtifacts()
flow_id = flow_test_lib.TestFlowHelper(
compatibility.GetName(collectors.ArtifactCollectorFlow),
client_mock=action_mocks.GlobClientMock(),
client_id=client_id,
artifact_list=["Quux"],
old_client_snapshot_fallback=True,
token=self.token)
results = flow_test_lib.GetFlowResults(client_id=client_id, flow_id=flow_id)
self.assertNotEmpty(results)
basenames = [os.path.basename(result.pathspec.path) for result in results]
self.assertCountEqual(basenames, ["thud", "norf"])
