def testExtAttrsCollection(self):
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
foo_filepath = temp.TempFilePath(dir=temp_dirpath)
filesystem_test_lib.SetExtAttr(
foo_filepath, name="user.quux", value="foo")
bar_filepath = temp.TempFilePath(dir=temp_dirpath)
filesystem_test_lib.SetExtAttr(
bar_filepath, name="user.quux", value="bar")
baz_filepath = temp.TempFilePath(dir=temp_dirpath)
filesystem_test_lib.SetExtAttr(
baz_filepath, name="user.quux", value="baz")
request = rdf_client_fs.FindSpec(
pathspec=rdf_paths.PathSpec(
path=temp_dirpath, pathtype=rdf_paths.PathSpec.PathType.OS),
path_glob="*",
collect_ext_attrs=True)
request.iterator.number = 100
hits = []
for response in self.RunAction(searching.Find, request):
if isinstance(response, rdf_client_fs.FindSpec):
hits.append(response.hit)
self.assertLen(hits, 3)
values = []
for hit in hits:
self.assertLen(hit.ext_attrs, 1)
values.append(hit.ext_attrs[0].value)
self.assertCountEqual(values, ["foo", "bar", "baz"])
def testMultipleResults(self):
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
with io.open(os.path.join(dirpath, "foo"), "wb") as filedesc:
del filedesc # Unused.
with io.open(os.path.join(dirpath, "bar"), "wb") as filedesc:
del filedesc # Unused.
with io.open(os.path.join(dirpath, "baz"), "wb") as filedesc:
del filedesc # Unused.
query = """
SELECT filename FROM file
WHERE directory = "{}"
ORDER BY filename;
""".format(dirpath)
with test_lib.ConfigOverrider({"Osquery.max_chunk_size": 3}):
results = _Query(query)
self.assertLen(results, 3)
for result in results:
self.assertEqual(result.table.query, query)
self.assertLen(result.table.header.columns, 1)
self.assertEqual(result.table.header.columns[0].name,
"filename")
self.assertEqual(list(results[0].table.Column("filename")),
["bar"])
self.assertEqual(list(results[1].table.Column("filename")),
["baz"])
self.assertEqual(list(results[2].table.Column("filename")),
["foo"])
def testTimestamp(self):
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
filepath = os.path.join(dirpath, "foo")
with open(filepath, mode="wb"):
pass
with open(filepath, mode="wb") as filedesc:
filedesc.write(b"quux")
with open(filepath, mode="rb") as filedesc:
_ = filedesc.read()
now_ns = time.time_ns()
entries = list(timeline.Walk(dirpath.encode("utf-8")))
self.assertLen(entries, 2)
self.assertEqual(entries[0].path, dirpath.encode("utf-8"))
self.assertEqual(entries[1].path, filepath.encode("utf-8"))
self.assertGreater(entries[1].ctime_ns, 0)
self.assertGreaterEqual(entries[1].mtime_ns, entries[1].ctime_ns)
self.assertGreaterEqual(entries[1].atime_ns, entries[1].mtime_ns)
self.assertLess(entries[1].atime_ns, now_ns)
def testNestedDirectories(self):
with temp.AutoTempDirPath(remove_non_empty=True) as root_dirpath:
foobar_dirpath = os.path.join(root_dirpath, "foo", "bar")
os.makedirs(foobar_dirpath)
foobaz_dirpath = os.path.join(root_dirpath, "foo", "baz")
os.makedirs(foobaz_dirpath)
quuxnorfthud_dirpath = os.path.join(root_dirpath, "quux", "norf", "thud")
os.makedirs(quuxnorfthud_dirpath)
entries = list(timeline.Walk(root_dirpath.encode("utf-8")))
self.assertLen(entries, 7)
paths = [_.path.decode("utf-8") for _ in entries]
self.assertCountEqual(paths, [
os.path.join(root_dirpath),
os.path.join(root_dirpath, "foo"),
os.path.join(root_dirpath, "foo", "bar"),
os.path.join(root_dirpath, "foo", "baz"),
os.path.join(root_dirpath, "quux"),
os.path.join(root_dirpath, "quux", "norf"),
os.path.join(root_dirpath, "quux", "norf", "thud"),
])
for entry in entries:
self.assertTrue(stat_mode.S_ISDIR(entry.mode))
def testMultipleResults(self):
row_count = 100
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
for i in range(row_count):
filepath = os.path.join(dirpath, "{:04}".format(i))
with io.open(filepath, "wb") as filedesc:
del filedesc # Unused.
query = """
SELECT filename FROM file
WHERE directory = "{}"
ORDER BY filename;
""".format(dirpath)
# Size limit is set so that each chunk should contain 2 rows.
with test_lib.ConfigOverrider({"Osquery.max_chunk_size": 10}):
results = self._RunFlow(query)
# Since each chunk is expected to have 2 rows, number of rows should be
# equal to twice the amount of chunks.
self.assertEqual(2 * len(results), row_count)
for i, result in enumerate(results):
table = result.table
self.assertEqual(table.query, query)
self.assertLen(table.header.columns, 1)
self.assertEqual(table.header.columns[0].name, "filename")
self.assertLen(table.rows, 2)
self.assertEqual(list(table.Column("filename")), [
"{:04}".format(2 * i),
"{:04}".format(2 * i + 1),
])
def SetUpTestFiles():
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
file_bar_path = os.path.join(temp_dirpath, "bar")
with open(file_bar_path, "wb") as fd:
fd.write(b"bar")
file_baz_path = os.path.join(temp_dirpath, "baz")
with open(file_baz_path, "wb") as fd:
fd.write(b"baz")
file_foo_path = os.path.join(temp_dirpath, "foo")
with open(file_foo_path, "wb") as fd:
fd.write(b"foo")
yield {
"bar":
TestFile(path=file_bar_path,
sha1=hashlib.sha1(b"bar").hexdigest()),
"baz":
TestFile(path=file_baz_path,
sha1=hashlib.sha1(b"baz").hexdigest()),
"foo":
TestFile(path=file_foo_path,
sha1=hashlib.sha1(b"foo").hexdigest()),
}
def testDir(self):
with temp.AutoTempDirPath() as dirpath:
filepath = temp.TempFilePath(dir=dirpath)
self.assertTrue(os.path.exists(filepath))
self.assertStartsWith(filepath, dirpath)
os.remove(filepath)
def testLs_EmptyDirectory(self):
data_store.REL_DB.WriteClientMetadata(
client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False)
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
fs_obj = fs.FileSystem(self._get_fake_api_client(),
jobs_pb2.PathSpec.OS)
stat_entries = fs_obj.ls(temp_dirpath)
self.assertEmpty(stat_entries)
def testGetFlagsSymlink(self):
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath, \
temp.AutoTempFilePath() as temp_filepath:
temp_linkpath = os.path.join(temp_dirpath, "foo")
os.symlink(temp_filepath, temp_linkpath)
stat = filesystem.Stat.FromPath(temp_linkpath, follow_symlink=False)
self.assertTrue(stat.IsSymlink())
self.assertEqual(stat.GetLinuxFlags(), 0)
self.assertEqual(stat.GetOsxFlags(), 0)
def testContents(self):
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
filepath = os.path.join(dirpath, "foo")
filesystem_test_lib.CreateFile(filepath, content=b"foobarbaz")
with io.open(filepath, "rb") as filedesc:
content = filedesc.read()
self.assertEqual(content, b"foobarbaz")
def testOldClientSnapshotFallbackUsesLatestApplicable(self):
rel_db = data_store.REL_DB
client_id = "C.0123456789abcdef"
rel_db.WriteClientMetadata(client_id, first_seen=rdfvalue.RDFDatetime.Now())
# Write some fake snapshot history.
kb_0 = rdf_client.KnowledgeBase(os="Linux", os_release="rel0")
snapshot_0 = rdf_objects.ClientSnapshot(
client_id=client_id, knowledge_base=kb_0)
rel_db.WriteClientSnapshot(snapshot_0)
kb_1 = rdf_client.KnowledgeBase(os="Linux", os_release="rel1")
snapshot_1 = rdf_objects.ClientSnapshot(
client_id=client_id, knowledge_base=kb_1)
rel_db.WriteClientSnapshot(snapshot_1)
kb_2 = rdf_client.KnowledgeBase(os="Linux")
snapshot_2 = rdf_objects.ClientSnapshot(
client_id=client_id, knowledge_base=kb_2)
rel_db.WriteClientSnapshot(snapshot_2)
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
filesystem_test_lib.CreateFile(os.path.join(dirpath, "rel0", "quux"))
filesystem_test_lib.CreateFile(os.path.join(dirpath, "rel1", "norf"))
# Write a fake artifact.
art = rdf_artifacts.Artifact(
name="Quux",
doc="Lorem ipsum.",
sources=[
rdf_artifacts.ArtifactSource(
type=rdf_artifacts.ArtifactSource.SourceType.DIRECTORY,
attributes={
"paths": [os.path.join(dirpath, "%%os_release%%", "*")],
}),
])
rel_db.WriteArtifact(art)
artifact_registry.REGISTRY.ReloadDatastoreArtifacts()
flow_id = flow_test_lib.TestFlowHelper(
compatibility.GetName(collectors.ArtifactCollectorFlow),
client_mock=action_mocks.GlobClientMock(),
client_id=client_id,
artifact_list=["Quux"],
old_client_snapshot_fallback=True,
token=self.token)
results = flow_test_lib.GetFlowResults(client_id=client_id, flow_id=flow_id)
self.assertNotEmpty(results)
basenames = [os.path.basename(result.pathspec.path) for result in results]
self.assertNotIn("quux", basenames)
self.assertIn("norf", basenames)
def testRemovesNonEmptyDirs(self):
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
self.assertTrue(os.path.exists(dirpath))
with io.open(os.path.join(dirpath, "foo"), "wb") as filedesc:
filedesc.write(b"foo")
with io.open(os.path.join(dirpath, "bar"), "wb") as filedesc:
filedesc.write(b"bar")
self.assertFalse(os.path.exists(dirpath))
def testUseExternalStores(self):
if not data_store.RelationalDBEnabled():
self.skipTest("Test uses relational filestore.")
with temp.AutoTempDirPath(remove_non_empty=True) as tempdir:
path = os.path.join(tempdir, "foo")
with io.open(path, "w") as fd:
fd.write("some content")
paths = [path]
action = rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.DOWNLOAD)
action.download.use_external_stores = False
with mock.patch.object(file_store.EXTERNAL_FILE_STORE, "AddFiles") as efs:
flow_id = flow_test_lib.TestFlowHelper(
compatibility.GetName(file_finder.FileFinder),
self.client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
action=action,
process_non_regular_files=True,
token=self.token)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(efs.call_count, 0)
# Change the file or the file finder will see that it was downloaded
# already and skip it.
with io.open(path, "w") as fd:
fd.write("some other content")
action.download.use_external_stores = True
with mock.patch.object(file_store.EXTERNAL_FILE_STORE, "AddFiles") as efs:
flow_id = flow_test_lib.TestFlowHelper(
compatibility.GetName(file_finder.FileFinder),
self.client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
action=action,
process_non_regular_files=True,
token=self.token)
results = flow_test_lib.GetFlowResults(self.client_id, flow_id)
self.assertLen(results, 1)
self.assertEqual(efs.call_count, 1)
def testWget_IsDirectory(self):
data_store.REL_DB.WriteClientMetadata(
client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False)
fs_obj = fs.FileSystem(self._get_fake_api_client(),
jobs_pb2.PathSpec.OS)
with flagsaver.flagsaver(grr_admin_ui_url=self.endpoint):
with temp.AutoTempDirPath() as temp_dir:
with self.assertRaises(Exception):
fs_obj.wget(temp_dir)
def testPathWithRedundantComponents(self):
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
os.makedirs(os.path.join(dirpath, "foo", "bar"))
redpath = os.path.join(dirpath, "foo", ".", "bar", "..", ".", "bar", "..")
entries = list(timeline.Walk(redpath.encode("utf-8")))
paths = [entry.path.decode("utf-8") for entry in entries]
self.assertLen(paths, 2)
self.assertEqual(paths[0], os.path.join(dirpath, "foo"))
self.assertEqual(paths[1], os.path.join(dirpath, "foo", "bar"))
def testModeLink(self):
with temp.AutoTempDirPath(remove_non_empty=True) as tempdir:
target = os.path.join(tempdir, "foo")
link = os.path.join(tempdir, "bar")
with open(target, mode="wb"):
pass
os.symlink(target, link)
result = statx.Get(link.encode("utf-8"))
self.assertTrue(stat.S_ISLNK(result.mode))
def testLs_EmptyDirectory(self):
data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID,
fleetspeak_enabled=False)
api_client = self._get_fake_api_client()
client = grr_colab.Client(api_client)
vfs_obj = vfs.VFS(api_client, jobs_pb2.PathSpec.OS)
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
client.ls(temp_dirpath)
stat_entries = vfs_obj.ls(temp_dirpath)
self.assertEmpty(stat_entries)
def testGetFileIsDirectory(self):
"""Tests that the flow raises when called on directory."""
client_mock = action_mocks.GetFileClientMock()
with temp.AutoTempDirPath() as temp_dir:
pathspec = rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS, path=temp_dir)
with self.assertRaises(RuntimeError):
flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
def testSingleFile(self) -> None:
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
filepath = os.path.join(dirpath, "foo")
filesystem_test_lib.CreateFile(filepath, content=b"foobar")
entries = list(self._Collect(dirpath.encode("utf-8")))
self.assertLen(entries, 2)
self.assertTrue(stat_mode.S_ISDIR(entries[0].mode))
self.assertEqual(entries[0].path, dirpath.encode("utf-8"))
self.assertTrue(stat_mode.S_ISREG(entries[1].mode))
self.assertEqual(entries[1].path, filepath.encode("utf-8"))
self.assertEqual(entries[1].size, 6)
def testWget_IsDirectory(self):
data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID,
fleetspeak_enabled=False)
api_client = self._get_fake_api_client()
client = grr_colab.Client(api_client)
vfs_obj = vfs.VFS(api_client, jobs_pb2.PathSpec.OS)
with flagsaver.flagsaver(grr_admin_ui_url=self.endpoint):
with temp.AutoTempDirPath() as temp_dir:
client.ls(temp_dir)
with self.assertRaises(ValueError):
vfs_obj.wget(temp_dir)
def testSingleFile(self):
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
filepath = os.path.join(dirpath, "foo")
_Touch(filepath, content=b"foobar")
entries = list(timeline.Walk(dirpath.encode("utf-8")))
self.assertLen(entries, 2)
self.assertTrue(stat_mode.S_ISDIR(entries[0].mode))
self.assertEqual(entries[0].path, dirpath.encode("utf-8"))
self.assertTrue(stat_mode.S_ISREG(entries[1].mode))
self.assertEqual(entries[1].path, filepath.encode("utf-8"))
self.assertEqual(entries[1].size, 6)
def testNlink(self):
with temp.AutoTempDirPath(remove_non_empty=True) as tempdir:
target = os.path.join(tempdir, "foo")
link_1 = os.path.join(tempdir, "bar")
link_2 = os.path.join(tempdir, "baz")
with open(target, mode="wb"):
pass
os.link(target, link_1)
os.link(target, link_2)
result = statx.Get(target.encode("utf-8"))
self.assertEqual(result.nlink, 3)
def testGetFlagsSocket(self):
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
temp_socketpath = os.path.join(temp_dirpath, "foo")
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
sock.bind(temp_socketpath)
stat = utils.Stat(temp_socketpath, follow_symlink=False)
self.assertTrue(stat.IsSocket())
self.assertEqual(stat.GetLinuxFlags(), 0)
self.assertEqual(stat.GetOsxFlags(), 0)
finally:
sock.close()
def testSocket(self):
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
temp_socketpath = os.path.join(temp_dirpath, "foo")
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
sock.bind(temp_socketpath)
stat = filesystem.Stat.FromPath(temp_socketpath, follow_symlink=False)
self.assertFalse(stat.IsDirectory())
self.assertFalse(stat.IsRegular())
self.assertTrue(stat.IsSocket())
self.assertFalse(stat.IsSymlink())
finally:
sock.close()
def testOpen_ReadLargeFile(self):
data_store.REL_DB.WriteClientMetadata(
client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False)
filename = 'foo'
size = 1024 * 1024
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
with io.open(os.path.join(temp_dirpath, filename),
'wb') as filedesc:
filedesc.write(b'a' * size)
fs_obj = fs.FileSystem(self._get_fake_api_client(),
jobs_pb2.PathSpec.OS)
with fs_obj.open(os.path.join(temp_dirpath, filename)) as filedesc:
self.assertEqual(len(filedesc.read()), size)
def testGlob_SingleFile(self):
data_store.REL_DB.WriteClientMetadata(
client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False)
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
os.mkdir(os.path.join(temp_dirpath, 'dir'))
os.mkdir(os.path.join(temp_dirpath, 'dir1'))
os.mkdir(os.path.join(temp_dirpath, 'dir2'))
fs_obj = fs.FileSystem(self._get_fake_api_client(),
jobs_pb2.PathSpec.OS)
stat_entries = fs_obj.glob(os.path.join(temp_dirpath, 'dir'))
self.assertLen(stat_entries, 1)
self.assertEqual(stat_entries[0].pathspec.path,
os.path.join(temp_dirpath, 'dir'))
def testRun(self):
file_count = 64
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
for idx in range(file_count):
temp_filepath = os.path.join(temp_dirpath, "foo{}".format(idx))
_Touch(temp_filepath,
content=os.urandom(random.randint(0, 1024)))
args = rdf_timeline.TimelineArgs()
args.root = temp_dirpath.encode("utf-8")
responses = self.RunAction(timeline.Timeline, args)
results: List[rdf_timeline.TimelineResult] = []
blobs: List[rdf_protodict.DataBlob] = []
# The test action runner is not able to distinguish between flow replies
# and responses sent to well-known flow handlers, so we have to do the
# filtering ourselves.
for response in responses:
if isinstance(response, rdf_timeline.TimelineResult):
results.append(response)
elif isinstance(response, rdf_protodict.DataBlob):
blobs.append(response)
else:
raise AssertionError(f"Unexpected response: f{response}")
self.assertNotEmpty(results)
self.assertNotEmpty(blobs)
blob_ids = []
for result in results:
blob_ids.extend(result.entry_batch_blob_ids)
for blob in blobs:
self.assertIn(hashlib.sha256(blob.data).digest(), blob_ids)
# Total number of entries should be one more than the file count because
# of the entry for the root folder.
total_entry_count = sum(result.entry_count for result in results)
self.assertEqual(total_entry_count, file_count + 1)
for result in results:
# The filesystem type should be the same for every result.
self.assertEqual(result.filesystem_type,
results[0].filesystem_type)
def testFgrep_NoMatches(self):
data_store.REL_DB.WriteClientMetadata(
client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False)
filename = 'foo'
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
with io.open(os.path.join(temp_dirpath, filename),
'wb') as filedesc:
filedesc.write(b'foo bar')
fs_obj = fs.FileSystem(self._get_fake_api_client(),
jobs_pb2.PathSpec.OS)
matches = fs_obj.fgrep(os.path.join(temp_dirpath, filename),
b'Foo')
self.assertLen(matches, 0)
def testMultipleFiles(self):
with temp.AutoTempDirPath(remove_non_empty=True) as dirpath:
foo_filepath = os.path.join(dirpath, "foo")
bar_filepath = os.path.join(dirpath, "bar")
baz_filepath = os.path.join(dirpath, "baz")
_Touch(foo_filepath)
_Touch(bar_filepath)
_Touch(baz_filepath)
entries = list(timeline.Walk(dirpath.encode("utf-8")))
self.assertLen(entries, 4)
paths = [_.path for _ in entries[1:]]
self.assertIn(foo_filepath.encode("utf-8"), paths)
self.assertIn(bar_filepath.encode("utf-8"), paths)
self.assertIn(baz_filepath.encode("utf-8"), paths)
def testRecursiveRegexMatch(self) -> None:
with temp.AutoTempDirPath(remove_non_empty=True) as temp_dir:
nested_dir = os.path.join(temp_dir, "a", "b", "c")
os.makedirs(nested_dir)
with open(os.path.join(nested_dir, "foo.txt"), "w") as f:
f.write("bar123")
results = _RunFileFinder(
rdf_file_finder.FileFinderArgs(
paths=[os.path.join(temp_dir, "**", "*")],
pathtype=rdf_paths.PathSpec.PathType.OS,
conditions=[
rdf_file_finder.FileFinderCondition.ContentsRegexMatch(
regex=b"bar[0-9]+"),
],
action=rdf_file_finder.FileFinderAction.Stat()))
self.assertLen(results, 1)
self.assertEqual(results[0].matches[0].data, b"bar123")
files.FlushHandleCache()
