def testExtAttrsCollection(self): with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: foo_filepath = temp.TempFilePath(dir=temp_dirpath) filesystem_test_lib.SetExtAttr( foo_filepath, name="user.quux", value="foo") bar_filepath = temp.TempFilePath(dir=temp_dirpath) filesystem_test_lib.SetExtAttr( bar_filepath, name="user.quux", value="bar") baz_filepath = temp.TempFilePath(dir=temp_dirpath) filesystem_test_lib.SetExtAttr( baz_filepath, name="user.quux", value="baz") request = rdf_client_fs.FindSpec( pathspec=rdf_paths.PathSpec( path=temp_dirpath, pathtype=rdf_paths.PathSpec.PathType.OS), path_glob="*", collect_ext_attrs=True) request.iterator.number = 100 hits = [] for response in self.RunAction(searching.Find, request): if isinstance(response, rdf_client_fs.FindSpec): hits.append(response.hit) self.assertLen(hits, 3) values = [] for hit in hits: self.assertLen(hit.ext_attrs, 1) values.append(hit.ext_attrs[0].value) self.assertCountEqual(values, ["foo", "bar", "baz"])
def testMultipleResults(self): with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: with io.open(os.path.join(dirpath, "foo"), "wb") as filedesc: del filedesc # Unused. with io.open(os.path.join(dirpath, "bar"), "wb") as filedesc: del filedesc # Unused. with io.open(os.path.join(dirpath, "baz"), "wb") as filedesc: del filedesc # Unused. query = """ SELECT filename FROM file WHERE directory = "{}" ORDER BY filename; """.format(dirpath) with test_lib.ConfigOverrider({"Osquery.max_chunk_size": 3}): results = _Query(query) self.assertLen(results, 3) for result in results: self.assertEqual(result.table.query, query) self.assertLen(result.table.header.columns, 1) self.assertEqual(result.table.header.columns[0].name, "filename") self.assertEqual(list(results[0].table.Column("filename")), ["bar"]) self.assertEqual(list(results[1].table.Column("filename")), ["baz"]) self.assertEqual(list(results[2].table.Column("filename")), ["foo"])
def testTimestamp(self): with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: filepath = os.path.join(dirpath, "foo") with open(filepath, mode="wb"): pass with open(filepath, mode="wb") as filedesc: filedesc.write(b"quux") with open(filepath, mode="rb") as filedesc: _ = filedesc.read() now_ns = time.time_ns() entries = list(timeline.Walk(dirpath.encode("utf-8"))) self.assertLen(entries, 2) self.assertEqual(entries[0].path, dirpath.encode("utf-8")) self.assertEqual(entries[1].path, filepath.encode("utf-8")) self.assertGreater(entries[1].ctime_ns, 0) self.assertGreaterEqual(entries[1].mtime_ns, entries[1].ctime_ns) self.assertGreaterEqual(entries[1].atime_ns, entries[1].mtime_ns) self.assertLess(entries[1].atime_ns, now_ns)
def testNestedDirectories(self): with temp.AutoTempDirPath(remove_non_empty=True) as root_dirpath: foobar_dirpath = os.path.join(root_dirpath, "foo", "bar") os.makedirs(foobar_dirpath) foobaz_dirpath = os.path.join(root_dirpath, "foo", "baz") os.makedirs(foobaz_dirpath) quuxnorfthud_dirpath = os.path.join(root_dirpath, "quux", "norf", "thud") os.makedirs(quuxnorfthud_dirpath) entries = list(timeline.Walk(root_dirpath.encode("utf-8"))) self.assertLen(entries, 7) paths = [_.path.decode("utf-8") for _ in entries] self.assertCountEqual(paths, [ os.path.join(root_dirpath), os.path.join(root_dirpath, "foo"), os.path.join(root_dirpath, "foo", "bar"), os.path.join(root_dirpath, "foo", "baz"), os.path.join(root_dirpath, "quux"), os.path.join(root_dirpath, "quux", "norf"), os.path.join(root_dirpath, "quux", "norf", "thud"), ]) for entry in entries: self.assertTrue(stat_mode.S_ISDIR(entry.mode))
def testMultipleResults(self): row_count = 100 with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: for i in range(row_count): filepath = os.path.join(dirpath, "{:04}".format(i)) with io.open(filepath, "wb") as filedesc: del filedesc # Unused. query = """ SELECT filename FROM file WHERE directory = "{}" ORDER BY filename; """.format(dirpath) # Size limit is set so that each chunk should contain 2 rows. with test_lib.ConfigOverrider({"Osquery.max_chunk_size": 10}): results = self._RunFlow(query) # Since each chunk is expected to have 2 rows, number of rows should be # equal to twice the amount of chunks. self.assertEqual(2 * len(results), row_count) for i, result in enumerate(results): table = result.table self.assertEqual(table.query, query) self.assertLen(table.header.columns, 1) self.assertEqual(table.header.columns[0].name, "filename") self.assertLen(table.rows, 2) self.assertEqual(list(table.Column("filename")), [ "{:04}".format(2 * i), "{:04}".format(2 * i + 1), ])
def SetUpTestFiles(): with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: file_bar_path = os.path.join(temp_dirpath, "bar") with open(file_bar_path, "wb") as fd: fd.write(b"bar") file_baz_path = os.path.join(temp_dirpath, "baz") with open(file_baz_path, "wb") as fd: fd.write(b"baz") file_foo_path = os.path.join(temp_dirpath, "foo") with open(file_foo_path, "wb") as fd: fd.write(b"foo") yield { "bar": TestFile(path=file_bar_path, sha1=hashlib.sha1(b"bar").hexdigest()), "baz": TestFile(path=file_baz_path, sha1=hashlib.sha1(b"baz").hexdigest()), "foo": TestFile(path=file_foo_path, sha1=hashlib.sha1(b"foo").hexdigest()), }
def testDir(self): with temp.AutoTempDirPath() as dirpath: filepath = temp.TempFilePath(dir=dirpath) self.assertTrue(os.path.exists(filepath)) self.assertStartsWith(filepath, dirpath) os.remove(filepath)
def testLs_EmptyDirectory(self): data_store.REL_DB.WriteClientMetadata( client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) stat_entries = fs_obj.ls(temp_dirpath) self.assertEmpty(stat_entries)
def testGetFlagsSymlink(self): with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath, \ temp.AutoTempFilePath() as temp_filepath: temp_linkpath = os.path.join(temp_dirpath, "foo") os.symlink(temp_filepath, temp_linkpath) stat = filesystem.Stat.FromPath(temp_linkpath, follow_symlink=False) self.assertTrue(stat.IsSymlink()) self.assertEqual(stat.GetLinuxFlags(), 0) self.assertEqual(stat.GetOsxFlags(), 0)
def testContents(self): with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: filepath = os.path.join(dirpath, "foo") filesystem_test_lib.CreateFile(filepath, content=b"foobarbaz") with io.open(filepath, "rb") as filedesc: content = filedesc.read() self.assertEqual(content, b"foobarbaz")
def testOldClientSnapshotFallbackUsesLatestApplicable(self): rel_db = data_store.REL_DB client_id = "C.0123456789abcdef" rel_db.WriteClientMetadata(client_id, first_seen=rdfvalue.RDFDatetime.Now()) # Write some fake snapshot history. kb_0 = rdf_client.KnowledgeBase(os="Linux", os_release="rel0") snapshot_0 = rdf_objects.ClientSnapshot( client_id=client_id, knowledge_base=kb_0) rel_db.WriteClientSnapshot(snapshot_0) kb_1 = rdf_client.KnowledgeBase(os="Linux", os_release="rel1") snapshot_1 = rdf_objects.ClientSnapshot( client_id=client_id, knowledge_base=kb_1) rel_db.WriteClientSnapshot(snapshot_1) kb_2 = rdf_client.KnowledgeBase(os="Linux") snapshot_2 = rdf_objects.ClientSnapshot( client_id=client_id, knowledge_base=kb_2) rel_db.WriteClientSnapshot(snapshot_2) with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: filesystem_test_lib.CreateFile(os.path.join(dirpath, "rel0", "quux")) filesystem_test_lib.CreateFile(os.path.join(dirpath, "rel1", "norf")) # Write a fake artifact. art = rdf_artifacts.Artifact( name="Quux", doc="Lorem ipsum.", sources=[ rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.DIRECTORY, attributes={ "paths": [os.path.join(dirpath, "%%os_release%%", "*")], }), ]) rel_db.WriteArtifact(art) artifact_registry.REGISTRY.ReloadDatastoreArtifacts() flow_id = flow_test_lib.TestFlowHelper( compatibility.GetName(collectors.ArtifactCollectorFlow), client_mock=action_mocks.GlobClientMock(), client_id=client_id, artifact_list=["Quux"], old_client_snapshot_fallback=True, token=self.token) results = flow_test_lib.GetFlowResults(client_id=client_id, flow_id=flow_id) self.assertNotEmpty(results) basenames = [os.path.basename(result.pathspec.path) for result in results] self.assertNotIn("quux", basenames) self.assertIn("norf", basenames)
def testRemovesNonEmptyDirs(self): with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: self.assertTrue(os.path.exists(dirpath)) with io.open(os.path.join(dirpath, "foo"), "wb") as filedesc: filedesc.write(b"foo") with io.open(os.path.join(dirpath, "bar"), "wb") as filedesc: filedesc.write(b"bar") self.assertFalse(os.path.exists(dirpath))
def testUseExternalStores(self): if not data_store.RelationalDBEnabled(): self.skipTest("Test uses relational filestore.") with temp.AutoTempDirPath(remove_non_empty=True) as tempdir: path = os.path.join(tempdir, "foo") with io.open(path, "w") as fd: fd.write("some content") paths = [path] action = rdf_file_finder.FileFinderAction( action_type=rdf_file_finder.FileFinderAction.Action.DOWNLOAD) action.download.use_external_stores = False with mock.patch.object(file_store.EXTERNAL_FILE_STORE, "AddFiles") as efs: flow_id = flow_test_lib.TestFlowHelper( compatibility.GetName(file_finder.FileFinder), self.client_mock, client_id=self.client_id, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, action=action, process_non_regular_files=True, token=self.token) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) self.assertLen(results, 1) self.assertEqual(efs.call_count, 0) # Change the file or the file finder will see that it was downloaded # already and skip it. with io.open(path, "w") as fd: fd.write("some other content") action.download.use_external_stores = True with mock.patch.object(file_store.EXTERNAL_FILE_STORE, "AddFiles") as efs: flow_id = flow_test_lib.TestFlowHelper( compatibility.GetName(file_finder.FileFinder), self.client_mock, client_id=self.client_id, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, action=action, process_non_regular_files=True, token=self.token) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) self.assertLen(results, 1) self.assertEqual(efs.call_count, 1)
def testWget_IsDirectory(self): data_store.REL_DB.WriteClientMetadata( client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with flagsaver.flagsaver(grr_admin_ui_url=self.endpoint): with temp.AutoTempDirPath() as temp_dir: with self.assertRaises(Exception): fs_obj.wget(temp_dir)
def testPathWithRedundantComponents(self): with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: os.makedirs(os.path.join(dirpath, "foo", "bar")) redpath = os.path.join(dirpath, "foo", ".", "bar", "..", ".", "bar", "..") entries = list(timeline.Walk(redpath.encode("utf-8"))) paths = [entry.path.decode("utf-8") for entry in entries] self.assertLen(paths, 2) self.assertEqual(paths[0], os.path.join(dirpath, "foo")) self.assertEqual(paths[1], os.path.join(dirpath, "foo", "bar"))
def testModeLink(self): with temp.AutoTempDirPath(remove_non_empty=True) as tempdir: target = os.path.join(tempdir, "foo") link = os.path.join(tempdir, "bar") with open(target, mode="wb"): pass os.symlink(target, link) result = statx.Get(link.encode("utf-8")) self.assertTrue(stat.S_ISLNK(result.mode))
def testLs_EmptyDirectory(self): data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) vfs_obj = vfs.VFS(api_client, jobs_pb2.PathSpec.OS) with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: client.ls(temp_dirpath) stat_entries = vfs_obj.ls(temp_dirpath) self.assertEmpty(stat_entries)
def testGetFileIsDirectory(self): """Tests that the flow raises when called on directory.""" client_mock = action_mocks.GetFileClientMock() with temp.AutoTempDirPath() as temp_dir: pathspec = rdf_paths.PathSpec( pathtype=rdf_paths.PathSpec.PathType.OS, path=temp_dir) with self.assertRaises(RuntimeError): flow_test_lib.TestFlowHelper(transfer.GetFile.__name__, client_mock, token=self.token, client_id=self.client_id, pathspec=pathspec)
def testSingleFile(self) -> None: with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: filepath = os.path.join(dirpath, "foo") filesystem_test_lib.CreateFile(filepath, content=b"foobar") entries = list(self._Collect(dirpath.encode("utf-8"))) self.assertLen(entries, 2) self.assertTrue(stat_mode.S_ISDIR(entries[0].mode)) self.assertEqual(entries[0].path, dirpath.encode("utf-8")) self.assertTrue(stat_mode.S_ISREG(entries[1].mode)) self.assertEqual(entries[1].path, filepath.encode("utf-8")) self.assertEqual(entries[1].size, 6)
def testWget_IsDirectory(self): data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) vfs_obj = vfs.VFS(api_client, jobs_pb2.PathSpec.OS) with flagsaver.flagsaver(grr_admin_ui_url=self.endpoint): with temp.AutoTempDirPath() as temp_dir: client.ls(temp_dir) with self.assertRaises(ValueError): vfs_obj.wget(temp_dir)
def testSingleFile(self): with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: filepath = os.path.join(dirpath, "foo") _Touch(filepath, content=b"foobar") entries = list(timeline.Walk(dirpath.encode("utf-8"))) self.assertLen(entries, 2) self.assertTrue(stat_mode.S_ISDIR(entries[0].mode)) self.assertEqual(entries[0].path, dirpath.encode("utf-8")) self.assertTrue(stat_mode.S_ISREG(entries[1].mode)) self.assertEqual(entries[1].path, filepath.encode("utf-8")) self.assertEqual(entries[1].size, 6)
def testNlink(self): with temp.AutoTempDirPath(remove_non_empty=True) as tempdir: target = os.path.join(tempdir, "foo") link_1 = os.path.join(tempdir, "bar") link_2 = os.path.join(tempdir, "baz") with open(target, mode="wb"): pass os.link(target, link_1) os.link(target, link_2) result = statx.Get(target.encode("utf-8")) self.assertEqual(result.nlink, 3)
def testGetFlagsSocket(self): with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: temp_socketpath = os.path.join(temp_dirpath, "foo") sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: sock.bind(temp_socketpath) stat = utils.Stat(temp_socketpath, follow_symlink=False) self.assertTrue(stat.IsSocket()) self.assertEqual(stat.GetLinuxFlags(), 0) self.assertEqual(stat.GetOsxFlags(), 0) finally: sock.close()
def testSocket(self): with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: temp_socketpath = os.path.join(temp_dirpath, "foo") sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: sock.bind(temp_socketpath) stat = filesystem.Stat.FromPath(temp_socketpath, follow_symlink=False) self.assertFalse(stat.IsDirectory()) self.assertFalse(stat.IsRegular()) self.assertTrue(stat.IsSocket()) self.assertFalse(stat.IsSymlink()) finally: sock.close()
def testOpen_ReadLargeFile(self): data_store.REL_DB.WriteClientMetadata( client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) filename = 'foo' size = 1024 * 1024 with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: with io.open(os.path.join(temp_dirpath, filename), 'wb') as filedesc: filedesc.write(b'a' * size) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with fs_obj.open(os.path.join(temp_dirpath, filename)) as filedesc: self.assertEqual(len(filedesc.read()), size)
def testGlob_SingleFile(self): data_store.REL_DB.WriteClientMetadata( client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: os.mkdir(os.path.join(temp_dirpath, 'dir')) os.mkdir(os.path.join(temp_dirpath, 'dir1')) os.mkdir(os.path.join(temp_dirpath, 'dir2')) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) stat_entries = fs_obj.glob(os.path.join(temp_dirpath, 'dir')) self.assertLen(stat_entries, 1) self.assertEqual(stat_entries[0].pathspec.path, os.path.join(temp_dirpath, 'dir'))
def testRun(self): file_count = 64 with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: for idx in range(file_count): temp_filepath = os.path.join(temp_dirpath, "foo{}".format(idx)) _Touch(temp_filepath, content=os.urandom(random.randint(0, 1024))) args = rdf_timeline.TimelineArgs() args.root = temp_dirpath.encode("utf-8") responses = self.RunAction(timeline.Timeline, args) results: List[rdf_timeline.TimelineResult] = [] blobs: List[rdf_protodict.DataBlob] = [] # The test action runner is not able to distinguish between flow replies # and responses sent to well-known flow handlers, so we have to do the # filtering ourselves. for response in responses: if isinstance(response, rdf_timeline.TimelineResult): results.append(response) elif isinstance(response, rdf_protodict.DataBlob): blobs.append(response) else: raise AssertionError(f"Unexpected response: f{response}") self.assertNotEmpty(results) self.assertNotEmpty(blobs) blob_ids = [] for result in results: blob_ids.extend(result.entry_batch_blob_ids) for blob in blobs: self.assertIn(hashlib.sha256(blob.data).digest(), blob_ids) # Total number of entries should be one more than the file count because # of the entry for the root folder. total_entry_count = sum(result.entry_count for result in results) self.assertEqual(total_entry_count, file_count + 1) for result in results: # The filesystem type should be the same for every result. self.assertEqual(result.filesystem_type, results[0].filesystem_type)
def testFgrep_NoMatches(self): data_store.REL_DB.WriteClientMetadata( client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: with io.open(os.path.join(temp_dirpath, filename), 'wb') as filedesc: filedesc.write(b'foo bar') fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) matches = fs_obj.fgrep(os.path.join(temp_dirpath, filename), b'Foo') self.assertLen(matches, 0)
def testMultipleFiles(self): with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: foo_filepath = os.path.join(dirpath, "foo") bar_filepath = os.path.join(dirpath, "bar") baz_filepath = os.path.join(dirpath, "baz") _Touch(foo_filepath) _Touch(bar_filepath) _Touch(baz_filepath) entries = list(timeline.Walk(dirpath.encode("utf-8"))) self.assertLen(entries, 4) paths = [_.path for _ in entries[1:]] self.assertIn(foo_filepath.encode("utf-8"), paths) self.assertIn(bar_filepath.encode("utf-8"), paths) self.assertIn(baz_filepath.encode("utf-8"), paths)
def testRecursiveRegexMatch(self) -> None: with temp.AutoTempDirPath(remove_non_empty=True) as temp_dir: nested_dir = os.path.join(temp_dir, "a", "b", "c") os.makedirs(nested_dir) with open(os.path.join(nested_dir, "foo.txt"), "w") as f: f.write("bar123") results = _RunFileFinder( rdf_file_finder.FileFinderArgs( paths=[os.path.join(temp_dir, "**", "*")], pathtype=rdf_paths.PathSpec.PathType.OS, conditions=[ rdf_file_finder.FileFinderCondition.ContentsRegexMatch( regex=b"bar[0-9]+"), ], action=rdf_file_finder.FileFinderAction.Stat())) self.assertLen(results, 1) self.assertEqual(results[0].matches[0].data, b"bar123") files.FlushHandleCache()