def testGlobWithStarStarRootPath(self): """Test ** expressions with root_path.""" # Add some usernames we can interpolate later. client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) kb = client.Get(client.Schema.KNOWLEDGE_BASE) kb.MergeOrAddUser(rdf_client.User(username="******")) kb.MergeOrAddUser(rdf_client.User(username="******")) client.Set(kb) client.Close() client_mock = action_mocks.GlobClientMock() # Glob for foo at a depth of 4. path = os.path.join("foo**4") root_path = rdf_paths.PathSpec(path=os.path.join( self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) root_path.Append(path="/", pathtype=rdf_paths.PathSpec.PathType.TSK) # Run the flow. for _ in flow_test_lib.TestFlowHelper( filesystem.Glob.__name__, client_mock, client_id=self.client_id, paths=[path], root_path=root_path, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token): pass output_path = self.client_id.Add("fs/tsk").Add( self.base_path.replace("\\", "/")).Add("test_img.dd/glob_test/a/b") children = [] fd = aff4.FACTORY.Open(output_path, token=self.token) for child in fd.ListChildren(): children.append(child.Basename()) # We should find some files. self.assertEqual(children, ["foo"])
def testGlobWildcardsAndTSK(self): client_mock = action_mocks.GlobClientMock() # This glob should find this file in test data: glob_test/a/b/foo. path = os.path.join(self.base_path, "test_IMG.dd", "glob_test", "a", "b", "FOO*") flow_test_lib.TestFlowHelper( filesystem.Glob.__name__, client_mock, client_id=self.client_id, paths=[path], pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token) output_path = self.client_id.Add("fs/tsk").Add( os.path.join(self.base_path, "test_img.dd", "glob_test", "a", "b")) fd = aff4.FACTORY.Open(output_path, token=self.token) children = list(fd.ListChildren()) self.assertEqual(len(children), 1) self.assertEqual(children[0].Basename(), "foo") aff4.FACTORY.Delete( self.client_id.Add("fs/tsk").Add(os.path.join(self.base_path)), token=self.token) # Specifying a wildcard for the image will not open it. path = os.path.join(self.base_path, "*.dd", "glob_test", "a", "b", "FOO*") flow_test_lib.TestFlowHelper( filesystem.Glob.__name__, client_mock, client_id=self.client_id, paths=[path], pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token) fd = aff4.FACTORY.Open(output_path, token=self.token) children = list(fd.ListChildren()) self.assertEqual(len(children), 0)
def testGlobWithStarStarRootPath(self): """Test ** expressions with root_path.""" users = [ rdf_client.User(username="******"), rdf_client.User(username="******") ] self.client_id = self.SetupClient(0, users=users) client_mock = action_mocks.GlobClientMock() # Glob for foo at a depth of 4. path = os.path.join("foo**4") root_path = rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) root_path.Append(path="/", pathtype=rdf_paths.PathSpec.PathType.TSK) # Run the flow. flow_test_lib.TestFlowHelper( compatibility.GetName(filesystem.Glob), client_mock, client_id=self.client_id, paths=[path], root_path=root_path, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token) if data_store.AFF4Enabled(): children = [] output_path = self.client_id.Add("fs/tsk").Add( self.base_path).Add("test_img.dd/glob_test/a/b") fd = aff4.FACTORY.Open(output_path, token=self.token) for child in fd.ListChildren(): children.append(child.Basename()) # We should find some files. self.assertEqual(children, ["foo"]) else: children = self._ListTestChildPathInfos( ["test_img.dd", "glob_test", "a", "b"]) self.assertLen(children, 1) self.assertEqual(children[0].components[-1], "foo")
def testGlob(self): """Test that glob works properly.""" users = [ rdf_client.User(username="******"), rdf_client.User(username="******") ] client_id = self.SetupClient(0, users=users) client_mock = action_mocks.GlobClientMock() # This glob selects all files which start with the username on this system. paths = [ os.path.join(self.base_path, "%%Users.username%%*"), os.path.join(self.base_path, "VFSFixture/var/*/wtmp") ] flow_test_lib.TestFlowHelper(compatibility.GetName(filesystem.Glob), client_mock, client_id=client_id, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token, check_flow_errors=False) expected_files = [ filename for filename in os.listdir(self.base_path) if filename.startswith("test") or filename.startswith("syslog") ] expected_files.append("VFSFixture") children = self._ListTestChildPathInfos( [], path_type=rdf_objects.PathInfo.PathType.OS) found_files = [child.components[-1] for child in children] self.assertCountEqual(expected_files, found_files) children = self._ListTestChildPathInfos( ["VFSFixture", "var", "log"], path_type=rdf_objects.PathInfo.PathType.OS) self.assertLen(children, 1) self.assertEqual(children[0].components[-1], "wtmp")
def testGlobDirectory(self): """Test that glob expands directories.""" self.client_id = self.SetupClient(0) # Add some usernames we can interpolate later. client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) kb = client.Get(client.Schema.KNOWLEDGE_BASE) kb.MergeOrAddUser( rdf_client.User(username="******", appdata="test_data/index.dat")) kb.MergeOrAddUser( rdf_client.User(username="******", appdata="test_data/History")) # This is a record which means something to the interpolation system. We # should not process this especially. kb.MergeOrAddUser(rdf_client.User(username="******", appdata="%%PATH%%")) client.Set(kb) client.Close() client_mock = action_mocks.GlobClientMock() # This glob selects all files which start with the username on this system. path = os.path.join(os.path.dirname(self.base_path), "%%users.appdata%%") # Run the flow. for _ in flow_test_lib.TestFlowHelper( filesystem.Glob.__name__, client_mock, client_id=self.client_id, paths=[path], token=self.token): pass path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat") aff4.FACTORY.Open(path, aff4_type=aff4_grr.VFSFile, token=self.token) path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat") aff4.FACTORY.Open(path, aff4_type=aff4_grr.VFSFile, token=self.token)
def testGlobWildcardOnImage(self): client_mock = action_mocks.GlobClientMock() # Specifying a wildcard for the image will not open it. path = os.path.join(self.base_path, "*.dd", "glob_test", "a", "b", "FOO*") flow_test_lib.TestFlowHelper( compatibility.GetName(filesystem.Glob), client_mock, client_id=self.client_id, paths=[path], pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token) if data_store.AFF4Enabled(): output_path = self.client_id.Add("fs/tsk").Add( os.path.join(self.base_path, "test_img.dd", "glob_test", "a", "b")) fd = aff4.FACTORY.Open(output_path, token=self.token) children = list(fd.ListChildren()) self.assertEmpty(children) else: children = self._ListTestChildPathInfos( ["test_img.dd", "glob_test", "a", "b"]) self.assertEmpty(children)
def testGlobWithWildcardsInsideTSKFileCaseInsensitive(self): client_mock = action_mocks.GlobClientMock() # This glob should find this file in test data: glob_test/a/b/foo. path = os.path.join("*", "a", "b", "FOO*") root_path = rdf_paths.PathSpec(path=os.path.join( self.base_path, "test_IMG.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) root_path.Append(path="/", pathtype=rdf_paths.PathSpec.PathType.TSK) # Run the flow. flow_test_lib.TestFlowHelper(compatibility.GetName(filesystem.Glob), client_mock, client_id=self.client_id, paths=[path], root_path=root_path, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token) children = self._ListTestChildPathInfos( ["test_img.dd", "glob_test", "a", "b"]) self.assertLen(children, 1) self.assertEqual(children[0].components[-1], "foo")
def testIllegalGlobAsync(self): # When running the flow asynchronously, we will not receive any errors from # the Start method, but the flow should still fail. paths = ["Test/%%Weird_illegal_attribute%%"] client_mock = action_mocks.GlobClientMock() # Run the flow. session_id = None # This should not raise here since the flow is run asynchronously. session_id = flow_test_lib.TestFlowHelper( filesystem.Glob.__name__, client_mock, client_id=self.client_id, check_flow_errors=False, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token, sync=False) fd = aff4.FACTORY.Open(session_id, token=self.token) self.assertIn("KnowledgeBaseInterpolationError", fd.context.backtrace) self.assertEqual("ERROR", str(fd.context.state))
def testGlob(self): """Test that glob works properly.""" client_id = self.SetupClient(0) # Add some usernames we can interpolate later. client = aff4.FACTORY.Open(client_id, mode="rw", token=self.token) kb = client.Get(client.Schema.KNOWLEDGE_BASE) kb.MergeOrAddUser(rdf_client.User(username="******")) kb.MergeOrAddUser(rdf_client.User(username="******")) client.Set(kb) client.Close() client_mock = action_mocks.GlobClientMock() # This glob selects all files which start with the username on this system. paths = [ os.path.join(self.base_path, "%%Users.username%%*"), os.path.join(self.base_path, "VFSFixture/var/*/wtmp") ] # Set iterator really low to force iteration. with utils.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2): for _ in flow_test_lib.TestFlowHelper( filesystem.Glob.__name__, client_mock, client_id=client_id, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token, sync=False, check_flow_errors=False): pass output_path = client_id.Add("fs/os").Add(self.base_path.replace("\\", "/")) children = [] fd = aff4.FACTORY.Open(output_path, token=self.token) for child in fd.ListChildren(): filename = child.Basename() if filename != "VFSFixture": children.append(filename) expected = [ filename for filename in os.listdir(self.base_path) if filename.startswith("test") or filename.startswith("syslog") ] self.assertTrue([x for x in expected if x.startswith("test")], "Need a file starting with 'test'" " in test_data for this test!") self.assertTrue([x for x in expected if x.startswith("syslog")], "Need a file starting with 'syslog'" " in test_data for this test!") self.assertItemsEqual(expected, children) children = [] fd = aff4.FACTORY.Open( output_path.Add("VFSFixture/var/log"), token=self.token) for child in fd.ListChildren(): children.append(child.Basename()) self.assertItemsEqual(children, ["wtmp"])
def testOldClientSnapshotFallbackIfInterpolationFails(self): rel_db = data_store.REL_DB client_id = "C.0123456789abcdef" rel_db.WriteClientMetadata(client_id, first_seen=rdfvalue.RDFDatetime.Now()) # Write some fake snapshot history. kb_0 = rdf_client.KnowledgeBase(os="Linux") kb_0.users = [ rdf_client.User(username="******"), rdf_client.User(username="******"), ] snapshot_0 = rdf_objects.ClientSnapshot( client_id=client_id, knowledge_base=kb_0) rel_db.WriteClientSnapshot(snapshot_0) kb_1 = rdf_client.KnowledgeBase(os="Linux") snapshot_1 = rdf_objects.ClientSnapshot( client_id=client_id, knowledge_base=kb_1) rel_db.WriteClientSnapshot(snapshot_1) kb_2 = rdf_client.KnowledgeBase(os="Linux") snapshot_2 = rdf_objects.ClientSnapshot( client_id=client_id, knowledge_base=kb_2) rel_db.WriteClientSnapshot(snapshot_2) with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: filesystem_test_lib.CreateFile( os.path.join(dirpath, "user1", "quux", "thud")) filesystem_test_lib.CreateFile( os.path.join(dirpath, "user2", "quux", "norf")) # Write a fake artifact. path = os.path.join(dirpath, "%%users.username%%", "quux", "*") art = rdf_artifacts.Artifact( name="Quux", doc="Lorem ipsum.", sources=[ rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.DIRECTORY, attributes={ "paths": [path], }), ]) rel_db.WriteArtifact(art) artifact_registry.REGISTRY.ReloadDatastoreArtifacts() flow_id = flow_test_lib.TestFlowHelper( compatibility.GetName(collectors.ArtifactCollectorFlow), client_mock=action_mocks.GlobClientMock(), client_id=client_id, artifact_list=["Quux"], old_client_snapshot_fallback=True, token=self.token) results = flow_test_lib.GetFlowResults(client_id=client_id, flow_id=flow_id) self.assertNotEmpty(results) basenames = [os.path.basename(result.pathspec.path) for result in results] self.assertCountEqual(basenames, ["thud", "norf"])