def testRecursiveListDirectoryDeep(self): """Test that RecursiveListDirectory lists files only up to max depth.""" client_mock = action_mocks.ListDirectoryClientMock() dir_components = ["dir1", "dir2", "dir3", "dir4", "dir5"] with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: os.makedirs(os.path.join(temp_dirpath, *dir_components)) pathspec = rdf_paths.PathSpec( path=temp_dirpath, pathtype=rdf_paths.PathSpec.PathType.OS) flow_id = flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.RecursiveListDirectory), client_mock, client_id=self.client_id, pathspec=pathspec, max_depth=3, token=self.token) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) self.assertLen(results, 3) dirs = [_.pathspec.Basename() for _ in results] self.assertCountEqual(dirs, ["dir1", "dir2", "dir3"])
def testUnicodeListDirectory(self): """Test that the ListDirectory flow works on unicode directories.""" client_mock = action_mocks.ListDirectoryClientMock() # Deliberately specify incorrect casing pb = rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) pb.Append(path=u"入乡随俗 海外春节别样过法", pathtype=rdf_paths.PathSpec.PathType.TSK) flow_test_lib.TestFlowHelper( filesystem.ListDirectory.__name__, client_mock, client_id=self.client_id, pathspec=pb, token=self.token) # Check the output file is created output_path = self.client_id.Add("fs/tsk").Add(pb.CollapsePath()) fd = aff4.FACTORY.Open(output_path, token=self.token) children = list(fd.OpenChildren()) self.assertEqual(len(children), 1) child = children[0] self.assertEqual( os.path.basename(utils.SmartUnicode(child.urn)), u"入乡随俗.txt")
def testListDirectory(self): """Test that the ListDirectory flow works.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec(path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) pb.Append(path="test directory", pathtype=rdf_paths.PathSpec.PathType.TSK) # Change the username so we get a notification about the flow termination. token = self.token.Copy() token.username = "******" with mock.patch.object(notification, "Notify") as mock_notify: flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.ListDirectory), client_mock, client_id=self.client_id, pathspec=pb, token=token) self.assertEqual(mock_notify.call_count, 1) args = list(mock_notify.mock_calls[0])[1] self.assertEqual(args[0], token.username) com = rdf_objects.UserNotification.Type.TYPE_VFS_LIST_DIRECTORY_COMPLETED self.assertEqual(args[1], com) self.assertIn(pb.path, args[2]) children = self._ListTestChildPathInfos(["test_img.dd"]) self.assertLen(children, 1) self.assertEqual(children[0].components[-1], "Test Directory")
def setUp(self): super(TestEndToEndTestFlow, self).setUp() self.SetupClients(1, system="Linux", os_version="14.04", arch="x86_64") install_time = rdfvalue.RDFDatetime.Now() user = "******" userobj = rdf_client.User(username=user) interface = rdf_client.Interface(ifname="eth0") self.client = aff4.FACTORY.Create( self.client_id, aff4_grr.VFSGRRClient, mode="rw", token=self.token, age=aff4.ALL_TIMES) kb = self.client.Get(self.client.Schema.KNOWLEDGE_BASE) kb.users.Append(userobj) self.client.Set(self.client.Schema.HOSTNAME("hostname")) self.client.Set(self.client.Schema.OS_RELEASE("debian")) self.client.Set(self.client.Schema.KERNEL("3.15-rc2")) self.client.Set(self.client.Schema.FQDN("hostname.example.com")) self.client.Set(self.client.Schema.INSTALL_DATE(install_time)) self.client.Set(self.client.Schema.KNOWLEDGE_BASE(kb)) self.client.Set(self.client.Schema.USERNAMES([user])) self.client.Set(self.client.Schema.INTERFACES([interface])) self.client.Flush() self.client_mock = action_mocks.ListDirectoryClientMock()
def testListDirectory(self): """Test that the ListDirectory flow works.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) pb.Append(path="test directory", pathtype=rdf_paths.PathSpec.PathType.TSK) for _ in flow_test_lib.TestFlowHelper( filesystem.ListDirectory.__name__, client_mock, client_id=self.client_id, pathspec=pb, token=self.token): pass # Check the output file is created output_path = self.client_id.Add("fs/tsk").Add(pb.first.path) fd = aff4.FACTORY.Open(output_path.Add("Test Directory"), token=self.token) children = list(fd.OpenChildren()) self.assertEqual(len(children), 1) child = children[0] # Check that the object is stored with the correct casing. self.assertEqual(child.urn.Basename(), "numbers.txt") # And the wrong object is not there self.assertRaises( IOError, aff4.FACTORY.Open, output_path.Add("test directory"), aff4_type=aff4_standard.VFSDirectory, token=self.token)
def testNotificationWhenListingRegistry(self): # Change the username so notifications get written. token = self.token.Copy() token.username = "******" acl_test_lib.CreateUser(token.username) with vfs_test_lib.RegistryVFSStubber(): client_id = self.SetupClient(0) pb = rdf_paths.PathSpec( path="/HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest", pathtype=rdf_paths.PathSpec.PathType.REGISTRY) client_mock = action_mocks.ListDirectoryClientMock() flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.ListDirectory), client_mock, client_id=client_id, pathspec=pb, token=token) notifications = data_store.REL_DB.ReadUserNotifications(token.username) self.assertLen(notifications, 1) n = notifications[0] self.assertEqual(n.reference.vfs_file.path_type, rdf_objects.PathInfo.PathType.REGISTRY) self.assertEqual(n.reference.vfs_file.path_components, ["HKEY_LOCAL_MACHINE", "SOFTWARE", "ListingTest"])
def testUnicodeListDirectory(self): """Test that the ListDirectory flow works on unicode directories.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec(path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) filename = "入乡随俗 海外春节别样过法" pb.Append(path=filename, pathtype=rdf_paths.PathSpec.PathType.TSK) flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.ListDirectory), client_mock, client_id=self.client_id, pathspec=pb, token=self.token) # Check the output file is created output_path = self.client_id.Add("fs/tsk").Add(pb.CollapsePath()) if data_store.AFF4Enabled(): fd = aff4.FACTORY.Open(output_path, token=self.token) children = list(fd.OpenChildren()) self.assertLen(children, 1) child = children[0] filename = child.urn.Basename() else: components = ["test_img.dd", filename] children = self._ListTestChildPathInfos(components) self.assertLen(children, 1) filename = children[0].components[-1] self.assertEqual(filename, "入乡随俗.txt")
def testListingRegistryDirectoryDoesNotYieldMtimes(self): with vfs_test_lib.RegistryVFSStubber(): client_id = self.SetupClient(0) pb = rdf_paths.PathSpec( path="/HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest", pathtype=rdf_paths.PathSpec.PathType.REGISTRY) output_path = client_id.Add("registry").Add(pb.first.path) aff4.FACTORY.Delete(output_path, token=self.token) client_mock = action_mocks.ListDirectoryClientMock() flow_test_lib.TestFlowHelper( filesystem.ListDirectory.__name__, client_mock, client_id=client_id, pathspec=pb, token=self.token) results = list( aff4.FACTORY.Open(output_path, token=self.token).OpenChildren()) self.assertEqual(len(results), 2) for result in results: st = result.Get(result.Schema.STAT) self.assertIsNone(st.st_mtime)
def testUnicodeListDirectory(self): """Test that the ListDirectory flow works on unicode directories.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec(path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) filename = "入乡随俗 海外春节别样过法" pb.Append(path=filename, pathtype=rdf_paths.PathSpec.PathType.TSK) flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.ListDirectory), client_mock, client_id=self.client_id, pathspec=pb, token=self.token) # Check the output file is created components = ["test_img.dd", filename] children = self._ListTestChildPathInfos(components) self.assertLen(children, 1) filename = children[0].components[-1] self.assertEqual(filename, "入乡随俗.txt")
def testListingRegistryDirectoryDoesNotYieldMtimes(self): with vfs_test_lib.RegistryVFSStubber(): client_id = self.SetupClient(0) pb = rdf_paths.PathSpec( path="/HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest", pathtype=rdf_paths.PathSpec.PathType.REGISTRY) client_mock = action_mocks.ListDirectoryClientMock() flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.ListDirectory), client_mock, client_id=client_id, pathspec=pb, token=self.token) if data_store.AFF4Enabled(): output_path = client_id.Add("registry").Add(pb.first.path) results = list( aff4.FACTORY.Open(output_path, token=self.token).OpenChildren()) self.assertLen(results, 2) for result in results: st = result.Get(result.Schema.STAT) self.assertIsNone(st.st_mtime) else: children = data_store.REL_DB.ListChildPathInfos( self.client_id.Basename(), rdf_objects.PathInfo.PathType.REGISTRY, ["HKEY_LOCAL_MACHINE", "SOFTWARE", "ListingTest"]) self.assertLen(children, 2) for child in children: self.assertIsNone(child.stat_entry.st_mtime)
def testListDirectory(self): """Test that the ListDirectory flow works.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec(path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) pb.Append(path="test directory", pathtype=rdf_paths.PathSpec.PathType.TSK) # Change the username so we get a notification about the flow termination. token = self.token.Copy() token.username = "******" with mock.patch.object(notification, "Notify") as mock_notify: flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.ListDirectory), client_mock, client_id=self.client_id, pathspec=pb, token=token) self.assertEqual(mock_notify.call_count, 1) args = list(mock_notify.mock_calls[0])[1] self.assertEqual(args[0], token.username) com = rdf_objects.UserNotification.Type.TYPE_VFS_LIST_DIRECTORY_COMPLETED self.assertEqual(args[1], com) self.assertIn(pb.path, args[2]) if data_store.AFF4Enabled(): # Check the output file is created output_path = self.client_id.Add("fs/tsk").Add(pb.first.path) fd = aff4.FACTORY.Open(output_path.Add("Test Directory"), token=self.token) children = list(fd.OpenChildren()) self.assertLen(children, 1) child = children[0] # Check that the object is stored with the correct casing. self.assertEqual(child.urn.Basename(), "numbers.txt") # And the wrong object is not there self.assertRaises(IOError, aff4.FACTORY.Open, output_path.Add("test directory"), aff4_type=aff4_standard.VFSDirectory, token=self.token) else: children = self._ListTestChildPathInfos(["test_img.dd"]) self.assertLen(children, 1) self.assertEqual(children[0].components[-1], "Test Directory")
def testListDirectoryOnFile(self): """OS ListDirectory on a file will raise.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec(path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) # Make sure the flow raises. with self.assertRaises(RuntimeError): flow_test_lib.TestFlowHelper(filesystem.ListDirectory.__name__, client_mock, client_id=self.client_id, pathspec=pb, token=self.token)
def testListDirectoryOnNonexistentDir(self): """Test that the ListDirectory flow works.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec(path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) pb.Append(path="doesnotexist", pathtype=rdf_paths.PathSpec.PathType.TSK) with self.assertRaises(RuntimeError): flow_test_lib.TestFlowHelper(filesystem.ListDirectory.__name__, client_mock, client_id=self.client_id, pathspec=pb, token=self.token)
def testListDirectoryOnNonexistentDir(self): """Test that the ListDirectory flow works.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) pb.Append(path="doesnotexist", pathtype=rdf_paths.PathSpec.PathType.TSK) with self.assertRaises(RuntimeError): with test_lib.SuppressLogs(): flow_test_lib.TestFlowHelper( compatibility.GetName(filesystem.ListDirectory), client_mock, client_id=self.client_id, pathspec=pb, creator=self.test_username)
def testSystemRootFallback(self): with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, vfs_test_lib.ClientVFSHandlerFixture): client_mock = action_mocks.ListDirectoryClientMock() session_id = flow_test_lib.TestFlowHelper( artifact_fallbacks.SystemRootSystemDriveFallbackFlow.__name__, client_mock, client_id=test_lib.TEST_CLIENT_ID, token=self.token, artifact_name="WindowsEnvironmentVariableSystemRoot") output_fd = flow.GRRFlow.ResultCollectionForFID(session_id) self.assertEqual(len(output_fd), 1) self.assertEqual(str(output_fd[0].registry_data.GetValue()), r"C:\WINDOWS")
def testListDirectoryOnFile(self): """OS ListDirectory on a file will raise.""" client_mock = action_mocks.ListDirectoryClientMock() pb = rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) # Make sure the flow raises. with self.assertRaises(RuntimeError): with test_lib.SuppressLogs(): flow_test_lib.TestFlowHelper( compatibility.GetName(filesystem.ListDirectory), client_mock, client_id=self.client_id, pathspec=pb, creator=self.test_username)
def testSystemRootFallback(self): client_id = self.SetupClient(0) with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, vfs_test_lib.ClientVFSHandlerFixture): client_mock = action_mocks.ListDirectoryClientMock() session_id = flow_test_lib.TestFlowHelper( artifact_fallbacks.SystemRootSystemDriveFallbackFlow.__name__, client_mock, client_id=client_id, creator=self.test_username, artifact_name="WindowsEnvironmentVariableSystemRoot") results = flow_test_lib.GetFlowResults(client_id, session_id) self.assertLen(results, 1) self.assertEqual(str(results[0].registry_data.GetValue()), r"C:\WINDOWS")
def testEndToEndTestsResultChecking(self): self.client_ids = [ "aff4:/C.6000000000000000", "aff4:/C.6000000000000001", "aff4:/C.6000000000000002" ] for clientid in self.client_ids: self._SetSummaries(clientid) self.client_mock = action_mocks.ListDirectoryClientMock() endtoend = system.EndToEndTests(None, token=self.token) endtoend.state.hunt_id = "aff4:/temphuntid" endtoend.state.client_ids = set(self.client_ids) endtoend.state.client_ids_failures = set() endtoend.state.client_ids_result_reported = set() # No results at all self.assertRaises(flow.FlowError, endtoend._CheckForSuccess, []) # Not enough client results endtoend.state.client_ids_failures = set() endtoend.state.client_ids_result_reported = set() self.assertRaises( flow.FlowError, endtoend._CheckForSuccess, [self._CreateResult(True, "aff4:/C.6000000000000001")]) # All clients succeeded endtoend.state.client_ids_failures = set() endtoend.state.client_ids_result_reported = set() endtoend._CheckForSuccess([ self._CreateResult(True, "aff4:/C.6000000000000000"), self._CreateResult(True, "aff4:/C.6000000000000001"), self._CreateResult(True, "aff4:/C.6000000000000002") ]) # All clients complete, but some failures endtoend.state.client_ids_failures = set() endtoend.state.client_ids_result_reported = set() self.assertRaises(flow.FlowError, endtoend._CheckForSuccess, [ self._CreateResult(True, "aff4:/C.6000000000000000"), self._CreateResult(False, "aff4:/C.6000000000000001"), self._CreateResult(False, "aff4:/C.6000000000000002") ])
def testEndToEndTests(self): self.client_ids = [ "aff4:/C.6000000000000000", "aff4:/C.6000000000000001", "aff4:/C.6000000000000002" ] for clientid in self.client_ids: self._SetSummaries(clientid) self.client_mock = action_mocks.ListDirectoryClientMock() with test_lib.ConfigOverrider( {"Test.end_to_end_client_ids": self.client_ids}): with utils.MultiStubber( (base.AutomatedTest, "classes", { "MockEndToEndTest": endtoend_mocks.MockEndToEndTest }), (system.EndToEndTests, "lifetime", 0)): # The test harness doesn't understand the callstate at a later time that # this flow is doing, so we need to disable check_flow_errors. for _ in flow_test_lib.TestFlowHelper( system.EndToEndTests.__name__, self.client_mock, client_id=self.client_id, check_flow_errors=False, token=self.token): pass hunt_test_lib.TestHuntHelperWithMultipleMocks( {}, check_flow_errors=False, token=self.token) hunt_ids = list( aff4.FACTORY.Open("aff4:/hunts", token=self.token).ListChildren()) # We have only created one hunt, and we should have started with a clean # aff4 space. self.assertEqual(len(hunt_ids), 1) hunt_obj = aff4.FACTORY.Open(hunt_ids[0], token=self.token, age=aff4.ALL_TIMES) self.assertItemsEqual(sorted(hunt_obj.GetClients()), sorted(self.client_ids))
def testListingRegistryDirectoryDoesNotYieldMtimes(self): with vfs_test_lib.RegistryVFSStubber(): client_id = self.SetupClient(0) pb = rdf_paths.PathSpec( path="/HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest", pathtype=rdf_paths.PathSpec.PathType.REGISTRY) client_mock = action_mocks.ListDirectoryClientMock() flow_test_lib.TestFlowHelper(compatibility.GetName( filesystem.ListDirectory), client_mock, client_id=client_id, pathspec=pb, token=self.token) children = data_store.REL_DB.ListChildPathInfos( self.client_id, rdf_objects.PathInfo.PathType.REGISTRY, ["HKEY_LOCAL_MACHINE", "SOFTWARE", "ListingTest"]) self.assertLen(children, 2) for child in children: self.assertIsNone(child.stat_entry.st_mtime)
def testRecursiveListDirectoryTrivial(self): """Test that RecursiveListDirectory lists files only up to max depth.""" client_mock = action_mocks.ListDirectoryClientMock() dir_components = ["dir1", "dir2"] with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: os.makedirs(os.path.join(temp_dirpath, *dir_components)) pathspec = rdf_paths.PathSpec( path=temp_dirpath, pathtype=rdf_paths.PathSpec.PathType.OS) flow_id = flow_test_lib.TestFlowHelper( compatibility.GetName(filesystem.RecursiveListDirectory), client_mock, client_id=self.client_id, pathspec=pathspec, max_depth=1, creator=self.test_username) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) self.assertLen(results, 1) self.assertEqual(results[0].pathspec.Basename(), "dir1")
def testNotificationWhenListingRegistry(self): # Change the username so notifications get written. token = self.token.Copy() token.username = "******" acl_test_lib.CreateUser(token.username) with vfs_test_lib.RegistryVFSStubber(): client_id = self.SetupClient(0) pb = rdf_paths.PathSpec( path="/HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest", pathtype=rdf_paths.PathSpec.PathType.REGISTRY) client_mock = action_mocks.ListDirectoryClientMock() flow_test_lib.TestFlowHelper( compatibility.GetName(filesystem.ListDirectory), client_mock, client_id=client_id, pathspec=pb, token=token) if data_store.RelationalDBEnabled(): notifications = data_store.REL_DB.ReadUserNotifications(token.username) self.assertLen(notifications, 1) n = notifications[0] self.assertEqual(n.reference.vfs_file.path_type, rdf_objects.PathInfo.PathType.REGISTRY) self.assertEqual(n.reference.vfs_file.path_components, ["HKEY_LOCAL_MACHINE", "SOFTWARE", "ListingTest"]) else: user = aff4.FACTORY.Open("aff4:/users/%s" % token.username, token=token) notifications = user.Get(user.Schema.PENDING_NOTIFICATIONS) self.assertLen(notifications, 1) expected_urn = ("aff4:/C.1000000000000000/registry/" "HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest") self.assertEqual(notifications[0].subject, expected_urn)
def testFlowExecution(self): client_mock = action_mocks.ListDirectoryClientMock() client_id = self.SetupClient(0) rollover = aff4.AUDIT_ROLLOVER_TIME.seconds # Set time to epoch + 20 intervals with test_lib.FakeTime(20 * rollover): flow_test_lib.TestFlowHelper( filesystem.ListDirectory.__name__, client_mock, client_id=client_id, pathspec=rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd/test directory"), pathtype=rdf_paths.PathSpec.PathType.OS), token=self.token) flow_test_lib.TestFlowHelper( filesystem.ListDirectory.__name__, client_mock, client_id=client_id, pathspec=rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd/glob_test"), pathtype=rdf_paths.PathSpec.PathType.OS), token=self.token) parentdir = aff4.FACTORY.Open("aff4:/audit/logs", aff4.AFF4Volume, mode="r", token=self.token) logs = list(parentdir.ListChildren()) self.assertEqual(len(logs), 1) log = aff4.CurrentAuditLog() stored_events = audit.AuditEventCollection(log) self.assertEqual(len(stored_events), 2) for event in stored_events: self.assertEqual(event.action, rdf_events.AuditEvent.Action.RUN_FLOW) self.assertEqual(event.flow_name, filesystem.ListDirectory.__name__) self.assertEqual(event.user, self.token.username) # Set time to epoch + 22 intervals with test_lib.FakeTime(22 * rollover): flow_test_lib.TestFlowHelper( filesystem.ListDirectory.__name__, client_mock, client_id=client_id, pathspec=rdf_paths.PathSpec( path=os.path.join(self.base_path, "test_img.dd/test directory"), pathtype=rdf_paths.PathSpec.PathType.OS), token=self.token) parentdir = aff4.FACTORY.Open("aff4:/audit/logs", aff4.AFF4Volume, mode="r", token=self.token) # Now we should have two collections logs = list(parentdir.ListChildren()) self.assertEqual(len(logs), 2) # One with two events stored_events = audit.AuditEventCollection(logs[0]) self.assertEqual(len(stored_events), 2) # The other with one stored_events = audit.AuditEventCollection(logs[1]) self.assertEqual(len(stored_events), 1)