def testMultiGetFileSetsFileHashAttributeWhenMultipleChunksDownloaded( self): client_mock = action_mocks.MultiGetFileClientMock() pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS, path=os.path.join( self.base_path, "test_img.dd")) args = transfer.MultiGetFileArgs(pathspecs=[pathspec]) for _ in test_lib.TestFlowHelper("MultiGetFile", client_mock, token=self.token, client_id=self.client_id, args=args): pass # Fix path for Windows testing. pathspec.path = pathspec.path.replace("\\", "/") # Test the AFF4 file that was created. urn = pathspec.AFF4Path(self.client_id) fd = aff4.FACTORY.Open(urn, token=self.token) fd_hash = fd.Get(fd.Schema.HASH) self.assertTrue(fd_hash) h = hashlib.sha256() with open(os.path.join(self.base_path, "test_img.dd"), "rb") as model_fd: h.update(model_fd.read()) self.assertEqual(fd_hash.sha256, h.digest())
def testMultiGetFileDeduplication(self): client_mock = action_mocks.MultiGetFileClientMock() pathspecs = [] # Make 10 files to download. for i in xrange(10): path = os.path.join(self.temp_dir, "test_%s.txt" % i) with open(path, "wb") as fd: fd.write("Hello") pathspecs.append( rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS, path=path)) # All those files are the same so the individual chunks should # only be downloaded once. By forcing maximum_pending_files=1, # there should only be a single TransferBuffer call. args = transfer.MultiGetFileArgs(pathspecs=pathspecs, maximum_pending_files=1) for _ in test_lib.TestFlowHelper("MultiGetFile", client_mock, token=self.token, client_id=self.client_id, args=args): pass self.assertEqual(client_mock.action_counts["TransferBuffer"], 1)
def testMultiGetFile(self): """Test MultiGetFile.""" client_mock = action_mocks.MultiGetFileClientMock() pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS, path=os.path.join( self.base_path, "test_img.dd")) args = transfer.MultiGetFileArgs(pathspecs=[pathspec, pathspec]) with test_lib.Instrument(transfer.MultiGetFile, "StoreStat") as storestat_instrument: for _ in test_lib.TestFlowHelper("MultiGetFile", client_mock, token=self.token, client_id=self.client_id, args=args): pass # We should only have called StoreStat once because the two paths # requested were identical. self.assertEqual(len(storestat_instrument.args), 1) # Fix path for Windows testing. pathspec.path = pathspec.path.replace("\\", "/") # Test the AFF4 file that was created. urn = pathspec.AFF4Path(self.client_id) fd1 = aff4.FACTORY.Open(urn, token=self.token) fd2 = open(pathspec.path, "rb") fd2.seek(0, 2) self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE))) self.CompareFDs(fd1, fd2)
def testMultiGetFileOfSpecialFiles(self): """Test that special /proc/ files are handled correctly. /proc/ files have the property that they are non seekable from their end (i.e. seeking them relative to the end is not supported). They also return an st_size of 0. For example: $ stat /proc/self/maps File: '/proc/self/maps' Size: 0 Blocks: 0 IO Block: 1024 regular empty file $ head /proc/self/maps 00400000-00409000 r-xp 00000000 fc:01 9180740 /usr/bin/head 00608000-00609000 r--p 00008000 fc:01 9180740 /usr/bin/head ... When we try to use the MultiGetFile flow, it deduplicates the files and since it thinks the file has a zero size, the flow will not download the file, and instead copy the zero size file into it. """ client_mock = action_mocks.MultiGetFileClientMock() # # Create a zero sized file. zero_sized_filename = os.path.join(self.temp_dir, "zero_size") with open(zero_sized_filename, "wb") as fd: pass pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS, path=zero_sized_filename) for _ in test_lib.TestFlowHelper("MultiGetFile", client_mock, token=self.token, file_size="1MiB", client_id=self.client_id, pathspecs=[pathspec]): pass # Now if we try to fetch a real /proc/ filename this will fail because the # filestore already contains the zero length file # aff4:/files/nsrl/da39a3ee5e6b4b0d3255bfef95601890afd80709. pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS, path="/proc/self/environ") for _ in test_lib.TestFlowHelper("MultiGetFile", client_mock, token=self.token, file_size=1024 * 1024, client_id=self.client_id, pathspecs=[pathspec]): pass data = open(pathspec.last.path, "rb").read() # Test the AFF4 file that was created - it should be empty since by default # we judge the file size based on its stat.st_size. urn = pathspec.AFF4Path(self.client_id) fd = aff4.FACTORY.Open(urn, token=self.token) self.assertEqual(fd.size, len(data)) self.assertMultiLineEqual(fd.read(len(data)), data)
def testMultiGetFileMultiFiles(self): """Test MultiGetFile downloading many files at once.""" client_mock = action_mocks.MultiGetFileClientMock() pathspecs = [] # Make 30 files to download. for i in xrange(30): path = os.path.join(self.temp_dir, "test_%s.txt" % i) with open(path, "wb") as fd: fd.write("Hello") pathspecs.append( rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS, path=path)) args = transfer.MultiGetFileArgs(pathspecs=pathspecs, maximum_pending_files=10) for session_id in test_lib.TestFlowHelper("MultiGetFile", client_mock, token=self.token, client_id=self.client_id, args=args): # Check up on the internal flow state. flow_obj = aff4.FACTORY.Open(session_id, mode="r", token=self.token) flow_state = flow_obj.state # All the pathspecs should be in this list. self.assertEqual(len(flow_state.indexed_pathspecs), 30) # At any one time, there should not be more than 10 files or hashes # pending. self.assertLessEqual(len(flow_state.pending_files), 10) self.assertLessEqual(len(flow_state.pending_hashes), 10) # When we finish there should be no pathspecs stored in the flow state. for flow_pathspec in flow_state.indexed_pathspecs: self.assertIsNone(flow_pathspec) for flow_request_data in flow_state.request_data_list: self.assertIsNone(flow_request_data) # Now open each file and make sure the data is there. for pathspec in pathspecs: urn = pathspec.AFF4Path(self.client_id) fd = aff4.FACTORY.Open(urn, token=self.token) self.assertEqual("Hello", fd.Read(100000))
def testMultiGetFileSizeLimit(self): client_mock = action_mocks.MultiGetFileClientMock() image_path = os.path.join(self.base_path, "test_img.dd") pathspec = rdf_paths.PathSpec( pathtype=rdf_paths.PathSpec.PathType.OS, path=image_path) # Read a bit more than one chunk (600 * 1024). expected_size = 750 * 1024 args = transfer.MultiGetFileArgs( pathspecs=[pathspec], file_size=expected_size) for _ in flow_test_lib.TestFlowHelper( transfer.MultiGetFile.__name__, client_mock, token=self.token, client_id=self.client_id, args=args): pass urn = pathspec.AFF4Path(self.client_id) blobimage = aff4.FACTORY.Open(urn, token=self.token) # Make sure a VFSBlobImage got written. self.assertTrue(isinstance(blobimage, aff4_grr.VFSBlobImage)) self.assertEqual(len(blobimage), expected_size) data = blobimage.read(100 * expected_size) self.assertEqual(len(data), expected_size) expected_data = open(image_path, "rb").read(expected_size) self.assertEqual(data, expected_data) hash_obj = blobimage.Get(blobimage.Schema.HASH) d = hashlib.sha1() d.update(expected_data) expected_hash = d.hexdigest() self.assertEqual(hash_obj.sha1, expected_hash)
def testRefreshFileStartsFlow(self): self.Open("/") self.Type("client_query", "C.0000000000000001") self.Click("client_query_submit") self.WaitUntilEqual(u"C.0000000000000001", self.GetText, "css=span[type=subject]") # Choose client 1. self.Click("css=td:contains('0001')") # Go to Browse VFS. self.Click("css=a:contains('Browse Virtual Filesystem')") self.Click("css=#_fs i.jstree-icon") self.Click("css=#_fs-os i.jstree-icon") self.Click("css=#_fs-os-c i.jstree-icon") # Test file versioning. self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads") self.Click("link=Downloads") # Select a file and start a flow by requesting a newer version. self.Click("css=tr:contains(\"a.txt\")") self.Click("css=li[heading=Download]") self.Click("css=button:contains(\"Collect from the client\")") # Create a new file version (that would have been created by the flow # otherwise) and finish the flow. with self.ACLChecksDisabled(): client_id = rdf_client.ClientURN("C.0000000000000001") fd = aff4.FACTORY.Open(client_id.Add("flows"), token=self.token) # Make sure that the flow has started (when button is clicked, the HTTP # API request is sent asynchronously). def MultiGetFileStarted(): return "MultiGetFile" in list(x.__class__.__name__ for x in fd.OpenChildren()) self.WaitUntil(MultiGetFileStarted) flows = list(fd.ListChildren()) client_mock = action_mocks.MultiGetFileClientMock() for flow_urn in flows: for _ in test_lib.TestFlowHelper(flow_urn, client_mock, client_id=client_id, check_flow_errors=False, token=self.token): pass time_in_future = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration( "1h") # We have to make sure that the new version will not be within a second # from the current one, otherwise the previous one and the new one will # be indistinguishable in the UI (as it has a 1s precision when # displaying versions). with test_lib.FakeTime(time_in_future): with aff4.FACTORY.Open( "aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt", aff4_type=aff4_grr.VFSFile, mode="rw", token=self.token) as fd: fd.Write("The newest version!") # Once the flow has finished, the file view should update and add the # newly created, latest version of the file to the list. The selected # option should still be "HEAD". self.WaitUntilContains("HEAD", self.GetText, "css=.version-dropdown > option[selected]") self.WaitUntilContains(gui_test_lib.DateTimeString(time_in_future), self.GetText, "css=.version-dropdown > option:nth(1)") # The file table should also update and display the new timestamp. self.WaitUntil( self.IsElementPresent, "css=grr-file-table tbody > tr td:contains(\"%s\")" % (gui_test_lib.DateTimeString(time_in_future))) # Make sure the file content has changed. self.Click("css=li[heading=TextView]") self.WaitUntilContains("The newest version!", self.GetText, "css=div.monospace pre") # Go to the flow management screen and check that there was a new flow. self.Click("css=a:contains('Manage launched flows')") self.Click("css=grr-flows-list tr:contains('MultiGetFile')") self.WaitUntilContains("MultiGetFile", self.GetText, "css=#main_bottomPane") self.WaitUntilContains( "c/Downloads/a.txt", self.GetText, "css=#main_bottomPane table > tbody td.proto_key:contains(\"Path\") " "~ td.proto_value")