def testTransfer(self): src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) content = "Content is some content\nwith something to say\n\n" src.write(content.encode()) dest = ButlerURI(self.makeS3Uri("test.txt")) self.assertFalse(dest.exists()) dest.transfer_from(src, transfer="copy") self.assertTrue(dest.exists()) dest2 = ButlerURI(self.makeS3Uri("copied.txt")) dest2.transfer_from(dest, transfer="copy") self.assertTrue(dest2.exists()) local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) local.transfer_from(dest2, transfer="copy") with open(local.ospath, "r") as fd: new_content = fd.read() self.assertEqual(new_content, content) with self.assertRaises(ValueError): dest2.transfer_from(local, transfer="symlink") b = dest.read() self.assertEqual(b.decode(), new_content) nbytes = 10 subset = dest.read(size=nbytes) self.assertEqual(len(subset), nbytes) # Extra byte comes back self.assertEqual(subset.decode(), content[:nbytes]) with self.assertRaises(FileExistsError): dest.transfer_from(src, transfer="copy") dest.transfer_from(src, transfer="copy", overwrite=True)
def testTransfer(self): src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) content = "Content is some content\nwith something to say\n\n" src.write(content.encode()) for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) dest.transfer_from(src, transfer=mode) self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") with open(dest.ospath, "r") as fh: new_content = fh.read() self.assertEqual(new_content, content) if mode in ("symlink", "relsymlink"): self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") with self.assertRaises(FileExistsError): dest.transfer_from(src, transfer=mode) dest.transfer_from(src, transfer=mode, overwrite=True) os.remove(dest.ospath) b = src.read() self.assertEqual(b.decode(), new_content) nbytes = 10 subset = src.read(size=nbytes) self.assertEqual(len(subset), nbytes) self.assertEqual(subset.decode(), content[:nbytes]) with self.assertRaises(ValueError): src.transfer_from(src, transfer="unknown")
def testFile(self): file = os.path.join(self.tmpdir, "test.txt") uri = ButlerURI(file) self.assertFalse(uri.exists(), f"{uri} should not exist") self.assertEqual(uri.ospath, file) content = "abcdefghijklmnopqrstuv\n" uri.write(content.encode()) self.assertTrue(os.path.exists(file), "File should exist locally") self.assertTrue(uri.exists(), f"{uri} should now exist") self.assertEqual(uri.read().decode(), content)
def saveUri(self, uri): """Save `QuantumGraph` to the specified URI. Parameters ---------- uri : `ButlerURI` or `str` URI to where the graph should be saved. """ buffer = self._buildSaveObject() butlerUri = ButlerURI(uri) if butlerUri.getExtension() not in (".qgraph"): raise TypeError( f"Can currently only save a graph in qgraph format not {uri}") butlerUri.write( buffer ) # type: ignore # Ignore because bytearray is safe to use in place of bytes
def testFile(self): file = os.path.join(self.tmpdir, "test.txt") uri = ButlerURI(file) self.assertFalse(uri.exists(), f"{uri} should not exist") self.assertEqual(uri.ospath, file) content = "abcdefghijklmnopqrstuv\n" uri.write(content.encode()) self.assertTrue(os.path.exists(file), "File should exist locally") self.assertTrue(uri.exists(), f"{uri} should now exist") self.assertEqual(uri.read().decode(), content) self.assertEqual(uri.size(), len(content.encode())) with self.assertRaises(FileNotFoundError): ButlerURI("file/not/there.txt").size() # Check that creating a URI from a URI returns the same thing uri2 = ButlerURI(uri) self.assertEqual(uri, uri2) self.assertEqual(id(uri), id(uri2))
class WebdavURITestCase(unittest.TestCase): def setUp(self): serverRoot = "www.not-exists.orgx" existingFolderName = "existingFolder" existingFileName = "existingFile" notExistingFileName = "notExistingFile" self.baseURL = ButlerURI( f"https://{serverRoot}", forceDirectory=True) self.existingFileButlerURI = ButlerURI( f"https://{serverRoot}/{existingFolderName}/{existingFileName}") self.notExistingFileButlerURI = ButlerURI( f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") self.existingFolderButlerURI = ButlerURI( f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) self.notExistingFolderButlerURI = ButlerURI( f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) # Need to declare the options responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"}) # Used by ButlerHttpURI.exists() responses.add(responses.HEAD, self.existingFileButlerURI.geturl(), status=200, headers={'Content-Length': '1024'}) responses.add(responses.HEAD, self.notExistingFileButlerURI.geturl(), status=404) # Used by ButlerHttpURI.read() responses.add(responses.GET, self.existingFileButlerURI.geturl(), status=200, body=str.encode("It works!")) responses.add(responses.GET, self.notExistingFileButlerURI.geturl(), status=404) # Used by ButlerHttpURI.write() responses.add(responses.PUT, self.existingFileButlerURI.geturl(), status=201) # Used by ButlerHttpURI.transfer_from() responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), method="COPY", headers={"Destination": self.existingFileButlerURI.geturl()}, status=201)) responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), method="COPY", headers={"Destination": self.notExistingFileButlerURI.geturl()}, status=201)) responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), method="MOVE", headers={"Destination": self.notExistingFileButlerURI.geturl()}, status=201)) # Used by ButlerHttpURI.remove() responses.add(responses.DELETE, self.existingFileButlerURI.geturl(), status=200) responses.add(responses.DELETE, self.notExistingFileButlerURI.geturl(), status=404) # Used by ButlerHttpURI.mkdir() responses.add(responses.HEAD, self.existingFolderButlerURI.geturl(), status=200, headers={'Content-Length': '1024'}) responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={'Content-Length': '1024'}) responses.add(responses.HEAD, self.notExistingFolderButlerURI.geturl(), status=404) responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), method="MKCOL", status=201)) responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), method="MKCOL", status=403)) @responses.activate def testExists(self): self.assertTrue(self.existingFileButlerURI.exists()) self.assertFalse(self.notExistingFileButlerURI.exists()) @responses.activate def testRemove(self): self.assertIsNone(self.existingFileButlerURI.remove()) with self.assertRaises(FileNotFoundError): self.notExistingFileButlerURI.remove() @responses.activate def testMkdir(self): # The mock means that we can't check this now exists self.notExistingFolderButlerURI.mkdir() # This should do nothing self.existingFolderButlerURI.mkdir() with self.assertRaises(ValueError): self.notExistingFileButlerURI.mkdir() @responses.activate def testRead(self): self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") with self.assertRaises(FileNotFoundError): self.notExistingFileButlerURI.read() @responses.activate def testWrite(self): self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) with self.assertRaises(FileExistsError): self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) @responses.activate def testTransfer(self): self.assertIsNone(self.notExistingFileButlerURI.transfer_from( src=self.existingFileButlerURI)) self.assertIsNone(self.notExistingFileButlerURI.transfer_from( src=self.existingFileButlerURI, transfer="move")) with self.assertRaises(FileExistsError): self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) with self.assertRaises(ValueError): self.notExistingFileButlerURI.transfer_from( src=self.existingFileButlerURI, transfer="unsupported") def testParent(self): self.assertEqual(self.existingFolderButlerURI.geturl(), self.notExistingFileButlerURI.parent().geturl()) self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl()) self.assertEqual(self.existingFileButlerURI.parent().geturl(), self.existingFileButlerURI.dirname().geturl())
def testWrite(self): s3write = ButlerURI(self.makeS3Uri("created.txt")) content = "abcdefghijklmnopqrstuv\n" s3write.write(content.encode()) self.assertEqual(s3write.read().decode(), content)
def testEscapes(self): """Special characters in file paths""" src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) self.assertFalse(src.scheme) src.write(b"Some content") self.assertTrue(src.exists()) # Use the internal API to force to a file file = src._force_to_file() self.assertTrue(file.exists()) self.assertIn("???", file.ospath) self.assertNotIn("???", file.path) file.updateFile("tests??.txt") self.assertNotIn("??.txt", file.path) file.write(b"Other content") self.assertEqual(file.read(), b"Other content") src.updateFile("tests??.txt") self.assertIn("??.txt", src.path) self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") # File URI and schemeless URI parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) child = ButlerURI("e/f/g.txt", forceAbsolute=False) self.assertEqual(child.relative_to(parent), "e/f/g.txt") child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") # Schemeless so should not quote dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) self.assertIn("???", dir.ospath) self.assertIn("???", dir.path) self.assertFalse(dir.scheme) # dir.join() morphs into a file scheme new = dir.join("test_j.txt") self.assertIn("???", new.ospath, f"Checking {new}") new.write(b"Content") new2name = "###/test??.txt" new2 = dir.join(new2name) self.assertIn("???", new2.ospath) new2.write(b"Content") self.assertTrue(new2.ospath.endswith(new2name)) self.assertEqual(new.read(), new2.read()) fdir = dir._force_to_file() self.assertNotIn("???", fdir.path) self.assertIn("???", fdir.ospath) self.assertEqual(fdir.scheme, "file") fnew = dir.join("test_jf.txt") fnew.write(b"Content") fnew2 = fdir.join(new2name) fnew2.write(b"Content") self.assertTrue(fnew2.ospath.endswith(new2name)) self.assertNotIn("###", fnew2.path) self.assertEqual(fnew.read(), fnew2.read()) # Test that children relative to schemeless and file schemes # still return the same unquoted name self.assertEqual(fnew2.relative_to(fdir), new2name) self.assertEqual(fnew2.relative_to(dir), new2name) self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") self.assertEqual(new2.relative_to(dir), new2name) # Check for double quoting plus_path = "/a/b/c+d/" with self.assertLogs(level="WARNING"): uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) self.assertEqual(uri.ospath, plus_path) # Check that # is not escaped for schemeless URIs hash_path = "/a/b#/c&d#xyz" hpos = hash_path.rfind("#") uri = ButlerURI(hash_path) self.assertEqual(uri.ospath, hash_path[:hpos]) self.assertEqual(uri.fragment, hash_path[hpos + 1:])
def testWalk(self): """Test that we can list an S3 bucket""" # Files we want to create expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt", "a/b/c/d/v.json") expected_uris = [ButlerURI(self.makeS3Uri(path)) for path in expected] for uri in expected_uris: # Doesn't matter what we write uri.write("123".encode()) # Find all the files in the a/ tree found = set(uri.path for uri in ButlerURI.findFileResources( [ButlerURI(self.makeS3Uri("a/"))])) self.assertEqual(found, {uri.path for uri in expected_uris}) # Find all the files in the a/ tree but group by folder found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], grouped=True) expected = (("/a/x.txt", "/a/y.txt", "/a/z.json"), ("/a/b/w.txt", ), ("/a/b/c/d/v.json", )) for got, expect in zip(found, expected): self.assertEqual(tuple(u.path for u in got), expect) # Find only JSON files found = set(uri.path for uri in ButlerURI.findFileResources( [ButlerURI(self.makeS3Uri("a/"))], file_filter=r"\.json$")) self.assertEqual( found, {uri.path for uri in expected_uris if uri.path.endswith(".json")}) # JSON files grouped by directory found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], file_filter=r"\.json$", grouped=True) expected = (("/a/z.json", ), ("/a/b/c/d/v.json", )) for got, expect in zip(found, expected): self.assertEqual(tuple(u.path for u in got), expect) # Check pagination works with large numbers of files. S3 API limits # us to 1000 response per list_objects call so create lots of files created = set() counter = 1 n_dir1 = 1100 while counter <= n_dir1: new = ButlerURI(self.makeS3Uri(f"test/file{counter:04d}.txt")) new.write(f"{counter}".encode()) created.add(str(new)) counter += 1 counter = 1 # Put some in a subdirectory to make sure we are looking in a # hierarchy. n_dir2 = 100 while counter <= n_dir2: new = ButlerURI( self.makeS3Uri(f"test/subdir/file{counter:04d}.txt")) new.write(f"{counter}".encode()) created.add(str(new)) counter += 1 found = ButlerURI.findFileResources( [ButlerURI(self.makeS3Uri("test/"))]) self.assertEqual({str(u) for u in found}, created) # Again with grouping. found = list( ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))], grouped=True)) self.assertEqual(len(found), 2) dir_1 = list(found[0]) dir_2 = list(found[1]) self.assertEqual(len(dir_1), n_dir1) self.assertEqual(len(dir_2), n_dir2)