def testTransfer(self):
src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
content = "Content is some content\nwith something to say\n\n"
src.write(content.encode())
dest = ButlerURI(self.makeS3Uri("test.txt"))
self.assertFalse(dest.exists())
dest.transfer_from(src, transfer="copy")
self.assertTrue(dest.exists())
dest2 = ButlerURI(self.makeS3Uri("copied.txt"))
dest2.transfer_from(dest, transfer="copy")
self.assertTrue(dest2.exists())
local = ButlerURI(os.path.join(self.tmpdir, "copied.txt"))
local.transfer_from(dest2, transfer="copy")
with open(local.ospath, "r") as fd:
new_content = fd.read()
self.assertEqual(new_content, content)
with self.assertRaises(ValueError):
dest2.transfer_from(local, transfer="symlink")
b = dest.read()
self.assertEqual(b.decode(), new_content)
nbytes = 10
subset = dest.read(size=nbytes)
self.assertEqual(len(subset), nbytes) # Extra byte comes back
self.assertEqual(subset.decode(), content[:nbytes])
with self.assertRaises(FileExistsError):
dest.transfer_from(src, transfer="copy")
dest.transfer_from(src, transfer="copy", overwrite=True)
def testTransfer(self):
src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
content = "Content is some content\nwith something to say\n\n"
src.write(content.encode())
for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"):
dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt"))
dest.transfer_from(src, transfer=mode)
self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
with open(dest.ospath, "r") as fh:
new_content = fh.read()
self.assertEqual(new_content, content)
if mode in ("symlink", "relsymlink"):
self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
with self.assertRaises(FileExistsError):
dest.transfer_from(src, transfer=mode)
dest.transfer_from(src, transfer=mode, overwrite=True)
os.remove(dest.ospath)
b = src.read()
self.assertEqual(b.decode(), new_content)
nbytes = 10
subset = src.read(size=nbytes)
self.assertEqual(len(subset), nbytes)
self.assertEqual(subset.decode(), content[:nbytes])
with self.assertRaises(ValueError):
src.transfer_from(src, transfer="unknown")
def testFile(self):
file = os.path.join(self.tmpdir, "test.txt")
uri = ButlerURI(file)
self.assertFalse(uri.exists(), f"{uri} should not exist")
self.assertEqual(uri.ospath, file)
content = "abcdefghijklmnopqrstuv\n"
uri.write(content.encode())
self.assertTrue(os.path.exists(file), "File should exist locally")
self.assertTrue(uri.exists(), f"{uri} should now exist")
self.assertEqual(uri.read().decode(), content)
def saveUri(self, uri):
"""Save `QuantumGraph` to the specified URI.
Parameters
----------
uri : `ButlerURI` or `str`
URI to where the graph should be saved.
"""
buffer = self._buildSaveObject()
butlerUri = ButlerURI(uri)
if butlerUri.getExtension() not in (".qgraph"):
raise TypeError(
f"Can currently only save a graph in qgraph format not {uri}")
butlerUri.write(
buffer
) # type: ignore # Ignore because bytearray is safe to use in place of bytes
def testFile(self):
file = os.path.join(self.tmpdir, "test.txt")
uri = ButlerURI(file)
self.assertFalse(uri.exists(), f"{uri} should not exist")
self.assertEqual(uri.ospath, file)
content = "abcdefghijklmnopqrstuv\n"
uri.write(content.encode())
self.assertTrue(os.path.exists(file), "File should exist locally")
self.assertTrue(uri.exists(), f"{uri} should now exist")
self.assertEqual(uri.read().decode(), content)
self.assertEqual(uri.size(), len(content.encode()))
with self.assertRaises(FileNotFoundError):
ButlerURI("file/not/there.txt").size()
# Check that creating a URI from a URI returns the same thing
uri2 = ButlerURI(uri)
self.assertEqual(uri, uri2)
self.assertEqual(id(uri), id(uri2))
class WebdavURITestCase(unittest.TestCase):
def setUp(self):
serverRoot = "www.not-exists.orgx"
existingFolderName = "existingFolder"
existingFileName = "existingFile"
notExistingFileName = "notExistingFile"
self.baseURL = ButlerURI(
f"https://{serverRoot}", forceDirectory=True)
self.existingFileButlerURI = ButlerURI(
f"https://{serverRoot}/{existingFolderName}/{existingFileName}")
self.notExistingFileButlerURI = ButlerURI(
f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}")
self.existingFolderButlerURI = ButlerURI(
f"https://{serverRoot}/{existingFolderName}", forceDirectory=True)
self.notExistingFolderButlerURI = ButlerURI(
f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True)
# Need to declare the options
responses.add(responses.OPTIONS,
self.baseURL.geturl(),
status=200, headers={"DAV": "1,2,3"})
# Used by ButlerHttpURI.exists()
responses.add(responses.HEAD,
self.existingFileButlerURI.geturl(),
status=200, headers={'Content-Length': '1024'})
responses.add(responses.HEAD,
self.notExistingFileButlerURI.geturl(),
status=404)
# Used by ButlerHttpURI.read()
responses.add(responses.GET,
self.existingFileButlerURI.geturl(),
status=200,
body=str.encode("It works!"))
responses.add(responses.GET,
self.notExistingFileButlerURI.geturl(),
status=404)
# Used by ButlerHttpURI.write()
responses.add(responses.PUT,
self.existingFileButlerURI.geturl(),
status=201)
# Used by ButlerHttpURI.transfer_from()
responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
method="COPY",
headers={"Destination": self.existingFileButlerURI.geturl()},
status=201))
responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
method="COPY",
headers={"Destination": self.notExistingFileButlerURI.geturl()},
status=201))
responses.add(responses.Response(url=self.existingFileButlerURI.geturl(),
method="MOVE",
headers={"Destination": self.notExistingFileButlerURI.geturl()},
status=201))
# Used by ButlerHttpURI.remove()
responses.add(responses.DELETE,
self.existingFileButlerURI.geturl(),
status=200)
responses.add(responses.DELETE,
self.notExistingFileButlerURI.geturl(),
status=404)
# Used by ButlerHttpURI.mkdir()
responses.add(responses.HEAD,
self.existingFolderButlerURI.geturl(),
status=200, headers={'Content-Length': '1024'})
responses.add(responses.HEAD,
self.baseURL.geturl(),
status=200, headers={'Content-Length': '1024'})
responses.add(responses.HEAD,
self.notExistingFolderButlerURI.geturl(),
status=404)
responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(),
method="MKCOL",
status=201))
responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(),
method="MKCOL",
status=403))
@responses.activate
def testExists(self):
self.assertTrue(self.existingFileButlerURI.exists())
self.assertFalse(self.notExistingFileButlerURI.exists())
@responses.activate
def testRemove(self):
self.assertIsNone(self.existingFileButlerURI.remove())
with self.assertRaises(FileNotFoundError):
self.notExistingFileButlerURI.remove()
@responses.activate
def testMkdir(self):
# The mock means that we can't check this now exists
self.notExistingFolderButlerURI.mkdir()
# This should do nothing
self.existingFolderButlerURI.mkdir()
with self.assertRaises(ValueError):
self.notExistingFileButlerURI.mkdir()
@responses.activate
def testRead(self):
self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!")
self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.")
with self.assertRaises(FileNotFoundError):
self.notExistingFileButlerURI.read()
@responses.activate
def testWrite(self):
self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content.")))
with self.assertRaises(FileExistsError):
self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False)
@responses.activate
def testTransfer(self):
self.assertIsNone(self.notExistingFileButlerURI.transfer_from(
src=self.existingFileButlerURI))
self.assertIsNone(self.notExistingFileButlerURI.transfer_from(
src=self.existingFileButlerURI,
transfer="move"))
with self.assertRaises(FileExistsError):
self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI)
with self.assertRaises(ValueError):
self.notExistingFileButlerURI.transfer_from(
src=self.existingFileButlerURI,
transfer="unsupported")
def testParent(self):
self.assertEqual(self.existingFolderButlerURI.geturl(),
self.notExistingFileButlerURI.parent().geturl())
self.assertEqual(self.baseURL.geturl(),
self.baseURL.parent().geturl())
self.assertEqual(self.existingFileButlerURI.parent().geturl(),
self.existingFileButlerURI.dirname().geturl())
def testWrite(self):
s3write = ButlerURI(self.makeS3Uri("created.txt"))
content = "abcdefghijklmnopqrstuv\n"
s3write.write(content.encode())
self.assertEqual(s3write.read().decode(), content)
def testEscapes(self):
"""Special characters in file paths"""
src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
self.assertFalse(src.scheme)
src.write(b"Some content")
self.assertTrue(src.exists())
# Use the internal API to force to a file
file = src._force_to_file()
self.assertTrue(file.exists())
self.assertIn("???", file.ospath)
self.assertNotIn("???", file.path)
file.updateFile("tests??.txt")
self.assertNotIn("??.txt", file.path)
file.write(b"Other content")
self.assertEqual(file.read(), b"Other content")
src.updateFile("tests??.txt")
self.assertIn("??.txt", src.path)
self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
# File URI and schemeless URI
parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
child = ButlerURI("e/f/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f/g.txt")
child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
# Schemeless so should not quote
dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
self.assertIn("???", dir.ospath)
self.assertIn("???", dir.path)
self.assertFalse(dir.scheme)
# dir.join() morphs into a file scheme
new = dir.join("test_j.txt")
self.assertIn("???", new.ospath, f"Checking {new}")
new.write(b"Content")
new2name = "###/test??.txt"
new2 = dir.join(new2name)
self.assertIn("???", new2.ospath)
new2.write(b"Content")
self.assertTrue(new2.ospath.endswith(new2name))
self.assertEqual(new.read(), new2.read())
fdir = dir._force_to_file()
self.assertNotIn("???", fdir.path)
self.assertIn("???", fdir.ospath)
self.assertEqual(fdir.scheme, "file")
fnew = dir.join("test_jf.txt")
fnew.write(b"Content")
fnew2 = fdir.join(new2name)
fnew2.write(b"Content")
self.assertTrue(fnew2.ospath.endswith(new2name))
self.assertNotIn("###", fnew2.path)
self.assertEqual(fnew.read(), fnew2.read())
# Test that children relative to schemeless and file schemes
# still return the same unquoted name
self.assertEqual(fnew2.relative_to(fdir), new2name)
self.assertEqual(fnew2.relative_to(dir), new2name)
self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}")
self.assertEqual(new2.relative_to(dir), new2name)
# Check for double quoting
plus_path = "/a/b/c+d/"
with self.assertLogs(level="WARNING"):
uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
self.assertEqual(uri.ospath, plus_path)
# Check that # is not escaped for schemeless URIs
hash_path = "/a/b#/c&d#xyz"
hpos = hash_path.rfind("#")
uri = ButlerURI(hash_path)
self.assertEqual(uri.ospath, hash_path[:hpos])
self.assertEqual(uri.fragment, hash_path[hpos + 1:])
def testWalk(self):
"""Test that we can list an S3 bucket"""
# Files we want to create
expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt",
"a/b/c/d/v.json")
expected_uris = [ButlerURI(self.makeS3Uri(path)) for path in expected]
for uri in expected_uris:
# Doesn't matter what we write
uri.write("123".encode())
# Find all the files in the a/ tree
found = set(uri.path for uri in ButlerURI.findFileResources(
[ButlerURI(self.makeS3Uri("a/"))]))
self.assertEqual(found, {uri.path for uri in expected_uris})
# Find all the files in the a/ tree but group by folder
found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))],
grouped=True)
expected = (("/a/x.txt", "/a/y.txt", "/a/z.json"), ("/a/b/w.txt", ),
("/a/b/c/d/v.json", ))
for got, expect in zip(found, expected):
self.assertEqual(tuple(u.path for u in got), expect)
# Find only JSON files
found = set(uri.path for uri in ButlerURI.findFileResources(
[ButlerURI(self.makeS3Uri("a/"))], file_filter=r"\.json$"))
self.assertEqual(
found,
{uri.path
for uri in expected_uris if uri.path.endswith(".json")})
# JSON files grouped by directory
found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))],
file_filter=r"\.json$",
grouped=True)
expected = (("/a/z.json", ), ("/a/b/c/d/v.json", ))
for got, expect in zip(found, expected):
self.assertEqual(tuple(u.path for u in got), expect)
# Check pagination works with large numbers of files. S3 API limits
# us to 1000 response per list_objects call so create lots of files
created = set()
counter = 1
n_dir1 = 1100
while counter <= n_dir1:
new = ButlerURI(self.makeS3Uri(f"test/file{counter:04d}.txt"))
new.write(f"{counter}".encode())
created.add(str(new))
counter += 1
counter = 1
# Put some in a subdirectory to make sure we are looking in a
# hierarchy.
n_dir2 = 100
while counter <= n_dir2:
new = ButlerURI(
self.makeS3Uri(f"test/subdir/file{counter:04d}.txt"))
new.write(f"{counter}".encode())
created.add(str(new))
counter += 1
found = ButlerURI.findFileResources(
[ButlerURI(self.makeS3Uri("test/"))])
self.assertEqual({str(u) for u in found}, created)
# Again with grouping.
found = list(
ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))],
grouped=True))
self.assertEqual(len(found), 2)
dir_1 = list(found[0])
dir_2 = list(found[1])
self.assertEqual(len(dir_1), n_dir1)
self.assertEqual(len(dir_2), n_dir2)
