def testButlerUri(self):
"""Tests whether ButlerURI instantiates correctly given different
arguments.
"""
# Root to use for relative paths
testRoot = "/tmp/"
# uriStrings is a list of tuples containing test string, forceAbsolute,
# forceDirectory as arguments to ButlerURI and scheme, netloc and path
# as expected attributes. Test asserts constructed equals to expected.
# 1) no determinable schemes (ensures schema and netloc are not set)
osRelFilePath = os.path.join(testRoot, "relative/file.ext")
uriStrings = [
("relative/file.ext", True, False, "", "", osRelFilePath),
("relative/file.ext", False, False, "", "", "relative/file.ext"),
("test/../relative/file.ext", True, False, "", "", osRelFilePath),
("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
("relative/dir", False, True, "", "", "relative/dir/")
]
# 2) implicit file scheme, tests absolute file and directory paths
uriStrings.extend((
("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'),
("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/"))
))
# 3) explicit file scheme, absolute and relative file and directory URI
posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
uriStrings.extend((
("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
("file:///absolute/directory", True, True, "file", "", "/absolute/directory/")
))
# 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
uriStrings.extend((
("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
("s3://bucketname/rootDir/relative/file.ext", True, False, "s3",
"bucketname", "/rootDir/relative/file.ext")
))
# 5) HTTPS scheme
uriStrings.extend((
("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"),
("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"),
("https://www.lsst.org/rootDir/relative/file.ext", True, False, "https",
"www.lsst.org", "/rootDir/relative/file.ext")
))
for uriInfo in uriStrings:
uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1],
forceDirectory=uriInfo[2])
with self.subTest(uri=uriInfo[0]):
self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
self.assertEqual(uri.path, uriInfo[5], "test path")
# test root becomes abspath(".") when not specified, note specific
# file:// scheme case
uriStrings = (
("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"),
("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext"))
)
for uriInfo in uriStrings:
uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
with self.subTest(uri=uriInfo[0]):
self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
self.assertEqual(uri.path, uriInfo[5], "test path")
# File replacement
uriStrings = (
("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
("relative/", "newfile.fits", "relative/newfile.fits"),
("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml")
)
for uriInfo in uriStrings:
uri = ButlerURI(uriInfo[0], forceAbsolute=False)
uri.updateFile(uriInfo[1])
with self.subTest(uri=uriInfo[0]):
self.assertEqual(uri.path, uriInfo[2])
# Check that schemeless can become file scheme
schemeless = ButlerURI("relative/path.ext")
filescheme = ButlerURI("/absolute/path.ext")
self.assertFalse(schemeless.scheme)
self.assertEqual(filescheme.scheme, "file")
self.assertNotEqual(type(schemeless), type(filescheme))
# Copy constructor
uri = ButlerURI("s3://amazon/datastore", forceDirectory=True)
uri2 = ButlerURI(uri)
self.assertEqual(uri, uri2)
uri = ButlerURI("file://amazon/datastore/file.txt")
uri2 = ButlerURI(uri)
self.assertEqual(uri, uri2)
# Copy constructor using subclass
uri3 = type(uri)(uri)
self.assertEqual(type(uri), type(uri3))
# Explicit copy
uri4 = copy.copy(uri3)
self.assertEqual(uri4, uri3)
uri4 = copy.deepcopy(uri3)
self.assertEqual(uri4, uri3)
def testEscapes(self):
"""Special characters in file paths"""
src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
self.assertFalse(src.scheme)
src.write(b"Some content")
self.assertTrue(src.exists())
# Use the internal API to force to a file
file = src._force_to_file()
self.assertTrue(file.exists())
self.assertIn("???", file.ospath)
self.assertNotIn("???", file.path)
file.updateFile("tests??.txt")
self.assertNotIn("??.txt", file.path)
file.write(b"Other content")
self.assertEqual(file.read(), b"Other content")
src.updateFile("tests??.txt")
self.assertIn("??.txt", src.path)
self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
# File URI and schemeless URI
parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
child = ButlerURI("e/f/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f/g.txt")
child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
# Schemeless so should not quote
dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
self.assertIn("???", dir.ospath)
self.assertIn("???", dir.path)
self.assertFalse(dir.scheme)
# dir.join() morphs into a file scheme
new = dir.join("test_j.txt")
self.assertIn("???", new.ospath, f"Checking {new}")
new.write(b"Content")
new2name = "###/test??.txt"
new2 = dir.join(new2name)
self.assertIn("???", new2.ospath)
new2.write(b"Content")
self.assertTrue(new2.ospath.endswith(new2name))
self.assertEqual(new.read(), new2.read())
fdir = dir._force_to_file()
self.assertNotIn("???", fdir.path)
self.assertIn("???", fdir.ospath)
self.assertEqual(fdir.scheme, "file")
fnew = dir.join("test_jf.txt")
fnew.write(b"Content")
fnew2 = fdir.join(new2name)
fnew2.write(b"Content")
self.assertTrue(fnew2.ospath.endswith(new2name))
self.assertNotIn("###", fnew2.path)
self.assertEqual(fnew.read(), fnew2.read())
# Test that children relative to schemeless and file schemes
# still return the same unquoted name
self.assertEqual(fnew2.relative_to(fdir), new2name)
self.assertEqual(fnew2.relative_to(dir), new2name)
self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}")
self.assertEqual(new2.relative_to(dir), new2name)
# Check for double quoting
plus_path = "/a/b/c+d/"
with self.assertLogs(level="WARNING"):
uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
self.assertEqual(uri.ospath, plus_path)
# Check that # is not escaped for schemeless URIs
hash_path = "/a/b#/c&d#xyz"
hpos = hash_path.rfind("#")
uri = ButlerURI(hash_path)
self.assertEqual(uri.ospath, hash_path[:hpos])
self.assertEqual(uri.fragment, hash_path[hpos + 1:])
def testButlerUri(self):
# Root to use for relative paths
testRoot = "/tmp/"
uriStrings = (
# Test string, forceAbsolute, scheme, netloc, path
# These are being tested with forceAbsolute=True
("file:///rootDir/absolute/file.ext", True, "file", "",
"/rootDir/absolute/file.ext"),
("/rootDir/absolute/file.ext", True, "file", "",
"/rootDir/absolute/file.ext"),
("/rootDir/absolute/file.ext", False, "file", "",
"/rootDir/absolute/file.ext"),
("/rootDir/absolute/", True, "file", "", "/rootDir/absolute/"),
("file:relative/file.ext", True, "file", "",
posixpath.join(testRoot, "relative/file.ext")),
("file:relative/directory/", True, "file", "",
posixpath.join(testRoot, "relative/directory/")),
("file://relative/file.ext", True, "file", "relative",
"/file.ext"),
("file:///absolute/directory/", True, "file", "",
"/absolute/directory/"),
("relative/file.ext", True, "", "",
os.path.join(testRoot, "relative/file.ext")),
("relative/file.ext", False, "", "", "relative/file.ext"),
("s3://bucketname/rootDir/relative/file.ext", True, "s3",
"bucketname", "/rootDir/relative/file.ext"),
("~/relative/file.ext", True, "file", "",
os.path.expanduser("~/relative/file.ext")),
("~/relative/file.ext", False, "file", "",
os.path.expanduser("~/relative/file.ext")),
("test/../relative/file.ext", True, "", "",
os.path.join(testRoot, "relative/file.ext")),
("test/../relative/file.ext", False, "", "", "relative/file.ext"),
)
for uriInfo in uriStrings:
uri = ButlerURI(uriInfo[0],
root=testRoot,
forceAbsolute=uriInfo[1])
with self.subTest(uri=uriInfo[0]):
self.assertEqual(uri.scheme, uriInfo[2], "test scheme")
self.assertEqual(uri.netloc, uriInfo[3], "test netloc")
self.assertEqual(uri.path, uriInfo[4], "test path")
# File replacement
uriStrings = (
("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
("relative/", "newfile.fits", "relative/newfile.fits"),
("isThisADirOrFile", "aFile.fits", "aFile.fits"),
("https://www.lsst.org/butler/", "butler.yaml",
"/butler/butler.yaml"),
("s3://amazon/datastore/", "butler.yaml",
"/datastore/butler.yaml"),
("s3://amazon/datastore/mybutler.yaml", "butler.yaml",
"/datastore/butler.yaml"),
)
for uriInfo in uriStrings:
uri = ButlerURI(uriInfo[0], forceAbsolute=False)
uri.updateFile(uriInfo[1])
with self.subTest(uri=uriInfo[0]):
self.assertEqual(uri.path, uriInfo[2])
