def testRelative(self):
"""Check that we can get subpaths back from two URIs"""
parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt"))
self.assertEqual(child.relative_to(parent), "dir1/file.txt")
not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt"))
self.assertFalse(not_child.relative_to(parent))
not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
self.assertFalse(child.relative_to(not_s3))
def testRelative(self):
"""Check that we can get subpaths back from two URIs"""
parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True)
child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True)
self.assertEqual(child.relative_to(parent), "dir1/file.txt")
not_child = ButlerURI("/a/b/dir1/file.txt")
self.assertFalse(not_child.relative_to(parent))
not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
self.assertFalse(child.relative_to(not_directory))
# Relative URIs
parent = ButlerURI("a/b/", forceAbsolute=False)
child = ButlerURI("a/b/c/d.txt", forceAbsolute=False)
self.assertFalse(child.scheme)
self.assertEqual(child.relative_to(parent), "c/d.txt")
# File URI and schemeless URI
parent = ButlerURI("file:/a/b/c/")
child = ButlerURI("e/f/g.txt", forceAbsolute=False)
# If the child is relative and the parent is absolute we assume
# that the child is a child of the parent unless it uses ".."
self.assertEqual(child.relative_to(parent), "e/f/g.txt")
child = ButlerURI("../e/f/g.txt", forceAbsolute=False)
self.assertFalse(child.relative_to(parent))
child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f/g.txt")
def testQuoting(self):
"""Check that quoting works."""
parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
subpath = "rootdir/dir1+/file?.txt"
child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath)))
self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
self.assertEqual(child.basename(), "file?.txt")
self.assertEqual(child.relativeToPathRoot, subpath)
self.assertIn("%", child.path)
self.assertEqual(child.unquoted_path, "/" + subpath)
def guessCollectionNames(self, instrument: Instrument, root: str) -> None:
"""Update `runName` and `chainName` with guesses that match Gen3 naming
conventions.
If `chainName` is not `None`, and `runName` is, `runName` will be set
from it. If `runName` is already set, nothing will be changed, and
if `chainName` is `None`, no chained collection will be created.
Parameters
----------
instrument : `Instrument`
Instrument object for the repository being converted.
root : `str`
Path to the root repository. If this is present at the start of
``self.path``, it will be stripped as part of generating the run
name.
Raises
------
ValueError
Raised if the appropriate collection names cannot be inferred.
"""
if self.runName is not None:
return
if self.chainName is None:
if os.path.isabs(self.path):
rerunURI = ButlerURI(self.path)
rootURI = ButlerURI(root)
chainName = rerunURI.relative_to(rootURI)
if chainName is None:
raise ValueError(
f"Cannot guess run name collection for rerun at '{self.path}': "
f"no clear relationship to root '{root}'."
)
else:
chainName = self.path
chainName, _ = _dropPrefix(chainName, "rerun/")
chainName, isPersonal = _dropPrefix(chainName, "private/")
if isPersonal:
chainName = f"u/{chainName}"
else:
chainName, _ = _dropPrefix(chainName, "shared/")
chainName = instrument.makeCollectionName("runs", chainName)
self.chainName = chainName
self.runName = f"{self.chainName}/direct"
def testEscapes(self):
"""Special characters in file paths"""
src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
self.assertFalse(src.scheme)
src.write(b"Some content")
self.assertTrue(src.exists())
# Use the internal API to force to a file
file = src._force_to_file()
self.assertTrue(file.exists())
self.assertIn("???", file.ospath)
self.assertNotIn("???", file.path)
file.updateFile("tests??.txt")
self.assertNotIn("??.txt", file.path)
file.write(b"Other content")
self.assertEqual(file.read(), b"Other content")
src.updateFile("tests??.txt")
self.assertIn("??.txt", src.path)
self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
# File URI and schemeless URI
parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
child = ButlerURI("e/f/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f/g.txt")
child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
# Schemeless so should not quote
dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
self.assertIn("???", dir.ospath)
self.assertIn("???", dir.path)
self.assertFalse(dir.scheme)
# dir.join() morphs into a file scheme
new = dir.join("test_j.txt")
self.assertIn("???", new.ospath, f"Checking {new}")
new.write(b"Content")
new2name = "###/test??.txt"
new2 = dir.join(new2name)
self.assertIn("???", new2.ospath)
new2.write(b"Content")
self.assertTrue(new2.ospath.endswith(new2name))
self.assertEqual(new.read(), new2.read())
fdir = dir._force_to_file()
self.assertNotIn("???", fdir.path)
self.assertIn("???", fdir.ospath)
self.assertEqual(fdir.scheme, "file")
fnew = dir.join("test_jf.txt")
fnew.write(b"Content")
fnew2 = fdir.join(new2name)
fnew2.write(b"Content")
self.assertTrue(fnew2.ospath.endswith(new2name))
self.assertNotIn("###", fnew2.path)
self.assertEqual(fnew.read(), fnew2.read())
# Test that children relative to schemeless and file schemes
# still return the same unquoted name
self.assertEqual(fnew2.relative_to(fdir), new2name)
self.assertEqual(fnew2.relative_to(dir), new2name)
self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}")
self.assertEqual(new2.relative_to(dir), new2name)
# Check for double quoting
plus_path = "/a/b/c+d/"
with self.assertLogs(level="WARNING"):
uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
self.assertEqual(uri.ospath, plus_path)
# Check that # is not escaped for schemeless URIs
hash_path = "/a/b#/c&d#xyz"
hpos = hash_path.rfind("#")
uri = ButlerURI(hash_path)
self.assertEqual(uri.ospath, hash_path[:hpos])
self.assertEqual(uri.fragment, hash_path[hpos + 1:])
