def setUpClass(cls):
"""Create a new butler once only."""
cls.storageClassFactory = StorageClassFactory()
cls.root = tempfile.mkdtemp(dir=TESTDIR)
data_ids = {
"instrument": [INSTRUMENT_NAME],
"detector": [0, 1, 2, 3, 4, 5],
"exposure": [11, 22],
}
configURI = ButlerURI("resource://spherex/configs",
forceDirectory=True)
butlerConfig = Config(configURI.join("butler.yaml"))
# in-memory db is being phased out
# butlerConfig["registry", "db"] = 'sqlite:///:memory:'
cls.creatorButler = makeTestRepo(
cls.root,
data_ids,
config=butlerConfig,
dimensionConfig=configURI.join("dimensions.yaml"))
for formatter in FORMATTERS:
datasetTypeName, storageClassName = (formatter["dataset_type"],
formatter["storage_class"])
storageClass = cls.storageClassFactory.getStorageClass(
storageClassName)
addDatasetType(cls.creatorButler, datasetTypeName, set(data_ids),
storageClass)
def testResource(self):
u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml")
self.assertTrue(u.exists(), f"Check {u} exists")
content = u.read().decode()
self.assertTrue(content.startswith("datastore:"))
truncated = u.read(size=9).decode()
self.assertEqual(truncated, "datastore")
d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True)
self.assertTrue(u.exists(), f"Check directory {d} exists")
j = d.join("datastore.yaml")
self.assertEqual(u, j)
self.assertFalse(j.dirLike)
self.assertFalse(d.join("not-there.yaml").exists())
def testMkdir(self):
tmpdir = ButlerURI(self.tmpdir)
newdir = tmpdir.join("newdir/seconddir")
newdir.mkdir()
self.assertTrue(newdir.exists())
newfile = newdir.join("temp.txt")
newfile.write("Data".encode())
self.assertTrue(newfile.exists())
def testHash(self):
"""Test that we can store URIs in sets and as keys."""
uri1 = ButlerURI(TESTDIR)
uri2 = uri1.join("test/")
s = {uri1, uri2}
self.assertIn(uri1, s)
d = {uri1: "1", uri2: "2"}
self.assertEqual(d[uri2], "2")
def testUriJoin(self):
uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False)
uri2 = uri.join("e/f/g.txt")
self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False)
uri2 = uri.join("e/f/g.txt")
self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True)
uri2 = uri.join("e/f/g.txt")
self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}")
uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True)
uri2 = uri.join("newpath/newfile.txt")
self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
uri = ButlerURI("s3://bucket/a/b/c/d/old.txt")
uri2 = uri.join("newpath/newfile.txt")
self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
def testParents(self):
"""Test of splitting and parent walking."""
parent = ButlerURI(self.tmpdir,
forceDirectory=True,
forceAbsolute=True)
child_file = parent.join("subdir/file.txt")
self.assertFalse(child_file.isdir())
child_subdir, file = child_file.split()
self.assertEqual(file, "file.txt")
self.assertTrue(child_subdir.isdir())
self.assertEqual(child_file.dirname(), child_subdir)
self.assertEqual(child_file.basename(), file)
self.assertEqual(child_file.parent(), child_subdir)
derived_parent = child_subdir.parent()
self.assertEqual(derived_parent, parent)
self.assertTrue(derived_parent.isdir())
self.assertEqual(child_file.parent().parent(), parent)
def testEscapes(self):
"""Special characters in file paths"""
src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
self.assertFalse(src.scheme)
src.write(b"Some content")
self.assertTrue(src.exists())
# Use the internal API to force to a file
file = src._force_to_file()
self.assertTrue(file.exists())
self.assertIn("???", file.ospath)
self.assertNotIn("???", file.path)
file.updateFile("tests??.txt")
self.assertNotIn("??.txt", file.path)
file.write(b"Other content")
self.assertEqual(file.read(), b"Other content")
src.updateFile("tests??.txt")
self.assertIn("??.txt", src.path)
self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
# File URI and schemeless URI
parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
child = ButlerURI("e/f/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f/g.txt")
child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
# Schemeless so should not quote
dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
self.assertIn("???", dir.ospath)
self.assertIn("???", dir.path)
self.assertFalse(dir.scheme)
# dir.join() morphs into a file scheme
new = dir.join("test_j.txt")
self.assertIn("???", new.ospath, f"Checking {new}")
new.write(b"Content")
new2name = "###/test??.txt"
new2 = dir.join(new2name)
self.assertIn("???", new2.ospath)
new2.write(b"Content")
self.assertTrue(new2.ospath.endswith(new2name))
self.assertEqual(new.read(), new2.read())
fdir = dir._force_to_file()
self.assertNotIn("???", fdir.path)
self.assertIn("???", fdir.ospath)
self.assertEqual(fdir.scheme, "file")
fnew = dir.join("test_jf.txt")
fnew.write(b"Content")
fnew2 = fdir.join(new2name)
fnew2.write(b"Content")
self.assertTrue(fnew2.ospath.endswith(new2name))
self.assertNotIn("###", fnew2.path)
self.assertEqual(fnew.read(), fnew2.read())
# Test that children relative to schemeless and file schemes
# still return the same unquoted name
self.assertEqual(fnew2.relative_to(fdir), new2name)
self.assertEqual(fnew2.relative_to(dir), new2name)
self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}")
self.assertEqual(new2.relative_to(dir), new2name)
# Check for double quoting
plus_path = "/a/b/c+d/"
with self.assertLogs(level="WARNING"):
uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
self.assertEqual(uri.ospath, plus_path)
# Check that # is not escaped for schemeless URIs
hash_path = "/a/b#/c&d#xyz"
hpos = hash_path.rfind("#")
uri = ButlerURI(hash_path)
self.assertEqual(uri.ospath, hash_path[:hpos])
self.assertEqual(uri.fragment, hash_path[hpos + 1:])
def testWalk(self):
"""Test ButlerURI.walk()."""
test_dir_uri = ButlerURI(TESTDIR)
file = test_dir_uri.join("config/basic/butler.yaml")
found = list(ButlerURI.findFileResources([file]))
self.assertEqual(found[0], file)
# Compare against the full local paths
expected = set(p for p in glob.glob(
os.path.join(TESTDIR, "config", "**"), recursive=True)
if os.path.isfile(p))
found = set(u.ospath for u in ButlerURI.findFileResources(
[test_dir_uri.join("config")]))
self.assertEqual(found, expected)
# Now solely the YAML files
expected_yaml = set(
glob.glob(os.path.join(TESTDIR, "config", "**", "*.yaml"),
recursive=True))
found = set(u.ospath for u in ButlerURI.findFileResources(
[test_dir_uri.join("config")], file_filter=r".*\.yaml$"))
self.assertEqual(found, expected_yaml)
# Now two explicit directories and a file
expected = set(
glob.glob(os.path.join(TESTDIR, "config", "**", "basic", "*.yaml"),
recursive=True))
expected.update(
set(
glob.glob(os.path.join(TESTDIR, "config", "**", "templates",
"*.yaml"),
recursive=True)))
expected.add(file.ospath)
found = set(
u.ospath
for u in ButlerURI.findFileResources([
file,
test_dir_uri.join("config/basic"),
test_dir_uri.join("config/templates")
],
file_filter=r".*\.yaml$"))
self.assertEqual(found, expected)
# Group by directory -- find everything and compare it with what
# we expected to be there in total. We expect to find 9 directories
# containing yaml files so make sure we only iterate 9 times.
found_yaml = set()
counter = 0
for uris in ButlerURI.findFileResources(
[file, test_dir_uri.join("config/")],
file_filter=r".*\.yaml$",
grouped=True):
found = set(u.ospath for u in uris)
if found:
counter += 1
found_yaml.update(found)
self.assertEqual(found_yaml, expected_yaml)
self.assertEqual(counter, 9)
# Grouping but check that single files are returned in a single group
# at the end
file2 = test_dir_uri.join("config/templates/templates-bad.yaml")
found = list(
ButlerURI.findFileResources(
[file, file2, test_dir_uri.join("config/dbAuth")],
grouped=True))
self.assertEqual(len(found), 2)
self.assertEqual(list(found[1]), [file, file2])
with self.assertRaises(ValueError):
list(file.walk())
