def setUpClass(cls): """Create a new butler once only.""" cls.storageClassFactory = StorageClassFactory() cls.root = tempfile.mkdtemp(dir=TESTDIR) data_ids = { "instrument": [INSTRUMENT_NAME], "detector": [0, 1, 2, 3, 4, 5], "exposure": [11, 22], } configURI = ButlerURI("resource://spherex/configs", forceDirectory=True) butlerConfig = Config(configURI.join("butler.yaml")) # in-memory db is being phased out # butlerConfig["registry", "db"] = 'sqlite:///:memory:' cls.creatorButler = makeTestRepo( cls.root, data_ids, config=butlerConfig, dimensionConfig=configURI.join("dimensions.yaml")) for formatter in FORMATTERS: datasetTypeName, storageClassName = (formatter["dataset_type"], formatter["storage_class"]) storageClass = cls.storageClassFactory.getStorageClass( storageClassName) addDatasetType(cls.creatorButler, datasetTypeName, set(data_ids), storageClass)
def testResource(self): u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") self.assertTrue(u.exists(), f"Check {u} exists") content = u.read().decode() self.assertTrue(content.startswith("datastore:")) truncated = u.read(size=9).decode() self.assertEqual(truncated, "datastore") d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) self.assertTrue(u.exists(), f"Check directory {d} exists") j = d.join("datastore.yaml") self.assertEqual(u, j) self.assertFalse(j.dirLike) self.assertFalse(d.join("not-there.yaml").exists())
def testMkdir(self): tmpdir = ButlerURI(self.tmpdir) newdir = tmpdir.join("newdir/seconddir") newdir.mkdir() self.assertTrue(newdir.exists()) newfile = newdir.join("temp.txt") newfile.write("Data".encode()) self.assertTrue(newfile.exists())
def testHash(self): """Test that we can store URIs in sets and as keys.""" uri1 = ButlerURI(TESTDIR) uri2 = uri1.join("test/") s = {uri1, uri2} self.assertIn(uri1, s) d = {uri1: "1", uri2: "2"} self.assertEqual(d[uri2], "2")
def testUriJoin(self): uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False) uri2 = uri.join("e/f/g.txt") self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False) uri2 = uri.join("e/f/g.txt") self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True) uri2 = uri.join("e/f/g.txt") self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}") uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True) uri2 = uri.join("newpath/newfile.txt") self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") uri = ButlerURI("s3://bucket/a/b/c/d/old.txt") uri2 = uri.join("newpath/newfile.txt") self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
def testParents(self): """Test of splitting and parent walking.""" parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) child_file = parent.join("subdir/file.txt") self.assertFalse(child_file.isdir()) child_subdir, file = child_file.split() self.assertEqual(file, "file.txt") self.assertTrue(child_subdir.isdir()) self.assertEqual(child_file.dirname(), child_subdir) self.assertEqual(child_file.basename(), file) self.assertEqual(child_file.parent(), child_subdir) derived_parent = child_subdir.parent() self.assertEqual(derived_parent, parent) self.assertTrue(derived_parent.isdir()) self.assertEqual(child_file.parent().parent(), parent)
def testEscapes(self): """Special characters in file paths""" src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) self.assertFalse(src.scheme) src.write(b"Some content") self.assertTrue(src.exists()) # Use the internal API to force to a file file = src._force_to_file() self.assertTrue(file.exists()) self.assertIn("???", file.ospath) self.assertNotIn("???", file.path) file.updateFile("tests??.txt") self.assertNotIn("??.txt", file.path) file.write(b"Other content") self.assertEqual(file.read(), b"Other content") src.updateFile("tests??.txt") self.assertIn("??.txt", src.path) self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") # File URI and schemeless URI parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) child = ButlerURI("e/f/g.txt", forceAbsolute=False) self.assertEqual(child.relative_to(parent), "e/f/g.txt") child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") # Schemeless so should not quote dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) self.assertIn("???", dir.ospath) self.assertIn("???", dir.path) self.assertFalse(dir.scheme) # dir.join() morphs into a file scheme new = dir.join("test_j.txt") self.assertIn("???", new.ospath, f"Checking {new}") new.write(b"Content") new2name = "###/test??.txt" new2 = dir.join(new2name) self.assertIn("???", new2.ospath) new2.write(b"Content") self.assertTrue(new2.ospath.endswith(new2name)) self.assertEqual(new.read(), new2.read()) fdir = dir._force_to_file() self.assertNotIn("???", fdir.path) self.assertIn("???", fdir.ospath) self.assertEqual(fdir.scheme, "file") fnew = dir.join("test_jf.txt") fnew.write(b"Content") fnew2 = fdir.join(new2name) fnew2.write(b"Content") self.assertTrue(fnew2.ospath.endswith(new2name)) self.assertNotIn("###", fnew2.path) self.assertEqual(fnew.read(), fnew2.read()) # Test that children relative to schemeless and file schemes # still return the same unquoted name self.assertEqual(fnew2.relative_to(fdir), new2name) self.assertEqual(fnew2.relative_to(dir), new2name) self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") self.assertEqual(new2.relative_to(dir), new2name) # Check for double quoting plus_path = "/a/b/c+d/" with self.assertLogs(level="WARNING"): uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) self.assertEqual(uri.ospath, plus_path) # Check that # is not escaped for schemeless URIs hash_path = "/a/b#/c&d#xyz" hpos = hash_path.rfind("#") uri = ButlerURI(hash_path) self.assertEqual(uri.ospath, hash_path[:hpos]) self.assertEqual(uri.fragment, hash_path[hpos + 1:])
def testWalk(self): """Test ButlerURI.walk().""" test_dir_uri = ButlerURI(TESTDIR) file = test_dir_uri.join("config/basic/butler.yaml") found = list(ButlerURI.findFileResources([file])) self.assertEqual(found[0], file) # Compare against the full local paths expected = set(p for p in glob.glob( os.path.join(TESTDIR, "config", "**"), recursive=True) if os.path.isfile(p)) found = set(u.ospath for u in ButlerURI.findFileResources( [test_dir_uri.join("config")])) self.assertEqual(found, expected) # Now solely the YAML files expected_yaml = set( glob.glob(os.path.join(TESTDIR, "config", "**", "*.yaml"), recursive=True)) found = set(u.ospath for u in ButlerURI.findFileResources( [test_dir_uri.join("config")], file_filter=r".*\.yaml$")) self.assertEqual(found, expected_yaml) # Now two explicit directories and a file expected = set( glob.glob(os.path.join(TESTDIR, "config", "**", "basic", "*.yaml"), recursive=True)) expected.update( set( glob.glob(os.path.join(TESTDIR, "config", "**", "templates", "*.yaml"), recursive=True))) expected.add(file.ospath) found = set( u.ospath for u in ButlerURI.findFileResources([ file, test_dir_uri.join("config/basic"), test_dir_uri.join("config/templates") ], file_filter=r".*\.yaml$")) self.assertEqual(found, expected) # Group by directory -- find everything and compare it with what # we expected to be there in total. We expect to find 9 directories # containing yaml files so make sure we only iterate 9 times. found_yaml = set() counter = 0 for uris in ButlerURI.findFileResources( [file, test_dir_uri.join("config/")], file_filter=r".*\.yaml$", grouped=True): found = set(u.ospath for u in uris) if found: counter += 1 found_yaml.update(found) self.assertEqual(found_yaml, expected_yaml) self.assertEqual(counter, 9) # Grouping but check that single files are returned in a single group # at the end file2 = test_dir_uri.join("config/templates/templates-bad.yaml") found = list( ButlerURI.findFileResources( [file, file2, test_dir_uri.join("config/dbAuth")], grouped=True)) self.assertEqual(len(found), 2) self.assertEqual(list(found[1]), [file, file2]) with self.assertRaises(ValueError): list(file.walk())