def testButlerUri(self): """Tests whether ButlerURI instantiates correctly given different arguments. """ # Root to use for relative paths testRoot = "/tmp/" # uriStrings is a list of tuples containing test string, forceAbsolute, # forceDirectory as arguments to ButlerURI and scheme, netloc and path # as expected attributes. Test asserts constructed equals to expected. # 1) no determinable schemes (ensures schema and netloc are not set) osRelFilePath = os.path.join(testRoot, "relative/file.ext") uriStrings = [ ("relative/file.ext", True, False, "", "", osRelFilePath), ("relative/file.ext", False, False, "", "", "relative/file.ext"), ("test/../relative/file.ext", True, False, "", "", osRelFilePath), ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), ("relative/dir", False, True, "", "", "relative/dir/") ] # 2) implicit file scheme, tests absolute file and directory paths uriStrings.extend(( ("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'), ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")) )) # 3) explicit file scheme, absolute and relative file and directory URI posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") uriStrings.extend(( ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/") )) # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) uriStrings.extend(( ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), ("s3://bucketname/rootDir/relative/file.ext", True, False, "s3", "bucketname", "/rootDir/relative/file.ext") )) # 5) HTTPS scheme uriStrings.extend(( ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"), ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"), ("https://www.lsst.org/rootDir/relative/file.ext", True, False, "https", "www.lsst.org", "/rootDir/relative/file.ext") )) for uriInfo in uriStrings: uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) with self.subTest(uri=uriInfo[0]): self.assertEqual(uri.scheme, uriInfo[3], "test scheme") self.assertEqual(uri.netloc, uriInfo[4], "test netloc") self.assertEqual(uri.path, uriInfo[5], "test path") # test root becomes abspath(".") when not specified, note specific # file:// scheme case uriStrings = ( ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"), ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")) ) for uriInfo in uriStrings: uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) with self.subTest(uri=uriInfo[0]): self.assertEqual(uri.scheme, uriInfo[3], "test scheme") self.assertEqual(uri.netloc, uriInfo[4], "test netloc") self.assertEqual(uri.path, uriInfo[5], "test path") # File replacement uriStrings = ( ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), ("relative/", "newfile.fits", "relative/newfile.fits"), ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml") ) for uriInfo in uriStrings: uri = ButlerURI(uriInfo[0], forceAbsolute=False) uri.updateFile(uriInfo[1]) with self.subTest(uri=uriInfo[0]): self.assertEqual(uri.path, uriInfo[2]) # Check that schemeless can become file scheme schemeless = ButlerURI("relative/path.ext") filescheme = ButlerURI("/absolute/path.ext") self.assertFalse(schemeless.scheme) self.assertEqual(filescheme.scheme, "file") self.assertNotEqual(type(schemeless), type(filescheme)) # Copy constructor uri = ButlerURI("s3://amazon/datastore", forceDirectory=True) uri2 = ButlerURI(uri) self.assertEqual(uri, uri2) uri = ButlerURI("file://amazon/datastore/file.txt") uri2 = ButlerURI(uri) self.assertEqual(uri, uri2) # Copy constructor using subclass uri3 = type(uri)(uri) self.assertEqual(type(uri), type(uri3)) # Explicit copy uri4 = copy.copy(uri3) self.assertEqual(uri4, uri3) uri4 = copy.deepcopy(uri3) self.assertEqual(uri4, uri3)
def testEscapes(self): """Special characters in file paths""" src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) self.assertFalse(src.scheme) src.write(b"Some content") self.assertTrue(src.exists()) # Use the internal API to force to a file file = src._force_to_file() self.assertTrue(file.exists()) self.assertIn("???", file.ospath) self.assertNotIn("???", file.path) file.updateFile("tests??.txt") self.assertNotIn("??.txt", file.path) file.write(b"Other content") self.assertEqual(file.read(), b"Other content") src.updateFile("tests??.txt") self.assertIn("??.txt", src.path) self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") # File URI and schemeless URI parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) child = ButlerURI("e/f/g.txt", forceAbsolute=False) self.assertEqual(child.relative_to(parent), "e/f/g.txt") child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") # Schemeless so should not quote dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) self.assertIn("???", dir.ospath) self.assertIn("???", dir.path) self.assertFalse(dir.scheme) # dir.join() morphs into a file scheme new = dir.join("test_j.txt") self.assertIn("???", new.ospath, f"Checking {new}") new.write(b"Content") new2name = "###/test??.txt" new2 = dir.join(new2name) self.assertIn("???", new2.ospath) new2.write(b"Content") self.assertTrue(new2.ospath.endswith(new2name)) self.assertEqual(new.read(), new2.read()) fdir = dir._force_to_file() self.assertNotIn("???", fdir.path) self.assertIn("???", fdir.ospath) self.assertEqual(fdir.scheme, "file") fnew = dir.join("test_jf.txt") fnew.write(b"Content") fnew2 = fdir.join(new2name) fnew2.write(b"Content") self.assertTrue(fnew2.ospath.endswith(new2name)) self.assertNotIn("###", fnew2.path) self.assertEqual(fnew.read(), fnew2.read()) # Test that children relative to schemeless and file schemes # still return the same unquoted name self.assertEqual(fnew2.relative_to(fdir), new2name) self.assertEqual(fnew2.relative_to(dir), new2name) self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") self.assertEqual(new2.relative_to(dir), new2name) # Check for double quoting plus_path = "/a/b/c+d/" with self.assertLogs(level="WARNING"): uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) self.assertEqual(uri.ospath, plus_path) # Check that # is not escaped for schemeless URIs hash_path = "/a/b#/c&d#xyz" hpos = hash_path.rfind("#") uri = ButlerURI(hash_path) self.assertEqual(uri.ospath, hash_path[:hpos]) self.assertEqual(uri.fragment, hash_path[hpos + 1:])
def testButlerUri(self): # Root to use for relative paths testRoot = "/tmp/" uriStrings = ( # Test string, forceAbsolute, scheme, netloc, path # These are being tested with forceAbsolute=True ("file:///rootDir/absolute/file.ext", True, "file", "", "/rootDir/absolute/file.ext"), ("/rootDir/absolute/file.ext", True, "file", "", "/rootDir/absolute/file.ext"), ("/rootDir/absolute/file.ext", False, "file", "", "/rootDir/absolute/file.ext"), ("/rootDir/absolute/", True, "file", "", "/rootDir/absolute/"), ("file:relative/file.ext", True, "file", "", posixpath.join(testRoot, "relative/file.ext")), ("file:relative/directory/", True, "file", "", posixpath.join(testRoot, "relative/directory/")), ("file://relative/file.ext", True, "file", "relative", "/file.ext"), ("file:///absolute/directory/", True, "file", "", "/absolute/directory/"), ("relative/file.ext", True, "", "", os.path.join(testRoot, "relative/file.ext")), ("relative/file.ext", False, "", "", "relative/file.ext"), ("s3://bucketname/rootDir/relative/file.ext", True, "s3", "bucketname", "/rootDir/relative/file.ext"), ("~/relative/file.ext", True, "file", "", os.path.expanduser("~/relative/file.ext")), ("~/relative/file.ext", False, "file", "", os.path.expanduser("~/relative/file.ext")), ("test/../relative/file.ext", True, "", "", os.path.join(testRoot, "relative/file.ext")), ("test/../relative/file.ext", False, "", "", "relative/file.ext"), ) for uriInfo in uriStrings: uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1]) with self.subTest(uri=uriInfo[0]): self.assertEqual(uri.scheme, uriInfo[2], "test scheme") self.assertEqual(uri.netloc, uriInfo[3], "test netloc") self.assertEqual(uri.path, uriInfo[4], "test path") # File replacement uriStrings = ( ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), ("relative/", "newfile.fits", "relative/newfile.fits"), ("isThisADirOrFile", "aFile.fits", "aFile.fits"), ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"), ) for uriInfo in uriStrings: uri = ButlerURI(uriInfo[0], forceAbsolute=False) uri.updateFile(uriInfo[1]) with self.subTest(uri=uriInfo[0]): self.assertEqual(uri.path, uriInfo[2])