def testFileExists(self):
self.assertTrue(
s3CheckFileExists(client=self.client,
bucket=self.bucketName,
path=self.fileName)[0])
self.assertFalse(
s3CheckFileExists(client=self.client,
bucket=self.bucketName,
path=self.fileName + "_NO_EXIST")[0])
datastoreRootUri = f"s3://{self.bucketName}/"
uri = f"s3://{self.bucketName}/{self.fileName}"
buri = ButlerURI(uri)
location = Location(datastoreRootUri, self.fileName)
self.assertTrue(s3CheckFileExists(client=self.client, path=buri)[0])
# just to make sure the overloaded keyword works correctly
self.assertTrue(s3CheckFileExists(buri, client=self.client)[0])
self.assertTrue(
s3CheckFileExists(client=self.client, path=location)[0])
# make sure supplying strings resolves correctly too
self.assertTrue(s3CheckFileExists(uri, client=self.client))
self.assertTrue(s3CheckFileExists(uri))
def setUp(self):
self.root = tempfile.mkdtemp(dir=TESTDIR)
self.root2 = tempfile.mkdtemp(dir=TESTDIR)
self.tmpConfigFile = ButlerURI(
os.path.join(self.root2, "something.yaml")).geturl()
Butler.makeRepo(self.root,
config=Config(self.configFile),
outfile=self.tmpConfigFile)
def checkFileExists(self, root, relpath):
"""Checks if file exists at a given path (relative to root).
Test testPutTemplates verifies actual physical existance of the files
in the requested location. For S3Datastore this test is equivalent to
`lsst.daf.butler.core.s3utils.s3checkFileExists` call.
"""
uri = ButlerURI(root)
client = boto3.client("s3")
return s3CheckFileExists(uri, client=client)[0]
def testConfigExistence(self):
c = Config(self.tmpConfigFile)
uri_config = ButlerURI(c["root"])
uri_expected = ButlerURI(self.root)
self.assertEqual(uri_config.geturl(), uri_expected.geturl())
self.assertNotIn(":", uri_config.path,
"Check for URI concatenated with normal path")
def setUp(self):
config = Config(self.configFile)
uri = ButlerURI(config[".datastore.datastore.root"])
self.bucketName = uri.netloc
if self.useTempRoot:
self.root = self.genRoot()
rooturi = f"s3://{self.bucketName}/{self.root}"
config.update({"datastore": {"datastore": {"root": rooturi}}})
# set up some fake credentials if they do not exist
self.usingDummyCredentials = setAwsEnvCredentials()
# MOTO needs to know that we expect Bucket bucketname to exist
# (this used to be the class attribute bucketName)
s3 = boto3.resource("s3")
s3.create_bucket(Bucket=self.bucketName)
self.datastoreStr = f"datastore={self.root}"
self.datastoreName = [f"S3Datastore@{rooturi}"]
Butler.makeRepo(rooturi, config=config, forceConfigRoot=False)
self.tmpConfigFile = os.path.join(rooturi, "butler.yaml")