def __init__(self, config, registry, butlerRoot=None):
super().__init__(config, registry)
if "root" not in self.config:
raise ValueError("No root directory specified in configuration")
# Name ourselves either using an explicit name or a name
# derived from the (unexpanded) root
if "name" in self.config:
self.name = self.config["name"]
else:
# We use the unexpanded root in the name to indicate that this
# datastore can be moved without having to update registry.
self.name = "{}@{}".format(type(self).__name__,
self.config["root"])
# Support repository relocation in config
# Existence of self.root is checked in subclass
self.root = replaceRoot(self.config["root"], butlerRoot)
self.locationFactory = LocationFactory(self.root)
self.formatterFactory = FormatterFactory()
# Now associate formatters with storage classes
self.formatterFactory.registerFormatters(self.config["formatters"],
universe=self.registry.dimensions)
# Read the file naming templates
self.templates = FileTemplates(self.config["templates"],
universe=self.registry.dimensions)
# Storage of paths and formatters, keyed by dataset_id
self.records = DatabaseDict.fromConfig(self.config["records"],
value=self.Record, key="dataset_id",
registry=registry)
def __init__(self, config, registry, butlerRoot=None):
super().__init__(config, registry)
if "root" not in self.config:
raise ValueError("No root directory specified in configuration")
# Name ourselves either using an explicit name or a name
# derived from the (unexpanded) root
if "name" in self.config:
self.name = self.config["name"]
else:
self.name = "POSIXDatastore@{}".format(self.config["root"])
# Support repository relocation in config
self.root = replaceRoot(self.config["root"], butlerRoot)
if not os.path.isdir(self.root):
if "create" not in self.config or not self.config["create"]:
raise ValueError(f"No valid root at: {self.root}")
safeMakeDir(self.root)
self.locationFactory = LocationFactory(self.root)
self.formatterFactory = FormatterFactory()
self.storageClassFactory = StorageClassFactory()
# Now associate formatters with storage classes
self.formatterFactory.registerFormatters(
self.config["formatters"], universe=self.registry.dimensions)
# Read the file naming templates
self.templates = FileTemplates(self.config["templates"],
universe=self.registry.dimensions)
# And read the constraints list
constraintsConfig = self.config.get("constraints")
self.constraints = Constraints(constraintsConfig,
universe=self.registry.dimensions)
# Storage of paths and formatters, keyed by dataset_id
types = {
"path": str,
"formatter": str,
"storage_class": str,
"file_size": int,
"checksum": str,
"dataset_id": int
}
lengths = {
"path": 256,
"formatter": 128,
"storage_class": 64,
"checksum": 128
}
self.records = DatabaseDict.fromConfig(self.config["records"],
types=types,
value=self.RecordTuple,
key="dataset_id",
lengths=lengths,
registry=registry)
def testValidation(self):
configRoot = os.path.join(TESTDIR, "config", "templates")
config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
templates = FileTemplates(config1, universe=self.universe)
entities = {}
entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX",
dataId={"instrument": "dummy", "physical_filter": "i",
"visit": 52})
with self.assertLogs(level="WARNING") as cm:
templates.validateTemplates(entities.values(), logFailures=True)
self.assertIn("Unchecked keys", cm.output[0])
self.assertIn("StorageClassX", cm.output[0])
entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
dataId={"instrument": "dummy", "physical_filter": "i"})
entities["StorageClassX"] = self.makeDatasetRef("storageClass",
storageClassName="StorageClassX",
dataId={"instrument": "dummy", "visit": 2})
entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs",
storageClassName="StorageClassX",
dataId={"instrument": "dummy",
"physical_filter": "i", "visit": 23},
conform=False)
entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
storageClassName="StorageClassX",
dataId={"physical_filter": "i",
"instrument": "SCUBA"})
entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
dataId={"physical_filter": "i", "instrument": "HSC"})
entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
storageClassName="StorageClassX",
dataId={"physical_filter": "i",
"instrument": "HSC"})
templates.validateTemplates(entities.values(), logFailures=True)
# Rerun but with a failure
entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
dataId={"band": "i"})
with self.assertRaises(FileTemplateValidationError):
with self.assertLogs(level="FATAL"):
templates.validateTemplates(entities.values(), logFailures=True)
def __init__(self, config, registry, butlerRoot=None):
super().__init__(config, registry)
if "root" not in self.config:
raise ValueError("No root directory specified in configuration")
# Name ourselves either using an explicit name or a name
# derived from the (unexpanded) root
if "name" in self.config:
self.name = self.config["name"]
else:
# We use the unexpanded root in the name to indicate that this
# datastore can be moved without having to update registry.
self.name = "{}@{}".format(
type(self).__name__, self.config["root"])
# Support repository relocation in config
# Existence of self.root is checked in subclass
self.root = replaceRoot(self.config["root"], butlerRoot)
self.locationFactory = LocationFactory(self.root)
self.formatterFactory = FormatterFactory()
# Now associate formatters with storage classes
self.formatterFactory.registerFormatters(
self.config["formatters"], universe=self.registry.dimensions)
# Read the file naming templates
self.templates = FileTemplates(self.config["templates"],
universe=self.registry.dimensions)
# Storage of paths and formatters, keyed by dataset_id
self._tableName = self.config["records", "table"]
try:
registry.registerOpaqueTable(self._tableName, self.makeTableSpec())
except ReadOnlyDatabaseError:
# If the database is read only and we just tried and failed to
# create a table, it means someone is trying to create a read-only
# butler client for an empty repo. That should be okay, as long
# as they then try to get any datasets before some other client
# creates the table. Chances are they'rejust validating
# configuration.
pass
# Determine whether checksums should be used
self.useChecksum = self.config.get("checksum", True)
def testSimpleConfig(self):
"""Test reading from config file"""
configRoot = os.path.join(TESTDIR, "config", "templates")
config1 = FileTemplatesConfig(
os.path.join(configRoot, "templates-nodefault.yaml"))
templates = FileTemplates(config1, universe=self.universe)
ref = self.makeDatasetRef("calexp")
tmpl = templates.getTemplate(ref)
self.assertIsInstance(tmpl, FileTemplate)
# This config file should not allow defaulting
ref2 = self.makeDatasetRef("unknown")
with self.assertRaises(KeyError):
templates.getTemplate(ref2)
# This should fall through the datasetTypeName check and use
# StorageClass instead
ref3 = self.makeDatasetRef("unknown2",
storageClassName="StorageClassX")
tmplSc = templates.getTemplate(ref3)
self.assertIsInstance(tmplSc, FileTemplate)
# Try with a component: one with defined formatter and one without
refWcs = self.makeDatasetRef("calexp.wcs")
refImage = self.makeDatasetRef("calexp.image")
tmplCalexp = templates.getTemplate(ref)
tmplWcs = templates.getTemplate(refWcs) # Should be special
tmpl_image = templates.getTemplate(refImage)
self.assertIsInstance(tmplCalexp, FileTemplate)
self.assertIsInstance(tmpl_image, FileTemplate)
self.assertIsInstance(tmplWcs, FileTemplate)
self.assertEqual(tmplCalexp, tmpl_image)
self.assertNotEqual(tmplCalexp, tmplWcs)
# Check dimensions lookup order.
# The order should be: dataset type name, dimension, storage class
# This one will not match name but might match storage class.
# It should match dimensions
refDims = self.makeDatasetRef("nomatch",
dataId={
"instrument": "LSST",
"physical_filter": "z"
},
storageClassName="StorageClassX")
tmplDims = templates.getTemplate(refDims)
self.assertIsInstance(tmplDims, FileTemplate)
self.assertNotEqual(tmplDims, tmplSc)
# Test that instrument overrides retrieve specialist templates
refPvi = self.makeDatasetRef("pvi")
refPviHsc = self.makeDatasetRef("pvi",
dataId={
"instrument": "HSC",
"physical_filter": "z"
})
refPviLsst = self.makeDatasetRef("pvi",
dataId={
"instrument": "LSST",
"physical_filter": "z"
})
tmplPvi = templates.getTemplate(refPvi)
tmplPviHsc = templates.getTemplate(refPviHsc)
tmplPviLsst = templates.getTemplate(refPviLsst)
self.assertEqual(tmplPvi, tmplPviLsst)
self.assertNotEqual(tmplPvi, tmplPviHsc)
# Have instrument match and dimensions look up with no name match
refNoPviHsc = self.makeDatasetRef("pvix",
dataId={
"instrument": "HSC",
"physical_filter": "z"
},
storageClassName="StorageClassX")
tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
self.assertNotEqual(tmplNoPviHsc, tmplDims)
self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
# Format config file with defaulting
config2 = FileTemplatesConfig(
os.path.join(configRoot, "templates-withdefault.yaml"))
templates = FileTemplates(config2, universe=self.universe)
tmpl = templates.getTemplate(ref2)
self.assertIsInstance(tmpl, FileTemplate)
# Format config file with bad format string
with self.assertRaises(FileTemplateValidationError):
FileTemplates(os.path.join(configRoot, "templates-bad.yaml"),
universe=self.universe)
# Config file with no defaulting mentioned
config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
templates = FileTemplates(config3, universe=self.universe)
with self.assertRaises(KeyError):
templates.getTemplate(ref2)
# Try again but specify a default in the constructor
default = "{run}/{datasetType}/{physical_filter}"
templates = FileTemplates(config3,
default=default,
universe=self.universe)
tmpl = templates.getTemplate(ref2)
self.assertEqual(tmpl.template, default)