def testValidation(self):
configRoot = os.path.join(TESTDIR, "config", "templates")
config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
templates = FileTemplates(config1, universe=self.universe)
entities = {}
entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX",
dataId={"instrument": "dummy", "physical_filter": "i",
"visit": 52})
with self.assertLogs(level="WARNING") as cm:
templates.validateTemplates(entities.values(), logFailures=True)
self.assertIn("Unchecked keys", cm.output[0])
self.assertIn("StorageClassX", cm.output[0])
entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
dataId={"instrument": "dummy", "physical_filter": "i"})
entities["StorageClassX"] = self.makeDatasetRef("storageClass",
storageClassName="StorageClassX",
dataId={"instrument": "dummy", "visit": 2})
entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs",
storageClassName="StorageClassX",
dataId={"instrument": "dummy",
"physical_filter": "i", "visit": 23},
conform=False)
entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
storageClassName="StorageClassX",
dataId={"physical_filter": "i",
"instrument": "SCUBA"})
entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
dataId={"physical_filter": "i", "instrument": "HSC"})
entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
storageClassName="StorageClassX",
dataId={"physical_filter": "i",
"instrument": "HSC"})
templates.validateTemplates(entities.values(), logFailures=True)
# Rerun but with a failure
entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
dataId={"band": "i"})
with self.assertRaises(FileTemplateValidationError):
with self.assertLogs(level="FATAL"):
templates.validateTemplates(entities.values(), logFailures=True)
def testSimpleConfig(self):
"""Test reading from config file"""
configRoot = os.path.join(TESTDIR, "config", "templates")
config1 = FileTemplatesConfig(
os.path.join(configRoot, "templates-nodefault.yaml"))
templates = FileTemplates(config1, universe=self.universe)
ref = self.makeDatasetRef("calexp")
tmpl = templates.getTemplate(ref)
self.assertIsInstance(tmpl, FileTemplate)
# This config file should not allow defaulting
ref2 = self.makeDatasetRef("unknown")
with self.assertRaises(KeyError):
templates.getTemplate(ref2)
# This should fall through the datasetTypeName check and use
# StorageClass instead
ref3 = self.makeDatasetRef("unknown2",
storageClassName="StorageClassX")
tmplSc = templates.getTemplate(ref3)
self.assertIsInstance(tmplSc, FileTemplate)
# Try with a component: one with defined formatter and one without
refWcs = self.makeDatasetRef("calexp.wcs")
refImage = self.makeDatasetRef("calexp.image")
tmplCalexp = templates.getTemplate(ref)
tmplWcs = templates.getTemplate(refWcs) # Should be special
tmpl_image = templates.getTemplate(refImage)
self.assertIsInstance(tmplCalexp, FileTemplate)
self.assertIsInstance(tmpl_image, FileTemplate)
self.assertIsInstance(tmplWcs, FileTemplate)
self.assertEqual(tmplCalexp, tmpl_image)
self.assertNotEqual(tmplCalexp, tmplWcs)
# Check dimensions lookup order.
# The order should be: dataset type name, dimension, storage class
# This one will not match name but might match storage class.
# It should match dimensions
refDims = self.makeDatasetRef("nomatch",
dataId={
"instrument": "LSST",
"physical_filter": "z"
},
storageClassName="StorageClassX")
tmplDims = templates.getTemplate(refDims)
self.assertIsInstance(tmplDims, FileTemplate)
self.assertNotEqual(tmplDims, tmplSc)
# Test that instrument overrides retrieve specialist templates
refPvi = self.makeDatasetRef("pvi")
refPviHsc = self.makeDatasetRef("pvi",
dataId={
"instrument": "HSC",
"physical_filter": "z"
})
refPviLsst = self.makeDatasetRef("pvi",
dataId={
"instrument": "LSST",
"physical_filter": "z"
})
tmplPvi = templates.getTemplate(refPvi)
tmplPviHsc = templates.getTemplate(refPviHsc)
tmplPviLsst = templates.getTemplate(refPviLsst)
self.assertEqual(tmplPvi, tmplPviLsst)
self.assertNotEqual(tmplPvi, tmplPviHsc)
# Have instrument match and dimensions look up with no name match
refNoPviHsc = self.makeDatasetRef("pvix",
dataId={
"instrument": "HSC",
"physical_filter": "z"
},
storageClassName="StorageClassX")
tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
self.assertNotEqual(tmplNoPviHsc, tmplDims)
self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
# Format config file with defaulting
config2 = FileTemplatesConfig(
os.path.join(configRoot, "templates-withdefault.yaml"))
templates = FileTemplates(config2, universe=self.universe)
tmpl = templates.getTemplate(ref2)
self.assertIsInstance(tmpl, FileTemplate)
# Format config file with bad format string
with self.assertRaises(FileTemplateValidationError):
FileTemplates(os.path.join(configRoot, "templates-bad.yaml"),
universe=self.universe)
# Config file with no defaulting mentioned
config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
templates = FileTemplates(config3, universe=self.universe)
with self.assertRaises(KeyError):
templates.getTemplate(ref2)
# Try again but specify a default in the constructor
default = "{run}/{datasetType}/{physical_filter}"
templates = FileTemplates(config3,
default=default,
universe=self.universe)
tmpl = templates.getTemplate(ref2)
self.assertEqual(tmpl.template, default)