def validateButlerConfiguration(root,
datasetTypes=None,
ignore=None,
quiet=False,
collection="validate"):
"""Validate a bulter configuration.
Parameters
----------
root : `str`
Butler root to validate.
datasetTypes : `list` of `str`
Dataset types to specifically check.
ignore : `list` of `str`
Dataset types to ignore.
quiet : `bool`, optional
If `True` report pass/fail but not details.
collection : `str`, optional
Collection to use. Sometimes this is needed to ensure that butler
can be instantiated properly.
Returns
-------
validates : `bool`
`True` if everything looks okay, `False` if there is a problem.
"""
logFailures = not quiet
# The collection does not matter for validation but if a run is specified
# in the configuration then it must be consistent with this collection
butler = Butler(config=root, collection=collection)
try:
butler.validateConfiguration(logFailures=logFailures,
datasetTypeNames=datasetTypes,
ignore=ignore)
except ValidationError:
return False
return True
action="append",
type=str,
help="DatasetType(s) to ignore for validation (can be comma-separated)"
)
args = parser.parse_args()
logFailures = True
if args.quiet:
logFailures = False
# Process any commas in dataset type or ignore list
ignore = processCommas(args.ignore)
datasetTypes = processCommas(args.datasettype)
exitStatus = 0
# The collection does not matter for validation but if a run is specified
# in the configuration then it must be consistent with this collection
butler = Butler(config=args.root, collection=args.collection)
try:
butler.validateConfiguration(logFailures=logFailures,
datasetTypeNames=datasetTypes,
ignore=ignore)
except ValidationError:
exitStatus = 1
else:
print("No problems encountered with configuration.")
sys.exit(exitStatus)
def testGetDatasetTypes(self):
butler = Butler(self.tmpConfigFile)
dimensions = butler.registry.dimensions.extract(
["instrument", "visit", "physical_filter"])
dimensionEntries = (("instrument", {
"instrument": "DummyCam"
}), ("instrument", {
"instrument": "DummyHSC"
}), ("instrument", {
"instrument": "DummyCamComp"
}), ("physical_filter", {
"instrument": "DummyCam",
"physical_filter": "d-r"
}), ("visit", {
"instrument": "DummyCam",
"visit": 42,
"physical_filter": "d-r"
}))
storageClass = self.storageClassFactory.getStorageClass(
"StructuredData")
# Add needed Dimensions
for name, value in dimensionEntries:
butler.registry.addDimensionEntry(name, value)
# When a DatasetType is added to the registry entries are created
# for each component. Need entries for each component in the test
# configuration otherwise validation won't work. The ones that
# are deliberately broken will be ignored later.
datasetTypeNames = {"metric", "metric2", "metric4", "metric33", "pvi"}
components = set()
for datasetTypeName in datasetTypeNames:
# Create and register a DatasetType
self.addDatasetType(datasetTypeName, dimensions, storageClass,
butler.registry)
for componentName in storageClass.components:
components.add(
DatasetType.nameWithComponent(datasetTypeName,
componentName))
fromRegistry = butler.registry.getAllDatasetTypes()
self.assertEqual({d.name
for d in fromRegistry},
datasetTypeNames | components)
# Now that we have some dataset types registered, validate them
butler.validateConfiguration(ignore=[
"test_metric_comp", "metric3", "calexp", "DummySC",
"datasetType.component"
])
# Add a new datasetType that will fail template validation
self.addDatasetType("test_metric_comp", dimensions, storageClass,
butler.registry)
if self.validationCanFail:
with self.assertRaises(ValidationError):
butler.validateConfiguration()
# Rerun validation but with a subset of dataset type names
butler.validateConfiguration(datasetTypeNames=["metric4"])
# Rerun validation but ignore the bad datasetType
butler.validateConfiguration(ignore=[
"test_metric_comp", "metric3", "calexp", "DummySC",
"datasetType.component"
])
