def validateKey(self, lookupKey, entity):
# Docstring is inherited from base class
# The key can be valid in either formatters or templates so we can
# only check the template if it exists
if lookupKey in self.templates:
try:
self.templates[lookupKey].validateTemplate(entity)
except FileTemplateValidationError as e:
raise DatastoreValidationError(e) from e
def __init__(self,
config: Union[Config, str],
bridgeManager: DatastoreRegistryBridgeManager,
butlerRoot: str = None):
super().__init__(config, bridgeManager)
# Scan for child datastores and instantiate them with the same registry
self.datastores = []
for c in self.config["datastores"]:
c = DatastoreConfig(c)
datastoreType = doImport(c["cls"])
datastore = datastoreType(c, bridgeManager, butlerRoot=butlerRoot)
log.debug("Creating child datastore %s", datastore.name)
self.datastores.append(datastore)
# Name ourself based on our children
if self.datastores:
# We must set the names explicitly
self._names = [d.name for d in self.datastores]
childNames = ",".join(self.names)
else:
childNames = "(empty@{})".format(time.time())
self._names = [childNames]
self.name = "{}[{}]".format(type(self).__qualname__, childNames)
# We declare we are ephemeral if all our child datastores declare
# they are ephemeral
isEphemeral = True
for d in self.datastores:
if not d.isEphemeral:
isEphemeral = False
break
self.isEphemeral = isEphemeral
# per-datastore override constraints
if "datastore_constraints" in self.config:
overrides = self.config["datastore_constraints"]
if len(overrides) != len(self.datastores):
raise DatastoreValidationError(
f"Number of registered datastores ({len(self.datastores)})"
" differs from number of constraints overrides"
f" {len(overrides)}")
self.datastoreConstraints = [
Constraints(c.get("constraints"),
universe=bridgeManager.universe) for c in overrides
]
else:
self.datastoreConstraints = (None, ) * len(self.datastores)
log.debug("Created %s (%s)", self.name,
("ephemeral" if self.isEphemeral else "permanent"))
def validateKey(self, lookupKey, entity):
# Docstring is inherited from base class
failures = []
for datastore in self.datastores:
try:
datastore.validateKey(lookupKey, entity)
except DatastoreValidationError as e:
failures.append(f"Datastore {self.name}: {e}")
if failures:
msg = ";\n".join(failures)
raise DatastoreValidationError(msg)
def validateConfiguration(self, entities, logFailures=False):
"""Validate some of the configuration for this datastore.
Parameters
----------
entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass`
Entities to test against this configuration. Can be differing
types.
logFailures : `bool`, optional
If `True`, output a log message for every validation error
detected.
Raises
------
DatastoreValidationError
Raised if there is a validation problem with a configuration.
All the problems are reported in a single exception.
Notes
-----
This method checks that all the supplied entities have valid file
templates and also have formatters defined.
"""
templateFailed = None
try:
self.templates.validateTemplates(entities, logFailures=logFailures)
except FileTemplateValidationError as e:
templateFailed = str(e)
formatterFailed = []
for entity in entities:
try:
self.formatterFactory.getFormatter(entity)
except KeyError as e:
formatterFailed.append(str(e))
if logFailures:
log.fatal("Formatter failure: %s", e)
if templateFailed or formatterFailed:
messages = []
if templateFailed:
messages.append(templateFailed)
if formatterFailed:
messages.append(",".join(formatterFailed))
msg = ";\n".join(messages)
raise DatastoreValidationError(msg)
def validateConfiguration(self,
entities: Iterable[Union[DatasetRef, DatasetType,
StorageClass]],
logFailures: bool = False) -> None:
"""Validate some of the configuration for this datastore.
Parameters
----------
entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass`
Entities to test against this configuration. Can be differing
types.
logFailures : `bool`, optional
If `True`, output a log message for every validation error
detected.
Raises
------
DatastoreValidationError
Raised if there is a validation problem with a configuration.
All the problems are reported in a single exception.
Notes
-----
This method checks each datastore in turn.
"""
# Need to catch each of the datastore outputs and ensure that
# all are tested.
failures = []
for datastore in self.datastores:
try:
datastore.validateConfiguration(entities,
logFailures=logFailures)
except DatastoreValidationError as e:
if logFailures:
log.critical("Datastore %s failed validation",
datastore.name)
failures.append(f"Datastore {self.name}: {e}")
if failures:
msg = ";\n".join(failures)
raise DatastoreValidationError(msg)