def testWildcardAccept(self):
# Accept everything
config = ConstraintsConfig({})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Accept everything
constraints = Constraints(None, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Accept everything explicitly
config = ConstraintsConfig({"accept": ["all"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Accept all instrument A but reject everything else
config = ConstraintsConfig({"accept": [{"instrument<A>": ["all"]}]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
def testAcceptReject(self):
# Reject everything except calexp
config = ConstraintsConfig({"accept": ["calexp"], "reject": ["all"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
# Accept everything except calexp
config = ConstraintsConfig({"reject": ["calexp"], "accept": ["all"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Reject pvi but explicitly accept pvi for instrument A
# Reject all instrument A but accept everything else
# The reject here is superfluous
config = ConstraintsConfig({"accept": [{"instrument<A>": ["pvi"]}], "reject": ["pvi"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
# Accept everything except pvi from other than instrument A
config = ConstraintsConfig({"accept": ["all", {"instrument<A>": ["pvi"]}], "reject": ["pvi"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
def __init__(self, config, registry, butlerRoot=None):
super().__init__(config, registry)
if "root" not in self.config:
raise ValueError("No root directory specified in configuration")
# Name ourselves either using an explicit name or a name
# derived from the (unexpanded) root
if "name" in self.config:
self.name = self.config["name"]
else:
self.name = "POSIXDatastore@{}".format(self.config["root"])
# Support repository relocation in config
self.root = replaceRoot(self.config["root"], butlerRoot)
if not os.path.isdir(self.root):
if "create" not in self.config or not self.config["create"]:
raise ValueError(f"No valid root at: {self.root}")
safeMakeDir(self.root)
self.locationFactory = LocationFactory(self.root)
self.formatterFactory = FormatterFactory()
self.storageClassFactory = StorageClassFactory()
# Now associate formatters with storage classes
self.formatterFactory.registerFormatters(
self.config["formatters"], universe=self.registry.dimensions)
# Read the file naming templates
self.templates = FileTemplates(self.config["templates"],
universe=self.registry.dimensions)
# And read the constraints list
constraintsConfig = self.config.get("constraints")
self.constraints = Constraints(constraintsConfig,
universe=self.registry.dimensions)
# Storage of paths and formatters, keyed by dataset_id
types = {
"path": str,
"formatter": str,
"storage_class": str,
"file_size": int,
"checksum": str,
"dataset_id": int
}
lengths = {
"path": 256,
"formatter": 128,
"storage_class": 64,
"checksum": 128
}
self.records = DatabaseDict.fromConfig(self.config["records"],
types=types,
value=self.RecordTuple,
key="dataset_id",
lengths=lengths,
registry=registry)
def __init__(self,
config: Union[Config, str],
bridgeManager: DatastoreRegistryBridgeManager,
butlerRoot: str = None):
super().__init__(config, bridgeManager)
# Scan for child datastores and instantiate them with the same registry
self.datastores = []
for c in self.config["datastores"]:
c = DatastoreConfig(c)
datastoreType = doImport(c["cls"])
datastore = datastoreType(c, bridgeManager, butlerRoot=butlerRoot)
log.debug("Creating child datastore %s", datastore.name)
self.datastores.append(datastore)
# Name ourself based on our children
if self.datastores:
# We must set the names explicitly
self._names = [d.name for d in self.datastores]
childNames = ",".join(self.names)
else:
childNames = "(empty@{})".format(time.time())
self._names = [childNames]
self.name = "{}[{}]".format(type(self).__qualname__, childNames)
# We declare we are ephemeral if all our child datastores declare
# they are ephemeral
isEphemeral = True
for d in self.datastores:
if not d.isEphemeral:
isEphemeral = False
break
self.isEphemeral = isEphemeral
# per-datastore override constraints
if "datastore_constraints" in self.config:
overrides = self.config["datastore_constraints"]
if len(overrides) != len(self.datastores):
raise DatastoreValidationError(
f"Number of registered datastores ({len(self.datastores)})"
" differs from number of constraints overrides"
f" {len(overrides)}")
self.datastoreConstraints = [
Constraints(c.get("constraints"),
universe=bridgeManager.universe) for c in overrides
]
else:
self.datastoreConstraints = (None, ) * len(self.datastores)
log.debug("Created %s (%s)", self.name,
("ephemeral" if self.isEphemeral else "permanent"))
def testSimpleAccept(self):
config = ConstraintsConfig({"accept": ["calexp", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
# Dimension accept
config = ConstraintsConfig(
{"accept": ["visit+physical_filter+instrument", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
config = ConstraintsConfig(
{"accept": ["visit+detector+instrument", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Only accept instrument A pvi
config = ConstraintsConfig({"accept": [{"instrument<A>": ["pvi"]}]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
# Accept PVI for instrument B but not instrument A
config = ConstraintsConfig(
{"accept": ["calexp", {
"instrument<B>": ["pvi"]
}]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
self.assertTrue(constraints.isAcceptable(self.pviB))
def __init__(self, config, registry=None, butlerRoot=None):
super().__init__(config, registry)
self.storageClassFactory = StorageClassFactory()
# Name ourselves with the timestamp the datastore
# was created.
self.name = "InMemoryDatastore@{}".format(time.time())
log.debug("Creating datastore %s", self.name)
# Storage of datasets, keyed by dataset_id
self.datasets = {}
# Records is distinct in order to track concrete composite components
# where we register multiple components for a single dataset.
self.records = {}
# And read the constraints list
constraintsConfig = self.config.get("constraints")
self.constraints = Constraints(constraintsConfig,
universe=self.registry.dimensions)
def testEdgeCases(self):
# Accept everything and reject everything
config = ConstraintsConfig({"accept": ["all"], "reject": ["all"]})
with self.assertRaises(ValidationError):
Constraints(config, universe=self.universe)
def testSimpleReject(self):
config = ConstraintsConfig({"reject": ["calexp", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
