def find(self, name: str) -> Optional[DatasetRecordStorage]:
# Docstring inherited from DatasetRecordStorageManager.
compositeName, componentName = DatasetType.splitDatasetTypeName(name)
storage = self._byName.get(compositeName)
if storage is not None and componentName is not None:
componentStorage = copy.copy(storage)
componentStorage.datasetType = storage.datasetType.makeComponentDatasetType(componentName)
return componentStorage
else:
return storage
def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass",
run="run2", conform=True):
"""Make a simple DatasetRef"""
if dataId is None:
dataId = self.dataId
# Pretend we have a parent if this looks like a composite
compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
parentStorageClass = DatasetType.PlaceholderParentStorageClass if componentName else None
datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()),
StorageClass(storageClassName),
parentStorageClass=parentStorageClass)
return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
def _makeDatasetRef(self, datasetTypeName, dimensions, storageClass, dataId, *, id=None, run=None,
conform=True):
# helper for makeDatasetRef
# Pretend we have a parent if this looks like a composite
compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
parentStorageClass = StorageClass("component") if componentName else None
datasetType = DatasetType(datasetTypeName, dimensions, storageClass,
parentStorageClass=parentStorageClass)
if id is None:
self.id += 1
id = self.id
if run is None:
run = "dummy"
return DatasetRef(datasetType, dataId, id=id, run=run, conform=conform)
def remove(self, name: str) -> None:
# Docstring inherited from DatasetRecordStorageManager.
compositeName, componentName = DatasetType.splitDatasetTypeName(name)
if componentName is not None:
raise ValueError(f"Cannot delete a dataset type of a component of a composite (given {name})")
# Delete the row
try:
self._db.delete(self._static.dataset_type, ["name"], {"name": name})
except sqlalchemy.exc.IntegrityError as e:
raise OrphanedRecordError(f"Dataset type {name} can not be removed."
" It is associated with datasets that must be removed first.") from e
# Now refresh everything -- removal is rare enough that this does
# not need to be fast.
self.refresh()
def makeDatasetTypesSet(connectionType, freeze=True):
"""Constructs a set of true `DatasetType` objects
Parameters
----------
connectionType : `str`
Name of the connection type to produce a set for, corresponds
to an attribute of type `list` on the connection class instance
freeze : `bool`, optional
If `True`, call `NamedValueSet.freeze` on the object returned.
Returns
-------
datasetTypes : `NamedValueSet`
A set of all datasetTypes which correspond to the input
connection type specified in the connection class of this
`PipelineTask`
Notes
-----
This function is a closure over the variables ``registry`` and
``taskDef``.
"""
datasetTypes = NamedValueSet()
for c in iterConnections(taskDef.connections, connectionType):
dimensions = set(getattr(c, 'dimensions', set()))
if "skypix" in dimensions:
try:
datasetType = registry.getDatasetType(c.name)
except LookupError as err:
raise LookupError(
f"DatasetType '{c.name}' referenced by "
f"{type(taskDef.connections).__name__} uses 'skypix' as a dimension "
f"placeholder, but does not already exist in the registry. "
f"Note that reference catalog names are now used as the dataset "
f"type name instead of 'ref_cat'.") from err
rest1 = set(
registry.dimensions.extract(dimensions -
set(["skypix"])).names)
rest2 = set(dim.name for dim in datasetType.dimensions
if not isinstance(dim, SkyPixDimension))
if rest1 != rest2:
raise ValueError(
f"Non-skypix dimensions for dataset type {c.name} declared in "
f"connections ({rest1}) are inconsistent with those in "
f"registry's version of this dataset ({rest2}).")
else:
# Component dataset types are not explicitly in the
# registry. This complicates consistency checks with
# registry and requires we work out the composite storage
# class.
registryDatasetType = None
try:
registryDatasetType = registry.getDatasetType(c.name)
except KeyError:
compositeName, componentName = DatasetType.splitDatasetTypeName(
c.name)
parentStorageClass = DatasetType.PlaceholderParentStorageClass \
if componentName else None
datasetType = c.makeDatasetType(
registry.dimensions,
parentStorageClass=parentStorageClass)
registryDatasetType = datasetType
else:
datasetType = c.makeDatasetType(
registry.dimensions,
parentStorageClass=registryDatasetType.
parentStorageClass)
if registryDatasetType and datasetType != registryDatasetType:
raise ValueError(
f"Supplied dataset type ({datasetType}) inconsistent with "
f"registry definition ({registryDatasetType}) "
f"for {taskDef.label}.")
datasetTypes.add(datasetType)
if freeze:
datasetTypes.freeze()
return datasetTypes
