def testConstructor(self):
"""Test that construction preserves and validates values.
"""
# Construct an unresolved ref.
ref = DatasetRef(self.datasetType, self.dataId)
self.assertEqual(ref.datasetType, self.datasetType)
self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe),
msg=ref.dataId)
self.assertIsInstance(ref.dataId, DataCoordinate)
# Constructing an unresolved ref with run and/or components should
# fail.
run = "somerun"
with self.assertRaises(ValueError):
DatasetRef(self.datasetType, self.dataId, run=run)
# Passing a data ID that is missing dimensions should fail.
with self.assertRaises(KeyError):
DatasetRef(self.datasetType, {"instrument": "DummyCam"})
# Constructing a resolved ref should preserve run as well as everything
# else.
ref = DatasetRef(self.datasetType, self.dataId, id=1, run=run)
self.assertEqual(ref.datasetType, self.datasetType)
self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe),
msg=ref.dataId)
self.assertIsInstance(ref.dataId, DataCoordinate)
self.assertEqual(ref.id, 1)
self.assertEqual(ref.run, run)
def splitByStateFlags(self,
dataIds: Optional[DataCoordinateSequence] = None,
*,
expanded: bool = True,
complete: bool = True,
minimal: bool = True) -> SplitByStateFlags:
"""Given a sequence of data IDs, generate new equivalent sequences
containing less information.
Parameters
----------
dataIds : `DataCoordinateSequence`, optional.
Data IDs to start from. Defaults to ``self.allDataIds``.
``dataIds.hasRecords()`` and ``dataIds.hasFull()`` must both return
`True`.
expanded : `bool`, optional
If `True` (default) include the original data IDs that contain all
information in the result.
complete : `bool`, optional
If `True` (default) include data IDs for which ``hasFull()``
returns `True` but ``hasRecords()`` does not.
minimal : `bool`, optional
If `True` (default) include data IDS that only contain values for
required dimensions, for which ``hasFull()`` may not return `True`.
Returns
-------
split : `SplitByStateFlags`
A dataclass holding the indicated data IDs in attributes that
correspond to the boolean keyword arguments.
"""
if dataIds is None:
dataIds = self.allDataIds
assert dataIds.hasFull() and dataIds.hasRecords()
result = SplitByStateFlags(expanded=dataIds)
if complete:
result.complete = DataCoordinateSequence([
DataCoordinate.standardize(e.full.byName(),
graph=dataIds.graph)
for e in result.expanded
],
graph=dataIds.graph)
self.assertTrue(result.complete.hasFull())
self.assertFalse(result.complete.hasRecords())
if minimal:
result.minimal = DataCoordinateSequence([
DataCoordinate.standardize(e.byName(), graph=dataIds.graph)
for e in result.expanded
],
graph=dataIds.graph)
self.assertEqual(result.minimal.hasFull(),
not dataIds.graph.implied)
self.assertFalse(result.minimal.hasRecords())
if not expanded:
result.expanded = None
return result
def visits(self):
butler = Butler(self.root, collections=[self.outputRun])
return {
DataCoordinate.standardize(instrument="MegaPrime",
visit=1038843,
universe=butler.registry.dimensions): [
DataCoordinate.standardize(instrument="MegaPrime",
exposure=1038843,
universe=butler.registry.dimensions)
]
}
def translate(self,
dataId2: dict,
*,
partial: bool = False,
log: Log) -> Optional[DataCoordinate]:
# Docstring inherited from PathElementHandler.
rawDataId3 = self._translator(dataId2, partial=partial, log=log)
if partial:
return DataCoordinate.standardize(
rawDataId3, universe=self._datasetType.dimensions.universe)
else:
return DataCoordinate.standardize(
rawDataId3, graph=self._datasetType.dimensions)
def translate(self, dataId2: dict, *, partial: bool = False
) -> Tuple[Optional[DataCoordinate], Optional[str]]:
# Docstring inherited from PathElementHandler.
rawDataId3, calibDate = self._translator(dataId2, partial=partial)
if partial:
return (
DataCoordinate.standardize(rawDataId3, universe=self._datasetType.dimensions.universe),
calibDate,
)
else:
return (
DataCoordinate.standardize(rawDataId3, graph=self._datasetType.dimensions),
calibDate
)
def handle(self, path: str, nextDataId2,
datasets: Mapping[DatasetType, Mapping[Optional[str], List[FileDataset]]], *,
predicate: Callable[[DataCoordinate], bool]):
dataId3, calibDate = self.translate(nextDataId2, partial=True)
def get_detectors(filename):
fitsData = lsst.afw.fits.Fits(filename, 'r')
# NOTE: The primary header (HDU=0) does not contain detector data.
detectors = []
for i in range(1, fitsData.countHdus()):
fitsData.setHdu(i)
metadata = fitsData.readMetadata()
detectors.append(metadata['CCDNUM'])
return detectors
if predicate(dataId3):
detectors = get_detectors(path)
refs = []
for detector in detectors:
newDataId3 = DataCoordinate.standardize(dataId3,
graph=self._datasetType.dimensions,
detector=detector)
refs.append(DatasetRef(self._datasetType, newDataId3))
datasets[self._datasetType][calibDate].append(
FileDataset(refs=refs, path=path, formatter=self._formatter)
)
def prep(self):
# Docstring inherited from RepoConverter.
self.task.log.info(f"Looking for skymaps in root {self.root}.")
for coaddName, datasetTypeName in SKYMAP_DATASET_TYPES.items():
if not self.task.isDatasetTypeIncluded(datasetTypeName):
continue
try:
exists = self.butler2.datasetExists(datasetTypeName)
except AttributeError:
# This mapper doesn't even define this dataset type.
continue
if not exists:
continue
instance = self.butler2.get(datasetTypeName)
name = self.task.useSkyMap(instance, datasetTypeName)
datasetType = DatasetType(datasetTypeName,
dimensions=["skymap"],
storageClass="SkyMap",
universe=self.task.universe)
dataId = DataCoordinate.standardize(skymap=name,
universe=self.task.universe)
struct = FoundSkyMap(name=name,
instance=instance,
coaddName=coaddName,
ref=DatasetRef(datasetType, dataId),
filename=self.butler2.getUri(datasetTypeName))
self._foundSkyMapsByCoaddName[coaddName] = struct
self.task.log.info("Found skymap %s in %s in %s.", name,
datasetTypeName, self.root)
super().prep()
def _calculate_dataset_info(self, header, filename):
"""Calculate a RawFileDatasetInfo from the supplied information.
Parameters
----------
header : `Mapping`
Header from the dataset.
filename : `str`
Filename to use for error messages.
Returns
-------
dataset : `RawFileDatasetInfo`
The region, dataId, and observation information associated with
this dataset.
"""
obsInfo = ObservationInfo(header)
dataId = DataCoordinate.standardize(instrument=obsInfo.instrument,
exposure=obsInfo.exposure_id,
detector=obsInfo.detector_num,
universe=self.universe)
if obsInfo.instrument != self.instrument.getName():
raise ValueError(f"Incorrect instrument (expected {self.instrument.getName()}, "
f"got {obsInfo.instrument}) for file {filename}.")
FormatterClass = self.instrument.getRawFormatter(dataId)
region = self._calculate_region_from_dataset_metadata(obsInfo, header, FormatterClass)
return RawFileDatasetInfo(obsInfo=obsInfo, region=region, dataId=dataId)
def getDataId(self, id: DatasetId) -> DataCoordinate:
"""Return DataId for a dataset.
Parameters
----------
id : `DatasetId`
Unique dataset identifier.
Returns
-------
dataId : `DataCoordinate`
DataId for the dataset.
"""
# This query could return multiple rows (one for each tagged collection
# the dataset is in, plus one for its run collection), and we don't
# care which of those we get.
sql = self._tags.select().where(
sqlalchemy.sql.and_(
self._tags.columns.dataset_id == id,
self._tags.columns.dataset_type_id ==
self._dataset_type_id)).limit(1)
row = self._db.query(sql).fetchone()
assert row is not None, "Should be guaranteed by caller and foreign key constraints."
return DataCoordinate.standardize(
{
dimension.name: row[dimension.name]
for dimension in self.datasetType.dimensions.required
},
graph=self.datasetType.dimensions)
def testSkyMapPacking(self):
"""Test that packing Tract+Patch into an integer in Gen3 works and is
self-consistent.
Note that this packing does *not* use the same algorithm as Gen2 and
hence generates different IDs, because the Gen2 algorithm is
problematically tied to the *default* SkyMap for a particular camera,
rather than the SkyMap actually used.
"""
# SkyMap used by ci_hsc has only one tract, so the test coverage in
# that area isn't great. That's okay because that's tested in SkyMap;
# what we care about here is that the converted repo has the necessary
# metadata to construct and use these packers at all.
for patch in [0, 43, 52]:
dataId = self.butler.registry.expandDataId(
skymap="discrete/ci_hsc", tract=0, patch=patch, band='r')
packer1 = self.butler.registry.dimensions.makePacker(
"tract_patch", dataId)
packer2 = self.butler.registry.dimensions.makePacker(
"tract_patch_band", dataId)
self.assertNotEqual(packer1.pack(dataId), packer2.pack(dataId))
self.assertEqual(
packer1.unpack(packer1.pack(dataId)),
DataCoordinate.standardize(dataId, graph=packer1.dimensions))
self.assertEqual(packer2.unpack(packer2.pack(dataId)), dataId)
self.assertEqual(packer1.pack(dataId, band='i'),
packer1.pack(dataId))
self.assertNotEqual(packer2.pack(dataId, band='i'),
packer2.pack(dataId))
def translate(self, dataId2: dict, *, partial: bool = False
) -> Tuple[Optional[DataCoordinate], Optional[str]]:
assert partial is True, "We always require partial, to ignore 'ccdnum'"
rawDataId3, calibDate = self._translator(dataId2, partial=partial)
return (
DataCoordinate.standardize(rawDataId3, universe=self._datasetType.dimensions.universe),
calibDate,
)
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"), "DATA"), writeable=False,
collections=["HSC/calib/2013-06-17", "HSC/runs/ci_hsc"])
# We need to provide a physical_filter value to fully identify a flat,
# but this still leaves the band as an implied value that this data ID
# doesn't know.
self.flatMinimalDataId = DataCoordinate.standardize(
instrument="HSC", detector=0, physical_filter="HSC-R",
universe=self.butler.registry.dimensions,
)
# For a calexp, the minimal data ID just has exposure and detector,
# so both band and physical_filter are implied and not known here.
self.calexpMinimalDataId = DataCoordinate.standardize(
instrument="HSC", detector=100, visit=903334,
universe=self.butler.registry.dimensions,
)
# Parameters with bbox to test that logic still works on subimage gets.
self.parameters = {"bbox": Box2I(Point2I(0, 0), Point2I(8, 7))}
def _makeDSRefVisit(self, dstype, visitId, universe):
return DatasetRef(datasetType=dstype,
dataId=DataCoordinate.standardize(
detector="X",
visit=visitId,
physical_filter='a',
abstract_filter='b',
instrument='TestInstrument',
universe=universe))
def _calculate_dataset_info(self, header, filename):
"""Calculate a RawFileDatasetInfo from the supplied information.
Parameters
----------
header : `Mapping`
Header from the dataset.
filename : `str`
Filename to use for error messages.
Returns
-------
dataset : `RawFileDatasetInfo`
The dataId, and observation information associated with this
dataset.
"""
# To ensure we aren't slowed down for no reason, explicitly
# list here the properties we need for the schema
# Use a dict with values a boolean where True indicates
# that it is required that we calculate this property.
ingest_subset = {
"altaz_begin": False,
"boresight_rotation_coord": False,
"boresight_rotation_angle": False,
"dark_time": False,
"datetime_begin": True,
"datetime_end": True,
"detector_num": True,
"exposure_group": False,
"exposure_id": True,
"exposure_time": True,
"instrument": True,
"tracking_radec": False,
"object": False,
"observation_counter": False,
"observation_id": True,
"observation_reason": False,
"observation_type": True,
"observing_day": False,
"physical_filter": True,
"science_program": False,
"visit_id": False,
}
obsInfo = ObservationInfo(
header,
pedantic=False,
filename=filename,
required={k
for k in ingest_subset if ingest_subset[k]},
subset=set(ingest_subset))
dataId = DataCoordinate.standardize(instrument=obsInfo.instrument,
exposure=obsInfo.exposure_id,
detector=obsInfo.detector_num,
universe=self.universe)
return RawFileDatasetInfo(obsInfo=obsInfo, dataId=dataId)
def unpack(self, packedId: int) -> DataCoordinate:
# Docstring inherited from DataIdPacker.unpack
d = {"skymap": self._skyMapName}
if self._filterMax is not None:
d["band"] = self.getFilterNameFromInt(packedId //
self._tractPatchMax)
packedId %= self._tractPatchMax
d["tract"] = packedId // self._patchMax
d["patch"] = packedId % self._patchMax
return DataCoordinate.standardize(d, graph=self.dimensions)
def unpack(self, packedId):
# Docstring inherited from DimensionPacker.unpack
observation, detector = divmod(packedId, self._detectorMax)
return DataCoordinate.standardize(
{
"instrument": self._instrumentName,
"detector": detector,
self._observationName: observation,
},
graph=self.dimensions
)
def testWithoutFilter(self):
dimensions = DimensionGraph(universe=self.universe,
names=["tract", "patch"])
dataId = DataCoordinate.standardize(skymap=self.fixed["skymap"],
tract=2,
patch=6,
universe=self.universe)
packer = SkyMapDimensionPacker(self.fixed, dimensions)
packedId = packer.pack(dataId)
self.assertLessEqual(packedId.bit_length(), packer.maxBits)
self.assertEqual(packer.unpack(packedId), dataId)
def _makeDSRefVisit(self, dstype, visitId, universe):
return DatasetRef(
datasetType=dstype,
dataId=DataCoordinate.standardize(
detector="X",
visit=visitId,
physical_filter="a",
band="b",
instrument="TestInstrument",
universe=universe,
),
)
def makeGraph(self, pipeline, collections, run, userQuery):
"""Create execution graph for a pipeline.
Parameters
----------
pipeline : `Pipeline`
Pipeline definition, task names/classes and their configs.
collections
Expressions representing the collections to search for input
datasets. May be any of the types accepted by
`lsst.daf.butler.CollectionSearch.fromExpression`.
run : `str`, optional
Name of the `~lsst.daf.butler.CollectionType.RUN` collection for
output datasets, if it already exists.
userQuery : `str`
String which defines user-defined selection for registry, should be
empty or `None` if there is no restrictions on data selection.
Returns
-------
graph : `QuantumGraph`
Raises
------
UserExpressionError
Raised when user expression cannot be parsed.
OutputExistsError
Raised when output datasets already exist.
Exception
Other exceptions types may be raised by underlying registry
classes.
"""
scaffolding = _PipelineScaffolding(pipeline, registry=self.registry)
instrument = pipeline.getInstrument()
if isinstance(instrument, str):
instrument = doImport(instrument)
if instrument is not None:
dataId = DataCoordinate.standardize(
instrument=instrument.getName(),
universe=self.registry.dimensions)
else:
dataId = DataCoordinate.makeEmpty(self.registry.dimensions)
with scaffolding.connectDataIds(self.registry, collections, userQuery,
dataId) as commonDataIds:
scaffolding.resolveDatasetRefs(self.registry,
collections,
run,
commonDataIds,
skipExisting=self.skipExisting)
return scaffolding.makeQuantumGraph()
def getDataId(self, id: int) -> DataCoordinate:
# Docstring inherited from DatasetRecordStorageManager.
# This query could return multiple rows (one for each tagged collection
# the dataset is in, plus one for its run collection), and we don't
# care which of those we get.
sql = self._dynamic.select().where(
sqlalchemy.sql.and_(
self._dynamic.columns.dataset_id == id,
self._dynamic.columns.dataset_type_id == self._dataset_type_id
)
).limit(1)
row = self._db.query(sql).fetchone()
assert row is not None, "Should be guaranteed by caller and foreign key constraints."
return DataCoordinate.standardize(
{dimension.name: row[dimension.name] for dimension in self.datasetType.dimensions.required},
graph=self.datasetType.dimensions
)
def _refFromConnection(butler: Butler, connection: DimensionedConnection,
dataId: DataId, **kwargs: Any) -> DatasetRef:
"""Create a DatasetRef for a connection in a collection.
Parameters
----------
butler : `lsst.daf.butler.Butler`
The collection to point to.
connection : `lsst.pipe.base.connectionTypes.DimensionedConnection`
The connection defining the dataset type to point to.
dataId
The data ID for the dataset to point to.
**kwargs
Additional keyword arguments used to augment or construct
a `~lsst.daf.butler.DataCoordinate`.
Returns
-------
ref : `lsst.daf.butler.DatasetRef`
A reference to a dataset compatible with ``connection``, with ID
``dataId``, in the collection pointed to by ``butler``.
"""
universe = butler.registry.dimensions
# DatasetRef only tests if required dimension is missing, but not extras
_checkDimensionsMatch(universe, set(connection.dimensions), dataId.keys())
dataId = DataCoordinate.standardize(dataId, **kwargs, universe=universe)
# skypix is a PipelineTask alias for "some spatial index", Butler doesn't
# understand it. Code copied from TaskDatasetTypes.fromTaskDef
if "skypix" in connection.dimensions:
datasetType = butler.registry.getDatasetType(connection.name)
else:
datasetType = connection.makeDatasetType(universe)
try:
butler.registry.getDatasetType(datasetType.name)
except KeyError:
raise ValueError(f"Invalid dataset type {connection.name}.")
try:
ref = DatasetRef(datasetType=datasetType, dataId=dataId)
return ref
except KeyError as e:
raise ValueError(
f"Dataset type ({connection.name}) and ID {dataId.byName()} not compatible."
) from e
def testReadingBadNewFileWithFullDataId(self):
"""If we try to read a new calexp with a full data ID, the reader
should check the filters in the file for consistency with the data ID
(and in this case, find them inconsistent, which should result in
warnings and returning what's in the data ID).
"""
self.skip_mock()
calexpBadDataId = DataCoordinate.standardize(
self.calexpMinimalDataId,
band="g",
physical_filter="HSC-G",
visit_system=0,
)
self.assertTrue(calexpBadDataId.hasFull())
# Some tests are only relevant when reading full calexps.
# By definition a disassembled exposure will have a correct
# filterlabel written out.
# In this situation the test becomes moot since the filterLabel
# formatter will not force a correct filter label into an
# incorrect filter label based on DataId.
_, components = self.butler.getURIs("calexp", calexpBadDataId)
if components:
raise unittest.SkipTest("Test not relevant because composite has been disassembled")
with self.assertWarns(Warning):
calexp = self.butler.get("calexp", calexpBadDataId)
with self.assertWarns(Warning):
calexpFilterLabel = self.butler.get("calexp.filter", calexpBadDataId)
self.assertEqual(calexp.getFilter(), calexpFilterLabel)
self.assertEqual(calexp.getFilter().bandLabel, calexpBadDataId["band"])
self.assertEqual(calexp.getFilter().physicalLabel, calexpBadDataId["physical_filter"])
self.assertEqual(calexpFilterLabel.bandLabel, calexpBadDataId["band"])
self.assertEqual(calexpFilterLabel.physicalLabel, calexpBadDataId["physical_filter"])
with self.assertWarns(Warning):
calexpSub = self.butler.get("calexp", calexpBadDataId, parameters=self.parameters)
self.assertEqual(calexp.getFilter(), calexpSub.getFilter())
def ref_from_connection(butler, connection, data_id):
"""Create a DatasetRef for a connection in a collection.
Parameters
----------
butler : `lsst.daf.butler.Butler`
The collection to point to.
connection : `lsst.pipe.base.connectionTypes.DimensionedConnection`
The connection defining the dataset type to point to.
data_id : `Mapping` [`str`] or `lsst.daf.butler.DataCoordinate`
The data ID for the dataset to point to.
Returns
-------
ref : `lsst.daf.butler.DatasetRef`
A reference to a dataset compatible with ``connection``, with ID
``data_id``, in the collection pointed to by ``butler``.
"""
universe = butler.registry.dimensions
data_id = DataCoordinate.standardize(data_id, universe=universe)
return DatasetRef(
datasetType=connection.makeDatasetType(universe),
dataId=data_id,
)
def pack_data_id(self, tract, patch, band=None):
"""Pack a skymap-based data ID into an integer.
Parameters
----------
tract : `int`
Integer ID for the tract.
patch : `tuple` (`int`) or `int`
Either a 2-element (x, y) tuple (Gen2 patch ID) or a single integer
(Gen3 patch ID, corresponding to the "sequential" patch index
methods in this package).
band : `str`, optional
If provided, a filter name present in
`SkyMapDimensionPacker.SUPPORTED_FILTERS` (which is aspirationally
a list of all Gen3 'bands', but in practice may be missing some;
see RFC-785). If not provided, the packing algorithm that does
not include the filter will be used.
Returns
-------
packed : `int`
Integer that corresponds to the data ID.
max_bits : `int`
Maximum number of bits that ``packed`` could have, assuming this
skymap and presence or absence of ``band``.
Notes
-----
This method uses a Gen3 `lsst.daf.butler.DimensionPacker` object under
the hood to guarantee consistency with pure Gen3 code, but it does not
require the caller to actually have a Gen3 butler available. It does,
however, require a filter value compatible with the Gen3 "band"
dimension.
This is a temporary interface intended to aid with the migration from
Gen2 to Gen3 middleware. It will be removed with the Gen2 middleware
or when DM-31924 provides a longer-term replacement, whichever comes
first. Pure Gen3 code should use `lsst.daf.butler.DataCoordinate.pack`
or other `lsst.daf.butler.DimensionPacker` interfaces.
"""
from lsst.daf.butler import DataCoordinate, DimensionUniverse
universe = DimensionUniverse()
dummy_skymap_name = "unimportant" # only matters to Gen3 registry
tract_info = self[tract]
patch_info = tract_info[patch]
nx, ny = tract_info.getNumPatches()
skymap_record = universe["skymap"].RecordClass(
name=dummy_skymap_name,
hash=self.getSha1(),
tract_max=len(self),
patch_nx_max=
nx, # assuming these are the same for all tracts for now
patch_ny_max=ny,
)
skymap_data_id = DataCoordinate.standardize(
skymap=dummy_skymap_name,
universe=universe,
).expanded(records={"skymap": skymap_record}, )
full_data_id = DataCoordinate.standardize(
skymap=dummy_skymap_name,
tract=tract_info.getId(),
patch=tract_info.getSequentialPatchIndex(patch_info),
universe=universe,
)
if band is None:
packer = universe.makePacker("tract_patch", skymap_data_id)
else:
packer = universe.makePacker("tract_patch_band", skymap_data_id)
full_data_id = DataCoordinate.standardize(full_data_id, band=band)
return packer.pack(full_data_id, returnMaxBits=True)
def writeCuratedCalibrations(self, butler):
"""Write human-curated calibration Datasets to the given Butler with
the appropriate validity ranges.
This is a temporary API that should go away once obs_ packages have
a standardized approach to this problem.
"""
# Write cameraGeom.Camera, with an infinite validity range.
datasetType = DatasetType("camera",
("instrument", "calibration_label"),
"Camera",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
unboundedDataId = addUnboundedCalibrationLabel(butler.registry,
self.getName())
camera = self.getCamera()
butler.put(camera, datasetType, unboundedDataId)
# Write brighter-fatter kernel, with an infinite validity range.
datasetType = DatasetType("bfKernel",
("instrument", "calibration_label"),
"NumpyArray",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
# Load and then put instead of just moving the file in part to ensure
# the version in-repo is written with Python 3 and does not need
# `encoding='latin1'` to be read.
bfKernel = self.getBrighterFatterKernel()
butler.put(bfKernel, datasetType, unboundedDataId)
# The following iterate over the values of the dictionaries returned by the transmission functions
# and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
# which do not give the proper notion of validity. As such unbounded calibration labels are used
# when inserting into the database. In the future these could and probably should be updated to
# properly account for what ranges are considered valid.
# Write optical transmissions
opticsTransmissions = getOpticsTransmission()
datasetType = DatasetType("transmission_optics",
("instrument", "calibration_label"),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in opticsTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, unboundedDataId)
# Write transmission sensor
sensorTransmissions = getSensorTransmission()
datasetType = DatasetType(
"transmission_sensor",
("instrument", "detector", "calibration_label"),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in sensorTransmissions.values():
if entry is None:
continue
for sensor, curve in entry.items():
dataId = DataCoordinate.standardize(unboundedDataId,
detector=sensor)
butler.put(curve, datasetType, dataId)
# Write filter transmissions
filterTransmissions = getFilterTransmission()
datasetType = DatasetType(
"transmission_filter",
("instrument", "physical_filter", "calibration_label"),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in filterTransmissions.values():
if entry is None:
continue
for band, curve in entry.items():
dataId = DataCoordinate.standardize(unboundedDataId,
physical_filter=band)
butler.put(curve, datasetType, dataId)
# Write atmospheric transmissions, this only as dimension of instrument as other areas will only
# look up along this dimension (ISR)
atmosphericTransmissions = getAtmosphereTransmission()
datasetType = DatasetType("transmission_atmosphere", ("instrument", ),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in atmosphericTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, {"instrument": self.getName()})
# Write defects with validity ranges taken from obs_subaru_data/hsc/defects
# (along with the defects themselves).
datasetType = DatasetType(
"defects", ("instrument", "detector", "calibration_label"),
"DefectsList",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc",
"defects")
camera = self.getCamera()
defectsDict = read_all(
defectPath,
camera)[0] # This method returns a dict plus the calib type
endOfTime = '20380119T031407'
dimensionRecords = []
datasetRecords = []
# First loop just gathers up the things we want to insert, so we
# can do some bulk inserts and minimize the time spent in transaction.
for det in defectsDict:
detector = camera[det]
times = sorted([k for k in defectsDict[det]])
defects = [defectsDict[det][time] for time in times]
times = times + [
parser.parse(endOfTime),
]
for defect, beginTime, endTime in zip(defects, times[:-1],
times[1:]):
md = defect.getMetadata()
calibrationLabel = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}"
dataId = DataCoordinate.standardize(
universe=butler.registry.dimensions,
instrument=self.getName(),
calibration_label=calibrationLabel,
detector=detector.getId(),
)
datasetRecords.append((defect, dataId))
dimensionRecords.append({
"instrument": self.getName(),
"name": calibrationLabel,
"datetime_begin": beginTime,
"datetime_end": endTime,
})
# Second loop actually does the inserts and filesystem writes.
with butler.transaction():
butler.registry.insertDimensionData("calibration_label",
*dimensionRecords)
# TODO: vectorize these puts, once butler APIs for that become
# available.
for defect, dataId in datasetRecords:
butler.put(defect, datasetType, dataId)
def writeAdditionalCuratedCalibrations(self,
butler,
collection=None,
labels=()):
# Register the CALIBRATION collection that adds validity ranges.
# This does nothing if it is already registered.
if collection is None:
collection = self.makeCalibrationCollectionName(*labels)
butler.registry.registerCollection(collection,
type=CollectionType.CALIBRATION)
# Register the RUN collection that holds these datasets directly. We
# only need one because all of these datasets have the same (unbounded)
# validity range right now.
run = self.makeUnboundedCalibrationRunName(*labels)
butler.registry.registerRun(run)
baseDataId = butler.registry.expandDataId(instrument=self.getName())
refs = []
# Write brighter-fatter kernel, with an infinite validity range.
datasetType = DatasetType("bfKernel", ("instrument", ),
"NumpyArray",
universe=butler.registry.dimensions,
isCalibration=True)
butler.registry.registerDatasetType(datasetType)
# Load and then put instead of just moving the file in part to ensure
# the version in-repo is written with Python 3 and does not need
# `encoding='latin1'` to be read.
bfKernel = self.getBrighterFatterKernel()
refs.append(butler.put(bfKernel, datasetType, baseDataId, run=run))
# The following iterate over the values of the dictionaries returned
# by the transmission functions and ignore the date that is supplied.
# This is due to the dates not being ranges but single dates,
# which do not give the proper notion of validity. As such unbounded
# calibration labels are used when inserting into the database.
# In the future these could and probably should be updated to
# properly account for what ranges are considered valid.
# Write optical transmissions
opticsTransmissions = getOpticsTransmission()
datasetType = DatasetType("transmission_optics", ("instrument", ),
"TransmissionCurve",
universe=butler.registry.dimensions,
isCalibration=True)
butler.registry.registerDatasetType(datasetType)
for entry in opticsTransmissions.values():
if entry is None:
continue
refs.append(butler.put(entry, datasetType, baseDataId, run=run))
# Write transmission sensor
sensorTransmissions = getSensorTransmission()
datasetType = DatasetType("transmission_sensor", (
"instrument",
"detector",
),
"TransmissionCurve",
universe=butler.registry.dimensions,
isCalibration=True)
butler.registry.registerDatasetType(datasetType)
for entry in sensorTransmissions.values():
if entry is None:
continue
for sensor, curve in entry.items():
dataId = DataCoordinate.standardize(baseDataId,
detector=sensor)
refs.append(butler.put(curve, datasetType, dataId, run=run))
# Write filter transmissions
filterTransmissions = getFilterTransmission()
datasetType = DatasetType("transmission_filter", (
"instrument",
"physical_filter",
),
"TransmissionCurve",
universe=butler.registry.dimensions,
isCalibration=True)
butler.registry.registerDatasetType(datasetType)
for entry in filterTransmissions.values():
if entry is None:
continue
for band, curve in entry.items():
dataId = DataCoordinate.standardize(baseDataId,
physical_filter=band)
refs.append(butler.put(curve, datasetType, dataId, run=run))
# Write atmospheric transmissions
atmosphericTransmissions = getAtmosphereTransmission()
datasetType = DatasetType("transmission_atmosphere", ("instrument", ),
"TransmissionCurve",
universe=butler.registry.dimensions,
isCalibration=True)
butler.registry.registerDatasetType(datasetType)
for entry in atmosphericTransmissions.values():
if entry is None:
continue
refs.append(
butler.put(entry,
datasetType, {"instrument": self.getName()},
run=run))
# Associate all datasets with the unbounded validity range.
butler.registry.certify(collection, refs, Timespan(begin=None,
end=None))
def setUp(self):
config = Config(
{
"version": 1,
"namespace": "pipe_base_test",
"skypix": {
"common": "htm7",
"htm": {
"class": "lsst.sphgeom.HtmPixelization",
"max_level": 24,
},
},
"elements": {
"A": {
"keys": [
{
"name": "id",
"type": "int",
}
],
"storage": {
"cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage",
},
},
"B": {
"keys": [
{
"name": "id",
"type": "int",
}
],
"storage": {
"cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage",
},
},
},
"packers": {},
}
)
universe = DimensionUniverse(config=config)
# need to make a mapping of TaskDef to set of quantum
quantumMap = {}
tasks = []
for task, label in (
(Dummy1PipelineTask, "R"),
(Dummy2PipelineTask, "S"),
(Dummy3PipelineTask, "T"),
(Dummy4PipelineTask, "U"),
):
config = task.ConfigClass()
taskDef = TaskDef(get_full_type_name(task), config, task, label)
tasks.append(taskDef)
quantumSet = set()
connections = taskDef.connections
for a, b in ((1, 2), (3, 4)):
if connections.initInputs:
initInputDSType = DatasetType(
connections.initInput.name,
tuple(),
storageClass=connections.initInput.storageClass,
universe=universe,
)
initRefs = [DatasetRef(initInputDSType, DataCoordinate.makeEmpty(universe))]
else:
initRefs = None
inputDSType = DatasetType(
connections.input.name,
connections.input.dimensions,
storageClass=connections.input.storageClass,
universe=universe,
)
inputRefs = [
DatasetRef(inputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe))
]
outputDSType = DatasetType(
connections.output.name,
connections.output.dimensions,
storageClass=connections.output.storageClass,
universe=universe,
)
outputRefs = [
DatasetRef(outputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe))
]
quantumSet.add(
Quantum(
taskName=task.__qualname__,
dataId=DataCoordinate.standardize({"A": a, "B": b}, universe=universe),
taskClass=task,
initInputs=initRefs,
inputs={inputDSType: inputRefs},
outputs={outputDSType: outputRefs},
)
)
quantumMap[taskDef] = quantumSet
self.tasks = tasks
self.quantumMap = quantumMap
self.qGraph = QuantumGraph(quantumMap, metadata=METADATA)
self.universe = universe
def ingestSimulated(repo,
locations,
regex,
output_run,
transfer="auto",
ingest_type="rawexp"):
"""Ingests raw frames into the butler registry
Parameters
----------
repo : `str`
URI to the repository.
locations : `list` [`str`]
Files to ingest and directories to search for files that match
``regex`` to ingest.
regex : `str`
Regex string used to find files in directories listed in locations.
output_run : `str`
The path to the location, the run, where datasets should be put.
transfer : `str` or None
The external data transfer type, by default "auto".
ingest_type : `str`
ingest product data type.
Raises
------
Exception
Raised if operations on configuration object fail.
Notes
-----
This method inserts all datasets for an exposure within a transaction,
guaranteeing that partial exposures are never ingested. The exposure
dimension record is inserted with `Registry.syncDimensionData` first
(in its own transaction), which inserts only if a record with the same
primary key does not already exist. This allows different files within
the same exposure to be incremented in different runs.
"""
butler = Butler(repo, writeable=True)
# make sure instrument and detector dimensions are populated
with butler.registry.transaction():
instrument_record = {
"name": "simulator",
"exposure_max": 600000,
"detector_max": 6,
"class_name": "spherex.instrument.SimulatorInstrument"
}
butler.registry.syncDimensionData("instrument", instrument_record)
for idx in range(1, 7):
detector_record = {
"instrument": "simulator",
"id": idx,
"full_name": f"array{idx}"
}
butler.registry.syncDimensionData("detector", detector_record)
dimension_universe = butler.registry.dimensions
datasetType = DatasetType(ingest_type,
dimension_universe.extract(
("instrument", "detector", "exposure")),
"SPHERExImage",
universe=dimension_universe)
# idempotent dataset type registration
butler.registry.registerDatasetType(datasetType)
# idempotent collection registration
run = f"{ingest_type}r" if (output_run is None) else output_run
butler.registry.registerCollection(run, type=CollectionType.RUN)
n_failed = 0
files = findFileResources(locations, regex)
# example: sim_exposure_000000_array_1.fits or
# sim_exposure_000000_array_2_dark_current.fits
pattern = re.compile(r"sim_exposure_(\d+)_array_(\d)[_,.]")
# do we want to group observations?
grp = datetime.date.today().strftime("%Y%m%d")
datasets = []
for file in files:
# parse exposure and detector ids from file name
m = pattern.search(file)
if m is None:
n_failed += 1
logging.error(f"{file} does not match simulator file pattern")
continue
else:
g = m.groups()
if len(g) != 2:
n_failed += 1
logging.error(
f"Unable to get exposure and detector from file name: {file}"
)
continue
else:
[exposure_id, detector_id] = list(map(int, g))
try:
exposure_record = {
"instrument": "simulator",
"id": exposure_id,
"name": f"{exposure_id:06d}",
"group_name": f"{grp}",
"timespan": Timespan(begin=None, end=None)
}
# idempotent insertion of individual dimension rows
butler.registry.syncDimensionData("exposure", exposure_record)
except Exception as e:
n_failed += 1
logging.error(
f"Unable to insert exposure record for file {file}: {e}")
continue
dataId = DataCoordinate.standardize(
instrument="simulator",
detector=detector_id,
exposure=exposure_id,
universe=butler.registry.dimensions)
ref = DatasetRef(datasetType, dataId=dataId)
datasets.append(
FileDataset(refs=ref, path=file, formatter=AstropyImageFormatter))
with butler.transaction():
butler.ingest(*datasets, transfer=transfer, run=run)
def makeQuantum(
task: PipelineTask,
butler: Butler,
dataId: DataId,
ioDataIds: Mapping[str, Union[DataId, Sequence[DataId]]],
) -> Quantum:
"""Create a Quantum for a particular data ID(s).
Parameters
----------
task : `lsst.pipe.base.PipelineTask`
The task whose processing the quantum represents.
butler : `lsst.daf.butler.Butler`
The collection the quantum refers to.
dataId: any data ID type
The data ID of the quantum. Must have the same dimensions as
``task``'s connections class.
ioDataIds : `collections.abc.Mapping` [`str`]
A mapping keyed by input/output names. Values must be data IDs for
single connections and sequences of data IDs for multiple connections.
Returns
-------
quantum : `lsst.daf.butler.Quantum`
A quantum for ``task``, when called with ``dataIds``.
"""
connections = task.config.ConnectionsClass(config=task.config)
try:
_checkDimensionsMatch(butler.registry.dimensions,
connections.dimensions, dataId.keys())
except ValueError as e:
raise ValueError("Error in quantum dimensions.") from e
inputs = defaultdict(list)
outputs = defaultdict(list)
for name in itertools.chain(connections.inputs,
connections.prerequisiteInputs):
try:
connection = connections.__getattribute__(name)
_checkDataIdMultiplicity(name, ioDataIds[name],
connection.multiple)
ids = _normalizeDataIds(ioDataIds[name])
for id in ids:
ref = _refFromConnection(butler, connection, id)
inputs[ref.datasetType].append(ref)
except (ValueError, KeyError) as e:
raise ValueError(f"Error in connection {name}.") from e
for name in connections.outputs:
try:
connection = connections.__getattribute__(name)
_checkDataIdMultiplicity(name, ioDataIds[name],
connection.multiple)
ids = _normalizeDataIds(ioDataIds[name])
for id in ids:
ref = _refFromConnection(butler, connection, id)
outputs[ref.datasetType].append(ref)
except (ValueError, KeyError) as e:
raise ValueError(f"Error in connection {name}.") from e
quantum = Quantum(
taskClass=type(task),
dataId=DataCoordinate.standardize(dataId,
universe=butler.registry.dimensions),
inputs=inputs,
outputs=outputs,
)
return quantum
def testStandardize(self):
"""Test constructing a DataCoordinate from many different kinds of
input via `DataCoordinate.standardize` and `DataCoordinate.subset`.
"""
for n in range(5):
dimensions = self.randomDimensionSubset()
dataIds = self.randomDataIds(n=1).subset(dimensions)
split = self.splitByStateFlags(dataIds)
for m, dataId in enumerate(split.chain()):
# Passing in any kind of DataCoordinate alone just returns
# that object.
self.assertIs(dataId, DataCoordinate.standardize(dataId))
# Same if we also explicitly pass the dimensions we want.
self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph))
# Same if we pass the dimensions and some irrelevant
# kwargs.
self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph, htm7=12))
# Test constructing a new data ID from this one with a
# subset of the dimensions.
# This is not possible for some combinations of
# dimensions if hasFull is False (see
# `DataCoordinate.subset` docs).
newDimensions = self.randomDimensionSubset(n=1, graph=dataId.graph)
if dataId.hasFull() or dataId.graph.required.issuperset(newDimensions.required):
newDataIds = [
dataId.subset(newDimensions),
DataCoordinate.standardize(dataId, graph=newDimensions),
DataCoordinate.standardize(dataId, graph=newDimensions, htm7=12),
]
for newDataId in newDataIds:
with self.subTest(newDataId=newDataId, type=type(dataId)):
commonKeys = dataId.keys() & newDataId.keys()
self.assertTrue(commonKeys)
self.assertEqual(
[newDataId[k] for k in commonKeys],
[dataId[k] for k in commonKeys],
)
# This should never "downgrade" from
# Complete to Minimal or Expanded to Complete.
if dataId.hasRecords():
self.assertTrue(newDataId.hasRecords())
if dataId.hasFull():
self.assertTrue(newDataId.hasFull())
# Start from a complete data ID, and pass its values in via several
# different ways that should be equivalent.
for dataId in split.complete:
# Split the keys (dimension names) into two random subsets, so
# we can pass some as kwargs below.
keys1 = set(self.rng.sample(list(dataId.graph.dimensions.names),
len(dataId.graph.dimensions)//2))
keys2 = dataId.graph.dimensions.names - keys1
newCompleteDataIds = [
DataCoordinate.standardize(dataId.full.byName(), universe=dataId.universe),
DataCoordinate.standardize(dataId.full.byName(), graph=dataId.graph),
DataCoordinate.standardize(DataCoordinate.makeEmpty(dataId.graph.universe),
**dataId.full.byName()),
DataCoordinate.standardize(DataCoordinate.makeEmpty(dataId.graph.universe),
graph=dataId.graph, **dataId.full.byName()),
DataCoordinate.standardize(**dataId.full.byName(), universe=dataId.universe),
DataCoordinate.standardize(graph=dataId.graph, **dataId.full.byName()),
DataCoordinate.standardize(
{k: dataId[k] for k in keys1},
universe=dataId.universe,
**{k: dataId[k] for k in keys2}
),
DataCoordinate.standardize(
{k: dataId[k] for k in keys1},
graph=dataId.graph,
**{k: dataId[k] for k in keys2}
),
]
for newDataId in newCompleteDataIds:
with self.subTest(dataId=dataId, newDataId=newDataId, type=type(dataId)):
self.assertEqual(dataId, newDataId)
self.assertTrue(newDataId.hasFull())