def read_qe_tester(self, per_amp):
if per_amp:
path_str = 'per_amp'
else:
path_str = 'per_detector'
files = glob.glob(os.path.join(self.qe_path, 'ccd00', path_str, '*'))
dest_path = os.path.join(self.tmp_dir_obj.name, 'trivial_camera',
'qe_curve', 'ccd00')
os.makedirs(dest_path)
dest_files = [
os.path.join(dest_path,
os.path.split(f)[1]) for f in files
]
for f, df in zip(files, dest_files):
os.symlink(f, df)
curves, data_type = read_all(
os.path.join(self.tmp_dir_obj.name, 'trivial_camera', 'qe_curve'),
self.cam)
self.assertEqual(len(curves.keys()), 1) # One sensor
self.assertEqual(data_type, 'qe_curve')
for s in curves:
self.assertEqual(len(curves[s].keys()), 2) # Two validity ranges
if per_amp:
for d in curves[s]:
self.assertEqual(len(curves[s][d].data), 2) # Two amps
def test_read_defects(self):
defects, data_type = read_all(self.defects_path, self.cam)
self.assertEqual(len(defects.keys()), 1) # One sensor
self.assertEqual(data_type, 'defects')
for s in defects:
self.assertEqual(len(defects[s].keys()), 2) # Two validity ranges
for d in defects[s]:
self.assertEqual(len(defects[s][d]), 4) # Four defects
def writeCuratedCalibrations(self, butler):
"""Write human-curated calibration Datasets to the given Butler with
the appropriate validity ranges.
This is a temporary API that should go away once obs_ packages have
a standardized approach to this problem.
"""
# Write cameraGeom.Camera, with an infinite validity range.
datasetType = DatasetType("camera",
("instrument", "calibration_label"),
"Camera",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
unboundedDataId = addUnboundedCalibrationLabel(butler.registry,
self.getName())
camera = self.getCamera()
butler.put(camera, datasetType, unboundedDataId)
# Write brighter-fatter kernel, with an infinite validity range.
datasetType = DatasetType("bfKernel",
("instrument", "calibration_label"),
"NumpyArray",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
# Load and then put instead of just moving the file in part to ensure
# the version in-repo is written with Python 3 and does not need
# `encoding='latin1'` to be read.
bfKernel = self.getBrighterFatterKernel()
butler.put(bfKernel, datasetType, unboundedDataId)
# The following iterate over the values of the dictionaries returned by the transmission functions
# and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
# which do not give the proper notion of validity. As such unbounded calibration labels are used
# when inserting into the database. In the future these could and probably should be updated to
# properly account for what ranges are considered valid.
# Write optical transmissions
opticsTransmissions = getOpticsTransmission()
datasetType = DatasetType("transmission_optics",
("instrument", "calibration_label"),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in opticsTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, unboundedDataId)
# Write transmission sensor
sensorTransmissions = getSensorTransmission()
datasetType = DatasetType(
"transmission_sensor",
("instrument", "detector", "calibration_label"),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in sensorTransmissions.values():
if entry is None:
continue
for sensor, curve in entry.items():
dataId = DataCoordinate.standardize(unboundedDataId,
detector=sensor)
butler.put(curve, datasetType, dataId)
# Write filter transmissions
filterTransmissions = getFilterTransmission()
datasetType = DatasetType(
"transmission_filter",
("instrument", "physical_filter", "calibration_label"),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in filterTransmissions.values():
if entry is None:
continue
for band, curve in entry.items():
dataId = DataCoordinate.standardize(unboundedDataId,
physical_filter=band)
butler.put(curve, datasetType, dataId)
# Write atmospheric transmissions, this only as dimension of instrument as other areas will only
# look up along this dimension (ISR)
atmosphericTransmissions = getAtmosphereTransmission()
datasetType = DatasetType("transmission_atmosphere", ("instrument", ),
"TransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in atmosphericTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, {"instrument": self.getName()})
# Write defects with validity ranges taken from obs_subaru_data/hsc/defects
# (along with the defects themselves).
datasetType = DatasetType(
"defects", ("instrument", "detector", "calibration_label"),
"DefectsList",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc",
"defects")
camera = self.getCamera()
defectsDict = read_all(
defectPath,
camera)[0] # This method returns a dict plus the calib type
endOfTime = '20380119T031407'
dimensionRecords = []
datasetRecords = []
# First loop just gathers up the things we want to insert, so we
# can do some bulk inserts and minimize the time spent in transaction.
for det in defectsDict:
detector = camera[det]
times = sorted([k for k in defectsDict[det]])
defects = [defectsDict[det][time] for time in times]
times = times + [
parser.parse(endOfTime),
]
for defect, beginTime, endTime in zip(defects, times[:-1],
times[1:]):
md = defect.getMetadata()
calibrationLabel = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}"
dataId = DataCoordinate.standardize(
universe=butler.registry.dimensions,
instrument=self.getName(),
calibration_label=calibrationLabel,
detector=detector.getId(),
)
datasetRecords.append((defect, dataId))
dimensionRecords.append({
"instrument": self.getName(),
"name": calibrationLabel,
"datetime_begin": beginTime,
"datetime_end": endTime,
})
# Second loop actually does the inserts and filesystem writes.
with butler.transaction():
butler.registry.insertDimensionData("calibration_label",
*dimensionRecords)
# TODO: vectorize these puts, once butler APIs for that become
# available.
for defect, dataId in datasetRecords:
butler.put(defect, datasetType, dataId)