def testSensors(self):
wavelengths = np.linspace(4000, 12000, 100)
point = Point2D(200, 10)
for sensors in makeTransmissionCurves.getSensorTransmission().values():
for i in range(112):
curve = sensors[i]
throughputs = curve.sampleAt(point, wavelengths)
siliconTransparent = wavelengths > 11000
self.assertTrue((throughputs[siliconTransparent] < 0.01).all())
midR = np.logical_and(wavelengths > 6000, wavelengths < 6300)
self.assertTrue((throughputs[midR] > 0.9).all())
def testSensors(self):
wavelengths = np.linspace(4000, 12000, 100)
point = Point2D(200, 10)
for sensors in makeTransmissionCurves.getSensorTransmission().values():
for i in range(112):
curve = sensors[i]
throughputs = curve.sampleAt(point, wavelengths)
siliconTransparent = wavelengths > 11000
self.assertTrue((throughputs[siliconTransparent] < 0.01).all())
midR = np.logical_and(wavelengths > 6000, wavelengths < 6300)
self.assertTrue((throughputs[midR] > 0.9).all())
def main(butler):
for start, nested in makeTransmissionCurves.getFilterTransmission().items():
for name, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_filter", filter=name)
for start, nested in makeTransmissionCurves.getSensorTransmission().items():
for ccd, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_sensor", ccd=ccd)
for start, curve in makeTransmissionCurves.getOpticsTransmission().items():
if curve is not None:
butler.put(curve, "transmission_optics")
for start, curve in makeTransmissionCurves.getAtmosphereTransmission().items():
if curve is not None:
butler.put(curve, "transmission_atmosphere")
def main(butler):
for start, nested in makeTransmissionCurves.getFilterTransmission().items(
):
for name, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_filter", filter=name)
for start, nested in makeTransmissionCurves.getSensorTransmission().items(
):
for ccd, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_sensor", ccd=ccd)
for start, curve in makeTransmissionCurves.getOpticsTransmission().items():
if curve is not None:
butler.put(curve, "transmission_optics")
for start, curve in makeTransmissionCurves.getAtmosphereTransmission(
).items():
if curve is not None:
butler.put(curve, "transmission_atmosphere")
def writeCuratedCalibrations(self, butler):
"""Write human-curated calibration Datasets to the given Butler with
the appropriate validity ranges.
This is a temporary API that should go away once obs_ packages have
a standardized approach to this problem.
"""
# Write cameraGeom.Camera, with an infinite validity range.
datasetType = DatasetType("camera",
("instrument", "calibration_label"),
"TablePersistableCamera",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
unboundedDataId = addUnboundedCalibrationLabel(butler.registry,
self.getName())
camera = self.getCamera()
butler.put(camera, datasetType, unboundedDataId)
# Write brighter-fatter kernel, with an infinite validity range.
datasetType = DatasetType("bfKernel",
("instrument", "calibration_label"),
"NumpyArray",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
# Load and then put instead of just moving the file in part to ensure
# the version in-repo is written with Python 3 and does not need
# `encoding='latin1'` to be read.
bfKernel = self.getBrighterFatterKernel()
butler.put(bfKernel, datasetType, unboundedDataId)
# The following iterate over the values of the dictionaries returned by the transmission functions
# and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
# which do not give the proper notion of validity. As such unbounded calibration labels are used
# when inserting into the database. In the future these could and probably should be updated to
# properly account for what ranges are considered valid.
# Write optical transmissions
opticsTransmissions = getOpticsTransmission()
datasetType = DatasetType("transmission_optics",
("instrument", "calibration_label"),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in opticsTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, unboundedDataId)
# Write transmission sensor
sensorTransmissions = getSensorTransmission()
datasetType = DatasetType(
"transmission_sensor",
("instrument", "detector", "calibration_label"),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in sensorTransmissions.values():
if entry is None:
continue
for sensor, curve in entry.items():
dataId = DataId(unboundedDataId, detector=sensor)
butler.put(curve, datasetType, dataId)
# Write filter transmissions
filterTransmissions = getFilterTransmission()
datasetType = DatasetType(
"transmission_filter",
("instrument", "physical_filter", "calibration_label"),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in filterTransmissions.values():
if entry is None:
continue
for band, curve in entry.items():
dataId = DataId(unboundedDataId, physical_filter=band)
butler.put(curve, datasetType, dataId)
# Write atmospheric transmissions, this only as dimension of instrument as other areas will only
# look up along this dimension (ISR)
atmosphericTransmissions = getAtmosphereTransmission()
datasetType = DatasetType("transmission_atmosphere", ("instrument", ),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in atmosphericTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, {"instrument": self.getName()})
# Write defects with validity ranges taken from obs_subaru_data/hsc/defects
# (along with the defects themselves).
datasetType = DatasetType(
"defects", ("instrument", "detector", "calibration_label"),
"DefectsList",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc",
"defects")
camera = self.getCamera()
defectsDict = read_all_defects(defectPath, camera)
endOfTime = '20380119T031407'
with butler.transaction():
for det in defectsDict:
detector = camera[det]
times = sorted([k for k in defectsDict[det]])
defects = [defectsDict[det][time] for time in times]
times = times + [
parser.parse(endOfTime),
]
for defect, beginTime, endTime in zip(defects, times[:-1],
times[1:]):
md = defect.getMetadata()
dataId = DataId(
universe=butler.registry.dimensions,
instrument=self.getName(),
calibration_label=
f"defect/{md['CALIBDATE']}/{md['DETECTOR']}")
dataId.entries["calibration_label"][
"valid_first"] = beginTime
dataId.entries["calibration_label"]["valid_last"] = endTime
butler.registry.addDimensionEntry("calibration_label",
dataId)
butler.put(defect,
datasetType,
dataId,
detector=detector.getId())
def writeCuratedCalibrations(self, butler):
"""Write human-curated calibration Datasets to the given Butler with
the appropriate validity ranges.
This is a temporary API that should go away once obs_ packages have
a standardized approach to this problem.
"""
# Write cameraGeom.Camera, with an infinite validity range.
datasetType = DatasetType("camera", ("instrument", "calibration_label"), "TablePersistableCamera")
butler.registry.registerDatasetType(datasetType)
unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
camera = self.getCamera()
butler.put(camera, datasetType, unboundedDataId)
# Write brighter-fatter kernel, with an infinite validity range.
datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray")
butler.registry.registerDatasetType(datasetType)
# Load and then put instead of just moving the file in part to ensure
# the version in-repo is written with Python 3 and does not need
# `encoding='latin1'` to be read.
bfKernel = self.getBrighterFatterKernel()
butler.put(bfKernel, datasetType, unboundedDataId)
# The following iterate over the values of the dictionaries returned by the transmission functions
# and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
# which do not give the proper notion of validity. As such unbounded calibration labels are used
# when inserting into the database. In the future these could and probably should be updated to
# properly account for what ranges are considered valid.
# Write optical transmissions
opticsTransmissions = getOpticsTransmission()
datasetType = DatasetType("transmission_optics",
("instrument", "calibration_label"),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in opticsTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, unboundedDataId)
# Write transmission sensor
sensorTransmissions = getSensorTransmission()
datasetType = DatasetType("transmission_sensor",
("instrument", "detector", "calibration_label"),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in sensorTransmissions.values():
if entry is None:
continue
for sensor, curve in entry.items():
dataId = DataId(unboundedDataId, detector=sensor)
butler.put(curve, datasetType, dataId)
# Write filter transmissions
filterTransmissions = getFilterTransmission()
datasetType = DatasetType("transmission_filter",
("instrument", "physical_filter", "calibration_label"),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in filterTransmissions.values():
if entry is None:
continue
for band, curve in entry.items():
dataId = DataId(unboundedDataId, physical_filter=band)
butler.put(curve, datasetType, dataId)
# Write atmospheric transmissions, this only as dimension of instrument as other areas will only
# look up along this dimension (ISR)
atmosphericTransmissions = getAtmosphereTransmission()
datasetType = DatasetType("transmission_atmosphere", ("instrument",),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in atmosphericTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, {"instrument": self.getName()})
# Write defects with validity ranges taken from obs_subaru/hsc/defects
# (along with the defects themselves).
datasetType = DatasetType("defects", ("instrument", "detector", "calibration_label"), "DefectsList")
butler.registry.registerDatasetType(datasetType)
defectPath = os.path.join(getPackageDir("obs_subaru"), "hsc", "defects")
dbPath = os.path.join(defectPath, "defectRegistry.sqlite3")
db = sqlite3.connect(dbPath)
db.row_factory = sqlite3.Row
sql = "SELECT path, ccd, validStart, validEnd FROM defect"
with butler.transaction():
for row in db.execute(sql):
dataId = DataId(universe=butler.registry.dimensions,
instrument=self.getName(),
calibration_label=f"defect/{row['path']}/{row['ccd']}")
dataId.entries["calibration_label"]["valid_first"] = readDateTime(row["validStart"])
dataId.entries["calibration_label"]["valid_last"] = readDateTime(row["validEnd"])
butler.registry.addDimensionEntry("calibration_label", dataId)
ref = butler.registry.addDataset(datasetType, dataId, run=butler.run, recursive=True,
detector=row['ccd'])
butler.datastore.ingest(os.path.join(defectPath, row["path"]), ref, transfer="copy")