def testFilters(self):
wavelengths = np.linspace(3000, 12000, 1000)
point = Point2D(1000, -500)
def check(curve, central, w1, w2):
# check that throughput within w1 of central is strictly greater
# than throughput outside w2 of central
throughput = curve.sampleAt(point, wavelengths)
mid = np.logical_and(wavelengths > central - w1,
wavelengths < central + w1)
outer = np.logical_or(wavelengths < central - w2,
wavelengths > central + w2)
self.assertGreater(throughput[mid].min(), throughput[outer].max())
for curves in makeTransmissionCurves.getFilterTransmission().values():
check(curves["NB0387"], 3870, 50, 100)
check(curves["NB0816"], 8160, 50, 100)
check(curves["NB0921"], 9210, 50, 100)
check(curves["HSC-G"], 4730, 500, 1500)
check(curves["HSC-R"], 6230, 500, 1500)
check(curves["HSC-R2"], 6230, 500, 1500)
check(curves["HSC-I"], 7750, 500, 1500)
check(curves["HSC-I2"], 7750, 500, 1500)
check(curves["HSC-Z"], 8923, 500, 1500)
check(curves["HSC-Y"], 10030, 500, 1500)
def main(butler):
for start, nested in makeTransmissionCurves.getFilterTransmission().items():
for name, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_filter", filter=name)
for start, nested in makeTransmissionCurves.getSensorTransmission().items():
for ccd, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_sensor", ccd=ccd)
for start, curve in makeTransmissionCurves.getOpticsTransmission().items():
if curve is not None:
butler.put(curve, "transmission_optics")
for start, curve in makeTransmissionCurves.getAtmosphereTransmission().items():
if curve is not None:
butler.put(curve, "transmission_atmosphere")
def main(butler):
for start, nested in makeTransmissionCurves.getFilterTransmission().items(
):
for name, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_filter", filter=name)
for start, nested in makeTransmissionCurves.getSensorTransmission().items(
):
for ccd, curve in nested.items():
if curve is not None:
butler.put(curve, "transmission_sensor", ccd=ccd)
for start, curve in makeTransmissionCurves.getOpticsTransmission().items():
if curve is not None:
butler.put(curve, "transmission_optics")
for start, curve in makeTransmissionCurves.getAtmosphereTransmission(
).items():
if curve is not None:
butler.put(curve, "transmission_atmosphere")
def testRadialDependence(self):
wavelengths = np.linspace(5000, 9000, 500)
radii = np.linspace(0, 20000, 50)
def computeMeanWavelengths(curve):
w = []
for radius in radii:
throughput = curve.sampleAt(Point2D(radius, 0), wavelengths)
w.append(np.dot(throughput, wavelengths) / throughput.sum())
return np.array(w)
for curves in makeTransmissionCurves.getFilterTransmission().values():
r = computeMeanWavelengths(curves['HSC-R'])
r2 = computeMeanWavelengths(curves['HSC-R2'])
i = computeMeanWavelengths(curves['HSC-I'])
i2 = computeMeanWavelengths(curves['HSC-I2'])
# i2 and r2 should have lower variance as a function of radius
self.assertLess(r2.std(), r.std())
self.assertLess(i2.std(), i.std())
# the mean wavelengths of the two r and two i filters should
# neverthess be close (within 100 angstroms)
self.assertLess(np.abs(r2.mean() - r.mean()), 100)
self.assertLess(np.abs(i2.mean() - i.mean()), 100)
def testRadialDependence(self):
wavelengths = np.linspace(5000, 9000, 500)
radii = np.linspace(0, 20000, 50)
def computeMeanWavelengths(curve):
w = []
for radius in radii:
throughput = curve.sampleAt(Point2D(radius, 0), wavelengths)
w.append(np.dot(throughput, wavelengths)/throughput.sum())
return np.array(w)
for curves in makeTransmissionCurves.getFilterTransmission().values():
r = computeMeanWavelengths(curves['HSC-R'])
r2 = computeMeanWavelengths(curves['HSC-R2'])
i = computeMeanWavelengths(curves['HSC-I'])
i2 = computeMeanWavelengths(curves['HSC-I2'])
# i2 and r2 should have lower variance as a function of radius
self.assertLess(r2.std(), r.std())
self.assertLess(i2.std(), i.std())
# the mean wavelengths of the two r and two i filters should
# neverthess be close (within 100 angstroms)
self.assertLess(np.abs(r2.mean() - r.mean()), 100)
self.assertLess(np.abs(i2.mean() - i.mean()), 100)
def testFilters(self):
wavelengths = np.linspace(3000, 12000, 1000)
point = Point2D(1000, -500)
def check(curve, central, w1, w2):
# check that throughput within w1 of central is strictly greater
# than throughput outside w2 of central
throughput = curve.sampleAt(point, wavelengths)
mid = np.logical_and(wavelengths > central - w1, wavelengths < central + w1)
outer = np.logical_or(wavelengths < central - w2, wavelengths > central + w2)
self.assertGreater(throughput[mid].min(), throughput[outer].max())
for curves in makeTransmissionCurves.getFilterTransmission().values():
check(curves["NB0387"], 3870, 50, 100)
check(curves["NB0816"], 8160, 50, 100)
check(curves["NB0921"], 9210, 50, 100)
check(curves["HSC-G"], 4730, 500, 1500)
check(curves["HSC-R"], 6230, 500, 1500)
check(curves["HSC-R2"], 6230, 500, 1500)
check(curves["HSC-I"], 7750, 500, 1500)
check(curves["HSC-I2"], 7750, 500, 1500)
check(curves["HSC-Z"], 8923, 500, 1500)
check(curves["HSC-Y"], 10030, 500, 1500)
def testRadialDependence(self):
wavelengths = np.linspace(3000, 11000, 500)
radii = np.linspace(0, 20000, 50)
def computeMeanWavelengths(curve):
w = []
for radius in radii:
throughput = curve.sampleAt(Point2D(radius, 0), wavelengths)
w.append(np.dot(throughput, wavelengths) / throughput.sum())
return np.array(w)
for curves in makeTransmissionCurves.getFilterTransmission().values():
r = computeMeanWavelengths(curves['HSC-R'])
r2 = computeMeanWavelengths(curves['HSC-R2'])
i = computeMeanWavelengths(curves['HSC-I'])
i2 = computeMeanWavelengths(curves['HSC-I2'])
# i2 and r2 should have lower variance as a function of radius
self.assertLess(r2.std(), r.std())
self.assertLess(i2.std(), i.std())
# the mean wavelengths of the two r and two i filters should
# neverthess be close (within 100 angstroms)
self.assertLess(np.abs(r2.mean() - r.mean()), 100)
self.assertLess(np.abs(i2.mean() - i.mean()), 100)
# Ensure that the nb vary but not too much
n387 = computeMeanWavelengths(curves['NB0387'])
self.assertLess(np.abs(n387.mean() - 3870.0), 20.0)
self.assertGreater(n387.std(), 0.0)
n816 = computeMeanWavelengths(curves['NB0816'])
self.assertLess(np.abs(n816.mean() - 8160.0), 20.0)
self.assertGreater(n816.std(), 0.0)
n921 = computeMeanWavelengths(curves['NB0921'])
self.assertLess(np.abs(n921.mean() - 9210.0), 20.0)
self.assertGreater(n921.std(), 0.0)
n1010 = computeMeanWavelengths(curves['NB1010'])
self.assertLess(np.abs(n1010.mean() - 10100.0), 20.0)
self.assertGreater(n1010.std(), 0.0)
def writeCuratedCalibrations(self, butler):
"""Write human-curated calibration Datasets to the given Butler with
the appropriate validity ranges.
This is a temporary API that should go away once obs_ packages have
a standardized approach to this problem.
"""
# Write cameraGeom.Camera, with an infinite validity range.
datasetType = DatasetType("camera",
("instrument", "calibration_label"),
"TablePersistableCamera",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
unboundedDataId = addUnboundedCalibrationLabel(butler.registry,
self.getName())
camera = self.getCamera()
butler.put(camera, datasetType, unboundedDataId)
# Write brighter-fatter kernel, with an infinite validity range.
datasetType = DatasetType("bfKernel",
("instrument", "calibration_label"),
"NumpyArray",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
# Load and then put instead of just moving the file in part to ensure
# the version in-repo is written with Python 3 and does not need
# `encoding='latin1'` to be read.
bfKernel = self.getBrighterFatterKernel()
butler.put(bfKernel, datasetType, unboundedDataId)
# The following iterate over the values of the dictionaries returned by the transmission functions
# and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
# which do not give the proper notion of validity. As such unbounded calibration labels are used
# when inserting into the database. In the future these could and probably should be updated to
# properly account for what ranges are considered valid.
# Write optical transmissions
opticsTransmissions = getOpticsTransmission()
datasetType = DatasetType("transmission_optics",
("instrument", "calibration_label"),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in opticsTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, unboundedDataId)
# Write transmission sensor
sensorTransmissions = getSensorTransmission()
datasetType = DatasetType(
"transmission_sensor",
("instrument", "detector", "calibration_label"),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in sensorTransmissions.values():
if entry is None:
continue
for sensor, curve in entry.items():
dataId = DataId(unboundedDataId, detector=sensor)
butler.put(curve, datasetType, dataId)
# Write filter transmissions
filterTransmissions = getFilterTransmission()
datasetType = DatasetType(
"transmission_filter",
("instrument", "physical_filter", "calibration_label"),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in filterTransmissions.values():
if entry is None:
continue
for band, curve in entry.items():
dataId = DataId(unboundedDataId, physical_filter=band)
butler.put(curve, datasetType, dataId)
# Write atmospheric transmissions, this only as dimension of instrument as other areas will only
# look up along this dimension (ISR)
atmosphericTransmissions = getAtmosphereTransmission()
datasetType = DatasetType("transmission_atmosphere", ("instrument", ),
"TablePersistableTransmissionCurve",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
for entry in atmosphericTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, {"instrument": self.getName()})
# Write defects with validity ranges taken from obs_subaru_data/hsc/defects
# (along with the defects themselves).
datasetType = DatasetType(
"defects", ("instrument", "detector", "calibration_label"),
"DefectsList",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc",
"defects")
camera = self.getCamera()
defectsDict = read_all_defects(defectPath, camera)
endOfTime = '20380119T031407'
with butler.transaction():
for det in defectsDict:
detector = camera[det]
times = sorted([k for k in defectsDict[det]])
defects = [defectsDict[det][time] for time in times]
times = times + [
parser.parse(endOfTime),
]
for defect, beginTime, endTime in zip(defects, times[:-1],
times[1:]):
md = defect.getMetadata()
dataId = DataId(
universe=butler.registry.dimensions,
instrument=self.getName(),
calibration_label=
f"defect/{md['CALIBDATE']}/{md['DETECTOR']}")
dataId.entries["calibration_label"][
"valid_first"] = beginTime
dataId.entries["calibration_label"]["valid_last"] = endTime
butler.registry.addDimensionEntry("calibration_label",
dataId)
butler.put(defect,
datasetType,
dataId,
detector=detector.getId())
def writeCuratedCalibrations(self, butler):
"""Write human-curated calibration Datasets to the given Butler with
the appropriate validity ranges.
This is a temporary API that should go away once obs_ packages have
a standardized approach to this problem.
"""
# Write cameraGeom.Camera, with an infinite validity range.
datasetType = DatasetType("camera", ("instrument", "calibration_label"), "TablePersistableCamera")
butler.registry.registerDatasetType(datasetType)
unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
camera = self.getCamera()
butler.put(camera, datasetType, unboundedDataId)
# Write brighter-fatter kernel, with an infinite validity range.
datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray")
butler.registry.registerDatasetType(datasetType)
# Load and then put instead of just moving the file in part to ensure
# the version in-repo is written with Python 3 and does not need
# `encoding='latin1'` to be read.
bfKernel = self.getBrighterFatterKernel()
butler.put(bfKernel, datasetType, unboundedDataId)
# The following iterate over the values of the dictionaries returned by the transmission functions
# and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
# which do not give the proper notion of validity. As such unbounded calibration labels are used
# when inserting into the database. In the future these could and probably should be updated to
# properly account for what ranges are considered valid.
# Write optical transmissions
opticsTransmissions = getOpticsTransmission()
datasetType = DatasetType("transmission_optics",
("instrument", "calibration_label"),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in opticsTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, unboundedDataId)
# Write transmission sensor
sensorTransmissions = getSensorTransmission()
datasetType = DatasetType("transmission_sensor",
("instrument", "detector", "calibration_label"),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in sensorTransmissions.values():
if entry is None:
continue
for sensor, curve in entry.items():
dataId = DataId(unboundedDataId, detector=sensor)
butler.put(curve, datasetType, dataId)
# Write filter transmissions
filterTransmissions = getFilterTransmission()
datasetType = DatasetType("transmission_filter",
("instrument", "physical_filter", "calibration_label"),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in filterTransmissions.values():
if entry is None:
continue
for band, curve in entry.items():
dataId = DataId(unboundedDataId, physical_filter=band)
butler.put(curve, datasetType, dataId)
# Write atmospheric transmissions, this only as dimension of instrument as other areas will only
# look up along this dimension (ISR)
atmosphericTransmissions = getAtmosphereTransmission()
datasetType = DatasetType("transmission_atmosphere", ("instrument",),
"TablePersistableTransmissionCurve")
butler.registry.registerDatasetType(datasetType)
for entry in atmosphericTransmissions.values():
if entry is None:
continue
butler.put(entry, datasetType, {"instrument": self.getName()})
# Write defects with validity ranges taken from obs_subaru/hsc/defects
# (along with the defects themselves).
datasetType = DatasetType("defects", ("instrument", "detector", "calibration_label"), "DefectsList")
butler.registry.registerDatasetType(datasetType)
defectPath = os.path.join(getPackageDir("obs_subaru"), "hsc", "defects")
dbPath = os.path.join(defectPath, "defectRegistry.sqlite3")
db = sqlite3.connect(dbPath)
db.row_factory = sqlite3.Row
sql = "SELECT path, ccd, validStart, validEnd FROM defect"
with butler.transaction():
for row in db.execute(sql):
dataId = DataId(universe=butler.registry.dimensions,
instrument=self.getName(),
calibration_label=f"defect/{row['path']}/{row['ccd']}")
dataId.entries["calibration_label"]["valid_first"] = readDateTime(row["validStart"])
dataId.entries["calibration_label"]["valid_last"] = readDateTime(row["validEnd"])
butler.registry.addDimensionEntry("calibration_label", dataId)
ref = butler.registry.addDataset(datasetType, dataId, run=butler.run, recursive=True,
detector=row['ccd'])
butler.datastore.ingest(os.path.join(defectPath, row["path"]), ref, transfer="copy")
