def read_qe_tester(self, per_amp): if per_amp: path_str = 'per_amp' else: path_str = 'per_detector' files = glob.glob(os.path.join(self.qe_path, 'ccd00', path_str, '*')) dest_path = os.path.join(self.tmp_dir_obj.name, 'trivial_camera', 'qe_curve', 'ccd00') os.makedirs(dest_path) dest_files = [ os.path.join(dest_path, os.path.split(f)[1]) for f in files ] for f, df in zip(files, dest_files): os.symlink(f, df) curves, data_type = read_all( os.path.join(self.tmp_dir_obj.name, 'trivial_camera', 'qe_curve'), self.cam) self.assertEqual(len(curves.keys()), 1) # One sensor self.assertEqual(data_type, 'qe_curve') for s in curves: self.assertEqual(len(curves[s].keys()), 2) # Two validity ranges if per_amp: for d in curves[s]: self.assertEqual(len(curves[s][d].data), 2) # Two amps
def test_read_defects(self): defects, data_type = read_all(self.defects_path, self.cam) self.assertEqual(len(defects.keys()), 1) # One sensor self.assertEqual(data_type, 'defects') for s in defects: self.assertEqual(len(defects[s].keys()), 2) # Two validity ranges for d in defects[s]: self.assertEqual(len(defects[s][d]), 4) # Four defects
def writeCuratedCalibrations(self, butler): """Write human-curated calibration Datasets to the given Butler with the appropriate validity ranges. This is a temporary API that should go away once obs_ packages have a standardized approach to this problem. """ # Write cameraGeom.Camera, with an infinite validity range. datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", universe=butler.registry.dimensions) butler.registry.registerDatasetType(datasetType) unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) camera = self.getCamera() butler.put(camera, datasetType, unboundedDataId) # Write brighter-fatter kernel, with an infinite validity range. datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray", universe=butler.registry.dimensions) butler.registry.registerDatasetType(datasetType) # Load and then put instead of just moving the file in part to ensure # the version in-repo is written with Python 3 and does not need # `encoding='latin1'` to be read. bfKernel = self.getBrighterFatterKernel() butler.put(bfKernel, datasetType, unboundedDataId) # The following iterate over the values of the dictionaries returned by the transmission functions # and ignore the date that is supplied. This is due to the dates not being ranges but single dates, # which do not give the proper notion of validity. As such unbounded calibration labels are used # when inserting into the database. In the future these could and probably should be updated to # properly account for what ranges are considered valid. # Write optical transmissions opticsTransmissions = getOpticsTransmission() datasetType = DatasetType("transmission_optics", ("instrument", "calibration_label"), "TransmissionCurve", universe=butler.registry.dimensions) butler.registry.registerDatasetType(datasetType) for entry in opticsTransmissions.values(): if entry is None: continue butler.put(entry, datasetType, unboundedDataId) # Write transmission sensor sensorTransmissions = getSensorTransmission() datasetType = DatasetType( "transmission_sensor", ("instrument", "detector", "calibration_label"), "TransmissionCurve", universe=butler.registry.dimensions) butler.registry.registerDatasetType(datasetType) for entry in sensorTransmissions.values(): if entry is None: continue for sensor, curve in entry.items(): dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor) butler.put(curve, datasetType, dataId) # Write filter transmissions filterTransmissions = getFilterTransmission() datasetType = DatasetType( "transmission_filter", ("instrument", "physical_filter", "calibration_label"), "TransmissionCurve", universe=butler.registry.dimensions) butler.registry.registerDatasetType(datasetType) for entry in filterTransmissions.values(): if entry is None: continue for band, curve in entry.items(): dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band) butler.put(curve, datasetType, dataId) # Write atmospheric transmissions, this only as dimension of instrument as other areas will only # look up along this dimension (ISR) atmosphericTransmissions = getAtmosphereTransmission() datasetType = DatasetType("transmission_atmosphere", ("instrument", ), "TransmissionCurve", universe=butler.registry.dimensions) butler.registry.registerDatasetType(datasetType) for entry in atmosphericTransmissions.values(): if entry is None: continue butler.put(entry, datasetType, {"instrument": self.getName()}) # Write defects with validity ranges taken from obs_subaru_data/hsc/defects # (along with the defects themselves). datasetType = DatasetType( "defects", ("instrument", "detector", "calibration_label"), "DefectsList", universe=butler.registry.dimensions) butler.registry.registerDatasetType(datasetType) defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc", "defects") camera = self.getCamera() defectsDict = read_all( defectPath, camera)[0] # This method returns a dict plus the calib type endOfTime = '20380119T031407' dimensionRecords = [] datasetRecords = [] # First loop just gathers up the things we want to insert, so we # can do some bulk inserts and minimize the time spent in transaction. for det in defectsDict: detector = camera[det] times = sorted([k for k in defectsDict[det]]) defects = [defectsDict[det][time] for time in times] times = times + [ parser.parse(endOfTime), ] for defect, beginTime, endTime in zip(defects, times[:-1], times[1:]): md = defect.getMetadata() calibrationLabel = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}" dataId = DataCoordinate.standardize( universe=butler.registry.dimensions, instrument=self.getName(), calibration_label=calibrationLabel, detector=detector.getId(), ) datasetRecords.append((defect, dataId)) dimensionRecords.append({ "instrument": self.getName(), "name": calibrationLabel, "datetime_begin": beginTime, "datetime_end": endTime, }) # Second loop actually does the inserts and filesystem writes. with butler.transaction(): butler.registry.insertDimensionData("calibration_label", *dimensionRecords) # TODO: vectorize these puts, once butler APIs for that become # available. for defect, dataId in datasetRecords: butler.put(defect, datasetType, dataId)