def testCuratedCalibrations(self):
"""Test that defects, the camera, and the brighter-fatter kernel were
added to the Gen3 registry.
"""
rawDatasetType = self.butler.registry.getDatasetType("raw")
cameraRef = None
bfKernelRef = None
rawRefs = list(
self.butler.registry.queryDatasets(rawDatasetType,
collections=["HSC/raw/all"
]).expanded())
self.assertEqual(len(rawRefs), 33)
for rawRef in rawRefs:
# Expand raw data ID to include implied dimensions (e.g.
# physical_filter from exposure).
for calibDatasetTypeName in ("camera", "bfKernel", "defects"):
with self.subTest(dataset=calibDatasetTypeName):
calibDatasetType = self.butler.registry.getDatasetType(
calibDatasetTypeName)
calibRef = self.butler.registry.findDataset(
calibDatasetType,
collections=["HSC/calib"],
dataId=rawRef.dataId,
timespan=rawRef.dataId.timespan)
# We should have exactly one calib of each type
self.assertIsNotNone(calibRef)
# Try getting those calibs to make sure the files
# themselves are where the Butler thinks they are. We
# defer that for camera and bfKernel, because there's only
# one of each of those.
if calibDatasetTypeName == "camera":
if cameraRef is None:
cameraRef = calibRef
else:
self.assertEqual(cameraRef, calibRef)
elif calibDatasetTypeName == "bfKernel":
if bfKernelRef is None:
bfKernelRef = calibRef
else:
self.assertEqual(bfKernelRef, calibRef)
else:
defects = self.butler.get(calibRef,
collections=calibRef.run)
self.assertIsInstance(defects, lsst.ip.isr.Defects)
instrument = HyperSuprimeCam()
cameraFromButler = self.butler.get(cameraRef,
collections=cameraRef.run)
cameraFromInstrument = instrument.getCamera()
self.assertEqual(len(cameraFromButler), len(cameraFromInstrument))
self.assertEqual(cameraFromButler.getName(),
cameraFromInstrument.getName())
self.assertFloatsEqual(
self.butler.get(bfKernelRef, collections=bfKernelRef.run),
instrument.getBrighterFatterKernel())
def makeDistortionData(self):
"""Make distortion data
The data format is a dict of detector name: ccdData, where
ccdData is a Struct containing these fields:
- serial: detector.getSerial
- cornerDict: a dict of pixPosKey, cornerData, where:
- pixPosKey: self.asKey(pixPos) where pixPos is pixel position
- cornerData is Struct contains these fields (all of
type lsst.geom.Point2D):
- pixPos: pixel position
- focalPlane: focal plane position computed from pixPos
- fieldAngle: fieldAngle position computed from focalPlane
- focalPlaneRoundTrip: focal plane position computed from
fieldAngle
- pixPosRoundTrip: pixel position computed from focalPlane
"""
hsc = HyperSuprimeCam()
camera = hsc.getCamera()
focalPlaneToFieldAngle = camera.getTransformMap().getTransform(
FOCAL_PLANE, FIELD_ANGLE)
data = {} # dict of detector name: CcdData
for detector in camera:
# for each corner of each CCD:
# - get pixel position
# - convert to focal plane coordinates using the detector and
# record it
# - convert to field angle (this is the conversion that uses
# HscDistortion) and record it
# - convert back to focal plane (testing inverse direction of
# HscDistortion) and record it
# - convert back to pixel position and record it; pixel <-> focal
# plane is affine so there is no reason to doubt the inverse
# transform, but there is no harm
pixelsToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
cornerDict = {}
for pixPos in detector.getCorners(PIXELS):
pixPos = pixPos
focalPlane = pixelsToFocalPlane.applyForward(pixPos)
fieldAngle = focalPlaneToFieldAngle.applyForward(focalPlane)
focalPlaneRoundTrip = focalPlaneToFieldAngle.applyInverse(
fieldAngle)
pixPosRoundTrip = pixelsToFocalPlane.applyInverse(focalPlane)
cornerDict[self.toKey(pixPos)] = Struct(
pixPos=pixPos,
focalPlane=focalPlane,
fieldAngle=fieldAngle,
focalPlaneRoundTrip=focalPlaneRoundTrip,
pixPosRoundTrip=pixPosRoundTrip,
)
data[detector.getName()] = Struct(serial=detector.getSerial(),
cornerDict=cornerDict)
return data
def makeTask(butler: Butler, *, continue_: bool = False, reruns: List[Rerun]):
instrument = HyperSuprimeCam()
config = ConvertRepoTask.ConfigClass()
instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, config)
config.relatedOnly = True
config.transfer = "symlink"
if not reruns:
# No reruns, so just include datasets we want from the root and calib
# repos (default is all datasets).
config.datasetIncludePatterns = [
"brightObjectMask", "flat", "bias", "dark", "fringe", "sky",
"ref_cat", "raw"
]
gaiaRefCat = "gaia_dr2_20200414"
if gaiaRefCat not in config.refCats:
config.refCats.append(gaiaRefCat)
config.datasetIgnorePatterns.append("*_camera")
config.datasetIgnorePatterns.append("yBackground")
config.datasetIgnorePatterns.append("fgcmLookUpTable")
config.fileIgnorePatterns.extend(["*.log", "*.png", "rerun*"])
config.doRegisterInstrument = not continue_
# Add a level of indirection to the collection that will hold bright object
# masks, so the repo can hold multiple versions of those.
defaultMaskCollection = instrument.makeCollectionName("masks")
s18aMaskCollection = instrument.makeCollectionName("masks", "S18A")
config.runsForced["brightObjectMask"] = s18aMaskCollection
butler.registry.registerRun(s18aMaskCollection)
butler.registry.registerCollection(defaultMaskCollection,
CollectionType.CHAINED)
butler.registry.setCollectionChain(defaultMaskCollection,
[s18aMaskCollection])
return ConvertRepoTask(config=config,
butler3=butler,
instrument=instrument)
import os.path
from lsst.obs.base.gen2to3 import ConvertRepoSkyMapConfig
from lsst.obs.subaru import HyperSuprimeCam
maskCollection = HyperSuprimeCam().makeCollectionName("masks")
config.runsForced["brightObjectMask"] = maskCollection
config.extraUmbrellaChildren.append(maskCollection)
config.skyMaps["hsc_rings_v1"] = ConvertRepoSkyMapConfig()
config.skyMaps["hsc_rings_v1"].load(os.path.join(os.path.dirname(__file__), "makeSkyMap.py"))
# If there's no skymap in the root repo, but some dataset defined on
# tracts/patches is present there (i.e. brightObjectMask), assume this
# skymap.
config.rootSkyMapName = "hsc_rings_v1"
config.refCats.append("ps1_pv3_3pi_20170110")
# ForcedPhotCoadd writes its configs to a filename that doesn't include a
# coaddName prefix, which means the conversion tools can't infer the right
# dataset type from the filename alone. Because the vast majority of HSC coadd
# processing is on "deep" coadds, we explicitly ignore the other
# <prefix>Coadd_forced_config datasets. Users who know what is in their own
# repositories can of course override.
config.datasetIgnorePatterns.extend(["dcrCoadd_forced_config",
"goodSeeingCoadd_forced_config",
"psfMatchedCoadd_forced_config"])
# Same problem, with assembleCoadd variant metadata; we assume
# "deep_compareWarpAssembleCoadd_metadata" is the one we want.
config.datasetIgnorePatterns.extend(["deep_assembleCoadd_metadata",
"deep_safeClipAssembleCoadd_metadata",
"deep_dcrAssembleCoadd_metadata",])
# This file contains overrides for obs.base.gen2to3.ConvertRepoTask to export
# the jointcal_* datasets that are in the root of the Gen2 repo into a special
# "HSC/external" RUN collection, since it doesn't make sense to put them any of
# the other RUNs generated from that conversion. This doesn't go in the
# obs_subaru config overrides because having those datasets in the root is
# unique to ci_hsc_gen2.
from lsst.obs.subaru import HyperSuprimeCam
collection = HyperSuprimeCam().makeCollectionName("external")
config.runs["jointcal_wcs"] = collection
config.runs["jointcal_photoCalib"] = collection
def setUp(self):
instrument = HyperSuprimeCam()
self.camera = instrument.getCamera()
def setUp(self):
hsc = HyperSuprimeCam()
self.camera = hsc.getCamera()
def displayCamera(args):
"""Display camera element according to command-line arguments.
Parameters
----------
args : `argparse.Namespace`
Command-line arguments to parse.
"""
hsc = HyperSuprimeCam()
camera = hsc.getCamera()
frame = 0
if args.showAmp:
frame = 0
for ampStr in args.showAmp:
if checkStr(ampStr, 'amp'):
ccd, amp = ampStr.split()
detector = camera[ccd]
amplifier = detector[amp]
disp = afwDisplay.Display(frame=frame)
cameraGeomUtils.showAmp(amplifier, display=disp)
frame += 1
if args.showCcd:
frame = 0
for ccdStr in args.showCcd:
if checkStr(ccdStr, 'ccd'):
detector = camera[ccdStr]
disp = afwDisplay.Display(frame=frame)
cameraGeomUtils.showCcd(detector, display=disp)
frame += 1
raftMap = {'0': [], '1': []}
for det in camera:
dName = det.getName()
if dName.startswith('1'):
raftMap['1'].append(dName)
elif dName.startswith('0'):
raftMap['0'].append(dName)
else:
raise RuntimeError("Did not recognize detector name")
if args.showRaft:
frame = 0
for raftStr in args.showRaft:
disp = afwDisplay.Display(frame)
if checkStr(raftStr, 'raft'):
detectorNameList = []
for detector in camera:
detName = detector.getName()
if detName in raftMap[raftStr.lower()]:
detectorNameList.append(detName)
cameraGeomUtils.showCamera(camera,
detectorNameList=detectorNameList,
display=disp,
binSize=4)
frame += 1
if args.showCamera:
disp = afwDisplay.Display(frame)
cameraGeomUtils.showCamera(camera,
display=disp,
binSize=args.cameraBinSize)
if args.plotFocalPlane:
cameraGeomUtils.plotFocalPlane(camera, 2., 2.)
from lsst.obs.subaru import HyperSuprimeCam
testCollection = HyperSuprimeCam.makeCollectionName('testdata')
extraCollection = HyperSuprimeCam.makeCollectionName('extra')
config.refCats.append("gaia_dr2_20200414")
config.runs["gaia_dr2_20200414"] = "refcats"
config.runs["calexp"] = testCollection
config.runs["src"] = testCollection
config.runs["sourceTable_visit"] = testCollection
config.runs["src_schema"] = testCollection
config.runs["icSrc_schema"] = testCollection
config.runs["packages"] = extraCollection
config.runs["singleFrameDriver_config"] = extraCollection
config.runs["skyCorr_config"] = extraCollection
config.runs["transformSourceTable_config"] = extraCollection
config.runs["writeSourceTable_config"] = extraCollection
#!/usr/bin/env python
# Create a gen3 exports.yaml file from an existing gen3 hsc repo.
# Requires that `convert_gen2_to_gen3_hsc.sh` has already been run.
import lsst.daf.butler as dafButler
from lsst.daf.butler import CollectionType
from lsst.obs.subaru import HyperSuprimeCam
butler = dafButler.Butler('hsc/butler.yaml')
collection = HyperSuprimeCam.makeCollectionName('testdata')
with butler.export(filename="hsc/exports.yaml") as export:
# Raws
rawCollection = HyperSuprimeCam.makeDefaultRawIngestRunName()
export.saveDatasets(
butler.registry.queryDatasets(collections=rawCollection,
datasetType='raw'))
# Datasets
export.saveDatasets(
butler.registry.queryDatasets(collections=collection, datasetType=...))
# Calibrations
for datasetTypeName in ('camera', 'transmission_optics',
'transmission_sensor', 'transmission_filter',
'transmission_atmosphere'):
export.saveDatasets(
butler.registry.queryDatasets(datasetTypeName, collections=...))
for collection in butler.registry.queryCollections(
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(
description='Plot contours of PSF quality')
parser.add_argument('--outputFile',
type=str,
help="File to write plot to",
default=None)
parser.add_argument('--noDistortion',
action="store_true",
help="Include distortion")
parser.add_argument('--names',
action="store_true",
help="Use CCD's names, not serials")
args = parser.parse_args()
hsc = HyperSuprimeCam()
camera = hsc.getCamera()
main(camera,
names=args.names,
sample=2,
outputFile=args.outputFile,
showDistortion=not args.noDistortion)
if not args.outputFile:
print("Hit any key to exit", end=' ')
input()