def getInfo(self, filename):
"""Get information about the image from the filename and its contents
Here, we open the image and parse the header, but one could also look at the filename itself
and derive information from that, or set values from the configuration.
@param filename Name of file to inspect
@return File properties; list of file properties for each extension
"""
md = readMetadata(filename, self.config.hdu)
phuInfo = self.getInfoFromMetadata(md)
if len(self.config.extnames) == 0:
# No extensions to worry about
return phuInfo, [phuInfo]
# Look in the provided extensions
extnames = set(self.config.extnames)
extnum = 0
infoList = []
while len(extnames) > 0:
extnum += 1
try:
md = readMetadata(filename, extnum)
except Exception as e:
self.log.warn("Error reading %s extensions %s: %s" % (filename, extnames, e))
break
ext = self.getExtensionName(md)
if ext in extnames:
hduInfo = self.getInfoFromMetadata(md, info=phuInfo.copy())
# We need the HDU number when registering MEF files.
hduInfo["hdu"] = extnum
infoList.append(hduInfo)
extnames.discard(ext)
return phuInfo, infoList
def makeitLsst(prefs, context, saveWcs=False, plot=dict()):
"""This is the python wrapper that reads lsst tables
"""
# Create an array of PSFs (one PSF for each extension)
if prefs.getVerboseType() != prefs.QUIET:
print("----- %d input catalogues:" % prefs.getNcat())
if saveWcs: # only needed for making plots
wcssList = []
fields = psfexLib.vectorField()
for cat in prefs.getCatalogs():
field = psfexLib.Field(cat)
wcss = []
wcssList.append(wcss)
with fits.open(cat):
# Hack: I want the WCS so I'll guess where the calexp is to be found
calexpFile = guessCalexp(cat)
md = readMetadata(calexpFile)
wcs = afwGeom.makeSkyWcs(md)
if not wcs:
cdMatrix = np.array([1.0, 0.0, 0.0, 1.0])
cdMatrix.shape = (2, 2)
wcs = afwGeom.makeSkyWcs(crpix=afwGeom.PointD(0, 0),
crval=afwGeom.SpherePoint(0.0, 0.0, afwGeom.degrees),
cdMatrix=cdMatrix)
naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar("NAXIS2")
# Find how many rows there are in the catalogue
md = readMetadata(cat)
field.addExt(wcs, naxis1, naxis2, md.getScalar("NAXIS2"))
if saveWcs:
wcss.append((wcs, naxis1, naxis2))
field.finalize()
fields.append(field)
fields[0].getNext() # number of extensions
prefs.getPsfStep()
sets = psfexLib.vectorSet()
for set in load_samplesLsst(prefs, context, plot=plot):
sets.append(set)
psfexLib.makeit(fields, sets)
ret = [[f.getPsfs() for f in fields], sets]
if saveWcs:
ret.append(wcssList)
return ret
def fromDir(cls, root, visit, **kwds):
ffp = {}
wcs = {}
fcrPattern = os.path.join(root, "fcr-%07d-*.fits" % visit) # meas_mosaic coords
wcsPattern = os.path.join(root, "wcs-%07d-*.fits" % visit) # LSST coords
start = fcrPattern.index("*")
for filename in glob.glob(fcrPattern):
ccd = int(filename[start:start+3])
md = readMetadata(filename)
ffp[ccd] = FluxFitParams(md)
for filename in glob.glob(wcsPattern):
ccd = int(filename[start:start+3])
md = readMetadata(filename)
wcs[ccd] = afwGeom.makeSkyWcs(md)
return CorrectionImageSource(ffp, wcs, **kwds)
def testReadMetadata(self):
im = afwImage.DecoratedImageF(self.fileForMetadata)
meta = readMetadata(self.fileForMetadata)
self.assertIn("NAXIS1", meta.names())
self.assertEqual(im.getWidth(), meta.getScalar("NAXIS1"))
self.assertEqual(im.getHeight(), meta.getScalar("NAXIS2"))
def processRawDir(rawDir, conn, done):
print(rawDir, "... started")
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
for fitsPath in glob.glob(os.path.join(rawDir, "*.fits*")):
m = re.search(r'raw_v(\d*)_f(.+)\.fits', fitsPath)
if not m:
sys.stderr.write("Warning: Unrecognized file: %r\n" % (fitsPath,))
nUnrecognized += 1
continue
visit, filterName = m.groups()
key = "%s_f%s" % (visit, filterName)
if key in done:
nSkipped += 1
continue
md = readMetadata(fitsPath)
expTime = md.getScalar("EXPTIME")
mjdObs = md.getScalar("MJD-OBS")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.TAI).toString(dafBase.DateTime.UTC)[:-1]
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?)""",
(visit, filterName, taiObs, expTime))
conn.commit()
nProcessed += 1
print("%s... %d processed, %d skipped, %d unrecognized" %
(rawDir, nProcessed, nSkipped, nUnrecognized))
def read(self, dirName="."):
"""Read self's pfsFiberTrace file from directory dirName"""
if not pyfits:
raise RuntimeError("I failed to import pyfits, so cannot read from disk")
fileName = PfsFiberTrace.fileNameFormat % (self.obsDate, self.visit0, self.arm, self.spectrograph)
self.metadata = afwFits.readMetadata(os.path.join(dirName, fileName), 0, True)
self.metadata.remove("COMMENT") # Added by FITS writer, not stripped (!)
allTracesMI = afwImage.MaskedImageF(os.path.join(dirName, fileName))
with pyfits.open(os.path.join(dirName, fileName)) as fd:
hdu = fd["ID_BOX"]
self.fiberId = hdu.data['FIBERID']
minX = hdu.data['MINX']
minY = hdu.data['MINY']
maxX = hdu.data['MAXX']
maxY = hdu.data['MAXY']
self.traces = []
x0 = 0
for i in range(len(self.fiberId)):
# bbox: BBox in full (i.e. data) image
bbox = afwGeom.BoxI(afwGeom.PointI(minX[i], minY[i]), afwGeom.PointI(maxX[i], maxY[i]))
# bboxAllTMI: BBox in allTracesMI
bboxAllTMI = afwGeom.BoxI(afwGeom.PointI(x0, bbox.getMinY()), bbox.getDimensions())
trace = allTracesMI[bboxAllTMI].clone()
trace.setXY0(bbox.getBegin())
self.traces.append(trace)
x0 += bbox.getWidth()
def testTransformWcsPixels(self):
filename = os.path.join(os.path.dirname(__file__),
'imgCharSources-v85501867-R01-S00.sipheader')
wcs1 = lsst.afw.geom.makeSkyWcs(readMetadata(filename))
s = makeRandomAffineTransform()
wcs2 = transformWcsPixels(wcs1, s)
crvalDeg = wcs1.getSkyOrigin().getPosition(lsst.afw.geom.degrees)
def t1a(p):
raDeg, decDeg = crvalDeg + lsst.afw.geom.Extent2D(p)
sky = lsst.afw.geom.SpherePoint(raDeg, decDeg, lsst.afw.geom.degrees)
return s(wcs1.skyToPixel(sky))
def t2a(p):
raDeg, decDeg = crvalDeg + lsst.afw.geom.Extent2D(p)
sky = lsst.afw.geom.SpherePoint(raDeg, decDeg, lsst.afw.geom.degrees)
return wcs2.skyToPixel(sky)
self.assertTransformsAlmostEqual(t1a, t2a)
def t1b(p):
sky = wcs1.pixelToSky(s.inverted()(p))
return sky.getPosition(lsst.afw.geom.degrees)
def t2b(p):
sky = wcs2.pixelToSky(p)
return sky.getPosition(lsst.afw.geom.degrees)
self.assertTransformsAlmostEqual(t1b, t2b)
def test1(self):
wcsfn = os.path.join(self.datadir, 'imsim-v85518312-fu-R43-S12.wcs2')
hdr = readMetadata(wcsfn)
wcs1 = afwGeom.makeSkyWcs(hdr)
crval = wcs1.getSkyOrigin()
cd = wcs1.getCdMatrix()
print(cd)
crval_p = lsst.geom.Point2D(crval.getLongitude().asDegrees(),
crval.getLatitude().asDegrees())
origin = wcs1.getPixelOrigin()
print(crval_p)
print(origin)
wcs2 = afwGeom.makeSkyWcs(crpix=origin, crval=crval, cdMatrix=cd)
for wcs in [wcs1, wcs2]:
print(wcs)
print('x, y, RA, Dec, pixscale("/pix), pixscale2')
for x, y in [(0, 0), (300, 0), (350, 0), (360, 0), (370, 0), (380, 0), (400, 0)]:
pixPos = lsst.geom.PointD(x, y)
radec = wcs.pixelToSky(pixPos)
ra = radec.getLongitude().asDegrees()
dec = radec.getLatitude().asDegrees()
pixscale = wcs.getPixelScale(pixPos).asArcseconds()
print(x, y, ra, dec, pixscale)
self.assertLess(abs(pixscale - 0.2), 1e-3)
def checkExtName(self, name, value, extNum):
filename = DATA + "[%s]" % name
header = readMetadata(filename)
self.assertEqual(header.getScalar("EXT_NUM"), extNum)
self.assertEqual(header.getScalar("EXTNAME").strip(), name)
image = afwImage.ImageI(filename, allowUnsafe=True)
self.assertEqual(image[0, 0, afwImage.LOCAL], value)
def writeAndRead(self, header):
"""Write the supplied header and read it back again.
"""
fitsFile = lsst.afw.fits.MemFileManager()
with lsst.afw.fits.Fits(fitsFile, "w") as fits:
fits.createEmpty()
fits.writeMetadata(header)
with lsst.afw.fits.Fits(fitsFile, "r") as fits:
metadata = fits.readMetadata()
return metadata
def makeCcdMosaic(dir, basename, e, c, aList, imageFactory=afwImage.MaskedImageF, verbose=0):
"""Return an image of all the specified amplifiers, aList, for the given CCD
E.g. sl = makeCcdMosaic("/lsst/DC3root/rlp1173", "v704897", 0, 3, range(8))
"""
try:
aList[0]
except TypeError:
aList = [aList]
for what in ("header", "data"):
if what == "header":
bbox = lsst.geom.Box2I()
ampBBox = {}
wcs = {}
else:
ccdImage = imageFactory(bbox.getWidth(), bbox.getHeight())
ccdImage.set(0)
ccdImage.setXY0(bbox.getLLC())
for a in aList:
filename = os.path.join(dir, "IPSD", "output", "sci", "%s-e%d" % (basename, e),
"%s-e%d-c%03d-a%02d.sci" % (basename, e, c, a))
if verbose and what == "header":
print(filename)
if what == "header":
md = readMetadata(filename + "_img.fits")
xy0 = lsst.geom.Point2I(md.getScalar("CRVAL1A"), md.getScalar("CRVAL2A"))
xy1 = xy0 + lsst.geom.Extent2I(md.getScalar("NAXIS1") - 1, md.getScalar("NAXIS2") - 1)
bbox.grow(xy0)
bbox.grow(xy1)
ampBBox[a] = lsst.geom.Box2I(xy0, xy1)
wcs[a] = afwGeom.makeSkyWcs(md)
else:
try:
data = imageFactory(filename + "_img.fits")
except Exception:
data = imageFactory(filename)
ampImage = ccdImage.Factory(ccdImage, ampBBox[a])
ampImage[:] = data
del ampImage
try:
ccdImage.getMask()
if 0 in wcs:
ccdImage = afwImage.ExposureF(ccdImage, wcs[0])
except AttributeError:
pass
return ccdImage
def testDM882(self):
"""Test that we can write a dotted header unit to a FITS file. See DM-882."""
self.dimage1.getMetadata().add("A.B.C.D", 12345)
tempdir = tempfile.mkdtemp()
testfile = os.path.join(tempdir, "test.fits")
try:
self.dimage1.writeFits(testfile)
meta = readMetadata(testfile)
self.assertEqual(meta.getScalar("A.B.C.D"), 12345)
finally:
shutil.rmtree(tempdir)
def processRun(runDir, conn, done):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
print(runDir, "... started", file=sys.stderr)
for fits in glob.iglob(
os.path.join(runDir, "*", "corr", "[1-6]", "fpC*.fit.gz")):
m = re.search(r'(\d+)/corr/([1-6])/fpC-(\d{6})-([ugriz])\2-(\d{4}).fit.gz', fits)
if not m:
print("Warning: Unrecognized file:", fits, file=sys.stderr)
nUnrecognized += 1
continue
(rerun, camcol, run, filter, field) = m.groups()
rerun = int(rerun)
camcol = int(camcol)
run = int(run)
field = int(field)
key = "%d_R%d_B%s_C%d_F%d" % (run, rerun, filter, camcol, field)
if key in done or rerun < 40:
nSkipped += 1
continue
md = readMetadata(fits)
date = md.getScalar("DATE-OBS")
if date.find("-") != -1:
(year, month, day) = md.getScalar("DATE-OBS").split("-")
else:
(day, month, year) = md.getScalar("DATE-OBS").split("/")
year = 1900 + int(year)
(hour, minute, second) = md.getScalar("TAIHMS").split(":")
seconds = float(second)
second = int(seconds)
taiObs = dafBase.DateTime(int(year), int(month), int(day), int(hour),
int(minute), second, dafBase.DateTime.TAI)
taiObs = dafBase.DateTime(taiObs.nsecs() +
int((seconds - second) * 1000000000), dafBase.DateTime.TAI)
taiObs = taiObs.toString(dafBase.DateTime.UTC)[:-1]
strip = "%d%s" % (md.getScalar('STRIPE'), md.getScalar('STRIP'))
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?)""",
(run, rerun, filter, camcol, field, taiObs, strip))
nProcessed += 1
if nProcessed % 100 == 0:
conn.commit()
conn.commit()
print(runDir,
"... %d processed, %d skipped, %d unrecognized" %
(nProcessed, nSkipped, nUnrecognized), file=sys.stderr)
def testRotateWcsPixelsBy90(self):
filename = os.path.join(os.path.dirname(__file__),
'imgCharSources-v85501867-R01-S00.sipheader')
wcs0 = lsst.afw.geom.makeSkyWcs(readMetadata(filename))
w, h = 11, 12
image0 = lsst.afw.image.ImageD(w, h)
x, y = np.meshgrid(np.arange(w), np.arange(h))
# Make a slowly-varying image of an asymmetric function
image0.getArray()[:, :] = (x/w)**2 + 0.5*(x/w)*(y/h) - 3.0*(y/h)**2
dimensions = image0.getBBox().getDimensions()
image1 = lsst.afw.math.rotateImageBy90(image0, 1)
wcs1 = rotateWcsPixelsBy90(wcs0, 1, dimensions)
image2 = lsst.afw.math.rotateImageBy90(image0, 2)
wcs2 = rotateWcsPixelsBy90(wcs0, 2, dimensions)
image3 = lsst.afw.math.rotateImageBy90(image0, 3)
wcs3 = rotateWcsPixelsBy90(wcs0, 3, dimensions)
bbox = image0.getBBox()
image0r = lsst.afw.image.ImageD(bbox)
image1r = lsst.afw.image.ImageD(bbox)
image2r = lsst.afw.image.ImageD(bbox)
image3r = lsst.afw.image.ImageD(bbox)
ctrl = lsst.afw.math.WarpingControl("nearest")
lsst.afw.math.warpImage(image0r, wcs0, image0, wcs0, ctrl)
lsst.afw.math.warpImage(image1r, wcs0, image1, wcs1, ctrl)
lsst.afw.math.warpImage(image2r, wcs0, image2, wcs2, ctrl)
lsst.afw.math.warpImage(image3r, wcs0, image3, wcs3, ctrl)
# warpImage doesn't seem to handle the first row and column,
# even with nearest-neighbor interpolation, so we have to
# ignore pixels it didn't know how to populate.
def compareFinite(ref, target):
finitPixels = np.isfinite(target.getArray())
self.assertGreater(finitPixels.sum(), 0.7*target.getArray().size)
self.assertFloatsAlmostEqual(
ref.getArray()[finitPixels],
target.getArray()[finitPixels],
rtol=1E-6
)
compareFinite(image0, image0r)
compareFinite(image0, image1r)
compareFinite(image0, image2r)
compareFinite(image0, image3r)
def testReadMetadata(self):
with lsst.utils.tests.getTempFilePath(".fits") as tmpFile:
self.exposureCrWcs.getMetadata().set("FRAZZLE", True)
# This will write the main metadata (inc. FRAZZLE) to the primary HDU, and the
# WCS to subsequent HDUs, along with INHERIT=T.
self.exposureCrWcs.writeFits(tmpFile)
# This should read the first non-empty HDU (i.e. it skips the primary), but
# goes back and reads it if it finds INHERIT=T. That should let us read
# frazzle and the Wcs from the PropertySet returned by
# testReadMetadata.
md = readMetadata(tmpFile)
wcs = afwGeom.makeSkyWcs(md, False)
self.assertPairsAlmostEqual(wcs.getPixelOrigin(), self.wcs.getPixelOrigin())
self.assertSpherePointsAlmostEqual(wcs.getSkyOrigin(), self.wcs.getSkyOrigin())
assert_allclose(wcs.getCdMatrix(), self.wcs.getCdMatrix(), atol=1e-10)
frazzle = md.getScalar("FRAZZLE")
self.assertTrue(frazzle)
def setUp(self):
np.random.seed(100)
self.tract = 8766
self.visit = 11506
self.ccds = {0: 49, 2: 50, 3: 101, 1: 102}
# Box is the same for all CCDs, since it's defined in CCD coordinates,
# which are rotated w.r.t. focal plane coordinates.
self.bbox = lsst.afw.geom.Box2I(
lsst.afw.geom.Point2I(0, 0),
lsst.afw.geom.Point2I(2047, 4175)
)
self.ffp = {}
self.wcs = {}
self.photoCalib = {}
self.dataRefs = {}
camera = {} # all we need from our mock camera is dict-like access to (Mock)Detectors.
calexpMetadata = lsst.daf.base.PropertyList()
for nQuarter, ccd in self.ccds.items():
fcrFilename = os.path.join(
DATA_DIR,
"%d/fcr-%07d-%03d.fits" % (self.tract, self.visit, ccd)
)
fcrMetadata = readMetadata(fcrFilename)
fcrPhotoCalib = lsst.afw.image.ExposureF(fcrFilename).getPhotoCalib()
self.ffp[ccd] = lsst.meas.mosaic.FluxFitParams(fcrMetadata)
wcsFilename = os.path.join(
DATA_DIR,
"%d/jointcal_wcs-%07d-%03d.fits" % (self.tract, self.visit, ccd)
)
self.wcs[ccd] = lsst.afw.geom.SkyWcs.readFits(wcsFilename)
photoCalibFilename = os.path.join(
DATA_DIR,
"%d/jointcal_photoCalib-%07d-%03d.fits" % (self.tract, self.visit, ccd)
)
self.photoCalib[ccd] = lsst.afw.image.PhotoCalib.readFits(photoCalibFilename)
camera[ccd] = MockDetector(MockOrientation(nQuarter))
self.dataRefs[ccd] = MockDataRef(visit=self.visit, tract=self.tract, ccd=ccd)
self.dataRefs[ccd].put(fcrMetadata, "fcr_md", )
self.dataRefs[ccd].put(fcrPhotoCalib, "fcr_photoCalib")
self.dataRefs[ccd].put(self.wcs[ccd], "jointcal_wcs")
self.dataRefs[ccd].put(calexpMetadata, "calexp_md")
self.dataRefs[ccd].put(camera, "camera")
self.dataRefs[ccd].put(self.bbox, "calexp_bbox")
def getCalibType(self, filename):
"""Return a a known calibration dataset type using
the observation type in the header keyword OBSTYPE
@param filename: Input filename
"""
md = readMetadata(filename, self.config.hdu)
if not md.exists("OBSTYPE"):
raise RuntimeError("Unable to find the required header keyword OBSTYPE in %s, hdu %d" %
(filename, self.config.hdu))
obstype = md.getScalar("OBSTYPE").strip().lower()
if "flat" in obstype:
obstype = "flat"
elif "zero" in obstype or "bias" in obstype:
obstype = "bias"
elif "dark" in obstype:
obstype = "dark"
elif "fringe" in obstype:
obstype = "fringe"
elif "sky" in obstype:
obstype = "sky"
return obstype
def processRaft(raftDir, conn, done):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
for fits in glob.glob(os.path.join(raftDir, "S[0-2][0-2]",
"imsim_*_R[0-4][0-4]_S[0-2][0-2]_C[01][0-7]_E00[01].fits*")):
m = re.search(r'v(\d+)-f(\w)/E00(\d)/R(\d)(\d)/S(\d)(\d)/' +
r'imsim_\1_R\4\5_S\6\7_C(\d)(\d)_E00\3\.fits', fits)
if not m:
print("Warning: Unrecognized file:", fits, file=sys.stderr)
nUnrecognized += 1
continue
(visit, filter, snap, raft1, raft2, sensor1, sensor2,
channel1, channel2) = m.groups()
key = "%s_F%s_E%s_R%s,%s_S%s,%s_C%s,%s" % (visit, filter,
snap, raft1, raft2, sensor1, sensor2, channel1, channel2)
if key in done:
nSkipped += 1
continue
md = readMetadata(fits)
expTime = md.getScalar("EXPTIME")
mjdObs = md.getScalar("MJD-OBS")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.TAI).toString(dafBase.DateTime.UTC)[:-1]
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?, ?)""",
(visit, filter, snap, "%s,%s" % (raft1, raft2),
"%s,%s" % (sensor1, sensor2),
"%s,%s" % (channel1, channel2), taiObs, expTime))
conn.commit()
nProcessed += 1
print(raftDir,
"... %d processed, %d skipped, %d unrecognized" %
(nProcessed, nSkipped, nUnrecognized), file=sys.stderr)
def setUp(self):
maskedImage = afwImage.MaskedImageF(inFilePathSmall)
maskedImageMD = readMetadata(inFilePathSmall)
self.smallExposure = afwImage.ExposureF(inFilePathSmall)
self.width = maskedImage.getWidth()
self.height = maskedImage.getHeight()
self.wcs = afwGeom.makeSkyWcs(maskedImageMD, False)
self.md = maskedImageMD
self.psf = DummyPsf(2.0)
self.detector = DetectorWrapper().detector
self.exposureBlank = afwImage.ExposureF()
self.exposureMiOnly = afwImage.makeExposure(maskedImage)
self.exposureMiWcs = afwImage.makeExposure(maskedImage, self.wcs)
# n.b. the (100, 100, ...) form
self.exposureCrWcs = afwImage.ExposureF(100, 100, self.wcs)
# test with ExtentI(100, 100) too
self.exposureCrOnly = afwImage.ExposureF(lsst.geom.ExtentI(100, 100))
afwImage.Filter.reset()
afwImage.FilterProperty.reset()
defineFilter("g", 470.0)
def test(self):
path = os.path.join(testPath, "data", "ticket2905.fits")
md = readMetadata(path)
value = md.get("INR-STR")
self.assertEqual(type(value), float)
self.assertEqual(value, 2.0e-5)
def extractMetadata(self, filename: str) -> RawFileData:
"""Extract and process metadata from a single raw file.
Parameters
----------
filename : `str`
Path to the file.
Returns
-------
data : `RawFileData`
A structure containing the metadata extracted from the file,
as well as the original filename. All fields will be populated,
but the `RawFileData.dataId` attribute will be a minimal
(unexpanded) `DataCoordinate` instance.
Notes
-----
Assumes that there is a single dataset associated with the given
file. Instruments using a single file to store multiple datasets
must implement their own version of this method.
"""
# We do not want to stop ingest if we are given a bad file.
# Instead return a RawFileData with no datasets and allow
# the caller to report the failure.
try:
# Manually merge the primary and "first data" headers here because
# we do not know in general if an input file has set INHERIT=T.
phdu = readMetadata(filename, 0)
header = merge_headers([phdu, readMetadata(filename)], mode="overwrite")
datasets = [self._calculate_dataset_info(header, filename)]
except Exception as e:
self.log.debug("Problem extracting metadata from %s: %s", filename, e)
# Indicate to the caller that we failed to read
datasets = []
FormatterClass = Formatter
instrument = None
if self.config.failFast:
raise RuntimeError(f"Problem extracting metadata from file {filename}") from e
else:
self.log.debug("Extracted metadata from file %s", filename)
# The data model currently assumes that whilst multiple datasets
# can be associated with a single file, they must all share the
# same formatter.
try:
instrument = Instrument.fromName(datasets[0].dataId["instrument"], self.butler.registry)
except LookupError as e:
self.log.warning("Instrument %s for file %s not known to registry",
datasets[0].dataId["instrument"], filename)
if self.config.failFast:
raise RuntimeError(f"Instrument {datasets[0].dataId['instrument']} for"
f" file {filename} not known to registry") from e
datasets = []
FormatterClass = Formatter
instrument = None
else:
FormatterClass = instrument.getRawFormatter(datasets[0].dataId)
return RawFileData(datasets=datasets, filename=filename,
FormatterClass=FormatterClass,
instrumentClass=instrument)
def testMakeWcs(self):
"""Test SipForwardTransform, SipReverseTransform and makeWcs
"""
filename = os.path.join(os.path.dirname(__file__),
'imgCharSources-v85501867-R01-S00.sipheader')
sipMetadata = readMetadata(filename)
# We're building an ICRS-based TAN-SIP using coefficients read from metadata
# so ignore the RADESYS in metadata (which is missing anyway, falling back to FK5)
sipMetadata.set("RADESYS", "ICRS")
crpix = lsst.geom.Point2D(
sipMetadata.getScalar("CRPIX1") - 1,
sipMetadata.getScalar("CRPIX2") - 1,
)
crval = lsst.geom.SpherePoint(
sipMetadata.getScalar("CRVAL1"),
sipMetadata.getScalar("CRVAL2"), lsst.geom.degrees,
)
cdLinearTransform = lsst.geom.LinearTransform(getCdMatrixFromMetadata(sipMetadata))
aArr = getSipMatrixFromMetadata(sipMetadata, "A")
bArr = getSipMatrixFromMetadata(sipMetadata, "B")
apArr = getSipMatrixFromMetadata(sipMetadata, "AP")
bpArr = getSipMatrixFromMetadata(sipMetadata, "BP")
abPoly = PolynomialTransform(aArr, bArr)
abRevPoly = PolynomialTransform(apArr, bpArr)
fwd = SipForwardTransform(crpix, cdLinearTransform, abPoly)
rev = SipReverseTransform(crpix, cdLinearTransform, abRevPoly)
wcsFromMakeWcs = lsst.meas.astrom.makeWcs(fwd, rev, crval)
wcsFromMetadata = lsst.afw.geom.makeSkyWcs(sipMetadata, strip=False)
# Check SipForwardTransform against a local implementation
localPixelToIwc = makeSipPixelToIwc(sipMetadata)
self.assertTransformsAlmostEqual(fwd, localPixelToIwc.applyForward, maxval=2000)
# Compare SipReverseTransform against a local implementation
# Use the forward direction first to get sensible inputs
localIwcToPixel = makeSipIwcToPixel(sipMetadata)
def fwdThenRev(p):
return rev(fwd(p))
def fwdThenLocalRev(p):
return localIwcToPixel.applyForward(fwd(p))
self.assertTransformsAlmostEqual(fwdThenRev, fwdThenLocalRev, maxval=2000)
# Check that SipReverseTransform is the inverse of SipForwardTransform;
# this is not perfect because the coefficients don't define a perfect inverse
def nullTransform(p):
return p
self.assertTransformsAlmostEqual(fwdThenRev, nullTransform, maxval=2000, atol=1e-3)
# Check SipForwardTransform against the one contained in wcsFromMakeWcs
# (Don't bother with the other direction because the WCS transform is iterative,
# so it doesn't tell us anything useful about SipReverseTransform
pixelToIwc = lsst.afw.geom.getPixelToIntermediateWorldCoords(wcsFromMetadata)
self.assertTransformsAlmostEqual(fwd, pixelToIwc.applyForward, maxval=2000)
# Check a WCS constructed from SipForwardTransform, SipReverseTransform
# against one constructed directly from the metadata
bbox = lsst.geom.Box2D(lsst.geom.Point2D(0, 0), lsst.geom.Extent2D(2000, 2000))
self.assertWcsAlmostEqualOverBBox(wcsFromMakeWcs, wcsFromMetadata, bbox)
def test(self):
path = os.path.join(testPath, "data", "ticket2905.fits")
md = readMetadata(path)
value = md.getScalar("INR-STR")
self.assertEqual(type(value), float)
self.assertEqual(value, 2.0e-5)
def testMakeWcs(self):
"""Test SipForwardTransform, SipReverseTransform and makeWcs
"""
filename = os.path.join(os.path.dirname(__file__),
'imgCharSources-v85501867-R01-S00.sipheader')
sipMetadata = readMetadata(filename)
# We're building an ICRS-based TAN-SIP using coefficients read from metadata
# so ignore the RADESYS in metadata (which is missing anyway, falling back to FK5)
sipMetadata.set("RADESYS", "ICRS")
crpix = lsst.afw.geom.Point2D(
sipMetadata.getScalar("CRPIX1") - 1,
sipMetadata.getScalar("CRPIX2") - 1,
)
crval = lsst.afw.geom.SpherePoint(
sipMetadata.getScalar("CRVAL1"),
sipMetadata.getScalar("CRVAL2"), lsst.afw.geom.degrees,
)
cdLinearTransform = lsst.afw.geom.LinearTransform(getCdMatrixFromMetadata(sipMetadata))
aArr = getSipMatrixFromMetadata(sipMetadata, "A")
bArr = getSipMatrixFromMetadata(sipMetadata, "B")
apArr = getSipMatrixFromMetadata(sipMetadata, "AP")
bpArr = getSipMatrixFromMetadata(sipMetadata, "BP")
abPoly = PolynomialTransform(aArr, bArr)
abRevPoly = PolynomialTransform(apArr, bpArr)
fwd = SipForwardTransform(crpix, cdLinearTransform, abPoly)
rev = SipReverseTransform(crpix, cdLinearTransform, abRevPoly)
wcsFromMakeWcs = lsst.meas.astrom.makeWcs(fwd, rev, crval)
wcsFromMetadata = lsst.afw.geom.makeSkyWcs(sipMetadata, strip=False)
# Check SipForwardTransform against a local implementation
localPixelToIwc = makeSipPixelToIwc(sipMetadata)
self.assertTransformsAlmostEqual(fwd, localPixelToIwc.applyForward, maxval=2000)
# Compare SipReverseTransform against a local implementation
# Use the forward direction first to get sensible inputs
localIwcToPixel = makeSipIwcToPixel(sipMetadata)
def fwdThenRev(p):
return rev(fwd(p))
def fwdThenLocalRev(p):
return localIwcToPixel.applyForward(fwd(p))
self.assertTransformsAlmostEqual(fwdThenRev, fwdThenLocalRev, maxval=2000)
# Check that SipReverseTransform is the inverse of SipForwardTransform;
# this is not perfect because the coefficients don't define a perfect inverse
def nullTransform(p):
return p
self.assertTransformsAlmostEqual(fwdThenRev, nullTransform, maxval=2000, atol=1e-3)
# Check SipForwardTransform against the one contained in wcsFromMakeWcs
# (Don't bother with the other direction because the WCS transform is iterative,
# so it doesn't tell us anything useful about SipReverseTransform
pixelToIwc = lsst.afw.geom.getPixelToIntermediateWorldCoords(wcsFromMetadata)
self.assertTransformsAlmostEqual(fwd, pixelToIwc.applyForward, maxval=2000)
# Check a WCS constructed from SipForwardTransform, SipReverseTransform
# against one constructed directly from the metadata
bbox = lsst.afw.geom.Box2D(lsst.afw.geom.Point2D(0, 0), lsst.afw.geom.Extent2D(2000, 2000))
self.assertWcsAlmostEqualOverBBox(wcsFromMakeWcs, wcsFromMetadata, bbox)
def bypass_raw_md(self, datasetType, pythonType, location, dataId):
"""Read metadata for raw image, adding fake Wcs"""
filename = location.getLocations()[0]
md = readMetadata(filename, 1) # 1 = PHU
return md
def checkBBoxFromMetadata(self, filename, expected, hdu=0):
metadata = afwFits.readMetadata(filename, hdu)
bbox = afwImage.bboxFromMetadata(metadata)
self.assertEqual(bbox, expected)
def checkBBoxFromMetadata(self, filename, expected, hdu=0):
metadata = afwFits.readMetadata(filename, hdu)
bbox = afwImage.bboxFromMetadata(metadata)
self.assertEqual(bbox, expected)
def checkExtNum(self, hdu, extNum):
if hdu is None:
hdu = DEFAULT_HDU
header = readMetadata(DATA, hdu)
self.assertEqual(header.getScalar("EXT_NUM"), extNum)
def processRun(runDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
print(runDir, "... started", file=sys.stderr)
for fits in glob.iglob(
os.path.join(runDir, "*", "corr", "[1-6]", "fpC*.fit.gz")):
m = re.search(r'(\d+)/corr/([1-6])/fpC-(\d{6})-([ugriz])\2-(\d{4}).fit.gz', fits)
if not m:
print("Warning: Unrecognized file:", fits, file=sys.stderr)
nUnrecognized += 1
continue
(rerun, camcol, run, filter, field) = m.groups()
rerun = int(rerun)
camcol = int(camcol)
run = int(run)
field = int(field)
key = "%d_R%d_B%s_C%d_F%d" % (run, rerun, filter, camcol, field)
if key in done or rerun < 40:
nSkipped += 1
continue
md = readMetadata(fits)
date = md.get("DATE-OBS")
if date.find("-") != -1:
(year, month, day) = md.get("DATE-OBS").split("-")
else:
(day, month, year) = md.get("DATE-OBS").split("/")
year = 1900 + int(year)
(hour, minute, second) = md.get("TAIHMS").split(":")
seconds = float(second)
second = int(seconds)
taiObs = dafBase.DateTime(int(year), int(month), int(day), int(hour),
int(minute), second, dafBase.DateTime.TAI)
taiObs = dafBase.DateTime(taiObs.nsecs() +
int((seconds - second) * 1000000000), dafBase.DateTime.TAI)
taiObs = taiObs.toString(dafBase.DateTime.UTC)[:-1]
strip = "%d%s" % (md.get('STRIPE'), md.get('STRIP'))
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?)""",
(run, rerun, filter, camcol, field, taiObs, strip))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = makeSkyWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"), md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
nProcessed += 1
if nProcessed % 100 == 0:
conn.commit()
conn.commit()
print(runDir,
"... %d processed, %d skipped, %d unrecognized" %
(nProcessed, nSkipped, nUnrecognized), file=sys.stderr)