def __init__(self, detectorFileName, ampFileNameList, inAmpCoords=True, plateScale=1.,
radialCoeffs=(0., 1.), clobberMetadata=False, doRaise=True):
''' @param[in] detectorFileName FITS file containing the detector description.
May use [] notation to specify an extension in an MEF.
@param[in] ampFileNameList List of FITS file names to use in building the amps.
May contain duplicate entries if the raw data are assembled.
@param[in] inAmpCoords Boolean, True if raw data are in amp coordinates, False if raw data
are assembled into pseudo detector pixel arrays
@param[in] plateScale Nominal platescale (arcsec/mm)
@param[in] radialCoeffs Radial distortion coefficients for a radial polynomial in normalized
units.
@param[in] clobberMetadata Clobber metadata from input files if overridden in the _sanitizeMetadata method
@param[in] doRaise Raise exception if not all non-defaulted keywords are defined? Default is True.
'''
self.inAmpCoords = inAmpCoords
self.defaultAmpMap = self._makeDefaultAmpMap()
self.defaultDetectorMap = self._makeDefaultDetectorMap()
self.detectorMetadata = afwImage.readMetadata(detectorFileName)
self._sanitizeHeaderMetadata(self.detectorMetadata, clobber=clobberMetadata)
self.ampMetadataList = []
self.detector = None
self.doRaise = doRaise
for fileName in ampFileNameList:
self.ampMetadataList.append(afwImage.readMetadata(fileName))
self._sanitizeHeaderMetadata(self.ampMetadataList[-1], clobber=clobberMetadata)
self.plateScale = plateScale
self.focalPlaneToPupil = self._makeRadialTransform(radialCoeffs)
def __init__(self,
detectorFileName,
ampFileNameList,
inAmpCoords=True,
plateScale=1.,
radialCoeffs=(0., 1.),
clobberMetadata=False,
doRaise=True):
''' @param[in] detectorFileName FITS file containing the detector description.
May use [] notation to specify an extension in an MEF.
@param[in] ampFileNameList List of FITS file names to use in building the amps.
May contain duplicate entries if the raw data are assembled.
@param[in] inAmpCoords Boolean, True if raw data are in amp coordinates, False if raw data
are assembled into pseudo detector pixel arrays
@param[in] plateScale Nominal platescale (arcsec/mm)
@param[in] radialCoeffs Radial distortion coefficients for a radial polynomial in normalized
units.
@param[in] clobberMetadata Clobber metadata from input files if overridden in the _sanitizeMetadata method
@param[in] doRaise Raise exception if not all non-defaulted keywords are defined? Default is True.
'''
self.inAmpCoords = inAmpCoords
self.defaultAmpMap = self._makeDefaultAmpMap()
self.defaultDetectorMap = self._makeDefaultDetectorMap()
self.detectorMetadata = afwImage.readMetadata(detectorFileName)
self._sanitizeHeaderMetadata(self.detectorMetadata,
clobber=clobberMetadata)
self.ampMetadataList = []
self.detector = None
self.doRaise = doRaise
for fileName in ampFileNameList:
self.ampMetadataList.append(afwImage.readMetadata(fileName))
self._sanitizeHeaderMetadata(self.ampMetadataList[-1],
clobber=clobberMetadata)
self.plateScale = plateScale
self.focalPlaneToPupil = self._makeRadialTransform(radialCoeffs)
def _get_catalog(self, dataset, **kwargs):
"""Load the catalogs from the butler."""
filenames = (self.butler.get(dataset + "_filename",
dataId, immediate=True)[0]
for dataId in self.dataIds[dataset])
try: # In recent stack version, metadata are in HDU 1
headers = (afwimage.readMetadata(fn, 1) for fn in filenames)
size = sum(md.get("NAXIS2") for md in headers)
except: # Older stack version
headers = (afwimage.readMetadata(fn, 2) for fn in filenames)
size = sum(md.get("NAXIS2") for md in headers)
cat = self.butler.get(dataset, self.dataIds[dataset][0],
flags=afwtable.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
self.from_butler['schema'] = cat.schema
catadic = {k: [] for k in sorted(self.dataIds[dataset][0].keys())}
catalog = afwtable.SourceCatalog(self.from_butler['schema'])
catalog.reserve(size)
# pbar = cutils.progressbar(len(self.dataIds[dataset]))
print("INFO: Looping over the dataids")
for i, dataid in enumerate(self.dataIds[dataset]):
cat = self.butler.get(dataset, dataid,
flags=afwtable.SOURCE_IO_NO_FOOTPRINTS)
catalog.extend(cat, deep=True)
for newkey in catadic:
catadic[newkey].extend([dataid[newkey]] * len(cat))
# pbar.update(i + 1)
# pbar.finish()
print("INFO: Merging the dictionnaries")
catadic.update(catalog.getColumnView().extract(*self.keys[dataset],
copy=True, ordered=True))
return catadic
def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId):
"""Work around for reading metadata from multi-HDU files.
afwImage.readMetadata() doesn't honour [hdu] suffixes in filenames.
Parameters
----------
datasetType : anything, unused
Unused
pythonType : anything, unused
Unused
location : `lsst.daf.persistence.ButlerLocation`
Butler location
dataId : anything, unused
Unused
Returns
-------
visitInfo : `lsst.afw.image.visitInfo`
The visitInfo
"""
import re
import lsst.afw.image as afwImage
fileName = location.getLocationsWithRoot()[0]
mat = re.search(r"\[(\d+)\]$", fileName)
if mat:
hdu = int(mat.group(1))
md = afwImage.readMetadata(fileName, hdu=hdu)
else:
md = afwImage.readMetadata(fileName) # or hdu = INT_MIN; -(1 << 31)
return afwImage.VisitInfo(md)
def _get_catalog(self, dataset, **kwargs):
"""Load the catalogs from the butler."""
filenames = (self.butler.get(dataset + "_filename",
dataId, immediate=True)[0]
for dataId in self.dataids[dataset])
try: # In recent stack version, metadata are in HDU 1
headers = (afwimage.readMetadata(fn, 1) for fn in filenames)
size = sum(md.get("NAXIS2") for md in headers)
except: # Older stack version
headers = (afwimage.readMetadata(fn, 2) for fn in filenames)
size = sum(md.get("NAXIS2") for md in headers)
cat = self.butler.get(dataset, self.dataids[dataset][0],
flags=afwtable.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
self.from_butler['schema'] = cat.schema
catadic = {k: [] for k in sorted(self.dataids[dataset][0].keys())}
catalog = afwtable.SourceCatalog(self.from_butler['schema'])
catalog.reserve(size)
pbar = cutils.progressbar(len(self.dataids[dataset]))
print("INFO: Looping over the dataids")
for i, dataid in enumerate(self.dataids[dataset]):
cat = self.butler.get(dataset, dataid,
flags=afwtable.SOURCE_IO_NO_FOOTPRINTS)
catalog.extend(cat, deep=True)
for newkey in catadic:
catadic[newkey].extend([dataid[newkey]] * len(cat))
pbar.update(i + 1)
pbar.finish()
print("INFO: Merging the dictionnaries")
catadic.update(catalog.getColumnView().extract(*self.keys[dataset],
copy=True, ordered=True))
# Clean memory before going further
# gc.collect()
return catadic
def getInfo(self, filename):
"""Get information about the image from the filename and its contents
Here, we open the image and parse the header, but one could also look at the filename itself
and derive information from that, or set values from the configuration.
@param filename Name of file to inspect
@return File properties; list of file properties for each extension
"""
md = afwImage.readMetadata(filename, self.config.hdu)
phuInfo = self.getInfoFromMetadata(md)
if len(self.config.extnames) == 0:
# No extensions to worry about
return phuInfo, [phuInfo]
# Look in the provided extensions
extnames = set(self.config.extnames)
extnum = 1
infoList = []
while len(extnames) > 0:
extnum += 1
try:
md = afwImage.readMetadata(filename, extnum)
except:
self.log.warn("Error reading %s extensions %s" % (filename, extnames))
break
ext = self.getExtensionName(md)
if ext in extnames:
infoList.append(self.getInfoFromMetadata(md, info=phuInfo.copy()))
extnames.discard(ext)
return phuInfo, infoList
def getInfo(self, filename):
"""Get information about the image from the filename and its contents
Here, we open the image and parse the header, but one could also look at the filename itself
and derive information from that, or set values from the configuration.
@param filename Name of file to inspect
@return File properties; list of file properties for each extension
"""
md = afwImage.readMetadata(filename, self.config.hdu)
phuInfo = self.getInfoFromMetadata(md)
if len(self.config.extnames) == 0:
# No extensions to worry about
return phuInfo, [phuInfo]
# Look in the provided extensions
extnames = set(self.config.extnames)
extnum = 0
infoList = []
while len(extnames) > 0:
extnum += 1
try:
md = afwImage.readMetadata(filename, extnum)
except:
self.log.warn("Error reading %s extensions %s" % (filename, extnames))
break
ext = self.getExtensionName(md)
if ext in extnames:
hduInfo = self.getInfoFromMetadata(md, info=phuInfo.copy())
# We need the HDU number when registering MEF files.
hduInfo["hdu"] = extnum
infoList.append(hduInfo)
extnames.discard(ext)
return phuInfo, infoList
def getInfo(self, filename, filetype):
"""
The science pixels, mask, and weight (inverse variance) are
stored in separate files each with a unique name but with a
common unique identifier EXPNUM in the FITS header. We have
to aggregate the 3 filenames for a given EXPNUM and return
this information along with that returned by the base class.
We expect a directory structure that looks like the following:
dqmask/ instcal/ wtmap/
The user creates the registry by running
ingestImagesDecam.py outputRepository --mode=link instcal/*fits
"""
if filetype == "instcal":
if self.expnumMapper is None:
self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename)))
# Note that phuInfo will have
# 'side': 'X', 'ccd': 0
phuInfo, infoList = super(DecamParseTask, self).getInfo(filename)
expnum = phuInfo["visit"]
phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix]
phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix]
phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix]
for idx, info in enumerate(infoList):
expnum = info["visit"]
info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix]
info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix]
info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix]
elif filetype == "raw":
md = afwImage.readMetadata(filename, self.config.hdu)
phuInfo = self.getInfoFromMetadata(md)
extnames = set(self.config.extnames)
extnum = 1
infoList = []
while len(extnames) > 0:
extnum += 1
try:
md = afwImage.readMetadata(filename, extnum)
except:
self.log.warn("Error reading %s extensions %s" % (filename, extnames))
break
ext = self.getExtensionName(md)
if ext in extnames:
info = self.getInfoFromMetadata(md, info=phuInfo.copy())
info['hdu'] = extnum - 1
info[self.instcalPrefix] = ""
info[self.dqmaskPrefix] = ""
info[self.wtmapPrefix] = ""
infoList.append(info)
extnames.discard(ext)
return phuInfo, infoList
def makeitLsst(prefs, context, saveWcs=False, plot=dict()):
"""This is the python wrapper that reads lsst tables"""
# Create an array of PSFs (one PSF for each extension)
if prefs.getVerboseType() != prefs.QUIET:
print("----- %d input catalogues:" % prefs.getNcat())
if saveWcs: # only needed for making plots
wcssList = []
fields = psfexLib.vectorField()
for cat in prefs.getCatalogs():
field = psfexLib.Field(cat)
wcss = []
wcssList.append(wcss)
with pyfits.open(cat):
# Hack: I want the WCS so I'll guess where the calexp is to be found
calexpFile = guessCalexp(cat)
md = afwImage.readMetadata(calexpFile)
wcs = afwImage.makeWcs(md)
if not wcs:
crval = afwCoord.makeCoord(afwCoord.ICRS,
0.0 * afwGeom.degrees,
0.0 * afwGeom.degrees)
wcs = afwImage.makeWcs(crval, afwGeom.PointD(0, 0), 1.0, 0, 0,
1.0)
naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
# Find how many rows there are in the catalogue
md = afwImage.readMetadata(cat)
field.addExt(wcs, naxis1, naxis2, md.get("NAXIS2"))
if saveWcs:
wcss.append((wcs, naxis1, naxis2))
field.finalize()
fields.append(field)
fields[0].getNext() # number of extensions
prefs.getPsfStep()
sets = psfexLib.vectorSet()
for set in load_samplesLsst(prefs, context, plot=plot):
sets.append(set)
psfexLib.makeit(fields, sets)
ret = [[f.getPsfs() for f in fields], sets]
if saveWcs:
ret.append(wcssList)
return ret
def makeitLsst(prefs, context, saveWcs=False, plot=dict()):
"""This is the python wrapper that reads lsst tables"""
# Create an array of PSFs (one PSF for each extension)
if prefs.getVerboseType() != prefs.QUIET:
print("----- %d input catalogues:" % prefs.getNcat())
if saveWcs: # only needed for making plots
wcssList = []
fields = psfexLib.vectorField()
for cat in prefs.getCatalogs():
field = psfexLib.Field(cat)
wcss = []
wcssList.append(wcss)
with pyfits.open(cat):
# Hack: I want the WCS so I'll guess where the calexp is to be found
calexpFile = guessCalexp(cat)
md = afwImage.readMetadata(calexpFile)
wcs = afwImage.makeWcs(md)
if not wcs:
crval = afwCoord.makeCoord(afwCoord.ICRS, 0.0*afwGeom.degrees, 0.0*afwGeom.degrees)
wcs = afwImage.makeWcs(crval, afwGeom.PointD(0, 0), 1.0, 0, 0, 1.0)
naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
# Find how many rows there are in the catalogue
md = afwImage.readMetadata(cat)
field.addExt(wcs, naxis1, naxis2, md.get("NAXIS2"))
if saveWcs:
wcss.append((wcs, naxis1, naxis2))
field.finalize()
fields.append(field)
fields[0].getNext() # number of extensions
prefs.getPsfStep()
sets = psfexLib.vectorSet()
for set in load_samplesLsst(prefs, context, plot=plot):
sets.append(set)
psfexLib.makeit(fields, sets)
ret = [[f.getPsfs() for f in fields], sets]
if saveWcs:
ret.append(wcssList)
return ret
def linearity(directory, infilebase, outfile, amps, x0, y0, boxsize):
#get list of files
files = glob.glob(directory+infilebase)
#write output file header
f=open(directory+outfile, 'w+')
f.write('amp\tx0\ty0\tboxsize\tmedian\texptime\n')
for filename in files:
#get exposure time from header
hdr = afwImage.readMetadata(filename, 1)
exptime = hdr.get('EXPTIME')
for amp in amps:
#define selected region
box = afwGeom.Box2I(afwGeom.Point2I(x0, y0), afwGeom.Extent2I(boxsize, boxsize))
#read in selected region of file
im = afwImage.ExposureF(filename, amp+1, box)
#get median of region of image
box_median = afwMath.makeStatistics(im.getMaskedImage(), afwMath.MEDIAN).getValue()
#write amp, box parameters, region median, and exptime to file
f.write(str(amp) + '\t' + str(x0) + '\t' + str(y0) + '\t' + str(boxsize) + '\t' + str(box_median) + '\t' + str(exptime) + '\n')
f.close()
def makeCcd(fileName, fromHeader=False, trim=True):
ccd = cameraGeom.Ccd(cameraGeom.Id(0))
a = 0 # one-less than the next HDU
while True: # while there are valid HDUs
a += 1
if fromHeader:
try:
hdu = 1 + a
md, channelNo = afwImage.readMetadata(fileName, hdu), None
except lsst.pex.exceptions.LsstCppException:
if hdu == 1: # an empty PDU
continue
break
if "TTYPE1" in md.names(): # not an image
break
else:
md, channelNo = None, a
#
# Add amp to the Ccd
#
try:
ccd.addAmp(makeAmp(md, channelNo, trim))
except StopIteration:
break
return ccd
def testFitsHeader(self):
"""Test that XY0 and crpix are written to the header as expected"""
#getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to
#compare to values stored in fits headers
parentCrpix = self.parent.getWcs().getPixelOrigin()
#Make a sub-image
x0, y0 = 20, 30
llc = afwGeom.Point2I(x0, y0)
bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50))
deep = False
subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep)
with utilsTests.getTempFilePath(".fits") as outFile:
subImg.writeFits(outFile)
hdr = afwImage.readMetadata(outFile)
def checkLtvHeader(hdr, name, value):
# Per DM-4133, LTVn headers are required to be floating point
self.assertTrue(hdr.exists(name), name + " not saved to FITS header")
self.assertIsInstance(hdr.get(name), numbers.Real, name + " is not numeric")
self.assertNotIsInstance(hdr.get(name), numbers.Integral, name + " is an int")
self.assertEqual(hdr.get(name), value, name + " has wrong value")
checkLtvHeader(hdr, "LTV1", -1*x0)
checkLtvHeader(hdr, "LTV2", -1*y0)
self.assertTrue( hdr.exists("CRPIX1"), "CRPIX1 not saved to fits header")
self.assertTrue( hdr.exists("CRPIX2"), "CRPIX2 not saved to fits header")
fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")]
self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"), parentCrpix[0]+1, 6, "CRPIX1 saved wrong")
self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"), parentCrpix[1]+1, 6, "CRPIX2 saved wrong")
def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId):
# Workaround until I can access the butler
instcalMap = self.map_instcal(dataId)
dqmaskMap = self.map_dqmask(dataId)
wtmapMap = self.map_wtmap(dataId)
rawMap = self.map_raw(dataId)
instcalType = getattr(afwImage,
instcalMap.getPythonType().split(".")[-1])
rawType = getattr(afwImage, rawMap.getPythonType().split(".")[-1])
dqmaskType = getattr(afwImage,
dqmaskMap.getPythonType().split(".")[-1])
wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1])
instcal = instcalType(instcalMap.getLocations()[0])
dqmask = dqmaskType(dqmaskMap.getLocations()[0])
wtmap = wtmapType(wtmapMap.getLocations()[0])
raw = rawType(rawMap.getLocations()[0])
mask = self.translate_dqmask(dqmask)
variance = self.translate_wtmap(wtmap)
mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask,
variance)
md = raw.getMetadata()
wcs = afwImage.makeWcs(md)
exp = afwImage.ExposureF(mi, wcs)
# Set the calib by hand; need to grab the zeroth extension
header = re.sub(r'[\[](\d+)[\]]$', "[0]", rawMap.getLocations()[0])
md0 = afwImage.readMetadata(header)
calib = afwImage.Calib()
calib.setExptime(md0.get("EXPTIME"))
calib.setFluxMag0(10**(0.4 * md0.get("MAGZERO")))
exp.setCalib(calib)
exp.setMetadata(md) # Do we need to remove WCS/calib info?
return exp
def readRawFile(fileName, dataId={}, detector=None):
"""Read a raw file from fileName, assembling it nicely.
Parameters
----------
filename : `str`
The fully-qualified filename.
dataId : `lsst.daf.persistence.DataId`
If provided, used to look up e.g. the filter.
detector : `lsst.afw.cameraGeom.Detector`
If provided, add this detector to the returned Exposure
Returns
-------
exposure : `lsst.afw.image.Exposure`
The assembled exposure from the supplied filename.
"""
class Info():
def __init__(self, obj):
self.obj = obj
amps = []
for hdu in range(1, 16+1):
exp = afwImage.makeExposure(afwImage.makeMaskedImage(afwImage.ImageF(fileName, hdu=hdu)))
exp.setDetector(detector)
amps.append(exp)
component_info = {}
component_info["raw_hdu"] = Info(afwImage.readMetadata(fileName, hdu=0))
component_info["raw_amp"] = Info(amps)
exp = assemble_raw(dataId, component_info, None)
return exp
def _standardizeMasterCal(self, datasetType, item, dataId, setFilter=False):
"""Standardize a MasterCal image obtained from NOAO archive into Exposure
These MasterCal images are MEF files with one HDU for each detector.
Some WCS header, eg CTYPE1, exists only in the zeroth extensionr,
so info in the zeroth header need to be copied over to metadata.
@param datasetType: Dataset type ("bias" or "flat")
@param item: The image read by the butler
@param dataId: Data identifier
@param setFilter: Whether to set the filter in the Exposure
@return (lsst.afw.image.Exposure) the standardized Exposure
"""
mi = afwImage.makeMaskedImage(item.getImage())
md = item.getMetadata()
masterCalMap = getattr(self, "map_" + datasetType)
masterCalPath = masterCalMap(dataId).getLocations()[0]
headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath)
md0 = afwImage.readMetadata(headerPath)
for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2',
'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'):
if kw in md0.paramNames() and kw not in md.paramNames():
md.add(kw, md0.get(kw))
wcs = afwImage.makeWcs(md, True)
exp = afwImage.makeExposure(mi, wcs)
exp.setMetadata(md)
return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter)
def testFitsHeader(self):
"""Test that XY0 and crpix are written to the header as expected"""
#getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to
#compare to values stored in fits headers
parentCrpix = self.parent.getWcs().getPixelOrigin()
#Make a sub-image
x0, y0 = 20, 30
llc = afwGeom.Point2I(x0, y0)
bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50))
deep = False
subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep)
outFile = "tmp.fits"
subImg.writeFits(outFile)
hdr = afwImage.readMetadata(outFile)
os.remove(outFile)
self.assertTrue(hdr.exists("LTV1"), "LTV1 not saved to fits header")
self.assertTrue(hdr.exists("LTV2"), "LTV2 not saved to fits header")
self.assertEqual(hdr.get("LTV1"), -1 * x0, "LTV1 has wrong value")
self.assertEqual(hdr.get("LTV2"), -1 * y0, "LTV1 has wrong value")
self.assertTrue(hdr.exists("CRPIX1"),
"CRPIX1 not saved to fits header")
self.assertTrue(hdr.exists("CRPIX2"),
"CRPIX2 not saved to fits header")
fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")]
self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"),
parentCrpix[0] + 1, 6, "CRPIX1 saved wrong")
self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"),
parentCrpix[1] + 1, 6, "CRPIX2 saved wrong")
def assembleImage(self, gains=None):
imDict = {}
hdus = {}
for i in range(self._namps):
hdu = i + 1
filename = '%s[%i]' % (self.infile, hdu)
md = afwImage.readMetadata(filename)
imDict[md.get('EXTNAME')] = afwImage.ImageF(filename)
hdus[md.get('EXTNAME')] = hdu
det = self.buildDetector()
assembleInput = {}
for amp in det:
im = imDict[amp.getName()]
oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox())
oscan = afwMath.makeStatistics(oscanim, afwMath.MEDIAN).getValue()
im -= oscan
if gains is not None:
im *= gains[hdus[amp.getName()]]
assembleInput[amp.getName()] = self.makeExposure(im)
assembleConfig = ipIsr.AssembleCcdTask.ConfigClass()
assembleConfig.doTrim = True
assembler = ipIsr.AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
return resultExp.getMaskedImage().getImage()
def testFitsHeader(self):
"""Test that XY0 and crpix are written to the header as expected"""
# getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to
# compare to values stored in fits headers
parentCrpix = self.parent.getWcs().getPixelOrigin()
# Make a sub-image
x0, y0 = 20, 30
llc = afwGeom.Point2I(x0, y0)
bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50))
deep = False
subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep)
with utilsTests.getTempFilePath(".fits") as outFile:
subImg.writeFits(outFile)
hdr = afwImage.readMetadata(outFile)
def checkLtvHeader(hdr, name, value):
# Per DM-4133, LTVn headers are required to be floating point
self.assertTrue(hdr.exists(name), name + " not saved to FITS header")
self.assertIsInstance(hdr.get(name), numbers.Real, name + " is not numeric")
self.assertNotIsInstance(hdr.get(name), numbers.Integral, name + " is an int")
self.assertEqual(hdr.get(name), value, name + " has wrong value")
checkLtvHeader(hdr, "LTV1", -1 * x0)
checkLtvHeader(hdr, "LTV2", -1 * y0)
self.assertTrue(hdr.exists("CRPIX1"), "CRPIX1 not saved to fits header")
self.assertTrue(hdr.exists("CRPIX2"), "CRPIX2 not saved to fits header")
fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")]
self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"), parentCrpix[0] + 1, 6, "CRPIX1 saved wrong")
self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"), parentCrpix[1] + 1, 6, "CRPIX2 saved wrong")
def getInfo(self, filename):
"""Get information about the image from the filename and its contents
@param filename Name of file to inspect
@return File properties; list of file properties for each extension
"""
#matches = re.search("^PFSA(\d{6})(\d)(\d).fits", filename)
matches = re.search("PF([JLXIASPF])([ABCDS])(\d{6})(\d)(\d).fits", filename)
if not matches:
matches = re.search("PF([JLXIASPF])([ABCDS])-(\d{6})(\d)(\d).fits", filename)
if not matches:
raise RuntimeError("Unable to interpret filename: %s" % filename)
site, category, visit, filterInt, spectrograph = matches.groups()
if int(spectrograph) > 4:
spectrograph = '4'
ccd = int(spectrograph)-1
filter = ''
if filterInt == '0':
filter = 'b'
elif filterInt == '1':
filter = 'r'
ccd += 4
elif filterInt == '2':
filter = 'n'
ccd += 8
else:
filter = 'm'
ccd += 4
arm = filter
header = afwImage.readMetadata(filename)
info = dict(site=site, category=category, visit=int(visit), filter=filter, arm=arm, spectrograph=int(spectrograph), ccd=int(ccd))
info = self.getInfoFromMetadata(header, info=info)
return info, [info]
def _get_catalog(self, dataset, **kwargs):
"""Load the catalogs from the butler."""
filenames = (self.butler.get(dataset + "_filename",
dataId,
immediate=True)[0]
for dataId in self.dataids[dataset])
headers = (afwimage.readMetadata(fn, 2) for fn in filenames)
size = sum(md.get("NAXIS2") for md in headers)
cat = self.butler.get(dataset,
self.dataids[dataset][0],
flags=afwtable.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
self.from_butler['schema'] = cat.schema
catadic = {k: [] for k in sorted(self.dataids[dataset][0].keys())}
catalog = afwtable.SourceCatalog(self.from_butler['schema'])
catalog.reserve(size)
pbar = progressbar(len(self.dataids[dataset]))
print "INFO: Looping over the dataids"
for i, dataid in enumerate(self.dataids[dataset]):
cat = self.butler.get(dataset,
dataid,
flags=afwtable.SOURCE_IO_NO_FOOTPRINTS)
catalog.extend(cat, deep=True)
for newkey, idk in zip(catadic,
sorted(self.dataids[dataset][0].keys())):
catadic[newkey].extend([dataid[idk]] * len(cat))
pbar.update(i + 1)
pbar.finish()
print "INFO: Merging the dictionnaries"
catadic.update(catalog.getColumnView().extract(*self.keys[dataset],
copy=True,
ordered=True))
# Clean memory before going further
gc.collect()
return catadic
def testFitsHeader(self):
"""Test that XY0 and crpix are written to the header as expected"""
#getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to
#compare to values stored in fits headers
parentCrpix = self.parent.getWcs().getPixelOrigin()
#Make a sub-image
x0, y0 = 20, 30
llc = afwGeom.Point2I(x0, y0)
bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50))
deep = False
subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep)
outFile = "tmp.fits"
subImg.writeFits(outFile)
hdr = afwImage.readMetadata(outFile)
os.remove(outFile)
self.assertTrue( hdr.exists("LTV1"), "LTV1 not saved to fits header")
self.assertTrue( hdr.exists("LTV2"), "LTV2 not saved to fits header")
self.assertEqual(hdr.get("LTV1"), -1*x0, "LTV1 has wrong value")
self.assertEqual(hdr.get("LTV2"), -1*y0, "LTV1 has wrong value")
self.assertTrue( hdr.exists("CRPIX1"), "CRPIX1 not saved to fits header")
self.assertTrue( hdr.exists("CRPIX2"), "CRPIX2 not saved to fits header")
fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")]
self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"), parentCrpix[0]+1, 6, "CRPIX1 saved wrong")
self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"), parentCrpix[1]+1, 6, "CRPIX2 saved wrong")
def setUp(self):
maskedImage = afwImage.MaskedImageF(inFilePathSmall)
maskedImageMD = afwImage.readMetadata(inFilePathSmall)
self.smallExposure = afwImage.ExposureF(inFilePathSmall)
self.width = maskedImage.getWidth()
self.height = maskedImage.getHeight()
self.wcs = afwImage.makeWcs(maskedImageMD)
self.psf = DummyPsf(2.0)
self.detector = DetectorWrapper().detector
self.exposureBlank = afwImage.ExposureF()
self.exposureMiOnly = afwImage.makeExposure(maskedImage)
self.exposureMiWcs = afwImage.makeExposure(maskedImage, self.wcs)
self.exposureCrWcs = afwImage.ExposureF(
100, 100, self.wcs) # n.b. the (100, 100, ...) form
self.exposureCrOnly = afwImage.ExposureF(afwGeom.ExtentI(
100, 100)) # test with ExtentI(100, 100) too
afwImage.Filter.reset()
afwImage.FilterProperty.reset()
filterPolicy = pexPolicy.Policy()
filterPolicy.add("lambdaEff", 470.0)
afwImage.Filter.define(afwImage.FilterProperty("g", filterPolicy))
def test1(self):
# This fails due to #1386
#wcsfn = os.path.join(self.datadir, 'imsim-v85518312-fu-R43-S12.wcs')
wcsfn = os.path.join(self.datadir, 'imsim-v85518312-fu-R43-S12.wcs2')
hdr = afwImage.readMetadata(wcsfn)
wcs1 = afwImage.makeWcs(hdr)
crval = wcs1.getSkyOrigin()
cd = wcs1.getCDMatrix()
print cd
crval_p = afwGeom.Point2D(crval.getLongitude().asDegrees(),
crval.getLatitude().asDegrees())
origin = wcs1.getPixelOrigin()
print crval_p
print origin
wcs2 = afwImage.Wcs(crval_p, origin, cd)
for wcs in [wcs1,wcs2]:
print wcs
print 'x, y, RA, Dec, pixscale("/pix), pixscale2'
for x,y in [(0,0),(300,0),(350,0),(360,0),(370,0),(380,0),(400,0)]:
radec = wcs.pixelToSky(x,y)
ra = radec.getLongitude().asDegrees()
dec = radec.getLatitude ().asDegrees()
pixscale = 3600. * sqrt(wcs.pixArea(afwGeom.Point2D(x,y)))
ps2 = wcs.pixelScale().asArcseconds()
print x,y,ra,dec,pixscale,ps2
self.assertTrue(abs(pixscale - 0.2) < 1e-3)
self.assertTrue(abs(ps2 - 0.2) < 1e-3)
def testReadMetadata(self):
im = afwImage.DecoratedImageF(self.fileForMetadata)
meta = afwImage.readMetadata(self.fileForMetadata)
self.assertIn("NAXIS1", meta.names())
self.assertEqual(im.getWidth(), meta.get("NAXIS1"))
self.assertEqual(im.getHeight(), meta.get("NAXIS2"))
def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId):
# Workaround until I can access the butler
instcalMap = self.map_instcal(dataId)
dqmaskMap = self.map_dqmask(dataId)
wtmapMap = self.map_wtmap(dataId)
rawMap = self.map_raw(dataId)
instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1])
rawType = getattr(afwImage, rawMap.getPythonType().split(".")[-1])
dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1])
wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1])
instcal = instcalType(instcalMap.getLocations()[0])
dqmask = dqmaskType(dqmaskMap.getLocations()[0])
wtmap = wtmapType(wtmapMap.getLocations()[0])
raw = rawType(rawMap.getLocations()[0])
mask = self.translate_dqmask(dqmask)
variance = self.translate_wtmap(wtmap)
mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance)
md = raw.getMetadata()
wcs = afwImage.makeWcs(md)
exp = afwImage.ExposureF(mi, wcs)
# Set the calib by hand; need to grab the zeroth extension
header = re.sub(r'[\[](\d+)[\]]$', "[0]", rawMap.getLocations()[0])
md0 = afwImage.readMetadata(header)
calib = afwImage.Calib()
calib.setExptime(md0.get("EXPTIME"))
calib.setFluxMag0(10**(0.4 * md0.get("MAGZERO")))
exp.setCalib(calib)
exp.setMetadata(md) # Do we need to remove WCS/calib info?
return exp
def _standardizeMasterCal(self,
datasetType,
item,
dataId,
setFilter=False):
"""Standardize a MasterCal image obtained from NOAO archive into Exposure
These MasterCal images are MEF files with one HDU for each detector.
Some WCS header, eg CTYPE1, exists only in the zeroth extensionr,
so info in the zeroth header need to be copied over to metadata.
@param datasetType: Dataset type ("bias" or "flat")
@param item: The image read by the butler
@param dataId: Data identifier
@param setFilter: Whether to set the filter in the Exposure
@return (lsst.afw.image.Exposure) the standardized Exposure
"""
mi = afwImage.makeMaskedImage(item.getImage())
md = item.getMetadata()
masterCalMap = getattr(self, "map_" + datasetType)
masterCalPath = masterCalMap(dataId).getLocations()[0]
headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath)
md0 = afwImage.readMetadata(headerPath)
for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2',
'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'):
if kw in md0.paramNames() and kw not in md.paramNames():
md.add(kw, md0.get(kw))
wcs = afwImage.makeWcs(md, True)
exp = afwImage.makeExposure(mi, wcs)
exp.setMetadata(md)
return self._standardizeExposure(self.calibrations[datasetType],
exp,
dataId,
filter=setFilter)
def processRaft(raftDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
for fits in glob.glob(
os.path.join(
raftDir, "S[0-2][0-2]",
"imsim_*_R[0-4][0-4]_S[0-2][0-2]_C[01][0-7]_E00[01].fits*")):
m = re.search(
r'v(\d+)-f(\w)/E00(\d)/R(\d)(\d)/S(\d)(\d)/' +
r'imsim_\1_R\4\5_S\6\7_C(\d)(\d)_E00\3\.fits', fits)
if not m:
print >> sys.stderr, "Warning: Unrecognized file:", fits
nUnrecognized += 1
continue
(visit, filter, snap, raft1, raft2, sensor1, sensor2, channel1,
channel2) = m.groups()
key = "%s_F%s_E%s_R%s,%s_S%s,%s_C%s,%s" % (visit, filter, snap, raft1,
raft2, sensor1, sensor2,
channel1, channel2)
if key in done:
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
expTime = md.get("EXPTIME")
mjdObs = md.get("MJD-OBS")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.TAI).toString(
dafBase.DateTime.UTC)[:-1]
conn.execute(
"""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?, ?)""",
(visit, filter, snap, "%s,%s" % (raft1, raft2), "%s,%s" %
(sensor1, sensor2), "%s,%s" %
(channel1, channel2), taiObs, expTime))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"),
md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
conn.commit()
nProcessed += 1
print >>sys.stderr, raftDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized)
def EventFromInputfile(inputfile,
datatypePolicy,
metadataPolicy,
topicName='triggerImageprocEvent',
hostName='lsst4.ncsa.uiuc.edu'):
"""generate a new file event for a given FITS file
@param inputfile the name of the FITS file
@param datatyepPolicy the policy describing the metadata transformation
@param metadataPolicy the policy describing the event metadata types
@param topicName the name of the topic to send event as
@param hostName the event broker hostname
"""
global exposureCount
exposureCount += 1
# For DC3a, inputfile is a .fits file on disk
metadata = afwImage.readMetadata(inputfile)
# First, transform the input metdata
transformMetadata(metadata, datatypePolicy, metadataPolicy, 'Keyword')
# To be consistent...
if not validateMetadata(metadata, metadataPolicy):
logger.log(logger.FATAL, 'Unable to create event from %s' % inputfile)
# Create event policy, using defaults from input metadata
event = dafBase.PropertySet()
event.copy('visitId', metadata, 'visitId')
event.copy('ccdId', metadata, 'ccdId')
event.copy('ampId', metadata, 'ampId')
event.copy('exposureId', metadata, 'exposureId')
event.copy('datasetId', metadata, 'datasetId')
event.copy('filter', metadata, 'filter')
event.copy('expTime', metadata, 'expTime')
event.copy('ra', metadata, 'ra')
event.copy('decl', metadata, 'decl')
event.copy('equinox', metadata, 'EQUINOX')
event.copy('airmass', metadata, 'airmass')
event.copy('dateObs', metadata, 'dateObs')
if event.getInt('exposureId') == 0:
eventTransmitter = ctrlEvents.EventTransmitter(hostName,
topicName + '0')
elif event.getInt('exposureId') == 1:
eventTransmitter = ctrlEvents.EventTransmitter(hostName,
topicName + '1')
logger.log(logger.INFO,
'Sending event for %s' % os.path.basename(inputfile))
if logger.sends(logger.DEBUG):
logger.log(logger.DEBUG, "Data Event data:\n%s" % event.toString())
elif logger.sends(VERB3):
logger.log(
VERB3, "Event data: datasetId=%s; ra=%f, dec=%f" %
(event.get("datasetId"), event.get("ra"), event.get("decl")))
eventTransmitter.publish(event)
return True
def fromDir(cls, root, visit, **kwds):
ffp = {}
wcs = {}
fcrPattern = os.path.join(root, "fcr-%07d-*.fits" %
visit) # meas_mosaic coords
wcsPattern = os.path.join(root,
"wcs-%07d-*.fits" % visit) # LSST coords
start = fcrPattern.index("*")
for filename in glob.glob(fcrPattern):
ccd = int(filename[start:start + 3])
md = afwImage.readMetadata(filename)
ffp[ccd] = FluxFitParams(md)
for filename in glob.glob(wcsPattern):
ccd = int(filename[start:start + 3])
md = afwImage.readMetadata(filename)
wcs[ccd] = afwImage.makeWcs(md)
return CorrectionImageSource(ffp, wcs, **kwds)
def std_dark(self, item, dataId):
exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
rawPath = self.map_raw(dataId).getLocations()[0]
headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath)
md0 = afwImage.readMetadata(headerPath)
visitInfo = self.makeRawVisitInfo(md0)
exp.getInfo().setVisitInfo(visitInfo)
return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False)
def __init__(self, infile, hdu):
self.header = None
try:
self.md = afwImage.readMetadata(infile, hdu)
except:
# This exception occurs when DM stack encounters a "." in
# a FITS header keyword.
self.header = fits.open(infile)[hdu-1].header
def checkExtName(self, name, value, extNum):
filename = DATA + "[%s]" % name
header = afwImage.readMetadata(filename)
self.assertEqual(header.get("EXT_NUM"), extNum)
self.assertEqual(header.get("EXTNAME").strip(), name)
image = afwImage.ImageI(filename)
self.assertEqual(image.get(0,0), value)
def testReadWcs(self):
"""Test reading a Wcs directly from a fits file"""
meta = afwImage.readMetadata(InputImagePath + "_img.fits")
wcs = afwImage.makeWcs(meta)
sky0 = wcs.pixelToSky(0.0, 0.0).getPosition()
sky1 = self.wcs.pixelToSky(0.0, 0.0).getPosition()
self.assertEqual(sky0, sky1)
def __init__(self, imfile, mask_files=(), ccdtemp_par=-100):
self.ccd = MaskedCCD(imfile, mask_files=mask_files)
self.md = afwImage.readMetadata(imfile, 0)
try:
ccdtemp = self.md.get('CCDTEMP')
except:
ccdtemp = ccdtemp_par
self.fe55_yield = Fe55Yield(ccdtemp).alpha()
self._footprint_signal = self._footprint_signal_spans
def testReadWcs(self):
"""Test reading a Wcs directly from a fits file"""
meta = afwImage.readMetadata(InputImagePath + ".fits")
wcs = afwImage.makeWcs(meta)
sky0 = wcs.pixelToSky(0.0, 0.0).getPosition()
sky1 = self.wcs.pixelToSky(0.0, 0.0).getPosition()
self.assertEqual(sky0, sky1)
def checkExtName(self, name, value, extNum):
filename = DATA + "[%s]" % name
header = afwImage.readMetadata(filename)
self.assertEqual(header.get("EXT_NUM"), extNum)
self.assertEqual(header.get("EXTNAME").strip(), name)
image = afwImage.ImageI(filename)
self.assertEqual(image.get(0,0), value)
def getInfo(self, filename):
"""Get information about the image from the filename and its contents
The information we want isn't in the headers, but in the filname.
Parameters
----------
filename : `str`
Name of file to inspect
Returns
-------
info : `dict`
File properties from the PHU.
infoList : `list` of `dict`
File properties from the extensions.
"""
self.log.debug('interpreting filename <%s>' % filename)
path, name = os.path.split(filename)
matches = re.search(r"^PF(%s)(%s)-?(\d{6})(\d)(\d)\.fits$" % (self.sites, self.categories), name)
if not matches:
raise RuntimeError("Unable to interpret filename: %s" % filename)
site, category, expId, spectrograph, armNum = matches.groups()
armNum = int(armNum)
spectrograph = int(spectrograph)
expId = int(expId, base=10)
# spectrograph 2 was called 9 at JHU, at least in the early days, so if we find
# a spectrograph 9 we re-assign its number to 2
if spectrograph == 9:
spectrograph = 2
self.log.info("Visit %06d has spectrograph == 9" % expId)
if spectrograph < 1 or spectrograph > self.nSpectrograph + 1:
raise RuntimeError('spectrograph (=%d) out of bounds 1..%d' % (spectrograph, self.nSpectrograph))
try:
arm = self.arms[armNum - 1] # 1-indexed
except IndexError as exc:
raise IndexError('armNum=%d out of bounds 1..%d' % (armNum, max(self.arms.keys()))) from exc
ccd = PfsMapper.computeDetectorId(spectrograph, arm)
self.log.debug('site = <%s>, category = <%s>, expId = <%s>, spectrograph = <%d>, armNum = <%d>, '
'arm = <%s>, ccd = <%d>' %
(site, category, expId, spectrograph, armNum, arm, ccd))
info = dict(site=site, category=category, expId=expId, visit=expId, filter=arm, arm=arm,
spectrograph=spectrograph, ccd=ccd)
if os.path.exists(filename):
header = afwImage.readMetadata(filename)
info = self.getInfoFromMetadata(header, info=info)
return info, [info]
def _listdir(self, path, prefix):
for file in os.listdir(path):
fileName = os.path.join(path, file)
md = afwImage.readMetadata(fileName)
if "EXPNUM" not in md.names():
return
expnum = md.get("EXPNUM")
if expnum not in self.expnumMapper.keys():
self.expnumMapper[expnum] = {self.instcalPrefix: None, self.wtmapPrefix: None, self.dqmaskPrefix: None}
self.expnumMapper[expnum][prefix] = fileName
def testMakeWcs(self):
path = eups.productDir("afw")
path = os.path.join(path, "tests", "data", "parent.fits")
fitsHdr = image.readMetadata(path)
wcs = image.makeWcs(fitsHdr)
c = wcs.pixelToSky(0, 0)
print type(c)
c.getPosition()
def testMakeWcs(self):
path= eups.productDir("afw")
path = os.path.join(path, "tests", "data", "parent.fits")
fitsHdr = image.readMetadata(path)
wcs = image.makeWcs(fitsHdr)
c = wcs.pixelToSky(0,0)
print type(c)
c.getPosition()
def testMakeWcs(self):
afwdataDir = lsst.utils.getPackageDir("afw")
path = os.path.join(afwdataDir, "tests", "data", "parent.fits")
fitsHdr = image.readMetadata(path)
wcs = image.makeWcs(fitsHdr)
c = wcs.pixelToSky(0, 0)
print(type(c))
c.getPosition()
def testXY0(self):
"""Test that XY0 values are handled correctly when building an exposure and also when
reading the WCS header directly. #2205"""
bbox = afwGeom.Box2I(afwGeom.Point2I(1000, 1000),
afwGeom.Extent2I(10, 10))
def makeWcs(crPixPos, crValDeg, projection):
ps = dafBase.PropertySet()
ctypes = [
("%-5s%3s" % (("RA", "DEC")[i], projection)).replace(" ", "-")
for i in range(2)
]
for i in range(2):
ip1 = i + 1
ps.add("CTYPE%1d" % (ip1, ), ctypes[i])
ps.add("CRPIX%1d" % (ip1, ), crPixPos[i])
ps.add("CRVAL%1d" % (ip1, ), crValDeg[i])
ps.add("RADECSYS", "ICRS")
ps.add("EQUINOX", 2000)
ps.add("CD1_1", -0.001)
ps.add("CD2_1", 0.0)
ps.add("CD1_2", 0.0)
ps.add("CD2_2", 0.001)
return afwImage.makeWcs(ps)
wcs = makeWcs(
crPixPos=(100.0, 100.0),
crValDeg=(10.0, 35.0),
projection="STG", # also fails for TAN
)
exposure = afwImage.ExposureF(bbox, wcs)
pixPos = afwGeom.Box2D(bbox).getMax()
if verbose:
print "XY0=", exposure.getXY0()
print "pixPos=", pixPos
skyPos = wcs.pixelToSky(pixPos)
with utilsTests.getTempFilePath(".fits") as tmpFile:
exposure.writeFits(tmpFile)
for useExposure in (False, True):
if useExposure:
unpExp = afwImage.ExposureF(tmpFile)
unpWcs = unpExp.getWcs()
else:
md = afwImage.readMetadata(tmpFile)
unpWcs = afwImage.makeWcs(md, False)
unpPixPos = unpWcs.skyToPixel(skyPos)
if verbose:
print "useExposure=%s; unp pixPos=%s" % (useExposure,
unpPixPos)
for i in range(2):
self.assertAlmostEqual(unpPixPos[i], 1009.5)
def testDM882(self):
"""Test that we can write a dotted header unit to a FITS file. See DM-882."""
self.dimage1.getMetadata().add("A.B.C.D", 12345)
tempdir = tempfile.mkdtemp()
testfile = os.path.join(tempdir, "test.fits")
try:
self.dimage1.writeFits(testfile)
meta = afwImage.readMetadata(testfile)
self.assertEqual(meta.get("A.B.C.D"), 12345)
finally:
shutil.rmtree(tempdir)
def checkEquinoxHeader(coordSysName, writeEquinox):
coordSys = getattr(afwCoord, coordSysName)
# We should get the same behaviour with both Wcs and TanWcs: check them both.
for dummyWcs in (makeWcs(coordSys=coordSys), afwImage.TanWcs.cast(makeWcs(coordSys=coordSys))):
dummyExposure = afwImage.ExposureF()
dummyExposure.setWcs(dummyWcs)
with utilsTests.getTempFilePath(".fits") as tmpFile:
dummyExposure.writeFits(tmpFile)
metadata = afwImage.readMetadata(tmpFile)
self.assertEqual(metadata.get("RADESYS"), coordSysName)
self.assertTrue(("EQUINOX" in metadata.names()) == writeEquinox)
def testReadMetadata(self):
if self.fileForMetadata:
im = afwImage.DecoratedImageF(self.fileForMetadata)
else:
print >> sys.stderr, "Warning: afwdata is not set up; not running the FITS metadata I/O tests"
return
meta = afwImage.readMetadata(self.fileForMetadata)
self.assertTrue("NAXIS1" in meta.names())
self.assertEqual(im.getWidth(), meta.get("NAXIS1"))
self.assertEqual(im.getHeight(), meta.get("NAXIS2"))
def testDM882(self):
"""Test that we can write a dotted header unit to a FITS file. See DM-882."""
self.dimage1.getMetadata().add("A.B.C.D", 12345)
tempdir = tempfile.mkdtemp()
testfile = os.path.join(tempdir, "test.fits")
try:
self.dimage1.writeFits(testfile)
meta = afwImage.readMetadata(testfile)
self.assertEqual(meta.get("A.B.C.D"), 12345)
finally:
shutil.rmtree(tempdir)
def testReadMetadata(self):
if self.fileForMetadata:
im = afwImage.DecoratedImageF(self.fileForMetadata)
else:
print >> sys.stderr, "Warning: afwdata is not set up; not running the FITS metadata I/O tests"
return
meta = afwImage.readMetadata(self.fileForMetadata)
self.assertTrue("NAXIS1" in meta.names())
self.assertEqual(im.getWidth(), meta.get("NAXIS1"))
self.assertEqual(im.getHeight(), meta.get("NAXIS2"))
def makeCcdMosaic(dir, basename, e, c, aList, imageFactory=afwImage.MaskedImageF, verbose=0):
"""Return an image of all the specified amplifiers, aList, for the given CCD
E.g. sl = makeCcdMosaic("/lsst/DC3root/rlp1173", "v704897", 0, 3, range(8))
"""
try:
aList[0]
except TypeError:
aList = [aList]
for what in ("header", "data"):
if what == "header":
bbox = afwGeom.Box2I()
ampBBox = {}
wcs = {}
else:
ccdImage = imageFactory(bbox.getWidth(), bbox.getHeight())
ccdImage.set(0)
ccdImage.setXY0(bbox.getLLC())
for a in aList:
filename = os.path.join(dir, "IPSD", "output", "sci", "%s-e%d" % (basename, e),
"%s-e%d-c%03d-a%02d.sci" % (basename, e, c, a))
if verbose and what == "header":
print filename
if what == "header":
md = afwImage.readMetadata(filename + "_img.fits")
xy0 = afwGeom.Point2I(md.get("CRVAL1A"), md.get("CRVAL2A"))
xy1 = xy0 + afwGeom.Extent2I(md.get("NAXIS1") - 1, md.get("NAXIS2") - 1)
bbox.grow(xy0)
bbox.grow(xy1)
ampBBox[a] = afwGeom.Box2I(xy0, xy1)
wcs[a] = afwImage.Wcs(md)
else:
try:
data = imageFactory(filename + "_img.fits")
except:
data = imageFactory(filename)
ampImage = ccdImage.Factory(ccdImage, ampBBox[a])
ampImage[:] = data
del ampImage
try:
ccdImage.getMask()
if wcs.has_key(0):
ccdImage = afwImage.ExposureF(ccdImage, wcs[0])
except AttributeError:
pass
return ccdImage
def readout_time(infile):
"""
Return readout time as an astropy.time.Time object.
It is computed from the PHDU keyword values as MJD-OBS + EXPTIME
"""
md = afwImage.readMetadata(infile, 1)
mjd_obs = astropy.time.Time(md.get('MJD-OBS'), format='mjd', scale='utc')
t_readout = astropy.time.Time(
mjd_obs.datetime + datetime.timedelta(seconds=md.get('EXPTIME')),
scale='utc')
return t_readout
def testXY0(self):
"""Test that XY0 values are handled correctly when building an exposure and also when
reading the WCS header directly. #2205"""
bbox = afwGeom.Box2I(afwGeom.Point2I(1000, 1000), afwGeom.Extent2I(10, 10))
def makeWcs(crPixPos, crValDeg, projection):
ps = dafBase.PropertySet()
ctypes = [("%-5s%3s" % (("RA", "DEC")[i], projection)).replace(" ", "-") for i in range(2)]
for i in range(2):
ip1 = i + 1
ps.add("CTYPE%1d" % (ip1,), ctypes[i])
ps.add("CRPIX%1d" % (ip1,), crPixPos[i])
ps.add("CRVAL%1d" % (ip1,), crValDeg[i])
ps.add("RADECSYS", "ICRS")
ps.add("EQUINOX", 2000)
ps.add("CD1_1", -0.001)
ps.add("CD2_1", 0.0)
ps.add("CD1_2", 0.0)
ps.add("CD2_2", 0.001)
return afwImage.makeWcs(ps)
wcs = makeWcs(
crPixPos = (100.0, 100.0),
crValDeg = (10.0, 35.0),
projection = "STG", # also fails for TAN
)
exposure = afwImage.ExposureF(bbox, wcs)
pixPos = afwGeom.Box2D(bbox).getMax()
if verbose:
print "XY0=", exposure.getXY0()
print "pixPos=", pixPos
skyPos = wcs.pixelToSky(pixPos)
tmpFile = "temp.fits"
try:
exposure.writeFits(tmpFile)
for useExposure in (False, True):
if useExposure:
unpExp = afwImage.ExposureF(tmpFile)
unpWcs = unpExp.getWcs()
else:
md = afwImage.readMetadata("temp.fits")
unpWcs = afwImage.makeWcs(md, False)
unpPixPos = unpWcs.skyToPixel(skyPos)
if verbose:
print "useExposure=%s; unp pixPos=%s" % (useExposure, unpPixPos)
for i in range(2):
self.assertAlmostEqual(unpPixPos[i], 1009.5)
finally:
os.remove(tmpFile)
def processRaft(raftDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
for fits in glob.glob(os.path.join(raftDir, "S[0-2][0-2]",
"imsim_*_R[0-4][0-4]_S[0-2][0-2]_C[01][0-7]_E00[01].fits*")):
import pdb; pdb.set_trace()
m = re.search(r'v(\d+)-f(\w)/E00(\d)/R(\d)(\d)/S(\d)(\d)/' +
r'imsim_\1_R\4\5_S\6\7_C(\d)(\d)_E00\3\.fits', fits)
if not m:
print >>sys.stderr, "Warning: Unrecognized file:", fits
nUnrecognized += 1
continue
(visit, filter, snap, raft1, raft2, sensor1, sensor2,
channel1, channel2) = m.groups()
key = "%s_F%s_E%s_R%s,%s_S%s,%s_C%s,%s" % (visit, filter,
snap, raft1, raft2, sensor1, sensor2, channel1, channel2)
if done.has_key(key):
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
expTime = md.get("EXPTIME")
mjdObs = md.get("MJD-OBS")
taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD,
dafBase.DateTime.TAI).toString()[:-1]
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?, ?, ?, ?, ?)""",
(visit, filter, snap, "%s,%s" % (raft1, raft2),
"%s,%s" % (sensor1, sensor2),
"%s,%s" % (channel1, channel2), taiObs, expTime))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"), md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
conn.commit()
nProcessed += 1
print >>sys.stderr, raftDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized)
def EventFromInputfile(inputfile,
datatypePolicy,
metadataPolicy,
topicName='triggerImageprocEvent',
hostName='newfield.as.arizona.edu'):
"""generate a new file event for a given FITS file
@param inputfile the name of the FITS file
@param datatyepPolicy the policy describing the metadata transformation
@param metadataPolicy the policy describing the event metadata types
@param topicName the name of the topic to send event as
@param hostName the event broker hostname
"""
global exposureCount
exposureCount += 1
# For mosphot, inputfile is a .fits file on disk
metadata = afwImage.readMetadata(inputfile)
# logger.log(logger.INFO,"Original metadata:\n" + metadata.toString())
# First, transform the input metdata
transformMetadata(metadata, datatypePolicy, metadataPolicy, 'Keyword')
# To be consistent...
if not validateMetadata(metadata, metadataPolicy):
logger.log(logger.FATAL, 'Unable to create event from %s' % inputfile)
# Create event policy, using defaults from input metadata
event = dafBase.PropertySet()
event.copy('exposureId', metadata, 'exposureId')
event.copy('datasetId', metadata, 'datasetId')
event.copy('filter', metadata, 'filter')
event.copy('expTime', metadata, 'expTime')
event.copy('ra', metadata, 'ra')
event.copy('decl', metadata, 'decl')
event.copy('equinox', metadata, 'equinox')
event.copy('airmass', metadata, 'airmass')
event.copy('dateObs', metadata, 'dateObs')
eventTransmitter = ctrlEvents.EventTransmitter(hostName, topicName)
logger.log(logger.INFO,
'Sending event for %s' % os.path.basename(inputfile))
if logger.sends(logger.DEBUG):
logger.log(logger.DEBUG, "Data Event data:\n%s" % event.toString())
elif logger.sends(VERB3):
logger.log(VERB3,
"Event data: datasetId=%s; ra=%f, dec=%f" %
(event.get("datasetId"), event.get("ra"), event.get("decl")))
eventTransmitter.publish(event)
return True
def fileNames(fileLoop):
"""This function searches for LSST FITS files with associated trimfiles. obss is a list
of the FITS subdirectories in the /astro/net/pogo3/.../obs/ directory. They start with a v,
then 9 numbers, then -fu, -fg, -fr, -fi, -fz, or -fy. Each subdirectory has an E000 path and an
E001 path; they simulate consecutive exposures of the same area of sky. Each of those has
a subdirectory, R22 (the central raft in the camera) containing one zipped eimage (compressed
FITS file). There are 5287 v...-f* directories. Load each directory in the obs directory
that starts with a 'v', name it and add '/E000/R22/' to it and call it the eDir. For this run
I only want images from the r filters. Not all of the files have trim files associated with
them, so I also skip the ones that don't."""
#ePath = '/astro/net/pogo3/rgibson/testSim/testPM/obs/v858247512-fr/E000/R22/eimage_858247512_R22_S11_E000.fits.gz'
ePath = '/astro/net/pogo3/rgibson/testSim/testPM/obs/v858903212-fr/E000/R22/eimage_858903212_R22_S11_E000.fits.gz'
#eFile = 'eimage_858247512_R22_S11_E000.fits.gz'
eFile = 'eimage_858903212_R22_S11_E000.fits.gz'
trimPath = None; metaPath = None; eEPath = None
trimDir = '/astro/net/pogo3/rgibson/testSim/testPM/trim/obsid'
t0 = eFile.split('_')
t1 = t0[1][0:-1]
metaPath = '%s%s/metadata_%s.dat' % (trimDir, t1, t1)
trimPath = '%s%s/pops/trim_%s_MSSTARS.dat.gz' % (trimDir, t1, t1)
eEPath = '%s%s/pops/trim_%s_EASTEREGGS.dat.gz' % (trimDir, t1, t1)
outDir = '/astro/net/pogo3/rgibson/testSim/testPM/out%s/' % eFile
# [0:-3] removes the ".gz"
newEFile = 'eImageTANSIPa' + eFile[0:-3]
newEPath = outDir + newEFile
fileLoop = 1
# Get the FITS image and find the positive footprints therein.
eIm = afwImage.ImageF(newEPath)
metaData = afwImage.readMetadata(newEPath)
wcs = afwImage.makeWcs(metaData)
eHDUList = pyfits.open(newEPath)
x0 = wcs.pixelToSky(0, 0).getPosition()[0]
y0 = wcs.pixelToSky(0, 0).getPosition()[1]
x1 = wcs.pixelToSky(0, 4096).getPosition()[0]
y1 = wcs.pixelToSky(0, 4096).getPosition()[1]
x2 = wcs.pixelToSky(4096, 4096).getPosition()[0]
y2 = wcs.pixelToSky(4096, 4096).getPosition()[1]
x3 = wcs.pixelToSky(4096, 0).getPosition()[0]
y3 = wcs.pixelToSky(4096, 0).getPosition()[1]
# You just want the minimum radius so you don't go off the edge of the image boundary.
deltax = min(abs(x2-x1), abs(x2-x0), abs(x3-x1), abs(x3-x0))
deltay = min(abs(y1-y0), abs(y2-y0), abs(y1-y3), abs(y2-y3))
radius = min(deltax/2., deltay/2.)
ra_center = wcs.pixelToSky(2048, 2048).getPosition()[0]
dec_center = wcs.pixelToSky(2048, 2048).getPosition()[1]
return ra_center, dec_center, radius, eIm, eHDUList, wcs, ePath, fileLoop
def writeDefects(self, maskFile, ccd, format=None):
"""Given a Mask, find the defects for each amp in the specified CCD (0-indexed).
If format is provided, it's expected to be used as "format % (ccd, amp)" to generate
a .paf filename to write the defects too. If it's "-", write to stdout"""
# Metadata should have validity range keywords, but it doesn't.
if False:
metaData = afwImage.readMetadata(maskFile, 0)
hdu = ccd + 2
im = afwImage.ImageF(maskFile, hdu)
im -= 1
im *= -1
mask = afwImage.MaskU(im.getBBox(afwImage.PARENT)); mask.set(0x0)
mask = afwImage.makeMaskedImage(im, mask)
for amp in self.ampList():
subMask = mask.Factory(mask, self.getTrimSecBBox(amp), afwImage.LOCAL)
ds = afwDetection.makeFootprintSet(subMask, afwDetection.Threshold(0.5), "INTRP")
if False:
ds9.setDefaultFrame(amp)
ds9.mtv(subMask)
ds9.dot("Amp %d" % amp, 0, 0)
if not format:
return
if format == "-":
fd = sys.stdout
else:
fd = open(format % (ccd, amp), "w")
print >> fd, """\
#<?cfg paf policy ?>
#
# Defects for CCD%03d amp %02d generated from %s, HDU %d
#
# Coordinates are trimmed and per-amp not per-CCD. If you change this, the ISR will have to change
#
# Generated by $HeadURL$
#
""" % (ccd, amp, maskFile, hdu),
for foot in ds.getFootprints():
afwDetectionUtils.writeFootprintAsDefects(fd, foot)
def testReadMetadata(self):
with utilsTests.getTempFilePath(".fits") as tmpFile:
self.exposureCrWcs.getMetadata().set("FRAZZLE", True)
# This will write the main metadata (inc. FRAZZLE) to the primary HDU, and the
# WCS to subsequent HDUs, along with INHERIT=T.
self.exposureCrWcs.writeFits(tmpFile)
# This should read the first non-empty HDU (i.e. it skips the primary), but
# goes back and reads it if it finds INHERIT=T. That should let us read
# frazzle and the Wcs from the PropertySet returned by readMetadata.
md = afwImage.readMetadata(tmpFile)
wcs = afwImage.makeWcs(md, True)
self.assertEqual(wcs.getPixelOrigin(), self.wcs.getPixelOrigin())
self.assertEqual(wcs.getSkyOrigin(), self.wcs.getSkyOrigin())
self.assert_(numpy.all(wcs.getCDMatrix() == self.wcs.getCDMatrix()))
frazzle = md.get("FRAZZLE")
self.assert_(frazzle is True)
def processBand(filterDir, conn, done, qsp):
nProcessed = 0
nSkipped = 0
nUnrecognized = 0
print >>sys.stderr, filterDir, "... started"
for fits in glob.iglob(
os.path.join(filterDir, "fpC*_ts_coaddNorm_NN.fit.gz")):
m = re.search(r'/([ugriz])/fpC-(\d{6})-\1(\d)-(\d{4})_ts_coaddNorm_NN.fit.gz', fits)
if not m:
print >>sys.stderr, "Warning: Unrecognized file:", fits
nUnrecognized += 1
continue
(filter, run, camcol, field) = m.groups()
camcol = int(camcol)
run = int(run)
field = int(field)
key = "%d_B%s_C%d_F%d" % (run, filter, camcol, field)
if done.has_key(key):
nSkipped += 1
continue
md = afwImage.readMetadata(fits)
conn.execute("""INSERT INTO raw VALUES
(NULL, ?, ?, ?, ?)""", (run, filter, camcol, field))
for row in conn.execute("SELECT last_insert_rowid()"):
id = row[0]
break
wcs = afwImage.makeWcs(md)
poly = skypix.imageToPolygon(wcs,
md.get("NAXIS1"), md.get("NAXIS2"),
padRad=0.000075) # about 15 arcsec
pix = qsp.intersect(poly)
for skyTileId in pix:
conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
(id, skyTileId))
nProcessed += 1
if nProcessed % 100 == 0:
conn.commit()
conn.commit()
print >>sys.stderr, filterDir, \
"... %d processed, %d skipped, %d unrecognized" % \
(nProcessed, nSkipped, nUnrecognized)
def AssembleImage(input_file, doGainCorrection = False, trim = True):
import os
imDict = {}
afilelist = []
dfilename = '%s[%s]' % (input_file, 0)
for i in range(16):
filename = '%s[%i]' % (input_file, i+1)
md = afwImage.readMetadata(filename)
afilelist.append(filename)
imDict[md.get('EXTNAME')] = afwImage.ImageF(filename)
db = TestCamDetectorBuilder(dfilename, afilelist, inAmpCoords=True, clobberMetadata=True)
det = db.buildDetector()
assembleInput = {}
for amp in det:
im = imDict[amp.getName()]
oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox())
oscan = numpy.median(oscanim.getArray())
imArr = im.getArray()
imArr -= oscan
#Calculate and correct for gain
if doGainCorrection:
# Buffer so edge rolloff doesn't interfere
buffImArr = imArr[30:-30][30:-30]
medCounts = numpy.median(buffImArr)
stdCounts = numpy.std(buffImArr)
gain = medCounts/stdCounts**2
imArr *= gain
assembleInput[amp.getName()] = db.makeExposure(im)
assembleConfig = AssembleCcdTask.ConfigClass()
if trim:
assembleConfig.doTrim = True
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
# camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=0)
return resultExp
else:
assembleConfig.doTrim = False
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
# camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=1)
return resultExp
