def testCalibFromCalibs(self):
"""Test creating a Calib from an array of Calibs"""
exptime = 20
mag0, mag0Sigma = 1.0, 0.01
time0 = dafBase.DateTime.now().get()
calibs = afwImage.vectorCalib()
ncalib = 3
for i in range(ncalib):
calib = afwImage.Calib()
calib.setMidTime(dafBase.DateTime(time0 + i))
calib.setExptime(exptime)
calib.setFluxMag0(mag0, mag0Sigma)
calibs.append(calib)
ocalib = afwImage.Calib(calibs)
self.assertEqual(ocalib.getExptime(), ncalib * exptime)
self.assertAlmostEqual(calibs[ncalib // 2].getMidTime().get(),
ocalib.getMidTime().get())
#
# Check that we can only merge Calibs with the same fluxMag0 values
#
calibs[0].setFluxMag0(1.001 * mag0, mag0Sigma)
self.assertRaises(pexExcept.InvalidParameterError,
lambda: afwImage.Calib(calibs))
def testCtorFromMetadata(self):
"""Test building a Calib from metadata"""
flux0, flux0Err = 1e12, 1e10
flux, fluxErr = 1000.0, 10.0
metadata = dafBase.PropertySet()
metadata.add("FLUXMAG0", flux0)
metadata.add("FLUXMAG0ERR", flux0Err)
self.calib = afwImage.Calib(metadata)
self.assertEqual(flux0, self.calib.getFluxMag0()[0])
self.assertEqual(flux0Err, self.calib.getFluxMag0()[1])
self.assertEqual(22.5, self.calib.getMagnitude(flux))
# Error just in flux
self.calib.setFluxMag0(flux0, 0)
self.assertAlmostEqual(
self.calib.getMagnitude(flux, fluxErr)[1],
2.5 / math.log(10) * fluxErr / flux)
# Check that we can clean up metadata
afwImage.stripCalibKeywords(metadata)
self.assertEqual(len(metadata.names()), 0)
def loadData(pixelScale=1.0):
"""Prepare the data we need to run the example"""
# Load sample input from disk
mypath = lsst.utils.getPackageDir('afwdata')
print(
" mypath = ",
mypath) # mssg -- error happens before i can even possibly print this
# sys.exit()
imFile = os.path.join(mypath, "CFHT", "D4",
"cal-53535-i-797722_small_1.fits")
exposure = afwImage.ExposureF(imFile)
# set the exposure time
calib = afwImage.Calib()
calib.setExptime(1.0)
exposure.setCalib(calib)
# add a filter
afwImage.Filter.define(afwImage.FilterProperty(FilterName, 600, True))
exposure.setFilter(afwImage.Filter(FilterName))
# and a trivial WCS (needed by MyAstrometryTask)
pixelScale /= 3600.0 # degrees per pixel
wcs = afwImage.makeWcs(Coord(PointD(15, 1)), PointD(0, 0), pixelScale, 0.0,
0.0, pixelScale)
exposure.setWcs(wcs)
return exposure
def buildExposure(self, dataRef):
image = dataRef.get(self.config.dataType + "image", immediate=True)
exposure = afwImage.ExposureF(image.getBBox(afwImage.PARENT))
exposure.getMaskedImage().getImage().getArray()[:, :] = image.getArray(
)
variance = self.computeVariance(image)
exposure.getMaskedImage().getVariance().set(variance)
self.algMetadata.set('noise_variance', variance)
exposure.setPsf(self.psf.run(dataRef, self.config.dataType))
pixelScale = self.config.PIXEL_SCALE * afwGeom.arcseconds
cdMatrix = makeCdMatrix(scale=pixelScale)
crpix = afwGeom.Point2D(0, 0)
crval = lsst.afw.coord.IcrsCoord(0. * afwGeom.degrees,
0. * afwGeom.degrees)
wcs = makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
exposure.setWcs(wcs)
calib = afwImage.Calib()
calib.setFluxMag0(1e12)
exposure.setCalib(calib)
return exposure
def testCtorFromMetadata(self):
"""Test building a Calib from metadata"""
isoDate = "1995-01-26T07:32:00.000000000Z"
exptime = 123.4
flux0, flux0Err = 1e12, 1e10
flux, fluxErr = 1000.0, 10.0
metadata = dafBase.PropertySet()
metadata.add("TIME-MID", isoDate)
metadata.add("EXPTIME", exptime)
metadata.add("FLUXMAG0", flux0)
metadata.add("FLUXMAG0ERR", flux0Err)
self.calib = afwImage.Calib(metadata)
self.assertEqual(isoDate, self.calib.getMidTime().toString())
self.assertAlmostEqual(self.calib.getMidTime().get(), 49743.3142245)
self.assertEqual(self.calib.getExptime(), exptime)
self.assertEqual(flux0, self.calib.getFluxMag0()[0])
self.assertEqual(flux0Err, self.calib.getFluxMag0()[1])
self.assertEqual(22.5, self.calib.getMagnitude(flux))
# Error just in flux
self.calib.setFluxMag0(flux0, 0)
self.assertAlmostEqual(
self.calib.getMagnitude(flux, fluxErr)[1],
2.5 / math.log(10) * fluxErr / flux)
#
# Check that we can clean up metadata
#
afwImage.stripCalibKeywords(metadata)
self.assertEqual(len(metadata.names()), 0)
def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId):
# Workaround until I can access the butler
instcalMap = self.map_instcal(dataId)
dqmaskMap = self.map_dqmask(dataId)
wtmapMap = self.map_wtmap(dataId)
rawMap = self.map_raw(dataId)
instcalType = getattr(afwImage,
instcalMap.getPythonType().split(".")[-1])
rawType = getattr(afwImage, rawMap.getPythonType().split(".")[-1])
dqmaskType = getattr(afwImage,
dqmaskMap.getPythonType().split(".")[-1])
wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1])
instcal = instcalType(instcalMap.getLocations()[0])
dqmask = dqmaskType(dqmaskMap.getLocations()[0])
wtmap = wtmapType(wtmapMap.getLocations()[0])
raw = rawType(rawMap.getLocations()[0])
mask = self.translate_dqmask(dqmask)
variance = self.translate_wtmap(wtmap)
mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask,
variance)
md = raw.getMetadata()
wcs = afwImage.makeWcs(md)
exp = afwImage.ExposureF(mi, wcs)
# Set the calib by hand; need to grab the zeroth extension
header = re.sub(r'[\[](\d+)[\]]$', "[0]", rawMap.getLocations()[0])
md0 = afwImage.readMetadata(header)
calib = afwImage.Calib()
calib.setExptime(md0.get("EXPTIME"))
calib.setFluxMag0(10**(0.4 * md0.get("MAGZERO")))
exp.setCalib(calib)
exp.setMetadata(md) # Do we need to remove WCS/calib info?
return exp
def _getCalibData(self, dataType, **id):
"""
Get the zero magnitude flux for the given data type and data id
It also returns the psf associated with the data id in case it's needed
later (e.g. when withSeeing=True)
"""
calexpType = self._getCalexpType(dataType)
if self.butler.datasetExists(calexpType, **id):
calexp = self.butler.get(calexpType, **id)
try:
calib = afwImage.Calib(calexp)
except NotImplementedError:
calib = calexp.getCalib()
else:
calexpType = dataType[:-4]
if calexpType[-1] == '_':
calexpType = calexpType[:-1]
#try:
calexp = self.butler.get(calexpType, **id)
calib = calexp.getCalib()
#except LsstCppException:
# fName = 'calexp-{0}-{1}-{2}.fits'.format(id['filter'], id['tract'], id['patch'])
# fName = os.path.join(self.dataRoot, 'deepCoadd', id['filter'], str(id['tract']), id['patch'], fName)
# calexp = afwImage.ExposureF(fName)
# calib = calexp.getCalib()
psf = calexp.getPsf()
fluxMag0, fluxMag0Err = calib.getFluxMag0()
if 'deepCoadd' in dataType:
exptime = self._getCoaddExpTime(calexp)
else:
exptime = calib.getExptime()
return fluxMag0, fluxMag0Err, psf, exptime
def getFluxFitParams(dataRef):
"""Retrieve the flux correction parameters determined by meas_mosaic
If the flux correction parameters do not exist, an exception will
be raised.
"""
calexp_md = dataRef.get("calexp_md", immediate=True)
hscRun = mosaicUtils.checkHscStack(calexp_md)
if hscRun is not None:
wcsHeader = dataRef.get("wcs_hsc_md", immediate=True)
ffpHeader = dataRef.get("fcr_hsc_md", immediate=True)
else:
wcsHeader = dataRef.get("wcs_md", immediate=True)
ffpHeader = dataRef.get("fcr_md", immediate=True)
calib = afwImage.Calib(ffpHeader)
ffp = FluxFitParams(ffpHeader)
wcs = getWcs(dataRef)
if hscRun is None:
detector = dataRef.get("camera")[dataRef.dataId["ccd"]]
nQuarter = detector.getOrientation().getNQuarter()
if nQuarter%4 != 0:
# Have to put this import here due to circular dependence in forcedPhotCcd.py in meas_base
import lsst.meas.astrom as measAstrom
dimensions = afwGeom.Extent2I(calexp_md.get("NAXIS1"), calexp_md.get("NAXIS2"))
wcs = measAstrom.rotateWcsPixelsBy90(wcs, nQuarter, dimensions)
return Struct(ffp=ffp, calib=calib, wcs=wcs)
def testVisitInfoFitsPersistence(self):
"""Test saving an exposure to FITS and reading it back in preserves (some) VisitInfo fields"""
exposureId = 5
exposureTime = 12.3
boresightRotAngle = 45.6 * afwGeom.degrees
weather = afwCoord.Weather(1.1, 2.2, 0.3)
visitInfo = afwImage.makeVisitInfo(
exposureId = exposureId,
exposureTime = exposureTime,
boresightRotAngle = boresightRotAngle,
weather = weather,
)
# Calib used to have exposure time and exposure date, so check for lack of interference
calib = afwImage.Calib(3.4)
exposureInfo = afwImage.ExposureInfo()
exposureInfo.setVisitInfo(visitInfo)
exposureInfo.setCalib(calib)
exposureInfo.setDetector(self.detector)
gFilter = afwImage.Filter("g")
exposureInfo.setFilter(gFilter)
maskedImage = afwImage.MaskedImageF(inFilePathSmall)
exposure = afwImage.ExposureF(maskedImage, exposureInfo)
with lsst.utils.tests.getTempFilePath(".fits") as tmpFile:
exposure.writeFits(tmpFile)
rtExposure = afwImage.ExposureF(tmpFile)
rtVisitInfo = rtExposure.getInfo().getVisitInfo()
self.assertEqual(rtVisitInfo.getWeather(), weather)
self.assertEqual(rtExposure.getCalib(), calib)
self.assertEqual(rtExposure.getFilter(), gFilter)
def testNullWarpExposure(self, interpLength=10):
"""Test that warpExposure maps an image onto itself.
Note:
- NO_DATA and off-CCD pixels must be ignored
- bad mask pixels get smeared out so we have to excluded all bad mask pixels
from the output image when comparing masks.
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw",
"SdssFilters.paf",
"tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
originalExposure = afwImage.ExposureF(originalExposurePath)
originalFilter = afwImage.Filter("i")
originalCalib = afwImage.Calib()
originalCalib.setFluxMag0(1.0e5, 1.0e3)
originalExposure.setFilter(originalFilter)
originalExposure.setCalib(originalCalib)
afwWarpedExposure = afwImage.ExposureF(originalExposure.getBBox(),
originalExposure.getWcs())
warpingControl = afwMath.WarpingControl("lanczos4", "", 0,
interpLength)
afwMath.warpExposure(afwWarpedExposure, originalExposure,
warpingControl)
if SAVE_FITS_FILES:
afwWarpedExposure.writeFits("afwWarpedExposureNull.fits")
self.assertEqual(afwWarpedExposure.getFilter().getName(),
originalFilter.getName())
self.assertEqual(afwWarpedExposure.getCalib().getFluxMag0(),
originalCalib.getFluxMag0())
afwWarpedMaskedImage = afwWarpedExposure.getMaskedImage()
afwWarpedMask = afwWarpedMaskedImage.getMask()
noDataBitMask = afwWarpedMask.getPlaneBitMask("NO_DATA")
afwWarpedMaskedImageArrSet = afwWarpedMaskedImage.getArrays()
afwWarpedMaskArr = afwWarpedMaskedImageArrSet[1]
# compare all non-DATA pixels of image and variance, but relax specs a bit
# because of minor noise introduced by bad pixels
noDataMaskArr = afwWarpedMaskArr & noDataBitMask
msg = "afw null-warped MaskedImage (all pixels, relaxed tolerance)"
self.assertMaskedImagesAlmostEqual(afwWarpedMaskedImage,
originalExposure.getMaskedImage(),
doMask=False,
skipMask=noDataMaskArr,
atol=1e-5,
msg=msg)
# compare good pixels (mask=0) of image, mask and variance using full
# tolerance
msg = "afw null-warped MaskedImage (good pixels, max tolerance)"
self.assertMaskedImagesAlmostEqual(afwWarpedMaskedImage,
originalExposure.getMaskedImage(),
skipMask=afwWarpedMask,
msg=msg)
def testCalibEquality(self):
self.assertEqual(self.calib, self.calib)
self.assertFalse(self.calib != self.calib)
calib2 = afwImage.Calib()
calib2.setExptime(12)
self.assertNotEqual(calib2, self.calib)
def testCalibEquality(self):
self.assertEqual(self.calib, self.calib)
# using assertFalse to directly test != operator
self.assertFalse(self.calib != self.calib)
calib2 = afwImage.Calib()
calib2.setFluxMag0(1200)
self.assertNotEqual(calib2, self.calib)
def __init__(self, *args, **kwargs):
"""Construct a ScaleZeroPointTask
"""
pipeBase.Task.__init__(self, *args, **kwargs)
#flux at mag=0 is 10^(zeroPoint/2.5) because m = -2.5*log10(F/F0)
fluxMag0 = 10**(0.4 * self.config.zeroPoint)
self._calib = afwImage.Calib()
self._calib.setFluxMag0(fluxMag0)
def _performTransform(self, transformClass, inCat, doExtend=True):
"""Operate on inCat with a transform of class transformClass"""
mapper = afwTable.SchemaMapper(inCat.schema)
config = SillyCentroidConfig()
transform = transformClass(config, self.pluginName, mapper)
outCat = afwTable.BaseCatalog(mapper.getOutputSchema())
if doExtend:
outCat.extend(inCat, mapper=mapper)
transform(inCat, outCat, makeWcs(), afwImage.Calib())
return outCat
def setUp(self):
'''Create two calibs, one with a valid zero-point and one without. Use these to create two UnitSystem
objects.
'''
self.mag2Flux = lambda m: 10.0**(m / 2.5)
self.flux2Mag = lambda f: 2.5 * np.log10(f)
calibNoZero = afwImage.Calib()
calibWithZero = afwImage.Calib()
calibWithZero.setFluxMag0(self.mag2Flux(25))
scale = 0.2 * afwGeom.arcseconds
wcs = afwGeom.makeSkyWcs(crpix=afwGeom.Point2D(),
crval=afwGeom.SpherePoint(
45.0, 45.0, afwGeom.degrees),
cdMatrix=afwGeom.makeCdMatrix(scale=scale))
self.unitNoZero = measModel.UnitSystem(wcs, calibNoZero)
self.unitWithZero = measModel.UnitSystem(wcs, calibWithZero)
def setUp(self):
'''Create two calibs, one with a valid zero-point and one without. Use these to create two UnitSystem
objects.
'''
self.mag2Flux = lambda m: 10.0**(m / 2.5)
self.flux2Mag = lambda f: 2.5 * np.log10(f)
calibNoZero = afwImage.Calib()
calibWithZero = afwImage.Calib()
calibWithZero.setFluxMag0(self.mag2Flux(25))
cdelt = (0.2 * afwGeom.arcseconds).asDegrees()
position = afwCoord.IcrsCoord(45.0 * afwGeom.degrees,
45.0 * afwGeom.degrees)
wcs = afwImage.makeWcs(position, afwGeom.Point2D(), cdelt, 0.0, 0.0,
cdelt)
self.unitNoZero = measModel.UnitSystem(wcs, calibNoZero)
self.unitWithZero = measModel.UnitSystem(wcs, calibWithZero)
def testSetMembers(self):
"""
Test that the MaskedImage and the WCS of an Exposure can be set.
"""
exposure = afwImage.ExposureF()
maskedImage = afwImage.MaskedImageF(inFilePathSmall)
exposure.setMaskedImage(maskedImage)
exposure.setWcs(self.wcs)
exposure.setDetector(self.detector)
exposure.setFilter(afwImage.Filter("g"))
self.assertEqual(exposure.getDetector().getName(),
self.detector.getName())
self.assertEqual(exposure.getDetector().getSerial(),
self.detector.getSerial())
self.assertEqual(exposure.getFilter().getName(), "g")
try:
exposure.getWcs()
except pexExcept.Exception as e:
print("caught expected exception (getWcs): %s" % e)
pass
#
# Test the Calib member. The Calib tests are in color.py, here we just check that it's in Exposure
#
calib = exposure.getCalib()
fluxMag0, FluxMag0Err = 1.1e12, 2.2e10
calib.setFluxMag0(fluxMag0, FluxMag0Err)
self.assertEqual(exposure.getCalib().getFluxMag0(),
(fluxMag0, FluxMag0Err))
#
# now check that we can set Calib
#
calib = afwImage.Calib()
fluxMag0_2, fluxMag0Err_2 = (511.1, 44.4)
calib.setFluxMag0(fluxMag0_2, fluxMag0Err_2)
exposure.setCalib(calib)
self.assertEqual(exposure.getCalib().getFluxMag0(),
(fluxMag0_2, fluxMag0Err_2))
#
# Psfs next
#
self.assertFalse(exposure.hasPsf())
exposure.setPsf(self.psf)
self.assertTrue(exposure.hasPsf())
exposure.setPsf(DummyPsf(1.0)) # we can reset the Psf
# Test that we can set the MaskedImage and WCS of an Exposure
# that already has both
self.exposureMiWcs.setMaskedImage(maskedImage)
exposure.setWcs(self.wcs)
def testApplyCppTransform(self):
"""Test that we can apply a simple C++ transform"""
inCat = self._generateCatalog()
sillyControl = testLib.SillyCentroidControl()
mapper = afwTable.SchemaMapper(inCat.schema)
sillyTransform = testLib.SillyTransform(sillyControl, self.pluginName, mapper)
outCat = afwTable.BaseCatalog(mapper.getOutputSchema())
outCat.extend(inCat, mapper=mapper)
self.assertEqual(len(inCat), len(outCat))
sillyTransform(inCat, outCat, makeWcs(), afwImage.Calib())
self._checkSillyOutputs(inCat, outCat)
def testCalibFromCalibs(self):
"""Test creating a Calib from an array of Calibs"""
mag0, mag0Sigma = 1.0, 0.01
calibs = afwImage.vectorCalib()
ncalib = 3
for i in range(ncalib):
calib = afwImage.Calib()
calib.setFluxMag0(mag0, mag0Sigma)
calibs.append(calib)
ocalib = afwImage.Calib(calibs)
# the following is surely incorrect; see DM-7619
self.assertEqual(ocalib.getFluxMag0(), (0.0, 0.0))
# Check that we can only merge Calibs with the same fluxMag0 values
calibs[0].setFluxMag0(1.001 * mag0, mag0Sigma)
with self.assertRaises(pexExcept.InvalidParameterError):
afwImage.Calib(calibs)
def scaleFromFluxMag0(self, fluxMag0):
"""Compute the scale for the specified fluxMag0
This is a wrapper around scaleFromCalib, which see for more information
@param[in] fluxMag0
@return a pipeBase.Struct containing:
- scale, as described in scaleFromCalib.
"""
calib = afwImage.Calib()
calib.setFluxMag0(fluxMag0)
return self.scaleFromCalib(calib)
def testBBox(self):
"""Test that the default bounding box includes all warped pixels
"""
kernelName = "lanczos2"
warper = afwMath.Warper(kernelName)
originalExposure, swarpedImage, swarpedWcs = self.getSwarpedImage(
kernelName=kernelName, useSubregion=True, useDeepCopy=False)
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw",
"SdssFilters.paf",
"tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
originalFilter = afwImage.Filter("i")
originalCalib = afwImage.Calib()
originalCalib.setFluxMag0(1.0e5, 1.0e3)
originalExposure.setFilter(originalFilter)
originalExposure.setCalib(originalCalib)
warpedExposure1 = warper.warpExposure(destWcs=swarpedWcs,
srcExposure=originalExposure)
# the default size must include all good pixels, so growing the bbox
# should not add any
warpedExposure2 = warper.warpExposure(destWcs=swarpedWcs,
srcExposure=originalExposure,
border=1)
# a bit of excess border is allowed, but surely not as much as 10 (in
# fact it is approx. 5)
warpedExposure3 = warper.warpExposure(destWcs=swarpedWcs,
srcExposure=originalExposure,
border=-10)
# assert that warpedExposure and warpedExposure2 have the same number of non-no_data pixels
# and that warpedExposure3 has fewer
noDataBitMask = afwImage.Mask.getPlaneBitMask("NO_DATA")
mask1Arr = warpedExposure1.getMaskedImage().getMask().getArray()
mask2Arr = warpedExposure2.getMaskedImage().getMask().getArray()
mask3Arr = warpedExposure3.getMaskedImage().getMask().getArray()
nGood1 = (mask1Arr & noDataBitMask == 0).sum()
nGood2 = (mask2Arr & noDataBitMask == 0).sum()
nGood3 = (mask3Arr & noDataBitMask == 0).sum()
self.assertEqual(nGood1, nGood2)
self.assertLess(nGood3, nGood1)
self.assertEqual(warpedExposure1.getFilter().getName(),
originalFilter.getName())
self.assertEqual(warpedExposure1.getCalib().getFluxMag0(),
originalCalib.getFluxMag0())
def converttsField(infile, filt, exptime=53.907456):
"""Extract data from a tsField table
@param[in] infile path to tsField FITS file
@param[in] filt index of filter in tsField FILTERS metadata entry
@param[in] exptime exposure time (sec)
@return a dict with the following entries:
- calib: an lsst.afw.Calib
- gain: gain as a float
- dateAvg: date of exposure at middle of exposure, as an lsst.daf.base.DateTime
- exptime: exposure time (sec)
- airmass: airmass
"""
ptr = pyfits.open(infile)
if ptr[0].header['NFIELDS'] != 1:
print("INVALID TSFIELD FILE")
sys.exit(1)
filts = ptr[0].header['FILTERS'].split()
idx = filts.index(filt)
mjdTaiStart = ptr[1].data.field('mjd')[0][
idx] # MJD(TAI) when row 0 was read
airmass = ptr[1].data.field("airmass")[0][idx]
gain = float(
ptr[1].data.field('gain')[0][idx]) # comes out as numpy.float32
aa = ptr[1].data.field('aa')[0][idx] # f0 = 10**(-0.4*aa) counts/second
aaErr = ptr[1].data.field('aaErr')[0][idx]
# Conversions
dateAvg = dafBase.DateTime(mjdTaiStart + 0.5 * exptime / 3600 / 24)
fluxMag0 = 10**(-0.4 * aa) * exptime
dfluxMag0 = fluxMag0 * 0.4 * np.log(10.0) * aaErr
calib = afwImage.Calib()
calib.setFluxMag0(fluxMag0, dfluxMag0)
ptr.close()
return TsField(
calib=calib,
gain=gain,
dateAvg=dateAvg,
exptime=exptime,
airmass=airmass,
)
def filterSources(sources, md, bright, flags=0x80):
if isinstance(sources, afwDet.PersistableSourceVector):
sources = sources.getSources()
calib = afwImage.Calib(md)
wcs = afwImage.makeWcs(md)
offsets = numpy.ndarray(len(sources))
outSources = afwDet.SourceSet()
for i, src in enumerate(sources):
try:
# psfMag = calib.getMagnitude(src.getPsfFlux())
apMag = calib.getMagnitude(src.getApFlux())
# modelMag = calib.getMagnitude(src.getModelFlux())
except:
continue
if bright is not None and apMag > bright:
continue
if src.getFlagForDetection() & flags == 0:
continue
# src.setPsfFlux(psfMag)
# src.setApFlux(apMag)
# src.setModelFlux(modelMag)
x, y = src.getXAstrom(), src.getYAstrom()
#if x < 5 or x > 2043 or y < 5 or y > 4172:
# src.setRa(4.32)
# src.setDec(0.0)
# continue
src.setRa(wcs.pixelToSky(x, y))
# x1, y1 = src.getXAstrom(), src.getYAstrom()
# sky = wcs.pixelToSky(x1, y1)
# pix = wcs.skyToPixel(sky)
# offsets[i] = math.hypot(pix.getX()-x1, pix.getY()-y1)
outSources.push_back(src)
# print offsets.mean(), offsets.std()
return outSources
def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId):
# Workaround until I can access the butler
instcalMap = self.map_instcal(dataId)
dqmaskMap = self.map_dqmask(dataId)
wtmapMap = self.map_wtmap(dataId)
instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1])
dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1])
wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1])
instcal = instcalType(instcalMap.getLocationsWithRoot()[0])
dqmask = dqmaskType(dqmaskMap.getLocationsWithRoot()[0])
wtmap = wtmapType(wtmapMap.getLocationsWithRoot()[0])
mask = self.translate_dqmask(dqmask)
variance = self.translate_wtmap(wtmap)
mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance)
md = instcal.getMetadata()
wcs = afwImage.makeWcs(md, True)
exp = afwImage.ExposureF(mi, wcs)
# Set the calib by hand; need to grab the zeroth extension
header = re.sub(r'[\[](\d+)[\]]$', "[0]", instcalMap.getLocationsWithRoot()[0])
md0 = afwImage.readMetadata(header)
calib = afwImage.Calib()
calib.setFluxMag0(10**(0.4 * md0.get("MAGZERO")))
exp.setCalib(calib)
exposureId = self._computeCcdExposureId(dataId)
visitInfo = self.makeRawVisitInfo(md=md0, exposureId=exposureId)
exp.getInfo().setVisitInfo(visitInfo)
for kw in ('LTV1', 'LTV2'):
md.remove(kw)
# Remove TPV keywords from the metadata
for kw in md.paramNames():
if re.match(r'PV\d_\d', kw):
md.remove(kw)
exp.setMetadata(md)
return exp
def writeFcr(self, butler, ccdIdList, ccdSet, filterName,
fexp, fchip, ffpSet):
for ccdId in ccdIdList:
iexp, ichip = self.decodeCcdExposureId(ccdId)
if not ichip in ccdSet.keys():
continue
x0 = 0.0
y0 = 0.0
newP = measMosaic.convertFluxFitParams(measMosaic.FluxFitParams(ffpSet[iexp]),
ccdSet[ichip], x0, y0)
metadata = measMosaic.metadataFromFluxFitParams(newP)
exp = afwImage.ExposureI(0,0)
exp.getMetadata().combine(metadata)
scale = fexp[iexp] * fchip[ichip]
calib = afwImage.Calib()
calib.setFluxMag0(1.0/scale)
exp.setCalib(calib)
exp.setFilter(afwImage.Filter(filterName))
try:
butler.put(exp, 'fcr', {'visit': iexp, 'ccd': ichip})
except Exception, e:
print "failed to write something: %s" % (e)
def extract_photometry(butler,
dataId,
forced_dataset,
filt,
object_id,
names_to_copy,
phot_type='base_PsfFlux'):
# Can grab filter, mjd from 'calexp_md' call on visit
md = butler.get('calexp_md', dataId=dataId, immediate=True)
mjd = md.get('MJD-OBS')
# filt = md.get('FILTER') # But that's not being set right now so we'll keep using f
this_measurement = butler.get(forced_dataset, dataId).asAstropy()
source_row, = np.where(this_measurement['objectId'] == object_id)
if len(source_row) != 1:
return None
# 'this_measurement' is a table, but we're only extracting the first entry from each column
new_row = {n: this_measurement[n][source_row] for n in names_to_copy}
# new_row['filter'] = dataId['filter']
new_row['filter'] = filt
new_row['mjd'] = mjd
# Calibrate to magnitudes
# The calibration information for the calexp
# should still apply to the difference image
calib = afwImage.Calib(md)
with afwImageUtils.CalibNoThrow():
new_row['%s_mag' % phot_type], new_row['%s_magSigma' % phot_type] = \
calib.getMagnitude(new_row['%s_flux' % phot_type],
new_row['%s_fluxSigma' % phot_type])
flux_mag_0, flux_magSigma_0 = calib.getFluxMag0()
flux_mag_25 = 10**(-0.4 * 25) * flux_mag_0
flux_norm = 1 / flux_mag_25
new_row['%s_flux_zp25' % phot_type] = \
flux_norm * new_row['%s_flux' % phot_type]
new_row['%s_fluxSigma_zp25' % phot_type] = \
flux_norm * new_row['%s_fluxSigma' % phot_type]
return new_row
class ExposureTestCase(unittest.TestCase):
"""
A test case for the Exposure Class
"""
def setUp(self):
maskedImage = afwImage.MaskedImageF(inFilePathSmall)
maskedImageMD = afwImage.readMetadata(inFilePathSmall)
self.smallExposure = afwImage.ExposureF(inFilePathSmall)
self.width = maskedImage.getWidth()
self.height = maskedImage.getHeight()
self.wcs = afwImage.makeWcs(maskedImageMD)
self.psf = DummyPsf(2.0)
self.exposureBlank = afwImage.ExposureF()
self.exposureMiOnly = afwImage.makeExposure(maskedImage)
self.exposureMiWcs = afwImage.makeExposure(maskedImage, self.wcs)
self.exposureCrWcs = afwImage.ExposureF(
100, 100, self.wcs) # n.b. the (100, 100, ...) form
self.exposureCrOnly = afwImage.ExposureF(afwGeom.ExtentI(
100, 100)) # test with ExtentI(100, 100) too
afwImage.Filter.reset()
afwImage.FilterProperty.reset()
filterPolicy = pexPolicy.Policy()
filterPolicy.add("lambdaEff", 470.0)
afwImage.Filter.define(afwImage.FilterProperty("g", filterPolicy))
def tearDown(self):
del self.smallExposure
del self.wcs
del self.psf
del self.exposureBlank
del self.exposureMiOnly
del self.exposureMiWcs
del self.exposureCrWcs
del self.exposureCrOnly
def testGetMaskedImage(self):
"""
Test to ensure a MaskedImage can be obtained from each
Exposure. An Exposure is required to have a MaskedImage,
therefore each of the Exposures should return a MaskedImage.
MaskedImage class should throw appropriate
lsst::pex::exceptions::NotFound if the MaskedImage can not be
obtained.
"""
maskedImageBlank = self.exposureBlank.getMaskedImage()
blankWidth = maskedImageBlank.getWidth()
blankHeight = maskedImageBlank.getHeight()
if blankWidth != blankHeight != 0:
self.fail("%s = %s != 0" % (blankWidth, blankHeight))
maskedImageMiOnly = self.exposureMiOnly.getMaskedImage()
miOnlyWidth = maskedImageMiOnly.getWidth()
miOnlyHeight = maskedImageMiOnly.getHeight()
self.assertAlmostEqual(miOnlyWidth, self.width)
self.assertAlmostEqual(miOnlyHeight, self.height)
# NOTE: Unittests for Exposures created from a MaskedImage and
# a WCS object are incomplete. No way to test the validity of
# the WCS being copied/created.
maskedImageMiWcs = self.exposureMiWcs.getMaskedImage()
miWcsWidth = maskedImageMiWcs.getWidth()
miWcsHeight = maskedImageMiWcs.getHeight()
self.assertAlmostEqual(miWcsWidth, self.width)
self.assertAlmostEqual(miWcsHeight, self.height)
maskedImageCrWcs = self.exposureCrWcs.getMaskedImage()
crWcsWidth = maskedImageCrWcs.getWidth()
crWcsHeight = maskedImageCrWcs.getHeight()
if crWcsWidth != crWcsHeight != 0:
self.fail("%s != %s != 0" % (crWcsWidth, crWcsHeight))
maskedImageCrOnly = self.exposureCrOnly.getMaskedImage()
crOnlyWidth = maskedImageCrOnly.getWidth()
crOnlyHeight = maskedImageCrOnly.getHeight()
if crOnlyWidth != crOnlyHeight != 0:
self.fail("%s != %s != 0" % (crOnlyWidth, crOnlyHeight))
# Check Exposure.getWidth() returns the MaskedImage's width
self.assertEqual(crOnlyWidth, self.exposureCrOnly.getWidth())
self.assertEqual(crOnlyHeight, self.exposureCrOnly.getHeight())
def testGetWcs(self):
"""
Test if a WCS can be obtained from each Exposure created with
a WCS.
Test that appropriate exceptions are thrown if a WCS is
requested from an Exposure that was not created with a WCS.
Python turns the pex::exceptions in the Exposure and
MaskedImage classes into IndexErrors.
The exposureBlank, exposureMiOnly, and exposureCrOnly
Exposures should throw a lsst::pex::exceptions::NotFound.
"""
self.assertTrue(not self.exposureBlank.getWcs())
self.assertTrue(not self.exposureMiOnly.getWcs())
# These two should pass
self.exposureMiWcs.getWcs()
self.exposureCrWcs.getWcs()
self.assertTrue(not self.exposureCrOnly.getWcs())
def testSetMembers(self):
"""
Test that the MaskedImage and the WCS of an Exposure can be set.
"""
exposure = afwImage.ExposureF()
maskedImage = afwImage.MaskedImageF(inFilePathSmall)
exposure.setMaskedImage(maskedImage)
exposure.setWcs(self.wcs)
exposure.setDetector(cameraGeom.Detector(cameraGeom.Id(666)))
exposure.setFilter(afwImage.Filter("g"))
self.assertEquals(exposure.getDetector().getId().getSerial(), 666)
self.assertEquals(exposure.getFilter().getName(), "g")
try:
exposure.getWcs()
except pexExcept.LsstCppException, e:
print "caught expected exception (getWcs): %s" % e
pass
#
# Test the Calib member. The Calib tests are in color.py, here we just check that it's in Exposure
#
calib = exposure.getCalib()
dt = 10
calib.setExptime(dt)
self.assertEqual(exposure.getCalib().getExptime(), dt)
#
# now check that we can set Calib
#
calib = afwImage.Calib()
dt = 666
calib.setExptime(dt)
exposure.setCalib(calib)
self.assertEqual(exposure.getCalib().getExptime(), dt)
#
# Psfs next
#
w, h = 11, 11
self.assertFalse(exposure.hasPsf())
exposure.setPsf(self.psf)
self.assertTrue(exposure.hasPsf())
exposure.setPsf(DummyPsf(1.0)) # we can reset the Psf
# Test that we can set the MaskedImage and WCS of an Exposure
# that already has both
self.exposureMiWcs.setMaskedImage(maskedImage)
exposure.setWcs(self.wcs)
def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius):
"""Load data from specific visit. Match with reference.
Parameters
----------
repo : string
The repository. This is generally the directory on disk
that contains the repository and mapper.
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to
reference. The `calexp` cpixel image is needed for the photometric
calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
Returns
-------
afw.table.GroupView
An object of matched catalog.
"""
# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
butler = dafPersist.Butler(repo)
dataset = 'src'
# 2016-02-08 MWV:
# I feel like I could be doing something more efficient with
# something along the lines of the following:
# dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds]
ccdKeyName = getCcdKeyName(dataIds[0])
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = SchemaMapper(schema)
mapper.addMinimalSchema(schema)
mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR'))
mapper.addOutputField(Field[float]('base_PsfFlux_mag',
'PSF magnitude'))
mapper.addOutputField(Field[float]('base_PsfFlux_magerr',
'PSF magnitude uncertainty'))
newSchema = mapper.getOutputSchema()
# Create an object that matches multiple catalogs with same schema
mmatch = MultiMatch(newSchema,
dataIdFormat={
'visit': np.int32,
ccdKeyName: np.int32
},
radius=matchRadius,
RecordClass=SimpleRecord)
# create the new extented source catalog
srcVis = SourceCatalog(newSchema)
for vId in dataIds:
try:
calexpMetadata = butler.get("calexp_md", vId, immediate=True)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping %s " % repr(vId))
continue
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping %s " % repr(vId))
continue
calib = afwImage.Calib(calexpMetadata)
oldSrc = butler.get('src', vId, immediate=True)
print(
len(oldSrc), "sources in ccd %s visit %s" %
(vId[ccdKeyName], vId["visit"]))
# create temporary catalog
tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
tmpCat.extend(oldSrc, mapper=mapper)
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \
/ tmpCat['base_PsfFlux_fluxSigma']
with afwImageUtils.CalibNoThrow():
_ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'],
tmpCat['base_PsfFlux_fluxSigma'])
tmpCat['base_PsfFlux_mag'][:] = _[0]
tmpCat['base_PsfFlux_magerr'][:] = _[1]
srcVis.extend(tmpCat, False)
mmatch.add(catalog=tmpCat, dataId=vId)
# Complete the match, returning a catalog that includes
# all matched sources with object IDs that can be used to group them.
matchCat = mmatch.finish()
# Create a mapping object that allows the matches to be manipulated
# as a mapping of object ID to catalog of sources.
allMatches = GroupView.build(matchCat)
return allMatches
def writeFcr(self, dataRefList):
self.log.info("Write Fcr ...")
M_LN10 = math.log(10)
dmag = list()
for m in self.matchVec:
if (m.good == True and m.mag != -9999 and m.jstar != -1
and m.mag0 != -9999 and m.mag_cat != -9999):
mag = m.mag
mag_cat = m.mag_cat
exp_cor = -2.5 * math.log10(self.fexp[m.iexp])
chip_cor = -2.5 * math.log10(self.fchip[m.ichip])
gain_cor = self.ffpSet[m.iexp].eval(m.u, m.v)
mag_cor = mag + exp_cor + chip_cor + gain_cor
dmag.append(mag_cor - mag_cat)
std, mean, n = mosaicUtils.clippedStd(numpy.array(dmag), 2.1)
for dataRef in dataRefList:
iexp = dataRef.dataId["visit"]
ichip = dataRef.dataId["ccd"]
try:
x0 = self.coeffSet[iexp].x0
y0 = self.coeffSet[iexp].y0
except:
x0 = 0.0
y0 = 0.0
newP = measMosaic.convertFluxFitParams(
measMosaic.FluxFitParams(self.ffpSet[iexp]),
self.ccdSet[ichip], x0, y0)
metadata = measMosaic.metadataFromFluxFitParams(newP)
exp = afwImage.ExposureI(0, 0)
exp.getMetadata().combine(metadata)
scale = self.fexp[iexp] * self.fchip[ichip]
calib = afwImage.Calib()
calib.setFluxMag0(1.0 / scale, 1.0 / scale * std * M_LN10 * 0.4)
exp.setCalib(calib)
try:
dataRef.put(exp, "fcr")
except Exception as e:
print("failed to write fcr: %s" % (e))
# Write the flux fit (including Jacobian) as a PhotoCalib for
# future compatibility with jointcal. This is redundant with
# the above, and should eventually supercede it.
detector = dataRef.get("camera")[dataRef.dataId["ccd"]]
nQuarter = detector.getOrientation().getNQuarter()
bbox = detector.getBBox()
try:
# Reading the Wcs we just wrote obviously isn't efficient, but
# it should be in the noise of the overall runtime and it
# saves us from doing a bunch of refactoring in a fragile
# package with no tests.
wcs = dataRef.get("jointcal_wcs")
except Exception as e:
print("failed to read Wcs for PhotoCalib: %s" % (e))
continue
instFluxMag0, instFluxMag0Err = calib.getFluxMag0()
bf = measMosaic.FluxFitBoundedField(bbox,
newP,
wcs,
zeroPoint=instFluxMag0,
nQuarter=nQuarter)
photoCalib = afwImage.PhotoCalib(1.0 / instFluxMag0,
instFluxMag0Err / instFluxMag0**2,
bf,
isConstant=False)
dataRef.put(photoCalib, "jointcal_photoCalib")
def makeCalib(self, zeroPoint):
calib = afwImage.Calib()
fluxMag0 = 10**(0.4 * zeroPoint)
calib.setFluxMag0(fluxMag0, 1.0)
return calib
