def setUp(self):
xy0 = Point2I(12345, 67890) # xy0 for image
dims = Extent2I(2345, 2345) # Dimensions of image
box = Box2I(xy0, dims) # Bounding box of image
sigma = 3.21 # PSF sigma
buffer = 4.0 # Buffer for star centers around edge
nSigmaForKernel = 5.0 # Number of PSF sigmas for kernel
sky = 12345.6 # Sky level
numStars = 100 # Number of stars
noise = np.sqrt(sky) * np.pi * sigma**2 # Poisson noise per PSF
faint = 1.0 * noise # Faintest level for star fluxes
bright = 100.0 * noise # Brightest level for star fluxes
starBox = Box2I(box) # Area on image in which we can put star centers
starBox.grow(-int(buffer * sigma))
scale = 1.0e-5 * degrees # Pixel scale
np.random.seed(12345)
stars = [(xx, yy, ff, sigma) for xx, yy, ff in zip(
np.random.uniform(starBox.getMinX(), starBox.getMaxX(), numStars),
np.random.uniform(starBox.getMinY(), starBox.getMaxY(), numStars),
np.linspace(faint, bright, numStars))]
self.exposure = plantSources(box, 2 * int(nSigmaForKernel * sigma) + 1,
sky, stars, True)
self.exposure.setWcs(
makeSkyWcs(crpix=Point2D(0, 0),
crval=SpherePoint(0, 0, degrees),
cdMatrix=makeCdMatrix(scale=scale)))
# Make a large area of extra background; we should be robust against it
# Unfortunately, some tuning is required here to get something challenging but not impossible:
# * A very large box will cause failures because the "extra" and the "normal" are reversed.
# * A small box will not be challenging because it's simple to clip out.
# * A large value will cause failures because it produces large edges in background-subtrction that
# broaden flux distributions.
# * A small value will not be challenging because it has little effect.
extraBox = Box2I(xy0 + Extent2I(345, 456),
Extent2I(1234, 1234)) # Box for extra background
extraValue = 0.5 * noise # Extra background value to add in
self.exposure.image[extraBox, PARENT] += extraValue
self.config = DynamicDetectionTask.ConfigClass()
self.config.skyObjects.nSources = 300
self.config.reEstimateBackground = False
self.config.doTempWideBackground = True
self.config.thresholdType = "pixel_stdev"
# Relative tolerance for tweak factor
# Not sure why this isn't smaller; maybe due to use of Poisson instead of Gaussian noise?
self.rtol = 0.1
def run(md, **kwds2):
kwds = extractCtorArgs(md)
kwds.update(kwds2)
gridShape = Extent2I(20, 20)
approx = SipApproximation(gridShape=gridShape, **kwds)
diffs = approx.computeMaxDeviation()
self.assertLess(diffs[0], 0.1)
self.assertLess(diffs[1], 0.1)
def run(md, **kwds2):
kwds = extractCtorArgs(md)
kwds['order'] = max(md["A_ORDER"], md["B_ORDER"], md["AP_ORDER"],
md["BP_ORDER"])
kwds.update(kwds2)
gridShape = Extent2I(10, 10)
approx = SipApproximation(gridShape=gridShape, **kwds)
diffs = approx.computeMaxDeviation()
self.compareSolution(md, approx)
self.assertLess(diffs[0], 1E-10)
self.assertLess(diffs[1], 1E-10)
def readFits(cls, path):
"""Read an external detrended image and create an LSST Exposure object
from it.
Any significant processing (e.g. pixel interpolation) should probably
be done in a ExternalIsrTask instead of here so it can be done once and
saved, instead of being done every time the image is loaded.
THIS METHOD IS INCOMPLETE; IT MUST BE MODIFIED ACCORDING TO THE
FORMAT OF THE DATA BEING LOADED.
"""
directory, filename = os.path.split(path)
match = cls.EXTERNAL_REGEX.match(filename)
camera = cls.getCameraFromVisit(match.group("visit"))
# Customize the code below based on the camera determined above.
# To support more than one camera it may be useful to delegate
# to other methods that are specific to certain cameras.
# Read the actual image in from the given path using e.g. astropy,
# and use it to fill in various arrays below.
bbox = Box2I(Point2I(0, 0), Extent2I(..., ...)) # width, height
result = ExposureF(bbox)
# main image, as a [y, x] numpy.float32 array
result.image.array = ...
# variance image, as a [y, x] numpy.float32 array
result.variance.array = ...
# This example includes masking NaN pixels as NO_DATA and pixels above
# 1E5 counts as SAT. External information about where bad pixels
# should be preferred when available, and obviously that saturation
# threshold is just an example (saturation should actually be
# determined before flat-fielding, of course).
# Interpolating these bad pixels is handled by ExternalIsrTask.
noDataBitMask = result.mask.getPlaneBitMask("NO_DATA")
satBitMask = result.mask.getPlaneBitMask("SAT")
result.mask.array |= noDataBitMask * np.isnan(result.image.array)
result.mask.array |= satBitMask * (result.image.array > 1E5)
# If you have a better guess at the PSF, we can find a way to use it.
# But it'd be a good idea to at least put this in with a guess at the
# seeing (RMS in pixels).
result.setPsf(SingleGaussianPsf(seeingRMS))
# Add a guess for the WCS, in this case assuming it's in the FITS
# header of the first HDU. Need to have something here, even if it
# isn't very good (e.g. whatever comes from the telescope).
metadata = readMetadata(filename)
wcs = SkyWcs(metadata)
result.setWcs(wcs)
return result
def extractCtorArgs(md):
wcs = makeSkyWcs(makePropertyListFromDict(md))
kwds = {
"pixelToIwc": getPixelToIntermediateWorldCoords(wcs),
"bbox":
Box2D(Box2I(Point2I(0, 0), Extent2I(md["NAXES1"], md["NAXES2"]))),
"crpix":
Point2D(md["CRPIX1"] - 1.0,
md["CRPIX2"] - 1.0), # -1 for LSST vs. FITS conventions
"cd": np.array([[md["CD1_1"], md["CD1_2"]], [md["CD2_1"],
md["CD2_2"]]]),
}
return kwds
def testBasics(self):
"""Test detection and measurement on simple synthesized data
"""
bbox = Box2I(Point2I(256, 100), Extent2I(128, 127))
minCounts = 5000
maxCounts = 50000
starSigma = 1.5
numX = 5
numY = 5
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11 # kernel width
sky = 2000
# create an exposure without a Wcs; add the Wcs later
exposure = plantSources(bbox=bbox,
kwid=kwid,
sky=sky,
coordList=coordList,
addPoissonNoise=True)
schema = SourceTable.makeMinimalSchema()
config = DetectAndMeasureTask.ConfigClass()
task = DetectAndMeasureTask(config=config, schema=schema)
butler = Butler(root=InputDir)
dataRef = butler.dataRef("calexp", dataId=dict(visit=1))
wcs = dataRef.get("raw").getWcs()
exposure.setWcs(wcs)
exposureIdInfo = dataRef.get("expIdInfo")
taskRes = task.run(exposure=exposure, exposureIdInfo=exposureIdInfo)
self.assertEqual(len(taskRes.sourceCat), numX * numY)
schema = taskRes.sourceCat.schema
centroidFlagKey = schema.find("slot_Centroid_flag").getKey()
parentKey = schema.find("parent").getKey()
psfFluxFlagKey = schema.find("slot_PsfFlux_flag").getKey()
psfFluxKey = schema.find("slot_PsfFlux_flux").getKey()
for src in taskRes.sourceCat:
self.assertFalse(src.get(centroidFlagKey)) # centroid found
self.assertEqual(src.get(parentKey), 0) # not debelended
self.assertFalse(src.get(psfFluxFlagKey)) # flux measured
self.assertGreater(src.get(psfFluxKey), 4000) # flux sane
def testBasics(self):
"""Test construction of a discrete sky map
"""
butler = Butler(inputs=self.inPath,
outputs={
'root': self.outPath,
'mode': 'rw'
})
coordList = [] # list of sky coords of all corners of all calexp
for dataId in (
dict(visit=1, filter="g"),
dict(visit=2, filter="g"),
dict(visit=3, filter="r"),
):
# TODO: pybind11 remove `immediate=True` once DM-9112 is resolved
rawImage = butler.get("raw", dataId, immediate=True)
# fake calexp by simply copying raw data; the task just cares about its bounding box
# (which is slightly larger for raw, but that doesn't matter for this test)
calexp = rawImage
butler.put(calexp, "calexp", dataId)
calexpWcs = calexp.getWcs()
calexpBoxD = Box2D(calexp.getBBox())
coordList += [
calexpWcs.pixelToSky(corner)
for corner in calexpBoxD.getCorners()
]
# use the calexp to make a sky map
retVal = MakeDiscreteSkyMapTask.parseAndRun(
args=[self.inPath, "--output", self.outPath, "--id", "filter=g^r"],
config=self.config,
doReturnResults=True,
)
self.assertEqual(len(retVal.resultList), 1)
skyMap = retVal.resultList[0].result.skyMap
self.assertEqual(type(skyMap), DiscreteSkyMap)
self.assertEqual(len(skyMap), 1)
tractInfo = skyMap[0]
self.assertEqual(tractInfo.getId(), 0)
self.assertEqual(tractInfo.getNumPatches(), Extent2I(3, 3))
tractWcs = tractInfo.getWcs()
tractBoxD = Box2D(tractInfo.getBBox())
for skyPoint in coordList:
self.assertTrue(tractBoxD.contains(tractWcs.skyToPixel(skyPoint)))
def setUp(self):
self.crpix = Point2D(100, 100)
self.crvalList = [
SpherePoint(0 * degrees, 45 * degrees),
SpherePoint(0.00001 * degrees, 45 * degrees),
SpherePoint(359.99999 * degrees, 45 * degrees),
SpherePoint(30 * degrees, 89.99999 * degrees),
SpherePoint(30 * degrees, -89.99999 * degrees),
]
self.orientationList = [
0 * degrees,
0.00001 * degrees,
-0.00001 * degrees,
-45 * degrees,
90 * degrees,
]
self.scale = 1.0 * arcseconds
self.tinyPixels = 1.0e-10
self.tinyAngle = 1.0e-10 * radians
self.bbox = Box2I(Point2I(-1000, -1000),
Extent2I(2000, 2000)) # arbitrary but reasonable
def run(md):
kwds = extractCtorArgs(md)
order = max(md["A_ORDER"], md["B_ORDER"], md["AP_ORDER"],
md["BP_ORDER"])
gridShape = Extent2I(5, 5)
a = np.zeros((order + 1, order + 1), dtype=float)
b = np.zeros((order + 1, order + 1), dtype=float)
ap = np.zeros((order + 1, order + 1), dtype=float)
bp = np.zeros((order + 1, order + 1), dtype=float)
for p, q in packedRange(order):
a[p, q] = md.get("A_%d_%d" % (p, q), 0.0)
b[p, q] = md.get("B_%d_%d" % (p, q), 0.0)
ap[p, q] = md.get("AP_%d_%d" % (p, q), 0.0)
bp[p, q] = md.get("BP_%d_%d" % (p, q), 0.0)
approx = SipApproximation(a=a,
b=b,
ap=ap,
bp=bp,
gridShape=gridShape,
**kwds)
self.compareSolution(md, approx)
diffs = approx.computeMaxDeviation()
self.assertLess(diffs[0], 1E-10)
self.assertLess(diffs[1], 1E-10)
bbox = kwds["bbox"]
pix1 = [
Point2D(x, y)
for x in np.linspace(bbox.getMinX(), bbox.getMaxX(),
gridShape.getX())
for y in np.linspace(bbox.getMinY(), bbox.getMaxY(),
gridShape.getY())
]
iwc1a = kwds["pixelToIwc"].applyForward(pix1)
pix2a = kwds["pixelToIwc"].applyInverse(iwc1a)
iwc1b = approx.applyForward(pix1)
assert_allclose(iwc1a, iwc1b, rtol=1E-9, atol=1E-12)
pix2b = approx.applyInverse(iwc1a)
assert_allclose(pix2a, pix2b, rtol=1E-9, atol=1E-12)
def checkTanWcs(self, crval, orientation, flipX):
"""Construct a pure TAN SkyWcs and check that it operates as specified
Parameters
----------
crval : `lsst.afw.geom.SpherePoint`
Desired reference sky position.
Must not be at either pole.
orientation : `lsst.afw.geom.Angle`
Position angle of focal plane +Y, measured from N through E.
At 0 degrees, +Y is along N and +X is along E/W if flipX false/true
At 90 degrees, +Y is along E and +X is along S/N if flipX false/true
flipX : `bool`
Flip x axis? See `orientation` for details.
Returns
-------
wcs : `lsst.afw.geom.SkyWcs`
The generated pure TAN SkyWcs
"""
cdMatrix = makeCdMatrix(scale=self.scale,
orientation=orientation,
flipX=flipX)
wcs = makeSkyWcs(crpix=self.crpix, crval=crval, cdMatrix=cdMatrix)
self.checkPersistence(wcs, bbox=self.bbox)
self.checkMakeFlippedWcs(wcs)
self.assertTrue(wcs.isFits)
self.assertEqual(wcs.isFlipped, bool(flipX))
xoffAng = 0 * degrees if flipX else 180 * degrees
pixelList = [
Point2D(self.crpix[0], self.crpix[1]),
Point2D(self.crpix[0] + 1, self.crpix[1]),
Point2D(self.crpix[0], self.crpix[1] + 1),
]
skyList = wcs.pixelToSky(pixelList)
# check pixels to sky
predSkyList = [
crval,
crval.offset(xoffAng - orientation, self.scale),
crval.offset(90 * degrees - orientation, self.scale),
]
self.assertSpherePointListsAlmostEqual(predSkyList, skyList)
self.assertSpherePointListsAlmostEqual(predSkyList,
wcs.pixelToSky(pixelList))
for pixel, predSky in zip(pixelList, predSkyList):
self.assertSpherePointsAlmostEqual(predSky, wcs.pixelToSky(pixel))
self.assertSpherePointsAlmostEqual(
predSky, wcs.pixelToSky(pixel[0], pixel[1]))
# check sky to pixels
self.assertPairListsAlmostEqual(pixelList, wcs.skyToPixel(skyList))
self.assertPairListsAlmostEqual(pixelList, wcs.skyToPixel(skyList))
for pixel, sky in zip(pixelList, skyList):
self.assertPairsAlmostEqual(pixel, wcs.skyToPixel(sky))
# self.assertPairsAlmostEqual(pixel, wcs.skyToPixel(sky[0], sky[1]))
# check CRVAL round trip
self.assertSpherePointsAlmostEqual(wcs.getSkyOrigin(),
crval,
maxSep=self.tinyAngle)
crpix = wcs.getPixelOrigin()
self.assertPairsAlmostEqual(crpix, self.crpix, maxDiff=self.tinyPixels)
self.assertFloatsAlmostEqual(wcs.getCdMatrix(), cdMatrix)
pixelScale = wcs.getPixelScale()
self.assertAnglesAlmostEqual(self.scale,
pixelScale,
maxDiff=self.tinyAngle)
pixelScale = wcs.getPixelScale(self.crpix)
self.assertAnglesAlmostEqual(self.scale,
pixelScale,
maxDiff=self.tinyAngle)
# check that getFitsMetadata can operate at high precision
# and has axis order RA, Dec
fitsMetadata = wcs.getFitsMetadata(True)
self.assertEqual(fitsMetadata.get("CTYPE1")[0:4], "RA--")
self.assertEqual(fitsMetadata.get("CTYPE2")[0:4], "DEC-")
# Compute a WCS with the pixel origin shifted by an arbitrary amount
# The resulting sky origin should not change
offset = Extent2D(500, -322) # arbitrary
shiftedWcs = wcs.copyAtShiftedPixelOrigin(offset)
self.assertTrue(shiftedWcs.isFits)
predShiftedPixelOrigin = self.crpix + offset
self.assertPairsAlmostEqual(shiftedWcs.getPixelOrigin(),
predShiftedPixelOrigin,
maxDiff=self.tinyPixels)
self.assertSpherePointsAlmostEqual(shiftedWcs.getSkyOrigin(),
crval,
maxSep=self.tinyAngle)
shiftedPixelList = [p + offset for p in pixelList]
shiftedSkyList = shiftedWcs.pixelToSky(shiftedPixelList)
self.assertSpherePointListsAlmostEqual(skyList,
shiftedSkyList,
maxSep=self.tinyAngle)
# Check that the shifted WCS can be round tripped as FITS metadata
shiftedMetadata = shiftedWcs.getFitsMetadata(precise=True)
shiftedWcsCopy = makeSkyWcs(shiftedMetadata)
shiftedBBox = Box2D(predShiftedPixelOrigin,
predShiftedPixelOrigin + Extent2I(2000, 2000))
self.assertWcsAlmostEqualOverBBox(shiftedWcs, shiftedWcsCopy,
shiftedBBox)
wcsCopy = SkyWcs.readString(wcs.writeString())
self.assertTrue(wcsCopy.isFits)
return wcs
def setUp(self):
product_dir = getPackageDir('obs_test')
data_dir = os.path.join(product_dir, 'data', 'input')
butler = lsst.daf.persistence.Butler(root=data_dir)
mapper = lsst.obs.test.TestMapper(root=data_dir)
dataIds = {
'raw': {
'visit': 1,
'filter': 'g'
},
'bias': {
'visit': 1
},
'flat': {
'visit': 1
},
'dark': unittest.SkipTest
}
self.setUp_tests(butler, mapper, dataIds)
ccdExposureId_bits = 41
exposureIds = {'raw': 1, 'bias': 1, 'flat': 1}
filters = {'raw': 'g', 'bias': '_unknown_', 'flat': 'g'}
exptimes = {'raw': 15.0, 'bias': 0.0, 'flat': 0.0}
detectorIds = {'raw': 0, 'bias': 0, 'flat': 0}
detector_names = {'raw': '0', 'bias': '0', 'flat': '0'}
detector_serials = {
'raw': '0000011',
'bias': '0000011',
'flat': '0000011'
}
dimensions = {
'raw': Extent2I(1026, 2000),
'bias': Extent2I(1018, 2000),
'flat': Extent2I(1018, 2000)
}
sky_origin = (79.24522, -9.702295)
raw_subsets = (({
'level': 'sensor',
'filter': 'g'
}, 2), ({
'level': 'sensor',
'visit': 1
}, 1), ({
'level': 'filter',
'visit': 1
}, 1), ({
'level': 'visit',
'filter': 'g'
}, 2))
linearizer_type = unittest.SkipTest
self.setUp_butler_get(ccdExposureId_bits=ccdExposureId_bits,
exposureIds=exposureIds,
filters=filters,
exptimes=exptimes,
detectorIds=detectorIds,
detector_names=detector_names,
detector_serials=detector_serials,
dimensions=dimensions,
sky_origin=sky_origin,
raw_subsets=raw_subsets,
linearizer_type=linearizer_type)
path_to_raw = os.path.join(data_dir, "raw", "raw_v1_fg.fits.gz")
keys = set(('filter', 'name', 'patch', 'tract', 'visit', 'pixel_id',
'subfilter', 'description', 'fgcmcycle'))
query_format = ["visit", "filter"]
queryMetadata = (
({
'visit': 1
}, [(1, 'g')]),
({
'visit': 2
}, [(2, 'g')]),
({
'visit': 3
}, [(3, 'r')]),
({
'filter': 'g'
}, [(1, 'g'), (2, 'g')]),
({
'filter': 'r'
}, [(3, 'r')]),
)
map_python_type = 'lsst.afw.image.DecoratedImageU'
map_cpp_type = 'DecoratedImageU'
map_storage_name = 'FitsStorage'
metadata_output_path = os.path.join('processCcd_metadata',
'v1_fg.boost')
raw_filename = 'raw_v1_fg.fits.gz'
default_level = 'visit'
raw_levels = (('skyTile', set(['filter'])), ('filter',
set(['filter', 'visit'])),
('visit', set(['filter', 'visit'])))
self.setUp_mapper(
output=data_dir,
path_to_raw=path_to_raw,
keys=keys,
query_format=query_format,
queryMetadata=queryMetadata,
metadata_output_path=metadata_output_path,
map_python_type=map_python_type,
map_cpp_type=map_cpp_type,
map_storage_name=map_storage_name,
raw_filename=raw_filename,
default_level=default_level,
raw_levels=raw_levels,
)
self.setUp_camera(
camera_name='test',
n_detectors=1,
first_detector_name='0',
plate_scale=20 * arcseconds,
)
super(TestObsTest, self).setUp()