def test_des_image():
"""Test the whole process with a DES CCD.
"""
import os
import fitsio
image_file = 'y1_test/DECam_00241238_01.fits.fz'
cat_file = 'y1_test/DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits'
orig_image = galsim.fits.read(image_file)
psf_file = os.path.join('output', 'pixel_des_psf.fits')
if __name__ == '__main__':
# These match what Gary used in fit_des.py
nstars = None
scale = 0.15
size = 41
else:
# These are faster and good enough for the unit tests.
nstars = 25
scale = 0.2
size = 21
stamp_size = 51
# The configuration dict with the right input fields for the file we're using.
start_sigma = 1.0 / 2.355 # TODO: Need to make this automatic somehow.
config = {
'input': {
'images': image_file,
'image_hdu': 1,
'weight_hdu': 3,
'badpix_hdu': 2,
'cats': cat_file,
'cat_hdu': 2,
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'sky_col': 'BACKGROUND',
'stamp_size': stamp_size,
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
},
'output': {
'file_name': psf_file,
},
'psf': {
'model': {
'type': 'PixelGrid',
'scale': scale,
'size': size,
'start_sigma': start_sigma,
},
'interp': {
'type': 'BasisPolynomial',
'order': 2,
},
},
}
if __name__ == '__main__': config['verbose'] = 3
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if True:
# Start by doing things manually:
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
# Largely copied from Gary's fit_des.py, but using the Piff input_handler to
# read the input files.
stars, wcs, pointing = piff.Input.process(config['input'],
logger=logger)
if nstars is not None:
stars = stars[:nstars]
# Make model, force PSF centering
model = piff.PixelGrid(scale=scale,
size=size,
interp=piff.Lanczos(3),
force_model_center=True,
start_sigma=start_sigma,
logger=logger)
# Interpolator will be zero-order polynomial.
# Find u, v ranges
interp = piff.BasisPolynomial(order=2, logger=logger)
# Make a psf
psf = piff.SimplePSF(model, interp)
psf.fit(stars, wcs, pointing, logger=logger)
# The difference between the images of the fitted stars and the originals should be
# consistent with noise. Keep track of how many don't meet that goal.
n_bad = 0 # chisq/dof > 2
n_marginal = 0 # chisq/dof > 1.1
n_good = 0 # chisq/dof <= 1.1
# Note: The 2 and 1.1 values here are very arbitrary!
for s in psf.stars:
fitted = psf.drawStar(s)
orig_stamp = orig_image[fitted.image.bounds] - s['sky']
fit_stamp = fitted.image
x0 = int(s['x'] + 0.5)
y0 = int(s['y'] + 0.5)
b = galsim.BoundsI(x0 - 3, x0 + 3, y0 - 3, y0 + 3)
#print('orig center = ',orig_stamp[b].array)
#print('flux = ',orig_stamp.array.sum())
#print('fit center = ',fit_stamp[b].array)
#print('flux = ',fit_stamp.array.sum())
flux = fitted.fit.flux
#print('max diff/flux = ',np.max(np.abs(orig_stamp.array-fit_stamp.array))/flux)
#np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
weight = s.weight # These should be 1/var_pix
resid = fit_stamp - orig_stamp
chisq = np.sum(resid.array**2 * weight.array)
print('chisq = ', chisq)
print('cf. star.chisq, dof = ', s.fit.chisq, s.fit.dof)
assert abs(chisq - s.fit.chisq) < 1.e-3 * chisq
if chisq > 2. * s.fit.dof:
n_bad += 1
elif chisq > 1.1 * s.fit.dof:
n_marginal += 1
else:
n_good += 1
# Check the convenience function that an end user would typically use
offset = s.center_to_offset(s.fit.center)
image = psf.draw(x=s['x'],
y=s['y'],
stamp_size=stamp_size,
flux=s.fit.flux,
offset=offset)
np.testing.assert_almost_equal(image.array,
fit_stamp.array,
decimal=4)
print('n_good, marginal, bad = ', n_good, n_marginal, n_bad)
# The real counts are 10 and 2. So this says make sure any updates to the code don't make
# things much worse.
assert n_marginal <= 12
assert n_bad <= 3
# Use piffify function
if __name__ == '__main__':
print('start piffify')
piff.piffify(config)
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'],
y=stars[0]['y'],
stamp_size=stamp_size,
flux=flux,
offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
# The first star happens to be a good one, so go ahead and test the arrays directly.
np.testing.assert_almost_equal(fit_stamp.array / flux,
orig_stamp.array / flux,
decimal=2)
# Test using the piffify executable
config['verbose'] = 0
with open('pixel_des.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
print('start piffify executable')
p = subprocess.Popen([piffify_exe, 'pixel_des.yaml'])
p.communicate()
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'],
y=stars[0]['y'],
stamp_size=stamp_size,
flux=flux,
offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
np.testing.assert_almost_equal(fit_stamp.array / flux,
orig_stamp.array / flux,
decimal=2)
def test_single_image():
"""Test the whole process with a single image.
Note: This test is based heavily on test_single_image in test_simple.py.
"""
import os
import fitsio
np_rng = np.random.RandomState(1234)
# Make the image
image = galsim.Image(2048, 2048, scale=0.2)
# The (x,y) values will be on a grid 5 x 5 stars with a random sub-pixel offset.
xvals = np.linspace(50., 1950., 5)
yvals = np.linspace(50., 1950., 5)
x_list, y_list = np.meshgrid(xvals, yvals)
x_list = x_list.flatten()
y_list = y_list.flatten()
x_list = x_list + (np_rng.rand(len(x_list)) - 0.5)
y_list = y_list + (np_rng.rand(len(x_list)) - 0.5)
print('x_list = ', x_list)
print('y_list = ', y_list)
# Range of fluxes from 100 to 15000
flux_list = 100. * np.exp(5. * np_rng.rand(len(x_list)))
print('fluxes range from ', np.min(flux_list), np.max(flux_list))
# Draw a Moffat PSF at each location on the image.
# Have the truth values vary quadratically across the image.
beta_fn = lambda x, y: 3.5 - 0.1 * (x / 1000) + 0.08 * (y / 1000)**2
fwhm_fn = lambda x, y: 0.9 + 0.05 * (x / 1000) - 0.03 * (
y / 1000) + 0.02 * (x / 1000) * (y / 1000)
e1_fn = lambda x, y: 0.02 - 0.01 * (x / 1000)
e2_fn = lambda x, y: -0.03 + 0.02 * (x / 1000)**2 - 0.01 * (y / 1000) * 2
for x, y, flux in zip(x_list, y_list, flux_list):
beta = beta_fn(x, y)
fwhm = fwhm_fn(x, y)
e1 = e1_fn(x, y)
e2 = e2_fn(x, y)
print(x, y, beta, fwhm, e1, e2)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta, flux=flux).shear(e1=e1,
e2=e2)
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
moffat.drawImage(image=image[bounds], offset=offset, method='no_pixel')
print('drew image')
# Write out the image to a file
image_file = os.path.join('data', 'pixel_moffat_image.fits')
image.write(image_file)
print('wrote image')
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('data', 'pixel_moffat_cat.fits')
fitsio.write(cat_file, data, clobber=True)
print('wrote catalog')
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file, stamp_size=32)
assert input.image_files == [image_file]
assert input.cat_files == [cat_file]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Make stars
orig_stars = input.makeStars()
assert len(orig_stars) == len(x_list)
assert orig_stars[0].image.array.shape == (32, 32)
# Make a test star, not at the location of any of the model stars to use for each of the
# below tests.
x0 = 1024 # Some random position, not where a star was originally.
y0 = 133
beta = beta_fn(x0, y0)
fwhm = fwhm_fn(x0, y0)
e1 = e1_fn(x0, y0)
e2 = e2_fn(x0, y0)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta).shear(e1=e1, e2=e2)
target_star = piff.Star.makeTarget(x=x0, y=y0, scale=image.scale)
test_im = galsim.ImageD(bounds=target_star.image.bounds, scale=image.scale)
moffat.drawImage(image=test_im, method='no_pixel', use_true_center=False)
print('made test star')
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if True:
# Process the star data
model = piff.PixelGrid(0.2, 16, start_sigma=0.9 / 2.355)
interp = piff.BasisPolynomial(order=2)
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
pointing = None # wcs is not Celestial here, so pointing needs to be None.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, {0: input.images[0].wcs}, pointing, logger=logger)
# Check that the interpolation is what it should be
print('target.flux = ', target_star.fit.flux)
test_star = psf.drawStar(target_star)
#print('test_im center = ',test_im[b].array)
#print('flux = ',test_im.array.sum())
#print('interp_im center = ',test_star.image[b].array)
#print('flux = ',test_star.image.array.sum())
#print('max diff = ',np.max(np.abs(test_star.image.array-test_im.array)))
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array, test_im.array, decimal=3)
# Round trip through a file
psf_file = os.path.join('output', 'pixel_psf.fits')
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.PixelGrid
assert type(psf.interp) is piff.BasisPolynomial
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array, test_im.array, decimal=3)
# Do the whole thing with the config parser
config = {
'input': {
'images': image_file,
'cats': cat_file,
'x_col': 'x',
'y_col': 'y',
'stamp_size': 48 # Bigger than we drew, but should still work.
},
'output': {
'file_name': psf_file
},
'psf': {
'model': {
'type': 'PixelGrid',
'scale': 0.2,
'size':
16, # Much smaller than the input stamps, but this is plenty here.
'start_sigma': 0.9 / 2.355
},
'interp': {
'type': 'BasisPolynomial',
'order': 2
},
},
}
if __name__ == '__main__':
print("Running piffify function")
piff.piffify(config)
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Test using the piffify executable
config['verbose'] = 0
with open('pixel_moffat.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
print("Running piffify executable")
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'pixel_moffat.yaml'])
p.communicate()
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
def test_undersamp_shift():
"""Next: fit PSF to undersampled, dithered data with variable centroids,
this time using chisq() and summing alpha,beta instead of fit() per star
"""
# Pixelized model with Lanczos 3 interpolation, slightly smaller than data
# than the data
pixinterp = piff.Lanczos(3)
influx = 150.
du = 0.5
mod = piff.PixelGrid(0.3,
25,
pixinterp,
start_sigma=1.3,
force_model_center=True)
# Make a sample star just so we can pass the initial PSF into interpolator
# Also store away a noiseless copy of the PSF, origin of focal plane
s0 = make_gaussian_data(1.0, 0., 0., influx, du=0.5)
s0 = mod.initialize(s0)
# Interpolator will be constant
interp = piff.BasisPolynomial(0)
# Draw stars on a 2d grid of "focal plane" with 0<=u,v<=1
positions = np.linspace(0., 1., 8)
stars = []
np_rng = np.random.RandomState(1234)
rng = galsim.BaseDeviate(1234)
for u in positions:
for v in positions:
# Nominal star centers move by +-1/2 pix, real centers another 1/2 pix
phase1 = (0.5 - np_rng.rand(2)) * du
phase2 = (0.5 - np_rng.rand(2)) * du
if u == 0. and v == 0.:
phase1 = phase2 = (0., 0.)
s = make_gaussian_data(1.0,
phase2[0],
phase2[1],
influx,
noise=0.1,
du=du,
fpu=u,
fpv=v,
nom_u0=phase1[0],
nom_v0=phase1[1],
rng=rng)
s = mod.initialize(s)
###print("phase:",phase2,'flux',s.fit.flux)###
stars.append(s)
# BasisInterp needs to be initialized before solving.
interp.initialize(stars)
oldchisq = 0.
# Iterate solution using mean of chisq
for iteration in range(10):
# Refit PSFs star by star:
stars = [mod.chisq(s) for s in stars]
# Solve for interpolated PSF function
interp.solve(stars)
# Refit and recenter all stars
stars = [mod.reflux(interp.interpolate(s)) for s in stars]
chisq = np.sum([s.fit.chisq for s in stars])
dof = np.sum([s.fit.dof for s in stars])
print('iteration', iteration, 'chisq=', chisq, 'dof=', dof)
if oldchisq > 0 and np.abs(oldchisq - chisq) < dof / 10.:
break
else:
oldchisq = chisq
# Now use the interpolator to produce a noiseless rendering
s1 = interp.interpolate(s0)
s1 = mod.reflux(s1)
print('Flux, ctr after interpolation: ', s1.fit.flux, s1.fit.center,
s1.fit.chisq)
# Less than 2 dp of accuracy here!
np.testing.assert_almost_equal(s1.fit.flux / influx, 1.0, decimal=1)
s1 = mod.draw(s1)
print('max image abs diff = ',
np.max(np.abs(s1.image.array - s0.image.array)))
print('max image abs value = ', np.max(np.abs(s0.image.array)))
peak = np.max(np.abs(s0.image.array))
np.testing.assert_almost_equal(s1.image.array / peak,
s0.image.array / peak,
decimal=1)
def do_undersamp_drift(fit_centers=False):
"""Draw stars whose size and position vary across FOV.
Fit to oversampled model with linear dependence across FOV.
Argument fit_centers decides whether we are letting the PSF model
center drift, or whether we re-fit the center positions of the stars.
"""
# Pixelized model with Lanczos 3 interpolation, slightly smaller than data
# than the data
pixinterp = piff.Lanczos(3)
influx = 150.
du = 0.5
mod = piff.PixelGrid(0.3,
25,
pixinterp,
start_sigma=1.3,
force_model_center=fit_centers)
# Make a sample star just so we can pass the initial PSF into interpolator
# Also store away a noiseless copy of the PSF, origin of focal plane
s0 = make_gaussian_data(1.0, 0., 0., influx, du=0.5)
s0 = mod.initialize(s0)
# Interpolator will be linear
interp = piff.BasisPolynomial(1)
# Draw stars on a 2d grid of "focal plane" with 0<=u,v<=1
positions = np.linspace(0., 1., 8)
stars = []
np_rng = np.random.RandomState(1234)
rng = galsim.BaseDeviate(1234)
for u in positions:
for v in positions:
# Nominal star centers move by +-1/2 pix
phase1 = (0.5 - np_rng.rand(2)) * du
phase2 = (0.5 - np_rng.rand(2)) * du
if u == 0. and v == 0.:
phase1 = phase2 = (0., 0.)
# PSF center will drift with v; size drifts with u
s = make_gaussian_data(1.0 + 0.1 * u,
0.,
0.5 * du * v,
influx,
noise=0.1,
du=du,
fpu=u,
fpv=v,
nom_u0=phase1[0],
nom_v0=phase1[1],
rng=rng)
s = mod.initialize(s)
###print("phase:",phase2,'flux',s.fit.flux)###
stars.append(s)
# BasisInterp needs to be initialized before solving.
interp.initialize(stars)
oldchisq = 0.
# Iterate solution using mean of chisq
for iteration in range(20):
# Refit PSFs star by star:
stars = [mod.chisq(s) for s in stars]
# Solve for interpolated PSF function
interp.solve(stars)
# Refit and recenter all stars
stars = [mod.reflux(interp.interpolate(s)) for s in stars]
chisq = np.sum([s.fit.chisq for s in stars])
dof = np.sum([s.fit.dof for s in stars])
print('iteration', iteration, 'chisq=', chisq, 'dof=', dof)
if oldchisq > 0 and np.abs(oldchisq - chisq) < dof / 10.:
break
else:
oldchisq = chisq
# Now use the interpolator to produce a noiseless rendering
s1 = interp.interpolate(s0)
s1 = mod.reflux(s1)
print('Flux, ctr after interpolation: ', s1.fit.flux, s1.fit.center,
s1.fit.chisq)
# Less than 2 dp of accuracy here!
np.testing.assert_almost_equal(s1.fit.flux / influx, 1.0, decimal=1)
s1 = mod.draw(s1)
print('max image abs diff = ',
np.max(np.abs(s1.image.array - s0.image.array)))
print('max image abs value = ', np.max(np.abs(s0.image.array)))
peak = np.max(np.abs(s0.image.array))
np.testing.assert_almost_equal(s1.image.array / peak,
s0.image.array / peak,
decimal=1)
def test_missing():
"""Next: fit mean PSF to multiple images, with missing pixels.
"""
# Pixelized model with Lanczos 3 interpolation, slightly smaller than data
# than the data
pixinterp = piff.Lanczos(3)
mod = piff.PixelGrid(0.5,
25,
pixinterp,
start_sigma=1.5,
force_model_center=False)
# Draw stars on a 2d grid of "focal plane" with 0<=u,v<=1
positions = np.linspace(0., 1., 4)
influx = 150.
stars = []
np_rng = np.random.RandomState(1234)
rng = galsim.BaseDeviate(1234)
for u in positions:
for v in positions:
# Draw stars in focal plane positions around a unit ring
s = make_gaussian_data(1.0,
0.,
0.,
influx,
noise=0.1,
du=0.5,
fpu=u,
fpv=v,
rng=rng)
s = mod.initialize(s)
# Kill 10% of each star's pixels
bad = np_rng.rand(*s.image.array.shape) < 0.1
s.weight.array[bad] = 0.
s.image.array[bad] = -999.
s = mod.reflux(s, fit_center=False) # Start with a sensible flux
stars.append(s)
# Also store away a noiseless copy of the PSF, origin of focal plane
s0 = make_gaussian_data(1.0, 0., 0., influx, du=0.5)
s0 = mod.initialize(s0)
if __name__ == "__main__":
interps = [piff.Polynomial(order=0), piff.BasisPolynomial(order=0)]
else:
# The Polynomial interpolator works, but it's slow. For the nosetests runs, skip it.
interps = [piff.BasisPolynomial(order=0)]
for interp in interps:
# Interpolator will be simple mean
interp = piff.Polynomial(order=0)
interp.initialize(stars)
oldchisq = 0.
# Iterate solution using interpolator
for iteration in range(40):
# Refit PSFs star by star:
for i, s in enumerate(stars):
stars[i] = mod.fit(s)
# Run the interpolator
interp.solve(stars)
# Install interpolator solution into each
# star, recalculate flux, report chisq
chisq = 0.
dof = 0
for i, s in enumerate(stars):
s = interp.interpolate(s)
s = mod.reflux(s)
chisq += s.fit.chisq
dof += s.fit.dof
stars[i] = s
###print(' chisq=',s.fit.chisq, 'dof=',s.fit.dof)
print('iteration', iteration, 'chisq=', chisq, 'dof=', dof)
if oldchisq > 0 and chisq < oldchisq and oldchisq - chisq < dof / 10.:
break
else:
oldchisq = chisq
# Now use the interpolator to produce a noiseless rendering
s1 = interp.interpolate(s0)
s1 = mod.reflux(s1)
print('Flux, ctr after interpolation: ', s1.fit.flux, s1.fit.center,
s1.fit.chisq)
# Less than 2 dp of accuracy here!
np.testing.assert_almost_equal(s1.fit.flux / influx, 1.0, decimal=1)
s1 = mod.draw(s1)
print('max image abs diff = ',
np.max(np.abs(s1.image.array - s0.image.array)))
print('max image abs value = ', np.max(np.abs(s0.image.array)))
peak = np.max(np.abs(s0.image.array))
np.testing.assert_almost_equal(s1.image.array / peak,
s0.image.array / peak,
decimal=1)