def test_gaussian():
gaussian = piff.Gaussian(include_pixel=False)
print('test gaussian')
star = make_empty_star()
# test gaussian alone
sigma = 1
g1 = -0.1
g2 = 0.05
model = piff.Optical(r0=0, sigma=sigma, template='des')
star = model.draw(star)
# insert assert statement about sigma
np.testing.assert_almost_equal(gaussian.fit(star).fit.params[0], sigma, 5)
# gaussian and shear
model = piff.Optical(r0=0, sigma=sigma, g1=g1, g2=g2, template='des')
star = model.draw(star)
params = gaussian.fit(star).fit.params
np.testing.assert_almost_equal(params[0], sigma, 5)
np.testing.assert_almost_equal(params[1], g1, 5)
np.testing.assert_almost_equal(params[2], g2, 5)
# now gaussian, shear, aberration, r0
star = make_empty_star(params=[0.5, 0.8, -0.7, 0.5, -0.2, 0.9, -1, 2.0])
model = piff.Optical(r0=0.1, sigma=sigma, g1=g1, g2=g2, template='des')
star = model.draw(star)
def generate_starlist(n_samples=500):
# create n_samples images from the 63 ccds and pixel coordinates
np_rng = np.random.RandomState(1234)
icens = np_rng.randint(100, 2048, n_samples)
jcens = np_rng.randint(100, 4096, n_samples)
ccdnums = np_rng.randint(1, 63, n_samples)
icenter = 1000
jcenter = 2000
# throw out any icens and jcens that are within 400 pixels of the center
conds = (np.abs(icens - icenter) > 400) | (np.abs(jcens - jcenter) > 400)
icens = icens[conds]
jcens = jcens[conds]
ccdnums = ccdnums[conds]
sigmas, g1s, g2s = psf_model(icens, jcens, icenter, jcenter)
# throw in a 2d polynomial function for sigma g1 and g2
# all sigma > 0, all g1 < 0, and g2 straddles.
star_list = [make_star(icen, jcen, ccdnum, sigma, g1, g2)
for icen, jcen, ccdnum, sigma, g1, g2
in zip(icens, jcens, ccdnums, sigmas, g1s, g2s)]
# load up model and draw the stars
model = piff.Gaussian(fastfit=True)
star_list = [model.draw(star) for star in star_list]
star_list = [model.initialize(star) for star in star_list]
star_list = [model.fit(star) for star in star_list]
return star_list, model
def test_Gaussian():
"""This is about the simplest possible model I could think of. It just uses the
HSM adaptive moments routine to measure the moments, and then it models the
PSF as a Gaussian.
"""
# Here is the true PSF
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
# Draw the PSF onto an image. Let's go ahead and give it a non-trivial WCS.
wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29)
image = galsim.Image(64,64, wcs=wcs)
# This is only going to come out right if we (unphysically) don't convolve by the pixel.
psf.drawImage(image, method='no_pixel')
# Make a StarData instance for this image
stardata = piff.StarData(image, image.true_center)
star = piff.Star(stardata, None)
# Fit the model from the image
model = piff.Gaussian(include_pixel=False)
star = model.initialize(star)
fit = model.fit(star).fit
print('True sigma = ',sigma,', model sigma = ',fit.params[0])
print('True g1 = ',g1,', model g1 = ',fit.params[1])
print('True g2 = ',g2,', model g2 = ',fit.params[2])
# This test is pretty accurate, since we didn't add any noise and didn't convolve by
# the pixel, so the image is very accurately a sheared Gaussian.
true_params = [ sigma, g1, g2 ]
np.testing.assert_almost_equal(fit.params[0], sigma, decimal=7)
np.testing.assert_almost_equal(fit.params[1], g1, decimal=7)
np.testing.assert_almost_equal(fit.params[2], g2, decimal=7)
np.testing.assert_almost_equal(fit.params, true_params, decimal=7)
# Now test running it via the config parser
config = {
'model' : {
'type' : 'Gaussian',
'include_pixel': False
}
}
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_Gaussian.log')
model = piff.Model.process(config['model'], logger)
fit = model.fit(star).fit
# Same tests.
np.testing.assert_almost_equal(fit.params[0], sigma, decimal=7)
np.testing.assert_almost_equal(fit.params[1], g1, decimal=7)
np.testing.assert_almost_equal(fit.params[2], g2, decimal=7)
np.testing.assert_almost_equal(fit.params, true_params, decimal=7)
def test_twodstats():
"""Make sure we can execute and print a readout of the plot
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
model = piff.Gaussian(fastfit=True)
interp = piff.Polynomial(order=1) # should find that order=1 is better
# create background model
stars, true_model = generate_starlist(100)
psf = piff.SimplePSF(model, interp)
psf.fit(stars, None, None)
# check the coeffs of sigma and g2, which are actually linear fits
# skip g1 since it is actually a 2d parabola
# factor of 0.263 is to account for going from pixel xy to wcs uv
np.testing.assert_almost_equal(psf.interp.coeffs[0].flatten(),
np.array([0.4, 0, 1. / (0.263 * 2048), 0]), decimal=4)
np.testing.assert_almost_equal(psf.interp.coeffs[2].flatten(),
np.array([-0.1 * 1000 / 2048, 0, 0.1 / (0.263 * 2048), 0]),
decimal=4)
stats = piff.TwoDHistStats(number_bins_u=5, number_bins_v=5, reducing_function='np.mean')
stats.compute(psf, stars, logger=logger)
# check the twodhists
# get the average value in the bin
u_i = 3
v_i = 3
icen = stats.twodhists['u'][v_i, u_i] / 0.263
jcen = stats.twodhists['v'][v_i, u_i] / 0.263
print('icen = ',icen)
print('jcen = ',jcen)
icenter = 1000
jcenter = 2000
# the average value in the bin should match up with the model for the average coordinates
sigma, g1, g2 = psf_model(icen, jcen, icenter, jcenter)
sigma_average = stats.twodhists['T'][v_i, u_i]
g1_average = stats.twodhists['g1'][v_i, u_i]
g2_average = stats.twodhists['g2'][v_i, u_i]
# assert equal to 4th decimal
print('sigma, g1, g2 = ',[sigma,g1,g2])
print('av sigma, g1, g2 = ',[sigma_average,g1_average,g2_average])
np.testing.assert_almost_equal([sigma, g1, g2], [sigma_average, g1_average, g2_average],
decimal=2)
# Test the plotting and writing
twodstats_file = os.path.join('output','twodstats.pdf')
stats.write(twodstats_file)
# repeat for whisker
stats = piff.WhiskerStats(number_bins_u=21, number_bins_v=21, reducing_function='np.mean')
stats.compute(psf, stars)
# Test the plotting and writing
twodstats_file = os.path.join('output','whiskerstats.pdf')
stats.write(twodstats_file)
def test_shearing():
print('test shearing')
# make sure if we put in common mode ellipticities that things change
star = make_empty_star()
g1 = 0
g2 = 0.05
model = piff.Optical(r0=0.1, g1=g1, g2=g2, template='des')
star = model.draw(star)
gaussian = piff.Gaussian(include_pixel=False)
star_gaussian = gaussian.fit(star)
np.testing.assert_almost_equal(star_gaussian.fit.params[1], g1, 5)
np.testing.assert_almost_equal(star_gaussian.fit.params[2], g2, 5)
def test_extra_interp():
# Test that specifying extra_interp_properties works properly
# TODO: This is a very bare bones test of the interface. There is basically no test of
# this functionality at all yet. TBD!
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29)
image = galsim.Image(64, 64, wcs=wcs)
psf.drawImage(image, method='no_pixel')
# use g-i color as an extra property for interpolation.
props = dict(gi_color=0.3)
print('props = ', props)
star = piff.Star(piff.StarData(image, image.true_center, properties=props),
None)
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
psf = piff.SimplePSF(model, interp, extra_interp_properties=['gi_color'])
assert psf.extra_interp_properties == ['gi_color']
# Note: Mean doesn't actually do anything useful with the extra properties, so this
# isn't really testing anything other than that the code doesn't completely break.
pointing = galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees)
psf.fit([star], wcs={0: wcs}, pointing=pointing)
# Not much of a check here. Just check that it actually draws something with flux ~= 1
im = psf.draw(x=5, y=7, gi_color=0.3)
np.testing.assert_allclose(im.array.sum(), 1.0, rtol=1.e-3)
# Check missing or extra properties
with np.testing.assert_raises(TypeError):
psf.draw(x=5, y=7)
with np.testing.assert_raises(TypeError):
psf.draw(x=5, y=7, gi_color=0.3, ri_color=3)
# No stars raises an error. (This can happen in practice if all stars are excluded on input.)
with np.testing.assert_raises(RuntimeError):
psf.fit([], wcs={0: wcs}, pointing=pointing)
# Also for SingleChipPSf
psf2 = piff.SingleChipPSF(psf, extra_interp_properties=['gi_color'])
assert psf2.extra_interp_properties == ['gi_color']
with np.testing.assert_raises(TypeError):
psf2.draw(x=5, y=7, chipnum=0)
def test_gaussian():
gaussian = piff.Gaussian(include_pixel=False)
print('test gaussian')
star = make_empty_star()
# test gaussian alone
sigma = 1.7
g1 = -0.1
g2 = 0.05
model = piff.Optical(r0=0, sigma=sigma, template='des')
star = model.draw(star)
# insert assert statement about sigma
# We don't expect sigma to match. But it should be the biggest component, so close-ish
np.testing.assert_allclose(gaussian.fit(star).fit.params[0],
sigma,
rtol=1.e-2)
# omitting atm params is equivalent
kwargs = piff.optical_model.optical_templates['des'].copy()
del kwargs['r0']
model2 = piff.Optical(sigma=sigma, **kwargs)
star2 = model2.draw(star)
assert star.image == star2.image
# gaussian and shear
model = piff.Optical(r0=0, sigma=sigma, g1=g1, g2=g2, template='des')
star = model.draw(star)
params = gaussian.fit(star).fit.params
np.testing.assert_allclose(gaussian.fit(star).fit.params[0],
sigma,
rtol=1.e-2)
# Shears are much closer, since the optical part is round.
np.testing.assert_allclose(params[1], g1, rtol=1.e-4)
np.testing.assert_allclose(params[2], g2, rtol=1.e-4)
# now gaussian, shear, aberration, r0
star = make_empty_star(params=[0.5, 0.8, -0.7, 0.5, -0.2, 0.9, -1, 2.0])
model = piff.Optical(r0=0.1, sigma=sigma, g1=g1, g2=g2, template='des')
star = model.draw(star)
def test_load_images():
"""Test the load_images function
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_load_image2.log')
# Same setup as test_single_image, but without flags
image = galsim.Image(2048, 2048, scale=0.26)
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y in zip(x_list, y_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
psf.drawImage(image[bounds],
center=galsim.PositionD(x, y),
method='no_pixel')
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
image_file = os.path.join('output', 'test_load_images_im.fits')
image.write(image_file)
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output', 'test_load_images_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Make star data
config = {'image_file_name': image_file, 'cat_file_name': cat_file}
orig_stars, wcs, pointing = piff.Input.process(config, logger)
# Fit these with a simple Mean, Gaussian
model = piff.Gaussian()
interp = piff.Mean()
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
psf_file = os.path.join('output', 'test_load_images_psf.fits')
psf.write(psf_file, logger)
# Read this file back in. It has the star data, but the images are blank.
psf2 = piff.read(psf_file, logger)
assert len(psf2.stars) == 10
for star in psf2.stars:
np.testing.assert_array_equal(star.image.array, 0.)
loaded_stars = piff.Star.load_images(psf2.stars, image_file)
for star, orig in zip(loaded_stars, psf.stars):
np.testing.assert_array_equal(star.image.array, orig.image.array)
# Can optionally supply sky to subtract
loaded_stars = piff.Star.load_images(psf2.stars, image_file, sky=10)
for star, orig in zip(loaded_stars, psf.stars):
np.testing.assert_array_equal(star.image.array, orig.image.array - 10)
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_single_image.log')
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
flag_list = [1, 1, 13, 1, 1, 4, 1, 1, 0, 1]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y, flag in zip(x_list, y_list, flag_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag & 4:
print('corrupting star at ', x, y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('output', 'simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
cat_file = os.path.join('output', 'simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
config = {'image_file_name': image_file, 'cat_file_name': cat_file}
input = piff.InputFiles(config, logger=logger)
assert input.image_file_name == [image_file]
assert input.cat_file_name == [cat_file]
# Check image
assert input.nimages == 1
image1, _, image_pos, _, _, _ = input.getRawImageData(0)
np.testing.assert_equal(image1.array, image.array)
# Check catalog
np.testing.assert_equal([pos.x for pos in image_pos], x_list)
np.testing.assert_equal([pos.y for pos in image_pos], y_list)
# Repeat, using flag columns this time.
config = {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': '1',
'skip_flag': '4',
'stamp_size': 48
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, _, image_pos, _, _, _ = input.getRawImageData(0)
assert len(image_pos) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48, 48)
# Process the star data
# can only compare to truth if include_pixel=False
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars]
interp.solve(fitted_stars)
print('mean = ', interp.mean)
# Check that the interpolation is what it should be
# Any position would work here.
chipnum = 0
x = 1024
y = 123
orig_wcs = input.getWCS()[chipnum]
orig_pointing = input.getPointing()
image_pos = galsim.PositionD(x, y)
world_pos = piff.StarData.calculateFieldPos(image_pos, orig_wcs,
orig_pointing)
u, v = world_pos.x, world_pos.y
stamp_size = config['stamp_size']
target = piff.Star.makeTarget(x=x,
y=y,
u=u,
v=v,
wcs=orig_wcs,
stamp_size=stamp_size,
pointing=orig_pointing)
true_params = [sigma, g1, g2]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Check default values of options
psf = piff.SimplePSF(model, interp)
assert psf.chisq_thresh == 0.1
assert psf.max_iter == 30
assert psf.outliers == None
assert psf.extra_interp_properties == []
# Now test running it via the config parser
psf_file = os.path.join('output', 'simple_psf.fits')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': 1,
'skip_flag': 4,
'stamp_size': stamp_size
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
'max_iter': 10,
'chisq_thresh': 0.2,
},
'output': {
'file_name': psf_file
},
}
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
interp = piff.Mean()
psf = piff.SimplePSF(model, interp, max_iter=10, chisq_thresh=0.2)
assert psf.chisq_thresh == 0.2
assert psf.max_iter == 10
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# test that drawStar and drawStarList work
test_star = psf.drawStar(target)
test_star_list = psf.drawStarList([target])[0]
np.testing.assert_equal(test_star.fit.params, test_star_list.fit.params)
np.testing.assert_equal(test_star.image.array, test_star_list.image.array)
# test copy_image property of drawStar and draw
for draw in [psf.drawStar, psf.model.draw]:
target_star_copy = psf.interp.interpolate(
piff.Star(target.data.copy(), target.fit.copy()))
# interp is so that when we do psf.model.draw we have fit.params to work with
test_star_copy = draw(target_star_copy, copy_image=True)
test_star_nocopy = draw(target_star_copy, copy_image=False)
# if we modify target_star_copy, then test_star_nocopy should be modified,
# but not test_star_copy
target_star_copy.image.array[0, 0] = 23456
assert test_star_nocopy.image.array[
0, 0] == target_star_copy.image.array[0, 0]
assert test_star_copy.image.array[0,
0] != target_star_copy.image.array[0,
0]
# however the other pixels SHOULD still be all the same value
assert test_star_nocopy.image.array[
1, 1] == target_star_copy.image.array[1, 1]
assert test_star_copy.image.array[1,
1] == target_star_copy.image.array[1,
1]
# test that draw works
test_image = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center)
# this image should be the same values as test_star
assert test_image == test_star.image
# test that draw does not copy the image
image_ref = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center,
image=test_image)
image_ref.array[0, 0] = 123456789
assert test_image.array[0, 0] == image_ref.array[0, 0]
assert test_star.image.array[0, 0] != test_image.array[0, 0]
assert test_star.image.array[1, 1] == test_image.array[1, 1]
# Round trip to a file
psf.write(psf_file, logger)
psf2 = piff.read(psf_file, logger)
assert type(psf2.model) is piff.Gaussian
assert type(psf2.interp) is piff.Mean
assert psf2.chisq == psf.chisq
assert psf2.last_delta_chisq == psf.last_delta_chisq
assert psf2.chisq_thresh == psf.chisq_thresh
assert psf2.max_iter == psf.max_iter
assert psf2.dof == psf.dof
assert psf2.nremoved == psf.nremoved
test_star = psf2.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf3 = piff.read(psf_file)
assert type(psf3.model) is piff.Gaussian
assert type(psf3.interp) is piff.Mean
assert psf3.chisq == psf.chisq
assert psf3.last_delta_chisq == psf.last_delta_chisq
assert psf3.chisq_thresh == psf.chisq_thresh
assert psf3.max_iter == psf.max_iter
assert psf3.dof == psf.dof
assert psf3.nremoved == psf.nremoved
test_star = psf3.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test using the piffify executable
os.remove(psf_file)
# This would be simpler as a direct assignment, but this once, test the way you would set
# this from the command line, which would call parse_variables.
piff.config.parse_variables(config, ['verbose=0'], logger=logger)
#config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
config2 = piff.config.read_config('simple.yaml')
assert config == config2
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
psf4 = piff.read(psf_file)
assert type(psf4.model) is piff.Gaussian
assert type(psf4.interp) is piff.Mean
assert psf4.chisq == psf.chisq
assert psf4.last_delta_chisq == psf.last_delta_chisq
assert psf4.chisq_thresh == psf.chisq_thresh
assert psf4.max_iter == psf.max_iter
assert psf4.dof == psf.dof
assert psf4.nremoved == psf.nremoved
test_star = psf4.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# With very low max_iter, we hit the warning about non-convergence
config['psf']['max_iter'] = 1
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert 'PSF fit did not converge' in cl.output
def test_var():
"""Check that the variance estimate in params_var is sane.
"""
# Here is the true PSF
scale = 1.3
g1 = 0.23
g2 = -0.17
du = 0.1
dv = 0.4
flux = 500
wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29)
noise = 0.2
gsobjs = [
galsim.Gaussian(sigma=1.0),
galsim.Kolmogorov(half_light_radius=1.0),
galsim.Moffat(half_light_radius=1.0, beta=3.0),
galsim.Moffat(half_light_radius=1.0, beta=2.5, trunc=3.0)
]
# Mix of centered = True/False,
# fastfit = True/False,
# include_pixel = True/False
models = [
piff.Gaussian(fastfit=False, include_pixel=False, centered=False),
piff.Kolmogorov(fastfit=True, include_pixel=True, centered=False),
piff.Moffat(fastfit=False, beta=4.8, include_pixel=True,
centered=True),
piff.Moffat(fastfit=True,
beta=2.5,
trunc=3.0,
include_pixel=False,
centered=True)
]
names = ['Gaussian', 'Kolmogorov', 'Moffat3', 'Moffat2.5']
for gsobj, model, name in zip(gsobjs, models, names):
print()
print("gsobj = ", gsobj)
print()
psf = gsobj.dilate(scale).shear(g1=g1, g2=g2).shift(du,
dv).withFlux(flux)
image = psf.drawImage(nx=64, ny=64, wcs=wcs, method='no_pixel')
weight = image.copy()
weight.fill(1 / noise**2)
# Save this one without noise.
image1 = image.copy()
image1.addNoise(galsim.GaussianNoise(sigma=noise))
# Make a StarData instance for this image
stardata = piff.StarData(image, image.true_center, weight)
star = piff.Star(stardata, None)
star = model.initialize(star)
fit = model.fit(star).fit
file_name = 'input/test_%s_var.npz' % name
print(file_name)
if not os.path.isfile(file_name):
num_runs = 1000
all_params = []
for i in range(num_runs):
image1 = image.copy()
image1.addNoise(galsim.GaussianNoise(sigma=noise))
sd = piff.StarData(image1, image1.true_center, weight)
s = piff.Star(sd, None)
try:
s = model.initialize(s)
s = model.fit(s)
except RuntimeError as e: # Occasionally hsm fails.
print('Caught ', e)
continue
print(s.fit.params)
all_params.append(s.fit.params)
var = np.var(all_params, axis=0)
np.savez(file_name, var=var)
var = np.load(file_name)['var']
print('params = ', fit.params)
print('empirical var = ', var)
print('piff estimate = ', fit.params_var)
print('ratio = ', fit.params_var / var)
print('max ratio = ', np.max(fit.params_var / var))
print('min ratio = ', np.min(fit.params_var / var))
print('mean ratio = ', np.mean(fit.params_var / var))
# Note: The fastfit=False estimates are better -- typically better than 10%
# The fastfit=True estimates are much rougher. Especially size. Need rtol=0.3.
np.testing.assert_allclose(fit.params_var, var, rtol=0.3)
def test_direct():
""" Simple test for directly instantiated Gaussian, Kolmogorov, and Moffat without going through
GSObjectModel explicitly.
"""
# Here is the true PSF
scale = 1.3
g1 = 0.23
g2 = -0.17
du = 0.1
dv = 0.4
gsobjs = [
galsim.Gaussian(sigma=1.0),
galsim.Kolmogorov(half_light_radius=1.0),
galsim.Moffat(half_light_radius=1.0, beta=3.0),
galsim.Moffat(half_light_radius=1.0, beta=2.5, trunc=3.0)
]
models = [
piff.Gaussian(fastfit=True, include_pixel=False),
piff.Kolmogorov(fastfit=True, include_pixel=False),
piff.Moffat(fastfit=True, beta=3.0, include_pixel=False),
piff.Moffat(fastfit=True, beta=2.5, trunc=3.0, include_pixel=False)
]
for gsobj, model in zip(gsobjs, models):
print()
print("gsobj = ", gsobj)
print()
psf = gsobj.dilate(scale).shear(g1=g1, g2=g2).shift(du, dv)
# Draw the PSF onto an image. Let's go ahead and give it a non-trivial WCS.
wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29)
image = galsim.Image(64, 64, wcs=wcs)
# This is only going to come out right if we (unphysically) don't convolve by the pixel.
psf.drawImage(image, method='no_pixel')
# Make a StarData instance for this image
stardata = piff.StarData(image, image.true_center)
star = piff.Star(stardata, None)
star = model.initialize(star)
# First try fastfit.
print('Fast fit')
fit = model.fit(star).fit
print('True scale = ', scale, ', model scale = ', fit.params[0])
print('True g1 = ', g1, ', model g1 = ', fit.params[1])
print('True g2 = ', g2, ', model g2 = ', fit.params[2])
print('True du = ', du, ', model du = ', fit.center[0])
print('True dv = ', dv, ', model dv = ', fit.center[1])
# This test is fairly accurate, since we didn't add any noise and didn't convolve by
# the pixel, so the image is very accurately a sheared GSObject.
# These tests are more strict above. The truncated Moffat included here but not there
# doesn't work quite as well.
np.testing.assert_allclose(fit.params[0], scale, rtol=1e-4)
np.testing.assert_allclose(fit.params[1], g1, rtol=0, atol=1e-5)
np.testing.assert_allclose(fit.params[2], g2, rtol=0, atol=1e-5)
np.testing.assert_allclose(fit.center[0], du, rtol=0, atol=1e-5)
np.testing.assert_allclose(fit.center[1], dv, rtol=0, atol=1e-5)
# Also need to test ability to serialize
outfile = os.path.join('output', 'gsobject_direct_test.fits')
with fitsio.FITS(outfile, 'rw', clobber=True) as f:
model.write(f, 'psf_model')
with fitsio.FITS(outfile, 'r') as f:
roundtrip_model = piff.GSObjectModel.read(f, 'psf_model')
assert model.__dict__ == roundtrip_model.__dict__
# repeat with fastfit=False
models = [
piff.Gaussian(fastfit=False, include_pixel=False),
piff.Kolmogorov(fastfit=False, include_pixel=False),
piff.Moffat(fastfit=False, beta=3.0, include_pixel=False),
piff.Moffat(fastfit=False, beta=2.5, trunc=3.0, include_pixel=False)
]
for gsobj, model in zip(gsobjs, models):
print()
print("gsobj = ", gsobj)
print()
psf = gsobj.dilate(scale).shear(g1=g1, g2=g2).shift(du, dv)
# Draw the PSF onto an image. Let's go ahead and give it a non-trivial WCS.
wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29)
image = galsim.Image(64, 64, wcs=wcs)
# This is only going to come out right if we (unphysically) don't convolve by the pixel.
psf.drawImage(image, method='no_pixel')
# Make a StarData instance for this image
stardata = piff.StarData(image, image.true_center)
star = piff.Star(stardata, None)
star = model.initialize(star)
print('Slow fit')
fit = model.fit(star).fit
print('True scale = ', scale, ', model scale = ', fit.params[0])
print('True g1 = ', g1, ', model g1 = ', fit.params[1])
print('True g2 = ', g2, ', model g2 = ', fit.params[2])
print('True du = ', du, ', model du = ', fit.center[0])
print('True dv = ', dv, ', model dv = ', fit.center[1])
# This test is fairly accurate, since we didn't add any noise and didn't convolve by
# the pixel, so the image is very accurately a sheared GSObject.
np.testing.assert_allclose(fit.params[0], scale, rtol=1e-5)
np.testing.assert_allclose(fit.params[1], g1, rtol=0, atol=1e-5)
np.testing.assert_allclose(fit.params[2], g2, rtol=0, atol=1e-5)
np.testing.assert_allclose(fit.center[0], du, rtol=0, atol=1e-5)
np.testing.assert_allclose(fit.center[1], dv, rtol=0, atol=1e-5)
# Also need to test ability to serialize
outfile = os.path.join('output', 'gsobject_direct_test.fits')
with fitsio.FITS(outfile, 'rw', clobber=True) as f:
model.write(f, 'psf_model')
with fitsio.FITS(outfile, 'r') as f:
roundtrip_model = piff.GSObjectModel.read(f, 'psf_model')
assert model.__dict__ == roundtrip_model.__dict__
def test_twodstats():
"""Make sure we can execute and print a readout of the plot
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
model = piff.Gaussian(fastfit=True)
interp = piff.Polynomial(order=1) # should find that order=1 is better
# create background model
stars, true_model = generate_starlist(100)
psf = piff.SimplePSF(model, interp)
psf.fit(stars, None, None)
stars = psf.stars # These have the right fit parameters
# check the coeffs of sigma and g2, which are actually linear fits
# skip g1 since it is actually a 2d parabola
# factor of 0.263 is to account for going from pixel xy to wcs uv
np.testing.assert_almost_equal(psf.interp.coeffs[0].flatten(),
np.array([0.4, 0, 1. / (0.263 * 2048), 0]),
decimal=4)
np.testing.assert_almost_equal(
psf.interp.coeffs[2].flatten(),
np.array([-0.1 * 1000 / 2048, 0, 0.1 / (0.263 * 2048), 0]),
decimal=4)
stats = piff.TwoDHistStats(nbins_u=5, nbins_v=5) # implicitly np.median
stats.compute(psf, stars, logger=logger)
# check the twodhists
# get the average value in the bin
u_i = 3
v_i = 3
icen = stats.twodhists['u'][v_i, u_i] / 0.263
jcen = stats.twodhists['v'][v_i, u_i] / 0.263
print('icen = ', icen)
print('jcen = ', jcen)
icenter = 1000
jcenter = 2000
# the average value in the bin should match up with the model for the average coordinates
sigma, g1, g2 = psf_model(icen, jcen, icenter, jcenter)
sigma_average = stats.twodhists['T'][v_i, u_i]
g1_average = stats.twodhists['g1'][v_i, u_i]
g2_average = stats.twodhists['g2'][v_i, u_i]
# assert equal to 4th decimal
print('sigma, g1, g2 = ', [sigma, g1, g2])
print('av sigma, g1, g2 = ', [sigma_average, g1_average, g2_average])
np.testing.assert_almost_equal([sigma, g1, g2],
[sigma_average, g1_average, g2_average],
decimal=2)
# Test the plotting and writing
twodstats_file = os.path.join('output', 'twodstats.pdf')
stats.write(twodstats_file)
with np.testing.assert_raises(ValueError):
stats.write() # If not given in constructor, must give file name here.
# repeat for whisker
stats = piff.WhiskerStats(nbins_u=21,
nbins_v=21,
reducing_function='np.mean')
stats.compute(psf, stars)
# Test the plotting and writing
whisker_file = os.path.join('output', 'whiskerstats.pdf')
stats.write(whisker_file)
with np.testing.assert_raises(ValueError):
stats.write()
# With large number of bins, many will have no objects. This is ok.
# Also, can use other np functions like max, std, instead to get different stats
# Not sure when these would be useful, but they are allowed.
# And, check usage where file_name is given in init.
twodstats_file2 = os.path.join('output', 'twodstats.pdf')
stats2 = piff.TwoDHistStats(nbins_u=50,
nbins_v=50,
reducing_function='np.std',
file_name=twodstats_file2)
with np.testing.assert_raises(RuntimeError):
stats2.write() # Cannot write before compute
stats2.compute(psf, stars, logger=logger)
stats2.write()
whisker_file2 = os.path.join('output', 'whiskerstats.pdf')
stats2 = piff.WhiskerStats(nbins_u=100,
nbins_v=100,
reducing_function='np.max',
file_name=whisker_file2)
with np.testing.assert_raises(RuntimeError):
stats2.write() # Cannot write before compute
stats2.compute(psf, stars)
stats2.write()
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
flag_list = [0, 0, 12, 0, 0, 1, 0, 0, 0, 0]
use_list = [1, 1, 1, 1, 1, 0, 1, 1, 0, 1]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y, flag, use in zip(x_list, y_list, flag_list, use_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag:
print('corrupting star at ', x, y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('data', 'simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2'), ('use', 'i2')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
data['use'] = use_list
cat_file = os.path.join('data', 'simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file)
assert input.image_files == [image_file]
assert input.cat_files == [cat_file]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Repeat, using flag and use columns this time.
input = piff.InputFiles(image_file,
cat_file,
flag_col='flag',
use_col='use',
stamp_size=48)
assert input.flag_col == 'flag'
assert input.use_col == 'use'
input.readImages()
input.readStarCatalogs()
assert len(input.cats[0]) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48, 48)
# Process the star data
# can only compare to truth if include_pixel=False
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars]
interp.solve(fitted_stars)
print('mean = ', interp.mean)
# Check that the interpolation is what it should be
target = piff.Star.makeTarget(x=1024,
y=123) # Any position would work here.
true_params = [sigma, g1, g2]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Now test running it via the config parser
psf_file = os.path.join('output', 'simple_psf.fits')
config = {
'input': {
'images': image_file,
'cats': cat_file,
'flag_col': 'flag',
'use_col': 'use',
'stamp_size': 48
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
},
'output': {
'file_name': psf_file
},
}
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(verbose=0)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Round trip to a file
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test using the piffify executable
os.remove(psf_file)
config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test that we can make rho statistics
min_sep = 1
max_sep = 100
bin_size = 0.1
stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size)
stats.compute(psf, orig_stars)
rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5]
for rho in rhos:
# Test the range of separations
radius = np.exp(rho.logr)
# last bin can be one bigger than max_sep
np.testing.assert_array_less(radius,
np.exp(np.log(max_sep) + bin_size))
np.testing.assert_array_less(min_sep, radius)
np.testing.assert_array_almost_equal(np.diff(rho.logr),
bin_size,
decimal=5)
# Test that the max absolute value of each rho isn't crazy
np.testing.assert_array_less(np.abs(rho.xip), 1)
# # Check that each rho isn't precisely zero. This means the sum of abs > 0
np.testing.assert_array_less(0, np.sum(np.abs(rho.xip)))
# Test the plotting and writing
rho_psf_file = os.path.join('output', 'simple_psf_rhostats.pdf')
stats.write(rho_psf_file)
# Test that we can make summary shape statistics, using HSM
shapeStats = piff.ShapeHistogramsStats()
shapeStats.compute(psf, orig_stars)
# test their characteristics
np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
shape_psf_file = os.path.join('output', 'simple_psf_shapestats.pdf')
shapeStats.write(shape_psf_file)
# Test that we can use the config parser for both RhoStats and ShapeHistogramsStats
config['output']['stats'] = [
{
'type': 'ShapeHistograms',
'file_name': shape_psf_file
},
{
'type': 'Rho',
'file_name': rho_psf_file
},
{
'type': 'TwoDHist',
'file_name': os.path.join('output',
'simple_psf_twodhiststats.pdf'),
'number_bins_u': 3,
'number_bins_v': 3,
},
{
'type': 'TwoDHist',
'file_name': os.path.join('output',
'simple_psf_twodhiststats_std.pdf'),
'reducing_function': 'np.std',
'number_bins_u': 3,
'number_bins_v': 3,
},
]
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
piff.piffify(config, logger)
# Test using the piffify executable
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
def test_chisq():
"""Test the Chisq outlier class
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_single_image.log')
# Make the image
image = galsim.Image(512, 512, scale=0.26)
nstars = 1000 # enough that there could be some overlaps. Also, some over the edge.
rng = np.random.RandomState(1234)
x = rng.random_sample(nstars) * 512
y = rng.random_sample(nstars) * 512
sigma = np.ones_like(x) * 0.4 # Most have this sigma
g1 = np.ones_like(x) * 0.023 # Most are pretty round.
g2 = np.ones_like(x) * 0.012
flux = np.exp(rng.normal(size=nstars))
print('flux range = ', np.min(flux), np.max(flux), np.median(flux),
np.mean(flux))
# Make a few intentionally wrong.
g1[35] = 0.29
g1[188] = -0.15
sigma[239] = 0.2
g2[347] = -0.15
sigma[551] = 1.3
g2[809] = 0.05
g1[922] = -0.03
# Draw a Gaussian PSF at each location on the image.
for i in range(nstars):
psf = galsim.Gaussian(sigma=sigma[i]).shear(g1=g1[i], g2=g2[i])
stamp = psf.drawImage(scale=0.26, center=galsim.PositionD(x[i], y[i]))
b = stamp.bounds & image.bounds
image[b] += stamp[b]
noise = 0.02
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=noise))
image_file = os.path.join('output', 'test_chisq_im.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x), dtype=dtype)
data['x'] = x
data['y'] = y
cat_file = os.path.join('output', 'test_chisq_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Read the catalog in as stars.
config = {
'image_file_name': image_file,
'cat_file_name': cat_file,
'noise': noise**2, # Variance here is sigma^2
'stamp_size': 15,
'use_partial': True,
}
input = piff.InputFiles(config, logger=logger)
stars = input.makeStars()
# Skip the solve step. Just give it the right answer and see what it finds for outliers
model = piff.Gaussian()
interp = piff.Mean()
interp.mean = np.array([0.4, 0.023, 0.012])
psf = piff.SimplePSF(model, interp)
stars = psf.interpolateStarList(stars)
stars = [psf.model.reflux(s, logger=logger) for s in stars]
outliers1 = piff.ChisqOutliers(nsigma=5)
stars1, nremoved1 = outliers1.removeOutliers(stars, logger=logger)
print('nremoved1 = ', nremoved1)
assert len(stars1) == len(stars) - nremoved1
# This is what nsigma=5 means in terms of probability
outliers2 = piff.ChisqOutliers(prob=5.733e-7)
stars2, nremoved2 = outliers2.removeOutliers(stars, logger=logger)
print('nremoved2 = ', nremoved2)
assert len(stars2) == len(stars) - nremoved2
assert nremoved1 == nremoved2
# The following is nearly equivalent for this particular data set.
# For dof=222 (what most of these have, this probability converts to
# thresh = 455.40143379
# or ndof = 2.0513578
# But note that when using the above prop or nsigma, the code uses a tailored threshold
# different for each star's particular dof, which varies (since some are off the edge).
outliers3 = piff.ChisqOutliers(thresh=455.401)
stars3, nremoved3 = outliers3.removeOutliers(stars, logger=logger)
print('nremoved3 = ', nremoved3)
assert len(stars3) == len(stars) - nremoved3
outliers4 = piff.ChisqOutliers(ndof=2.05136)
stars4, nremoved4 = outliers4.removeOutliers(stars, logger=logger)
print('nremoved4 = ', nremoved4)
assert len(stars4) == len(stars) - nremoved4
assert nremoved3 == nremoved4
# Regression tests. If these change, make sure we understand why.
assert nremoved1 == nremoved2 == 58
assert nremoved3 == nremoved4 == 16 # Much less, since edge objects aren't being removed
# nearly as often as when they have a custom thresh.
# Can't provide multiple thresh specifications
np.testing.assert_raises(TypeError,
piff.ChisqOutliers,
nsigma=5,
prob=1.e-3)
np.testing.assert_raises(TypeError,
piff.ChisqOutliers,
nsigma=5,
thresh=100)
np.testing.assert_raises(TypeError, piff.ChisqOutliers, nsigma=5, ndof=3)
np.testing.assert_raises(TypeError,
piff.ChisqOutliers,
prob=1.e-3,
thresh=100)
np.testing.assert_raises(TypeError, piff.ChisqOutliers, prob=1.e-3, ndof=3)
np.testing.assert_raises(TypeError, piff.ChisqOutliers, thresh=100, ndof=3)
# Need to specifiy it somehow.
np.testing.assert_raises(TypeError, piff.ChisqOutliers)
