def test_twodstats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_twodstats_config.log')
image_file = os.path.join('output', 'test_stats_image.fits')
cat_file = os.path.join('output', 'test_stats_cat.fits')
psf_file = os.path.join('output', 'test_twodstats.fits')
twodhist_file = os.path.join('output', 'test_twodhiststats.pdf')
twodhist_std_file = os.path.join('output', 'test_twodhiststats_std.pdf')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'stamp_size': 48
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
},
'output': {
'file_name':
psf_file,
'stats': [
{
'type': 'TwoDHist',
'file_name': twodhist_file,
'nbins_u': 3,
'nbins_v': 3,
},
{
'type': 'TwoDHist',
'file_name': twodhist_std_file,
'reducing_function': 'np.std',
'nbins_u': 3,
'nbins_v': 3,
},
]
}
}
piff.piffify(config, logger)
assert os.path.isfile(twodhist_file)
assert os.path.isfile(twodhist_std_file)
# repeat with plotify function
os.remove(twodhist_file)
os.remove(twodhist_std_file)
piff.plotify(config, logger)
assert os.path.isfile(twodhist_file)
assert os.path.isfile(twodhist_std_file)
def test_sizemag_plot():
"""Check a size-magnitude plot.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = piff.config.setup_logger(
log_file='output/test_sizemag_plot.log')
config = piff.config.read_config('sizemag.yaml')
file_name = os.path.join('output',
config['output']['stats'][0]['file_name'])
# Some modifications to speed it up a bit.
config['select'] = {
'type': 'Properties',
'where': '(CLASS_STAR > 0.9) & (MAG_AUTO < 13)',
'hsm_size_reject': 4,
'min_snr': 50,
}
config['psf']['interp'] = {'type': 'Mean'}
config['psf']['outliers']['nsigma'] = 10
del config['output']['stats'][1:]
# Run via piffify
piff.piffify(config, logger)
assert os.path.isfile(file_name)
# repeat with plotify function
os.remove(file_name)
piff.plotify(config, logger)
assert os.path.isfile(file_name)
def test_shapestats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_shapestats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_shapestats.fits')
shape_file = os.path.join('output','test_shapestats.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'ShapeHistograms',
'file_name': shape_file
},
]
},
}
piff.piffify(config, logger)
assert os.path.isfile(shape_file)
# repeat with plotify function
os.remove(shape_file)
piff.plotify(config, logger)
assert os.path.isfile(shape_file)
# Test ShapeHistogramStats directly
psf = piff.read(psf_file)
shapeStats = piff.ShapeHistogramsStats()
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
shapeStats.compute(psf, orig_stars)
# test their characteristics
sigma = 1.3 # (copied from setup())
g1 = 0.23
g2 = -0.17
np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
def test_invalid_config():
# Test a few invalid uses of the config parsing.
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_invalid_config.log')
image_file = os.path.join('output','simple_image.fits')
cat_file = os.path.join('output','simple_cat.fits')
psf_file = os.path.join('output','simple_psf.fits')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'flag_col' : 'flag',
'use_flag' : 1,
'skip_flag' : 4,
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False},
'interp' : { 'type' : 'Mean' },
'max_iter' : 10,
'chisq_thresh' : 0.2,
},
'output' : { 'file_name' : psf_file },
}
# Invalid variable specification
with np.testing.assert_raises(ValueError):
piff.parse_variables(config, ['verbose:0'], logger=logger)
# process needs both input and psf
with np.testing.assert_raises(ValueError):
piff.process(config={'input':config['input']}, logger=logger)
with np.testing.assert_raises(ValueError):
piff.process(config={'psf':config['psf']}, logger=logger)
# piffify also needs output
with np.testing.assert_raises(ValueError):
piff.piffify(config={'input':config['input']}, logger=logger)
with np.testing.assert_raises(ValueError):
piff.piffify(config={'psf':config['psf']}, logger=logger)
with np.testing.assert_raises(ValueError):
piff.piffify(config={'input':config['input'], 'psf':config['psf']}, logger=logger)
# plotify doesn't need psf, but needs a 'file_name' in output
with np.testing.assert_raises(ValueError):
piff.plotify(config={'input':config['input']}, logger=logger)
with np.testing.assert_raises(ValueError):
piff.plotify(config={'input':config['input'], 'output':{}}, logger=logger)
# Error if missing either model or interp
config2 = copy.deepcopy(config)
del config2['psf']['model']
with np.testing.assert_raises(ValueError):
piff.piffify(config2, logger)
config2['psf']['model'] = config['psf']['model']
del config2['psf']['interp']
with np.testing.assert_raises(ValueError):
piff.piffify(config2, logger)
def test_twodstats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_twodstats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_twodstats.fits')
twodhist_file = os.path.join('output','test_twodhiststats.pdf')
twodhist_std_file = os.path.join('output','test_twodhiststats_std.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'TwoDHist',
'file_name': twodhist_file,
'number_bins_u': 3,
'number_bins_v': 3,
},
{
'type': 'TwoDHist',
'file_name': twodhist_std_file,
'reducing_function': 'np.std',
'number_bins_u': 3,
'number_bins_v': 3,
},
]
}
}
piff.piffify(config, logger)
assert os.path.isfile(twodhist_file)
assert os.path.isfile(twodhist_std_file)
# repeat with plotify function
os.remove(twodhist_file)
os.remove(twodhist_std_file)
piff.plotify(config, logger)
assert os.path.isfile(twodhist_file)
assert os.path.isfile(twodhist_std_file)
def test_single_image():
"""Test the whole process with a single image.
Note: This test is based heavily on test_single_image in test_simple.py.
"""
import os
import fitsio
np_rng = np.random.RandomState(1234)
# Make the image
image = galsim.Image(2048, 2048, scale=0.2)
# The (x,y) values will be on a grid 5 x 5 stars with a random sub-pixel offset.
xvals = np.linspace(50., 1950., 5)
yvals = np.linspace(50., 1950., 5)
x_list, y_list = np.meshgrid(xvals, yvals)
x_list = x_list.flatten()
y_list = y_list.flatten()
x_list = x_list + (np_rng.rand(len(x_list)) - 0.5)
y_list = y_list + (np_rng.rand(len(x_list)) - 0.5)
print('x_list = ',x_list)
print('y_list = ',y_list)
# Range of fluxes from 100 to 15000
flux_list = 100. * np.exp(5. * np_rng.rand(len(x_list)))
print('fluxes range from ',np.min(flux_list),np.max(flux_list))
# Draw a Moffat PSF at each location on the image.
# Have the truth values vary quadratically across the image.
beta_fn = lambda x,y: 3.5 - 0.1*(x/1000) + 0.08*(y/1000)**2
fwhm_fn = lambda x,y: 0.9 + 0.05*(x/1000) - 0.03*(y/1000) + 0.02*(x/1000)*(y/1000)
e1_fn = lambda x,y: 0.02 - 0.01*(x/1000)
e2_fn = lambda x,y: -0.03 + 0.02*(x/1000)**2 - 0.01*(y/1000)*2
for x,y,flux in zip(x_list, y_list, flux_list):
beta = beta_fn(x,y)
fwhm = fwhm_fn(x,y)
e1 = e1_fn(x,y)
e2 = e2_fn(x,y)
print(x,y,beta,fwhm,e1,e2)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta, flux=flux).shear(e1=e1, e2=e2)
bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32))
offset = galsim.PositionD( x-int(x)-0.5 , y-int(y)-0.5 )
moffat.drawImage(image=image[bounds], offset=offset, method='no_pixel')
print('drew image')
# Add sky level and noise
sky_level = 1000
noise_sigma = 0.1 # Not much noise to keep this an easy test.
image += sky_level
image.addNoise(galsim.GaussianNoise(sigma=noise_sigma))
# Write out the image to a file
image_file = os.path.join('output','pixel_moffat_image.fits')
image.write(image_file)
print('wrote image')
# Write out the catalog to a file
dtype = [ ('x','f8'), ('y','f8') ]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output','pixel_moffat_cat.fits')
fitsio.write(cat_file, data, clobber=True)
print('wrote catalog')
# Use InputFiles to read these back in
config = { 'image_file_name': image_file,
'cat_file_name': cat_file,
'stamp_size': 32,
'noise' : noise_sigma**2,
'sky' : sky_level,
}
input = piff.InputFiles(config)
assert input.image_file_name == [image_file]
assert input.cat_file_name == [cat_file]
# Check image
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
assert len(input.image_pos) == 1
assert len(input.image_pos[0]) == len(x_list)
np.testing.assert_equal([pos.x for pos in input.image_pos[0]], x_list)
np.testing.assert_equal([pos.y for pos in input.image_pos[0]], y_list)
# Make stars
orig_stars = input.makeStars()
assert len(orig_stars) == len(x_list)
assert orig_stars[0].image.array.shape == (32,32)
# Make a test star, not at the location of any of the model stars to use for each of the
# below tests.
x0 = 1024 # Some random position, not where a star was originally.
y0 = 133
beta = beta_fn(x0,y0)
fwhm = fwhm_fn(x0,y0)
e1 = e1_fn(x0,y0)
e2 = e2_fn(x0,y0)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta).shear(e1=e1, e2=e2)
target_star = piff.Star.makeTarget(x=x0, y=y0, scale=image.scale)
test_im = galsim.ImageD(bounds=target_star.image.bounds, scale=image.scale)
moffat.drawImage(image=test_im, method='no_pixel', use_true_center=False)
print('made test star')
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
order = 2
else:
logger = None
order = 1
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
psf_file = os.path.join('output','pixel_psf.fits')
if __name__ == '__main__':
# Process the star data
model = piff.PixelGrid(0.2, 16, start_sigma=0.9/2.355)
interp = piff.BasisPolynomial(order=order)
pointing = None # wcs is not Celestial here, so pointing needs to be None.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, {0:input.images[0].wcs}, pointing, logger=logger)
# Check that the interpolation is what it should be
print('target.flux = ',target_star.fit.flux)
test_star = psf.drawStar(target_star)
print('flux = ', test_im.array.sum(), test_star.image.array.sum())
print('max diff = ',np.max(np.abs(test_star.image.array-test_im.array)))
np.testing.assert_almost_equal(test_star.image.array/2, test_im.array/2, decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array/2, test_im.array/2, decimal=3)
# Round trip through a file
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.PixelGrid
assert type(psf.interp) is piff.BasisPolynomial
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array/2, test_im.array/2, decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array/2., test_im.array/2., decimal=3)
# Do the whole thing with the config parser
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'x_col' : 'x',
'y_col' : 'y',
'noise' : noise_sigma**2,
'sky' : sky_level,
'stamp_size' : 48 # Bigger than we drew, but should still work.
},
'output' : {
'file_name' : psf_file
},
'psf' : {
'model' : {
'type' : 'PixelGrid',
'scale' : 0.2,
'size' : 16, # Much smaller than the input stamps, but this is plenty here.
'start_sigma' : 0.9/2.355
},
'interp' : {
'type' : 'BasisPolynomial',
'order' : order
},
},
}
if __name__ == '__main__':
config['verbose'] = 2
else:
config['verbose'] = 0
print("Running piffify function")
piff.piffify(config)
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
print("Max abs diff = ",np.max(np.abs(test_star.image.array - test_im.array)))
np.testing.assert_almost_equal(test_star.image.array/2., test_im.array/2., decimal=3)
# Test using the piffify executable
with open('pixel_moffat.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
print("Running piffify executable")
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'pixel_moffat.yaml'] )
p.communicate()
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array/2., test_im.array/2., decimal=3)
# test copy_image property of drawStar and draw
for draw in [psf.drawStar, psf.model.draw]:
target_star_copy = psf.interp.interpolate(piff.Star(target_star.data.copy(), target_star.fit.copy())) # interp is so that when we do psf.model.draw we have fit.params to work with
test_star_copy = draw(target_star_copy, copy_image=True)
test_star_nocopy = draw(target_star_copy, copy_image=False)
# if we modify target_star_copy, then test_star_nocopy should be modified, but not test_star_copy
target_star_copy.image.array[0,0] = 23456
assert test_star_nocopy.image.array[0,0] == target_star_copy.image.array[0,0]
assert test_star_copy.image.array[0,0] != target_star_copy.image.array[0,0]
# however the other pixels SHOULD still be all the same value
assert test_star_nocopy.image.array[1,1] == target_star_copy.image.array[1,1]
assert test_star_copy.image.array[1,1] == target_star_copy.image.array[1,1]
# check that drawing onto an image does not return a copy
image = psf.draw(x=x0, y=y0)
image_reference = psf.draw(x=x0, y=y0, image=image)
image_reference.array[0,0] = 123456
assert image.array[0,0] == image_reference.array[0,0]
def test_bad_hsm():
"""Test that stats don't break when all stars end up being flagged with hsm errors.
"""
image_file = os.path.join('input','DECam_00241238_01.fits.fz')
cat_file = os.path.join('input',
'DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits')
psf_file = os.path.join('output','bad_hsm.fits')
twodhist_file = os.path.join('output','bad_hsm_twod.pdf')
whisker_file = os.path.join('output','bad_hsm_whisk.pdf')
rho_file = os.path.join('output','bad_hsm_rho.pdf')
shape_file = os.path.join('output','bad_hsm_shape.pdf')
star_file = os.path.join('output','bad_hsm_star.pdf')
hsm_file = os.path.join('output','bad_hsm_hsm.fits')
sizemag_file = os.path.join('output','bad_hsm_sizemag.png')
stamp_size = 25
# The configuration dict with the right input fields for the file we're using.
config = {
'input' : {
'nstars': 8,
'image_file_name' : image_file,
'image_hdu' : 1,
'weight_hdu' : 3,
'badpix_hdu' : 2,
'cat_file_name' : cat_file,
'cat_hdu' : 2,
# These next two are intentionally backwards. The PixelGrid will find some kind
# of solution, but it will be complex garbage, and hsm will fail for them.
'x_col' : 'YWIN_IMAGE',
'y_col' : 'XWIN_IMAGE',
'sky_col' : 'BACKGROUND',
'stamp_size' : stamp_size,
'ra' : 'TELRA',
'dec' : 'TELDEC',
'gain' : 'GAINA',
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'TwoDHist',
'file_name': twodhist_file,
},
{
'type': 'Whisker',
'file_name': whisker_file,
},
{ # Note: stats doesn't have to be a list.
'type': 'Rho',
'file_name': rho_file
},
{
'type': 'ShapeHist',
'file_name': shape_file,
},
{
'type': 'Star',
'file_name': star_file,
},
{
'type': 'SizeMag',
'file_name': sizemag_file,
},
{
'type': 'HSMCatalog',
'file_name': hsm_file,
},
],
},
'psf' : {
'model' : {
'type' : 'PixelGrid',
'scale' : 0.3,
'size' : 10,
},
'interp' : { 'type' : 'Mean' },
'outliers' : {
'type' : 'Chisq',
'nsigma' : 0.05 # This will throw out all but 1, which adds an additional
# test of Star stats when nstars < nplot
}
},
}
if __name__ == '__main__':
logger = piff.config.setup_logger(1)
else:
config['verbose'] = 0
logger = None
for f in [twodhist_file, rho_file, shape_file, star_file, hsm_file]:
if os.path.exists(f):
os.remove(f)
piff.piffify(config, logger=logger)
# Confirm that all but one star was rejected, since that was part of the intent of this test.
psf = piff.read(psf_file)
print('stars = ',psf.stars)
print('nremoved = ',psf.nremoved)
assert len(psf.stars) == 1
assert psf.nremoved == 7 # There were 8 to start.
for f in [twodhist_file, rho_file, shape_file, star_file, sizemag_file, hsm_file]:
assert os.path.exists(f)
# Check hsm file with bad measurements
# The one star that was left still fails hsm measurement here.
data = fitsio.read(hsm_file)
for col in ['ra', 'dec', 'x', 'y', 'u', 'v',
'T_data', 'g1_data', 'g2_data',
'T_model', 'g1_model', 'g2_model',
'flux', 'reserve', 'flag_truth', 'flag_model']:
assert len(data[col]) == 1
print('flag_truth = ',data['flag_truth'])
print('flag_model = ',data['flag_model'])
np.testing.assert_array_equal(data['flag_truth'], 7)
np.testing.assert_array_equal(data['flag_model'], 7)
def test_reserve():
"""Test the reserve_frac option.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_single_reserve.log')
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
true_params = [sigma, g1, g2]
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y in zip(x_list, y_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
psf.drawImage(image[bounds],
center=galsim.PositionD(x, y),
method='no_pixel')
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('output', 'test_simple_reserve_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output', 'test_simple_reserve_cat.fits')
fitsio.write(cat_file, data, clobber=True)
psf_file = os.path.join('output', 'test_simple_reserve_psf.fits')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'reserve_frac': 0.2,
'stamp_size': 32,
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
},
'output': {
'file_name': psf_file
},
}
piff.piffify(config, logger)
psf = piff.read(psf_file)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
print('chisq = ', psf.chisq)
print('dof = ', psf.dof)
nreserve = len([s for s in psf.stars if s.is_reserve])
ntot = len(psf.stars)
print('reserve = %s/%s' % (nreserve, ntot))
assert nreserve == 2
assert ntot == 10
print('dof =? ', (32 * 32 - 6) * (ntot - nreserve))
assert psf.dof == (32 * 32 - 6) * (ntot - nreserve)
for star in psf.stars:
# Fits should be good for both reserve and non-reserve stars
np.testing.assert_almost_equal(star.fit.params, true_params, decimal=4)
def test_yaml():
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_gp.log')
# Take DES test image, and test doing a psf run with GP interpolator
# Use config parser:
psf_file = os.path.join('output', 'gp_psf.fits')
config = {
'input': {
# These can be regular strings
'image_file_name': 'input/DECam_00241238_01.fits.fz',
# Or any GalSim str value type. e.g. FormattedStr
'cat_file_name': {
'type': 'FormattedStr',
'format':
'%s/DECam_%08d_%02d_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits',
'items': [
'input', # dir
241238, # expnum
1 # chipnum
]
},
# What hdu is everything in?
'image_hdu': 1,
'badpix_hdu': 2,
'weight_hdu': 3,
'cat_hdu': 2,
# What columns in the catalog have things we need?
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
'sky_col': 'BACKGROUND',
# How large should the postage stamp cutouts of the stars be?
'stamp_size': 21,
},
'psf': {
'model': {
'type': 'GSObjectModel',
'fastfit': True,
'gsobj': 'galsim.Gaussian(sigma=1.0)'
},
'interp': {
'type': 'GPInterp',
'keys': ['u', 'v'],
'optimizer': 'none',
'kernel': 'RBF(200.0)'
}
},
'output': {
'file_name': psf_file
},
}
piff.piffify(config, logger)
psf = piff.read(psf_file)
assert type(psf.model) is piff.GSObjectModel
assert type(psf.interp) is piff.GPInterp
print('nstars = ', len(psf.stars))
target = psf.stars[17]
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
target.fit.params,
decimal=3)
# This should also work if the target doesn't have a fit yet.
print('interpolate ', piff.Star(target.data, None))
test_star = psf.interp.interpolate(piff.Star(target.data, None))
np.testing.assert_almost_equal(test_star.fit.params,
target.fit.params,
decimal=3)
def test_single_image():
"""Test the whole process with a single image.
Note: This test is based heavily on test_single_image in test_simple.py.
"""
import os
import fitsio
np_rng = np.random.RandomState(1234)
# Make the image
image = galsim.Image(2048, 2048, scale=0.2)
# The (x,y) values will be on a grid 5 x 5 stars with a random sub-pixel offset.
xvals = np.linspace(50., 1950., 5)
yvals = np.linspace(50., 1950., 5)
x_list, y_list = np.meshgrid(xvals, yvals)
x_list = x_list.flatten()
y_list = y_list.flatten()
x_list = x_list + (np_rng.rand(len(x_list)) - 0.5)
y_list = y_list + (np_rng.rand(len(x_list)) - 0.5)
print('x_list = ', x_list)
print('y_list = ', y_list)
# Range of fluxes from 100 to 15000
flux_list = 100. * np.exp(5. * np_rng.rand(len(x_list)))
print('fluxes range from ', np.min(flux_list), np.max(flux_list))
# Draw a Moffat PSF at each location on the image.
# Have the truth values vary quadratically across the image.
beta_fn = lambda x, y: 3.5 - 0.1 * (x / 1000) + 0.08 * (y / 1000)**2
fwhm_fn = lambda x, y: 0.9 + 0.05 * (x / 1000) - 0.03 * (
y / 1000) + 0.02 * (x / 1000) * (y / 1000)
e1_fn = lambda x, y: 0.02 - 0.01 * (x / 1000)
e2_fn = lambda x, y: -0.03 + 0.02 * (x / 1000)**2 - 0.01 * (y / 1000) * 2
for x, y, flux in zip(x_list, y_list, flux_list):
beta = beta_fn(x, y)
fwhm = fwhm_fn(x, y)
e1 = e1_fn(x, y)
e2 = e2_fn(x, y)
print(x, y, beta, fwhm, e1, e2)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta, flux=flux).shear(e1=e1,
e2=e2)
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
moffat.drawImage(image=image[bounds], offset=offset, method='no_pixel')
print('drew image')
# Write out the image to a file
image_file = os.path.join('data', 'pixel_moffat_image.fits')
image.write(image_file)
print('wrote image')
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('data', 'pixel_moffat_cat.fits')
fitsio.write(cat_file, data, clobber=True)
print('wrote catalog')
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file, stamp_size=32)
assert input.image_files == [image_file]
assert input.cat_files == [cat_file]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Make stars
orig_stars = input.makeStars()
assert len(orig_stars) == len(x_list)
assert orig_stars[0].image.array.shape == (32, 32)
# Make a test star, not at the location of any of the model stars to use for each of the
# below tests.
x0 = 1024 # Some random position, not where a star was originally.
y0 = 133
beta = beta_fn(x0, y0)
fwhm = fwhm_fn(x0, y0)
e1 = e1_fn(x0, y0)
e2 = e2_fn(x0, y0)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta).shear(e1=e1, e2=e2)
target_star = piff.Star.makeTarget(x=x0, y=y0, scale=image.scale)
test_im = galsim.ImageD(bounds=target_star.image.bounds, scale=image.scale)
moffat.drawImage(image=test_im, method='no_pixel', use_true_center=False)
print('made test star')
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if True:
# Process the star data
model = piff.PixelGrid(0.2, 16, start_sigma=0.9 / 2.355)
interp = piff.BasisPolynomial(order=2)
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
pointing = None # wcs is not Celestial here, so pointing needs to be None.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, {0: input.images[0].wcs}, pointing, logger=logger)
# Check that the interpolation is what it should be
print('target.flux = ', target_star.fit.flux)
test_star = psf.drawStar(target_star)
#print('test_im center = ',test_im[b].array)
#print('flux = ',test_im.array.sum())
#print('interp_im center = ',test_star.image[b].array)
#print('flux = ',test_star.image.array.sum())
#print('max diff = ',np.max(np.abs(test_star.image.array-test_im.array)))
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array, test_im.array, decimal=3)
# Round trip through a file
psf_file = os.path.join('output', 'pixel_psf.fits')
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.PixelGrid
assert type(psf.interp) is piff.BasisPolynomial
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array, test_im.array, decimal=3)
# Do the whole thing with the config parser
config = {
'input': {
'images': image_file,
'cats': cat_file,
'x_col': 'x',
'y_col': 'y',
'stamp_size': 48 # Bigger than we drew, but should still work.
},
'output': {
'file_name': psf_file
},
'psf': {
'model': {
'type': 'PixelGrid',
'scale': 0.2,
'size':
16, # Much smaller than the input stamps, but this is plenty here.
'start_sigma': 0.9 / 2.355
},
'interp': {
'type': 'BasisPolynomial',
'order': 2
},
},
}
if __name__ == '__main__':
print("Running piffify function")
piff.piffify(config)
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Test using the piffify executable
config['verbose'] = 0
with open('pixel_moffat.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
print("Running piffify executable")
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'pixel_moffat.yaml'])
p.communicate()
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
def test_yaml():
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_gp.log')
# Take DES test image, and test doing a psf run with GP interpolator
# Use config parser:
for gp_piff in ['GPInterp', 'GPInterp2pcf']:
psf_file = os.path.join('output','gp_psf.fits')
config = {
'input' : {
# These can be regular strings
'image_file_name' : 'input/DECam_00241238_01.fits.fz',
# Or any GalSim str value type. e.g. FormattedStr
'cat_file_name' : {
'type': 'FormattedStr',
'format': '%s/DECam_%08d_%02d_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits',
'items': [
'input', # dir
241238, # expnum
1 # chipnum
]
},
# What hdu is everything in?
'image_hdu' : 1,
'badpix_hdu' : 2,
'weight_hdu' : 3,
'cat_hdu' : 2,
# What columns in the catalog have things we need?
'x_col' : 'XWIN_IMAGE',
'y_col' : 'YWIN_IMAGE',
'ra' : 'TELRA',
'dec' : 'TELDEC',
'gain' : 'GAINA',
'sky_col' : 'BACKGROUND',
# How large should the postage stamp cutouts of the stars be?
'stamp_size' : 21,
},
'psf' : {
'model' : { 'type' : 'GSObjectModel',
'fastfit' : True,
'gsobj' : 'galsim.Gaussian(sigma=1.0)' },
'interp' : { 'type' : gp_piff,
'keys' : ['u', 'v'],
'kernel' : 'RBF(200.0)',
'optimize' : False,}
},
'output' : { 'file_name' : psf_file },
}
if __name__ != '__main__':
config['input']['nstars'] = 25
piff.piffify(config, logger)
psf = piff.read(psf_file)
assert type(psf.model) is piff.GSObjectModel
assert type(psf.interp) is piff.GPInterp or type(psf.interp) is piff.GPInterp2pcf
print('nstars = ',len(psf.stars))
target = psf.stars[17]
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, target.fit.params, decimal=3)
# This should also work if the target doesn't have a fit yet.
print('interpolate ',piff.Star(target.data,None))
test_star = psf.interp.interpolate(piff.Star(target.data,None))
np.testing.assert_almost_equal(test_star.fit.params, target.fit.params, decimal=3)
def test_parallel():
# Run the same test as test_single, but using nproc
wcs1 = galsim.TanWCS(
galsim.AffineTransform(0.26, 0.05, -0.08, -0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees))
wcs2 = galsim.TanWCS(
galsim.AffineTransform(0.25, -0.02, 0.01, 0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees))
field_center = galsim.CelestialCoord(0 * galsim.degrees,
-25 * galsim.degrees)
if __name__ == '__main__':
nstars = 20 # per ccd
else:
nstars = 6 # per ccd
rng = np.random.RandomState(1234)
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra1, dec1 = wcs1.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data1 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im1 = drawImage(2048, 2048, wcs1, x, y, e1, e2, s)
im1.write('output/test_parallel_im1.fits')
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra2, dec2 = wcs2.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
# Same functions of u,v, but using the positions on chip 2
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data2 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im2 = drawImage(2048, 2048, wcs2, x, y, e1, e2, s)
im2.write('output/test_parallel_im2.fits')
ra12 = np.concatenate([ra1, ra2])
dec12 = np.concatenate([dec1, dec2])
data12 = np.array(list(zip(ra12, dec12)),
dtype=[('ra', float), ('dec', float)])
fitsio.write('output/test_parallel.fits', data12, clobber=True)
# im3 is blank. Will give errors trying to measure PSF from it.
im3 = galsim.Image(2048, 2048, wcs=wcs2)
im3.write('output/test_parallel_im3.fits')
psf_file = os.path.join('output', 'test_single.fits')
config = {
'input': {
# A third way to input these same file names. Use GalSim config values and
# explicitly specify the number of images to read
'nimages': 2,
'image_file_name': {
'type': 'FormattedStr',
'format': '%s/test_parallel_im%d.fits',
'items': ['output', '$image_num+1'],
},
'cat_file_name': 'output/test_parallel.fits',
'chipnum': '$image_num+1',
'ra_col': 'ra',
'dec_col': 'dec',
'ra_units': 'rad',
'dec_units': 'rad',
'nproc': -1,
},
'psf': {
'type': 'SingleChip',
'model': {
'type': 'Moffat',
'beta': 2.5,
},
'interp': {
'type': 'Polynomial',
'order': 2,
},
'nproc': 2,
},
'output': {
'file_name': psf_file,
},
}
with CaptureLog(level=2) as cl:
piff.piffify(config, logger=cl.logger)
psf = piff.read(psf_file)
for chipnum, data, wcs in [(1, data1, wcs1), (2, data2, wcs2)]:
for k in range(nstars):
x = data['x'][k]
y = data['y'][k]
e1 = data['e1'][k]
e2 = data['e2'][k]
s = data['s'][k]
image_pos = galsim.PositionD(x, y)
star = piff.Star.makeTarget(x=x,
y=y,
wcs=wcs,
stamp_size=48,
pointing=field_center,
chipnum=chipnum)
star = psf.drawStar(star)
np.testing.assert_almost_equal(star.fit.params, [s, e1, e2],
decimal=6)
# Finally, check that the logger properly captures the subprocess logs
with CaptureLog(level=2) as cl:
psf = piff.process(config, cl.logger)
#print('with nproc=2, log = ',cl.output)
assert "Processing catalog 1" in cl.output
assert "Processing catalog 2" in cl.output
assert "Building solution for chip 1" in cl.output
assert "Building solution for chip 2" in cl.output
# Check that errors in the solution get properly reported.
config['input']['nimages'] = 3
with CaptureLog(level=2) as cl:
psf = piff.process(config, cl.logger)
assert "Removed 6 stars in initialize" in cl.output
assert "No stars. Cannot find PSF model." in cl.output
assert "Solutions failed for chipnums: [3]" in cl.output
# Check that errors in the multiprocessing input get properly reported.
config['input']['ra_col'] = 'invalid'
with CaptureLog(level=2) as cl:
with np.testing.assert_raises(ValueError):
psf = piff.process(config, cl.logger)
assert "ra_col = invalid is not a column" in cl.output
# With nproc=1, the error is raised directly.
config['input']['nproc'] = 1
config['verbose'] = 0
with np.testing.assert_raises(ValueError):
psf = piff.process(config)
# But just the input error. Not the one in fitting.
config['psf']['nproc'] = 1
config['input']['ra_col'] = 'ra'
config['verbose'] = 1
with CaptureLog(level=1) as cl:
psf = piff.process(config, logger=cl.logger)
assert "No stars. Cannot find PSF model." in cl.output
assert "Ignoring this failure and continuing on." in cl.output
# pixmappy = os.path.join(pixmappy_dir, 'zone%03d.astro'%zone) # wcs = pixmappy.GalSimWCS(pixmappy_file, exp=expnum, ccdnum=ccdnum, default_color=0) cat_file = '/Users/rebeccachen/Desktop/Piff_work/y3_test/'+str(expnum)+'_c'+str_ccdnum+'.fits' psf_file = '/Users/rebeccachen/Desktop/Piff_work/y3_test/D00'+str(expnum)+'_r_c'+str_ccdnum+'_r2277p01_gaiamatch.piff' config = piff.read_config(piff_config) config['input']['image_file_name'] = img_file config['input']['cat_file_name'] = cat_file config['output']['file_name'] = psf_file # config['input']['wcs']['file_name'] = pixmappy # config['input']['wcs']['exp'] = expnum # config['input']['wcs']['ccdnum'] = ccdnum piff.piffify(config, logger) #gaia matches run for i in [x for x in range(1, 63) if (x != 31 and x!= 61 and x!=2 and x!=5)]: ccdnum = i str_ccdnum = "%.2d" % ccdnum img_file = '/Users/rebeccachen/Desktop/Piff_work/y3_test/D00'+str(expnum)+'_r_c'+str_ccdnum+'_r2277p01_immasked.fits' # f = fitsio.FITS(img_file) # header_list = f[hdu].read_header_list() # header_list = [ d for d in header_list if 'CONTINUE' not in d['name'] ] # h = fitsio.FITSHDR(header_list) # detpos = h['DETPOS'].strip() # dp = detpos[ccdnum] # wz = np.where((which_zone['expnum'] == expnum) & (which_zone['detpos'] == dp))[0][0]
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
flag_list = [0, 0, 12, 0, 0, 1, 0, 0, 0, 0]
use_list = [1, 1, 1, 1, 1, 0, 1, 1, 0, 1]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y, flag, use in zip(x_list, y_list, flag_list, use_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag:
print('corrupting star at ', x, y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('data', 'simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2'), ('use', 'i2')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
data['use'] = use_list
cat_file = os.path.join('data', 'simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file)
assert input.image_files == [image_file]
assert input.cat_files == [cat_file]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Repeat, using flag and use columns this time.
input = piff.InputFiles(image_file,
cat_file,
flag_col='flag',
use_col='use',
stamp_size=48)
assert input.flag_col == 'flag'
assert input.use_col == 'use'
input.readImages()
input.readStarCatalogs()
assert len(input.cats[0]) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48, 48)
# Process the star data
# can only compare to truth if include_pixel=False
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars]
interp.solve(fitted_stars)
print('mean = ', interp.mean)
# Check that the interpolation is what it should be
target = piff.Star.makeTarget(x=1024,
y=123) # Any position would work here.
true_params = [sigma, g1, g2]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Now test running it via the config parser
psf_file = os.path.join('output', 'simple_psf.fits')
config = {
'input': {
'images': image_file,
'cats': cat_file,
'flag_col': 'flag',
'use_col': 'use',
'stamp_size': 48
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
},
'output': {
'file_name': psf_file
},
}
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(verbose=0)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Round trip to a file
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test using the piffify executable
os.remove(psf_file)
config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test that we can make rho statistics
min_sep = 1
max_sep = 100
bin_size = 0.1
stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size)
stats.compute(psf, orig_stars)
rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5]
for rho in rhos:
# Test the range of separations
radius = np.exp(rho.logr)
# last bin can be one bigger than max_sep
np.testing.assert_array_less(radius,
np.exp(np.log(max_sep) + bin_size))
np.testing.assert_array_less(min_sep, radius)
np.testing.assert_array_almost_equal(np.diff(rho.logr),
bin_size,
decimal=5)
# Test that the max absolute value of each rho isn't crazy
np.testing.assert_array_less(np.abs(rho.xip), 1)
# # Check that each rho isn't precisely zero. This means the sum of abs > 0
np.testing.assert_array_less(0, np.sum(np.abs(rho.xip)))
# Test the plotting and writing
rho_psf_file = os.path.join('output', 'simple_psf_rhostats.pdf')
stats.write(rho_psf_file)
# Test that we can make summary shape statistics, using HSM
shapeStats = piff.ShapeHistogramsStats()
shapeStats.compute(psf, orig_stars)
# test their characteristics
np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
shape_psf_file = os.path.join('output', 'simple_psf_shapestats.pdf')
shapeStats.write(shape_psf_file)
# Test that we can use the config parser for both RhoStats and ShapeHistogramsStats
config['output']['stats'] = [
{
'type': 'ShapeHistograms',
'file_name': shape_psf_file
},
{
'type': 'Rho',
'file_name': rho_psf_file
},
{
'type': 'TwoDHist',
'file_name': os.path.join('output',
'simple_psf_twodhiststats.pdf'),
'number_bins_u': 3,
'number_bins_v': 3,
},
{
'type': 'TwoDHist',
'file_name': os.path.join('output',
'simple_psf_twodhiststats_std.pdf'),
'reducing_function': 'np.std',
'number_bins_u': 3,
'number_bins_v': 3,
},
]
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
piff.piffify(config, logger)
# Test using the piffify executable
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
def test_depr_select():
# A bunch of keys used to be allowed in input, but now belong in select.
# Check that the old way still works, but gives a warning.
# This is the input from test_stars in test_input.py
dir = 'input'
image_file = 'test_input_image_00.fits'
cat_file = 'test_input_cat_00.fits'
psf_file = os.path.join('output','test_depr_select.fits')
config = {
'input' : {
'dir' : dir,
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'weight_hdu' : 1,
'sky_col' : 'sky',
'gain_col' : 'gain',
'use_partial' : True,
'nstars': 15, # Just to make the test faster
},
'select': {
'max_snr' : 200,
'min_snr' : 20,
'hsm_size_reject' : 20,
'max_edge_frac': 0.25,
'stamp_center_size': 10,
'max_mask_pixels' : 20,
'reserve_frac' : 0.1,
'seed': 1234,
},
'psf': {
'model' : {'type': 'Gaussian'},
'interp' : {'type': 'Mean'},
},
'output' : { 'file_name' : psf_file },
}
logger = piff.config.setup_logger(log_file='output/test_depr_select.log')
# This is the new API
print('config = ',config)
piff.piffify(config, logger)
psf1 = piff.read(psf_file)
im1 = psf1.draw(x=23, y=34, stamp_size=16)
print('len1 = ',len(psf1.stars))
# This is the old API
config['input'].update(config['select'])
del config['select']
config = galsim.config.CleanConfig(config)
print('config = ',config)
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert "WARNING: Items [" in cl.output
assert "] should now be in the 'select' field of the config file." in cl.output
psf2 = piff.read(psf_file)
print('len2 = ',len(psf2.stars))
assert len(psf1.stars) == len(psf2.stars)
im2 = psf2.draw(x=23, y=34, stamp_size=16)
np.testing.assert_allclose(im2.array, im1.array)
# Also ok for some items to be in select, but erroneously put some in input.
config['input']['min_snr'] = config['select'].pop('min_snr')
config['input']['max_snr'] = config['select'].pop('max_snr')
config = galsim.config.CleanConfig(config)
print('config = ',config)
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert "WARNING: Items ['max_snr', 'min_snr'] should now be in the 'select' field of the config file." in cl.output
psf3 = piff.read(psf_file)
print('len3 = ',len(psf3.stars))
assert len(psf1.stars) == len(psf3.stars)
im3 = psf3.draw(x=23, y=34, stamp_size=16)
np.testing.assert_allclose(im3.array, im1.array)
def test_des_image():
"""Test the whole process with a DES CCD.
"""
import os
import fitsio
image_file = 'input/DECam_00241238_01.fits.fz'
cat_file = 'input/DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits'
orig_image = galsim.fits.read(image_file)
psf_file = os.path.join('output','pixel_des_psf.fits')
if __name__ == '__main__':
# These match what Gary used in fit_des.py
nstars = None
scale = 0.15
size = 31
order = 2
nsigma = 4
else:
# These are faster and good enough for the unit tests.
nstars = 25
scale = 0.26
size = 15
order = 1
nsigma = 1. # This needs to be low to make sure we do test outlier rejection here.
stamp_size = 25
# The configuration dict with the right input fields for the file we're using.
start_sigma = 1.0/2.355 # TODO: Need to make this automatic somehow.
config = {
'input' : {
'nstars': nstars,
'image_file_name' : image_file,
'image_hdu' : 1,
'weight_hdu' : 3,
'badpix_hdu' : 2,
'cat_file_name' : cat_file,
'cat_hdu' : 2,
'x_col' : 'XWIN_IMAGE',
'y_col' : 'YWIN_IMAGE',
'sky_col' : 'BACKGROUND',
'stamp_size' : stamp_size,
'ra' : 'TELRA',
'dec' : 'TELDEC',
'gain' : 'GAINA',
# Test explicitly specifying the wcs (although it is the same here as what is in the
# image anyway).
'wcs' : {
'type': 'Fits',
'file_name': image_file
}
},
'output' : {
'file_name' : psf_file,
},
'psf' : {
'model' : {
'type' : 'PixelGrid',
'scale' : scale,
'size' : size,
'interp' : 'Lanczos(5)',
'start_sigma' : start_sigma,
},
'interp' : {
'type' : 'BasisPolynomial',
'order' : order,
},
'outliers' : {
'type' : 'Chisq',
'nsigma' : nsigma,
'max_remove' : 3
}
},
}
if __name__ == '__main__':
config['verbose'] = 2
else:
config['verbose'] = 0
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if __name__ == '__main__':
# Start by doing things manually:
logger = piff.config.setup_logger(2)
# Largely copied from Gary's fit_des.py, but using the Piff input_handler to
# read the input files.
stars, wcs, pointing = piff.Input.process(config['input'], logger=logger)
if nstars is not None:
stars = stars[:nstars]
# Make model, force PSF centering
model = piff.PixelGrid(scale=scale, size=size, interp=piff.Lanczos(3),
force_model_center=True, start_sigma=start_sigma,
logger=logger)
# Interpolator will be zero-order polynomial.
# Find u, v ranges
interp = piff.BasisPolynomial(order=order, logger=logger)
# Make a psf
psf = piff.SimplePSF(model, interp)
psf.fit(stars, wcs, pointing, logger=logger)
# The difference between the images of the fitted stars and the originals should be
# consistent with noise. Keep track of how many don't meet that goal.
n_bad = 0 # chisq/dof > 2
n_marginal = 0 # chisq/dof > 1.1
n_good = 0 # chisq/dof <= 1.1
# Note: The 2 and 1.1 values here are very arbitrary!
for s in psf.stars:
fitted = psf.drawStar(s)
orig_stamp = orig_image[fitted.image.bounds] - s['sky']
fit_stamp = fitted.image
x0 = int(s['x']+0.5)
y0 = int(s['y']+0.5)
b = galsim.BoundsI(x0-3,x0+3,y0-3,y0+3)
#print('orig center = ',orig_stamp[b].array)
#print('flux = ',orig_stamp.array.sum())
#print('fit center = ',fit_stamp[b].array)
#print('flux = ',fit_stamp.array.sum())
flux = fitted.fit.flux
#print('max diff/flux = ',np.max(np.abs(orig_stamp.array-fit_stamp.array))/flux)
#np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
weight = s.weight # These should be 1/var_pix
resid = fit_stamp - orig_stamp
chisq = np.sum(resid.array**2 * weight.array)
print('chisq = ',chisq)
print('cf. star.chisq, dof = ',s.fit.chisq, s.fit.dof)
assert abs(chisq - s.fit.chisq) < 1.e-3 * chisq
if chisq > 2. * s.fit.dof:
n_bad += 1
elif chisq > 1.1 * s.fit.dof:
n_marginal += 1
else:
n_good += 1
# Check the convenience function that an end user would typically use
offset = s.center_to_offset(s.fit.center)
image = psf.draw(x=s['x'], y=s['y'], stamp_size=stamp_size,
flux=s.fit.flux, offset=offset)
np.testing.assert_almost_equal(image.array, fit_stamp.array, decimal=4)
print('n_good, marginal, bad = ',n_good,n_marginal,n_bad)
# The real counts are 10 and 2. So this says make sure any updates to the code don't make
# things much worse.
assert n_marginal <= 12
assert n_bad <= 3
# Use piffify function
print('start piffify')
piff.piffify(config)
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'], y=stars[0]['y'], stamp_size=stamp_size,
flux=flux, offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
# The first star happens to be a good one, so go ahead and test the arrays directly.
np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
# Test using the piffify executable
with open('pixel_des.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
print('start piffify executable')
p = subprocess.Popen( [piffify_exe, 'pixel_des.yaml'] )
p.communicate()
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'], y=stars[0]['y'], stamp_size=stamp_size,
flux=flux, offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ]
y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ]
flag_list = [ 0, 0, 12, 0, 0, 1, 0, 0, 0, 0 ]
use_list = [ 1, 1, 1, 1, 1, 0, 1, 1, 0, 1 ]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x,y,flag,use in zip(x_list, y_list, flag_list, use_list):
bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32))
offset = galsim.PositionD( x-int(x)-0.5 , y-int(y)-0.5 )
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag:
print('corrupting star at ',x,y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
# Write out the image to a file
image_file = os.path.join('data','simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [ ('x','f8'), ('y','f8'), ('flag','i2'), ('use','i2') ]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
data['use'] = use_list
cat_file = os.path.join('data','simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file)
assert input.image_files == [ image_file ]
assert input.cat_files == [ cat_file ]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Repeat, using flag and use columns this time.
input = piff.InputFiles(image_file, cat_file, flag_col='flag', use_col='use', stamp_size=48)
assert input.flag_col == 'flag'
assert input.use_col == 'use'
input.readImages()
input.readStarCatalogs()
assert len(input.cats[0]) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48,48)
# Process the star data
model = piff.Gaussian()
interp = piff.Mean()
fitted_stars = [ model.fit(star) for star in orig_stars ]
interp.solve(fitted_stars)
print('mean = ',interp.mean)
# Check that the interpolation is what it should be
target = piff.Star.makeTarget(x=1024, y=123) # Any position would work here.
true_params = [ sigma, g1, g2 ]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Now test running it via the config parser
psf_file = os.path.join('output','simple_psf.fits')
config = {
'input' : {
'images' : image_file,
'cats' : cat_file,
'flag_col' : 'flag',
'use_col' : 'use',
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian' },
'interp' : { 'type' : 'Mean' },
},
'output' : { 'file_name' : psf_file },
}
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=3)
else:
logger = piff.config.setup_logger(verbose=1)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Round trip to a file
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Test using the piffify executable
os.remove(psf_file)
with open('simple.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'simple.yaml'] )
p.communicate()
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Test that we can make rho statistics
min_sep = 1
max_sep = 100
bin_size = 0.1
stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size)
stats.compute(psf, orig_stars)
rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5]
for rho in rhos:
# Test the range of separations
radius = np.exp(rho.logr)
# last bin can be one bigger than max_sep
np.testing.assert_array_less(radius, np.exp(np.log(max_sep) + bin_size))
np.testing.assert_array_less(min_sep, radius)
np.testing.assert_array_almost_equal(np.diff(rho.logr), bin_size, decimal=5)
# Test that the max absolute value of each rho isn't crazy
np.testing.assert_array_less(np.abs(rho.xip), 1)
# Check that each rho isn't precisely zero. This means the sum of abs > 0
np.testing.assert_array_less(0, np.sum(np.abs(rho.xip)))
# Test the plotting and writing
rho_psf_file = os.path.join('output','simple_psf_rhostats.png')
stats.write(rho_psf_file)
# Test that we can make summary shape statistics, using HSM
shapeStats = piff.ShapeHistogramsStats()
shapeStats.compute(psf, orig_stars)
# test their characteristics
np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
shape_psf_file = os.path.join('output','simple_psf_shapestats.png')
shapeStats.write(shape_psf_file)
# Test that we can use the config parser for both RhoStats and ShapeHistogramsStats
config['output']['stats'] = [
{
'type': 'ShapeHistograms',
'file_name': shape_psf_file
},
{
'type': 'Rho',
'file_name': rho_psf_file
},
]
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
piff.piffify(config)
# Test using the piffify executable
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
with open('simple.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'simple.yaml'] )
p.communicate()
def test_des_image():
"""Test the whole process with a DES CCD.
"""
import os
import fitsio
image_file = 'y1_test/DECam_00241238_01.fits.fz'
cat_file = 'y1_test/DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits'
orig_image = galsim.fits.read(image_file)
psf_file = os.path.join('output', 'pixel_des_psf.fits')
if __name__ == '__main__':
# These match what Gary used in fit_des.py
nstars = None
scale = 0.15
size = 41
else:
# These are faster and good enough for the unit tests.
nstars = 25
scale = 0.2
size = 21
stamp_size = 51
# The configuration dict with the right input fields for the file we're using.
start_sigma = 1.0 / 2.355 # TODO: Need to make this automatic somehow.
config = {
'input': {
'images': image_file,
'image_hdu': 1,
'weight_hdu': 3,
'badpix_hdu': 2,
'cats': cat_file,
'cat_hdu': 2,
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'sky_col': 'BACKGROUND',
'stamp_size': stamp_size,
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
},
'output': {
'file_name': psf_file,
},
'psf': {
'model': {
'type': 'PixelGrid',
'scale': scale,
'size': size,
'start_sigma': start_sigma,
},
'interp': {
'type': 'BasisPolynomial',
'order': 2,
},
},
}
if __name__ == '__main__': config['verbose'] = 3
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if True:
# Start by doing things manually:
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
# Largely copied from Gary's fit_des.py, but using the Piff input_handler to
# read the input files.
stars, wcs, pointing = piff.Input.process(config['input'],
logger=logger)
if nstars is not None:
stars = stars[:nstars]
# Make model, force PSF centering
model = piff.PixelGrid(scale=scale,
size=size,
interp=piff.Lanczos(3),
force_model_center=True,
start_sigma=start_sigma,
logger=logger)
# Interpolator will be zero-order polynomial.
# Find u, v ranges
interp = piff.BasisPolynomial(order=2, logger=logger)
# Make a psf
psf = piff.SimplePSF(model, interp)
psf.fit(stars, wcs, pointing, logger=logger)
# The difference between the images of the fitted stars and the originals should be
# consistent with noise. Keep track of how many don't meet that goal.
n_bad = 0 # chisq/dof > 2
n_marginal = 0 # chisq/dof > 1.1
n_good = 0 # chisq/dof <= 1.1
# Note: The 2 and 1.1 values here are very arbitrary!
for s in psf.stars:
fitted = psf.drawStar(s)
orig_stamp = orig_image[fitted.image.bounds] - s['sky']
fit_stamp = fitted.image
x0 = int(s['x'] + 0.5)
y0 = int(s['y'] + 0.5)
b = galsim.BoundsI(x0 - 3, x0 + 3, y0 - 3, y0 + 3)
#print('orig center = ',orig_stamp[b].array)
#print('flux = ',orig_stamp.array.sum())
#print('fit center = ',fit_stamp[b].array)
#print('flux = ',fit_stamp.array.sum())
flux = fitted.fit.flux
#print('max diff/flux = ',np.max(np.abs(orig_stamp.array-fit_stamp.array))/flux)
#np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
weight = s.weight # These should be 1/var_pix
resid = fit_stamp - orig_stamp
chisq = np.sum(resid.array**2 * weight.array)
print('chisq = ', chisq)
print('cf. star.chisq, dof = ', s.fit.chisq, s.fit.dof)
assert abs(chisq - s.fit.chisq) < 1.e-3 * chisq
if chisq > 2. * s.fit.dof:
n_bad += 1
elif chisq > 1.1 * s.fit.dof:
n_marginal += 1
else:
n_good += 1
# Check the convenience function that an end user would typically use
offset = s.center_to_offset(s.fit.center)
image = psf.draw(x=s['x'],
y=s['y'],
stamp_size=stamp_size,
flux=s.fit.flux,
offset=offset)
np.testing.assert_almost_equal(image.array,
fit_stamp.array,
decimal=4)
print('n_good, marginal, bad = ', n_good, n_marginal, n_bad)
# The real counts are 10 and 2. So this says make sure any updates to the code don't make
# things much worse.
assert n_marginal <= 12
assert n_bad <= 3
# Use piffify function
if __name__ == '__main__':
print('start piffify')
piff.piffify(config)
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'],
y=stars[0]['y'],
stamp_size=stamp_size,
flux=flux,
offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
# The first star happens to be a good one, so go ahead and test the arrays directly.
np.testing.assert_almost_equal(fit_stamp.array / flux,
orig_stamp.array / flux,
decimal=2)
# Test using the piffify executable
config['verbose'] = 0
with open('pixel_des.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
print('start piffify executable')
p = subprocess.Popen([piffify_exe, 'pixel_des.yaml'])
p.communicate()
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'],
y=stars[0]['y'],
stamp_size=stamp_size,
flux=flux,
offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
np.testing.assert_almost_equal(fit_stamp.array / flux,
orig_stamp.array / flux,
decimal=2)
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_single_image.log')
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
flag_list = [1, 1, 13, 1, 1, 4, 1, 1, 0, 1]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y, flag in zip(x_list, y_list, flag_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag & 4:
print('corrupting star at ', x, y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('output', 'simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
cat_file = os.path.join('output', 'simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
config = {'image_file_name': image_file, 'cat_file_name': cat_file}
input = piff.InputFiles(config, logger=logger)
assert input.image_file_name == [image_file]
assert input.cat_file_name == [cat_file]
# Check image
assert input.nimages == 1
image1, _, image_pos, _, _, _ = input.getRawImageData(0)
np.testing.assert_equal(image1.array, image.array)
# Check catalog
np.testing.assert_equal([pos.x for pos in image_pos], x_list)
np.testing.assert_equal([pos.y for pos in image_pos], y_list)
# Repeat, using flag columns this time.
config = {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': '1',
'skip_flag': '4',
'stamp_size': 48
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, _, image_pos, _, _, _ = input.getRawImageData(0)
assert len(image_pos) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48, 48)
# Process the star data
# can only compare to truth if include_pixel=False
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars]
interp.solve(fitted_stars)
print('mean = ', interp.mean)
# Check that the interpolation is what it should be
# Any position would work here.
chipnum = 0
x = 1024
y = 123
orig_wcs = input.getWCS()[chipnum]
orig_pointing = input.getPointing()
image_pos = galsim.PositionD(x, y)
world_pos = piff.StarData.calculateFieldPos(image_pos, orig_wcs,
orig_pointing)
u, v = world_pos.x, world_pos.y
stamp_size = config['stamp_size']
target = piff.Star.makeTarget(x=x,
y=y,
u=u,
v=v,
wcs=orig_wcs,
stamp_size=stamp_size,
pointing=orig_pointing)
true_params = [sigma, g1, g2]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Check default values of options
psf = piff.SimplePSF(model, interp)
assert psf.chisq_thresh == 0.1
assert psf.max_iter == 30
assert psf.outliers == None
assert psf.extra_interp_properties == []
# Now test running it via the config parser
psf_file = os.path.join('output', 'simple_psf.fits')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': 1,
'skip_flag': 4,
'stamp_size': stamp_size
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
'max_iter': 10,
'chisq_thresh': 0.2,
},
'output': {
'file_name': psf_file
},
}
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
interp = piff.Mean()
psf = piff.SimplePSF(model, interp, max_iter=10, chisq_thresh=0.2)
assert psf.chisq_thresh == 0.2
assert psf.max_iter == 10
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# test that drawStar and drawStarList work
test_star = psf.drawStar(target)
test_star_list = psf.drawStarList([target])[0]
np.testing.assert_equal(test_star.fit.params, test_star_list.fit.params)
np.testing.assert_equal(test_star.image.array, test_star_list.image.array)
# test copy_image property of drawStar and draw
for draw in [psf.drawStar, psf.model.draw]:
target_star_copy = psf.interp.interpolate(
piff.Star(target.data.copy(), target.fit.copy()))
# interp is so that when we do psf.model.draw we have fit.params to work with
test_star_copy = draw(target_star_copy, copy_image=True)
test_star_nocopy = draw(target_star_copy, copy_image=False)
# if we modify target_star_copy, then test_star_nocopy should be modified,
# but not test_star_copy
target_star_copy.image.array[0, 0] = 23456
assert test_star_nocopy.image.array[
0, 0] == target_star_copy.image.array[0, 0]
assert test_star_copy.image.array[0,
0] != target_star_copy.image.array[0,
0]
# however the other pixels SHOULD still be all the same value
assert test_star_nocopy.image.array[
1, 1] == target_star_copy.image.array[1, 1]
assert test_star_copy.image.array[1,
1] == target_star_copy.image.array[1,
1]
# test that draw works
test_image = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center)
# this image should be the same values as test_star
assert test_image == test_star.image
# test that draw does not copy the image
image_ref = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center,
image=test_image)
image_ref.array[0, 0] = 123456789
assert test_image.array[0, 0] == image_ref.array[0, 0]
assert test_star.image.array[0, 0] != test_image.array[0, 0]
assert test_star.image.array[1, 1] == test_image.array[1, 1]
# Round trip to a file
psf.write(psf_file, logger)
psf2 = piff.read(psf_file, logger)
assert type(psf2.model) is piff.Gaussian
assert type(psf2.interp) is piff.Mean
assert psf2.chisq == psf.chisq
assert psf2.last_delta_chisq == psf.last_delta_chisq
assert psf2.chisq_thresh == psf.chisq_thresh
assert psf2.max_iter == psf.max_iter
assert psf2.dof == psf.dof
assert psf2.nremoved == psf.nremoved
test_star = psf2.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf3 = piff.read(psf_file)
assert type(psf3.model) is piff.Gaussian
assert type(psf3.interp) is piff.Mean
assert psf3.chisq == psf.chisq
assert psf3.last_delta_chisq == psf.last_delta_chisq
assert psf3.chisq_thresh == psf.chisq_thresh
assert psf3.max_iter == psf.max_iter
assert psf3.dof == psf.dof
assert psf3.nremoved == psf.nremoved
test_star = psf3.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test using the piffify executable
os.remove(psf_file)
# This would be simpler as a direct assignment, but this once, test the way you would set
# this from the command line, which would call parse_variables.
piff.config.parse_variables(config, ['verbose=0'], logger=logger)
#config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
config2 = piff.config.read_config('simple.yaml')
assert config == config2
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
psf4 = piff.read(psf_file)
assert type(psf4.model) is piff.Gaussian
assert type(psf4.interp) is piff.Mean
assert psf4.chisq == psf.chisq
assert psf4.last_delta_chisq == psf.last_delta_chisq
assert psf4.chisq_thresh == psf.chisq_thresh
assert psf4.max_iter == psf.max_iter
assert psf4.dof == psf.dof
assert psf4.nremoved == psf.nremoved
test_star = psf4.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# With very low max_iter, we hit the warning about non-convergence
config['psf']['max_iter'] = 1
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert 'PSF fit did not converge' in cl.output
def setup():
np.random.seed(42)
if __name__ == '__main__':
nimages = 20
stars_per_image = 500
else:
nimages = 2
stars_per_image = 50
# Delete any existing image files
for image_file in glob.glob(os.path.join('output','test_mean_image_*.fits')):
os.remove(image_file)
for k in range(nimages):
print(k)
image = galsim.Image(2048, 2048, scale=0.26)
x_list = [np.random.uniform(0, 2048)]
y_list = [np.random.uniform(0, 2048)]
i=0
while i < stars_per_image:
x = np.random.uniform(0, 2048)
y = np.random.uniform(0, 2048)
D = np.sqrt((np.array(x_list)-x)**2 + (np.array(y_list)-y)**2)
#avoid 2 stars on the same stamp
if np.all(D > 60):
x_list.append(x)
y_list.append(y)
i+=1
coord = np.array([x_list,y_list]).T
params = make_average(coord=coord)
psfs = []
for x, y, hlr, g1, g2 in zip(x_list, y_list, params['hlr'], params['g1'], params['g2']):
psf = galsim.Kolmogorov(half_light_radius=hlr, flux=1.).shear(g1=g1, g2=g2)
bounds = galsim.BoundsI(int(x-21), int(x+22), int(y-21), int(y+22))
if not image.bounds.includes(bounds): continue
offset = galsim.PositionD( x-int(x)-0.5 , y-int(y)-0.5 )
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
image_file = os.path.join('output','test_mean_image_%02i.fits'%k)
image.write(image_file)
dtype = [ ('x','f8'), ('y','f8') ]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output','test_mean_cat_%02i.fits'%k)
fitsio.write(cat_file, data, clobber=True)
image_file = os.path.join('output','test_mean_image_%02i.fits'%k)
cat_file = os.path.join('output','test_mean_cat_%02i.fits'%k)
psf_file = os.path.join('output','test_mean_%02i.piff'%k)
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Kolmogorov',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Polynomial',
'order' : 2}
},
'output' : {
'file_name' : psf_file
}
}
if __name__ == '__main__':
config['verbose'] = 2
else:
config['verbose'] = 0
piff.piffify(config)
def setup():
np.random.seed(42)
if __name__ == '__main__':
nimages = 20
stars_per_image = 500
else:
nimages = 2
stars_per_image = 50
# Delete any existing image files
for image_file in glob.glob(
os.path.join('output', 'test_mean_image_*.fits')):
os.remove(image_file)
for k in range(nimages):
print(k)
image = galsim.Image(2048, 2048, scale=0.26)
x_list = [np.random.uniform(0, 2048)]
y_list = [np.random.uniform(0, 2048)]
i = 0
while i < stars_per_image:
x = np.random.uniform(0, 2048)
y = np.random.uniform(0, 2048)
D = np.sqrt((np.array(x_list) - x)**2 + (np.array(y_list) - y)**2)
#avoid 2 stars on the same stamp
if np.all(D > 60):
x_list.append(x)
y_list.append(y)
i += 1
coord = np.array([x_list, y_list]).T
params = make_average(coord=coord)
psfs = []
for x, y, hlr, g1, g2 in zip(x_list, y_list, params['hlr'],
params['g1'], params['g2']):
psf = galsim.Kolmogorov(half_light_radius=hlr,
flux=1.).shear(g1=g1, g2=g2)
bounds = galsim.BoundsI(int(x - 21), int(x + 22), int(y - 21),
int(y + 22))
if not image.bounds.includes(bounds): continue
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds],
method='no_pixel',
offset=offset)
image_file = os.path.join('output', 'test_mean_image_%02i.fits' % k)
image.write(image_file)
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output', 'test_mean_cat_%02i.fits' % k)
fitsio.write(cat_file, data, clobber=True)
image_file = os.path.join('output', 'test_mean_image_%02i.fits' % k)
cat_file = os.path.join('output', 'test_mean_cat_%02i.fits' % k)
psf_file = os.path.join('output', 'test_mean_%02i.piff' % k)
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'stamp_size': 48
},
'psf': {
'model': {
'type': 'Kolmogorov',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Polynomial',
'order': 2
}
},
'output': {
'file_name': psf_file
}
}
if __name__ == '__main__':
config['verbose'] = 2
else:
config['verbose'] = 0
piff.piffify(config)
def test_hsmcatalog():
"""Test HSMCatalog stats type.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_hsmcatalog.log')
image_file = os.path.join('output', 'test_stats_image.fits')
cat_file = os.path.join('output', 'test_stats_cat.fits')
psf_file = os.path.join('output', 'test_starstats.fits')
hsm_file = os.path.join('output', 'test_hsmcatalog.fits')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'stamp_size': 48,
'reserve_frac': 0.2,
'seed': 123
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
},
'output': {
'file_name': psf_file,
'stats': [{
'type': 'HSMCatalog',
'file_name': hsm_file,
}]
}
}
piff.piffify(config, logger)
assert os.path.isfile(hsm_file)
data = fitsio.read(hsm_file)
for col in [
'ra', 'dec', 'x', 'y', 'u', 'v', 'T_data', 'g1_data', 'g2_data',
'T_model', 'g1_model', 'g2_model', 'flux', 'reserve'
]:
assert len(data[col]) == 10
true_data = fitsio.read(cat_file)
np.testing.assert_allclose(data['x'], true_data['x'])
np.testing.assert_allclose(data['y'], true_data['y'])
np.testing.assert_allclose(data['flux'], 123.45, atol=0.001)
print('reserve = ', data['reserve'])
print('nreserve = ', np.sum(data['reserve']))
print('ntot = ', len(data['reserve']))
assert np.sum(data['reserve']) == int(0.2 * len(data['reserve']))
np.testing.assert_allclose(data['T_model'], data['T_data'], rtol=1.e-4)
np.testing.assert_allclose(data['g1_model'], data['g1_data'], rtol=1.e-4)
np.testing.assert_allclose(data['g2_model'], data['g2_data'], rtol=1.e-4)
image = galsim.fits.read(image_file)
world = [
image.wcs.toWorld(galsim.PositionD(x, y))
for x, y in zip(data['x'], data['y'])
]
np.testing.assert_allclose(data['ra'], [w.ra.deg for w in world],
rtol=1.e-4)
np.testing.assert_allclose(data['dec'], [w.dec.deg for w in world],
rtol=1.e-4)
# Repeat with non-Celestial WCS
wcs = galsim.AffineTransform(0.26, 0.05, -0.08, -0.24,
galsim.PositionD(1024, 1024))
config['input']['wcs'] = wcs
piff.piffify(config, logger)
data = fitsio.read(hsm_file)
np.testing.assert_array_equal(data['ra'], 0.)
np.testing.assert_array_equal(data['dec'], 0.)
world = [
wcs.toWorld(galsim.PositionD(x, y))
for x, y in zip(data['x'], data['y'])
]
np.testing.assert_allclose(data['u'], [w.x for w in world], rtol=1.e-4)
np.testing.assert_allclose(data['v'], [w.y for w in world], rtol=1.e-4)
# Use class directly, rather than through config.
psf = piff.PSF.read(psf_file)
stars, _, _ = piff.Input.process(config['input'])
hsmcat = piff.stats.HSMCatalogStats()
with np.testing.assert_raises(RuntimeError):
hsmcat.write('dummy') # Cannot write before compute
hsmcat.compute(psf, stars)
hsm_file2 = os.path.join('output', 'test_hsmcatalog2.fits')
with np.testing.assert_raises(ValueError):
hsmcat.write() # Must supply file_name if not given in constructor
hsmcat.write(hsm_file2)
data2 = fitsio.read(hsm_file2)
for key in data.dtype.names:
np.testing.assert_allclose(data2[key], data[key], rtol=1.e-5)
def test_rhostats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_rhostats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_rhostats.fits')
rho_file = os.path.join('output','test_rhostats.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : { # Note: stats doesn't have to be a list.
'type': 'Rho',
'file_name': rho_file
}
},
}
piff.piffify(config, logger)
assert os.path.isfile(rho_file)
# repeat with plotify function
os.remove(rho_file)
piff.plotify(config, logger)
assert os.path.isfile(rho_file)
# Test rho statistics directly.
min_sep = 1
max_sep = 100
bin_size = 0.1
psf = piff.read(psf_file)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size)
stats.compute(psf, orig_stars)
rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5]
for rho in rhos:
# Test the range of separations
radius = np.exp(rho.logr)
np.testing.assert_array_less(radius, max_sep)
np.testing.assert_array_less(min_sep, radius)
# bin_size is reduced slightly to get integer number of bins
assert rho.bin_size < bin_size
assert np.isclose(rho.bin_size, bin_size, rtol=0.1)
np.testing.assert_array_almost_equal(np.diff(rho.logr), rho.bin_size, decimal=5)
# Test that the max absolute value of each rho isn't crazy
np.testing.assert_array_less(np.abs(rho.xip), 1)
# # Check that each rho isn't precisely zero. This means the sum of abs > 0
np.testing.assert_array_less(0, np.sum(np.abs(rho.xip)))
# Test using the piffify executable
os.remove(rho_file)
config['verbose'] = 0
with open('rho.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'rho.yaml'] )
p.communicate()
assert os.path.isfile(rho_file)
# Test using the plotify executable
os.remove(rho_file)
plotify_exe = get_script_name('plotify')
p = subprocess.Popen( [plotify_exe, 'rho.yaml'] )
p.communicate()
assert os.path.isfile(rho_file)
# test running plotify with dir in config, with no logger, and with a modules specification.
# (all to improve test coverage)
config['output']['dir'] = '.'
config['modules'] = [ 'custom_wcs' ]
os.remove(rho_file)
piff.plotify(config)
assert os.path.isfile(rho_file)
def test_bad_hsm():
"""Test that stats don't break when all stars end up being flagged with hsm errors.
"""
image_file = os.path.join('input', 'DECam_00241238_01.fits.fz')
cat_file = os.path.join(
'input',
'DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits')
psf_file = os.path.join('output', 'bad_hsm.fits')
twodhist_file = os.path.join('output', 'bad_hsm_twod.pdf')
whisker_file = os.path.join('output', 'bad_hsm_whisk.pdf')
rho_file = os.path.join('output', 'bad_hsm_rho.pdf')
shape_file = os.path.join('output', 'bad_hsm_shape.pdf')
star_file = os.path.join('output', 'bad_hsm_star.pdf')
hsm_file = os.path.join('output', 'bad_hsm_hsm.fits')
stamp_size = 25
# The configuration dict with the right input fields for the file we're using.
config = {
'input': {
'nstars': 8,
'image_file_name': image_file,
'image_hdu': 1,
'weight_hdu': 3,
'badpix_hdu': 2,
'cat_file_name': cat_file,
'cat_hdu': 2,
# These next two are intentionally backwards. The PixelGrid will find some kind
# of solution, but it will be complex garbage, and hsm will fail for them.
'x_col': 'YWIN_IMAGE',
'y_col': 'XWIN_IMAGE',
'sky_col': 'BACKGROUND',
'stamp_size': stamp_size,
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
},
'output': {
'file_name':
psf_file,
'stats': [
{
'type': 'TwoDHist',
'file_name': twodhist_file,
},
{
'type': 'Whisker',
'file_name': whisker_file,
},
{ # Note: stats doesn't have to be a list.
'type': 'Rho',
'file_name': rho_file
},
{
'type': 'ShapeHist',
'file_name': shape_file,
},
{
'type': 'Star',
'file_name': star_file,
},
{
'type': 'HSMCatalog',
'file_name': hsm_file,
},
],
},
'psf': {
'model': {
'type': 'PixelGrid',
'scale': 0.3,
'size': 10,
},
'interp': {
'type': 'Mean'
},
'outliers': {
'type': 'Chisq',
'nsigma':
1.e-3, # This will throw out all but 1, which adds an additional
# test of Star stats when nstars < nplot
}
},
}
if __name__ == '__main__':
logger = piff.config.setup_logger(1)
else:
config['verbose'] = 0
logger = None
for f in [twodhist_file, rho_file, shape_file, star_file, hsm_file]:
if os.path.exists(f):
os.remove(f)
piff.piffify(config, logger=logger)
for f in [twodhist_file, rho_file, shape_file, star_file, hsm_file]:
assert os.path.exists(f)
def test_starstats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_starstats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_starstats.fits')
star_file = os.path.join('output', 'test_starstats.pdf')
star_noadjust_file = os.path.join('output', 'test_starstats_noadjust.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'Star',
'file_name': star_file,
'number_plot': 5,
'adjust_stars': True,
}
]
}
}
piff.piffify(config, logger)
assert os.path.isfile(star_file)
# repeat with plotify function
os.remove(star_file)
piff.plotify(config, logger)
assert os.path.isfile(star_file)
# check default number_plot
psf = piff.read(psf_file)
starStats = piff.StarStats()
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
starStats.compute(psf, orig_stars)
assert starStats.number_plot == len(starStats.stars)
assert starStats.number_plot == len(starStats.models)
assert starStats.number_plot == len(starStats.indices)
np.testing.assert_array_equal(starStats.stars[2].image.array,
orig_stars[starStats.indices[2]].image.array)
# check number_plot = 6
starStats = piff.StarStats(number_plot=6)
starStats.compute(psf, orig_stars)
assert len(starStats.stars) == 6
# check number_plot >> len(stars)
starStats = piff.StarStats(number_plot=1000000)
starStats.compute(psf, orig_stars)
assert len(starStats.stars) == len(orig_stars)
# if use all stars, no randomness
np.testing.assert_array_equal(starStats.stars[3].image.array, orig_stars[3].image.array)
np.testing.assert_array_equal(starStats.indices, np.arange(len(orig_stars)))
# check number_plot = 0
starStats = piff.StarStats(number_plot=0)
starStats.compute(psf, orig_stars)
assert len(starStats.stars) == len(orig_stars)
# if use all stars, no randomness
np.testing.assert_array_equal(starStats.stars[3].image.array, orig_stars[3].image.array)
np.testing.assert_array_equal(starStats.indices, np.arange(len(orig_stars)))
# rerun with adjust stars and see if it did the right thing
# first with starstats == False
starStats = piff.StarStats(number_plot=0, adjust_stars=False)
starStats.compute(psf, orig_stars, logger=logger)
fluxs_noadjust = np.array([s.fit.flux for s in starStats.stars])
ds_noadjust = np.array([s.fit.center for s in starStats.stars])
# check that fluxes 1
np.testing.assert_array_equal(fluxs_noadjust, 1)
# check that ds are 0
np.testing.assert_array_equal(ds_noadjust, 0)
# now with starstats == True
starStats = piff.StarStats(number_plot=0, adjust_stars=True)
starStats.compute(psf, orig_stars, logger=logger)
fluxs_adjust = np.array([s.fit.flux for s in starStats.stars])
ds_adjust = np.array([s.fit.center for s in starStats.stars])
# copy the right values from setup()
dx = 0.31
dy = -0.32
flux = 123.45
# compare fluxes
np.testing.assert_allclose(fluxs_adjust, flux, rtol=1e-4)
# compare dx and dy, keeping in mind that ds_adjust is dx/y * 0.26 (scale)
dx_adjust = ds_adjust[:, 0] / 0.26
dy_adjust = ds_adjust[:, 1] / 0.26
np.testing.assert_allclose(dx_adjust, dx, rtol=1e-4)
np.testing.assert_allclose(dy_adjust, dy, rtol=1e-4)
# do once with adjust_stars = False to graphically demonstrate
config['output']['stats'][0]['file_name'] = star_noadjust_file
config['output']['stats'][0]['adjust_stars'] = False
piff.plotify(config, logger)
assert os.path.isfile(star_noadjust_file)
def test_shapestats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_shapestats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_shapestats.fits')
shape_file = os.path.join('output','test_shapestats.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'ShapeHist',
'file_name': shape_file
},
]
},
}
piff.piffify(config, logger)
assert os.path.isfile(shape_file)
# repeat with plotify function
os.remove(shape_file)
piff.plotify(config, logger)
assert os.path.isfile(shape_file)
# Test ShapeHistStats directly
psf = piff.read(psf_file)
shapeStats = piff.ShapeHistStats(nbins=5) # default is sqrt(nstars)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
with np.testing.assert_raises(RuntimeError):
shapeStats.write() # Cannot write before compute
shapeStats.compute(psf, orig_stars)
shapeStats.plot(histtype='bar', log=True) # can supply additional args for matplotlib
# test their characteristics
sigma = 1.3 # (copied from setup())
T = 2*sigma**2
g1 = 0.23
g2 = -0.17
np.testing.assert_array_almost_equal(T, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(T, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
