def test_yaml():
# Take DES test image, and test doing a psf run with GP interpolator
# Use config parser:
psf_file = os.path.join('output', 'gp_psf.fits')
config = {
'input': {
'images': 'y1_test/DECam_00241238_01.fits.fz',
'cats':
'y1_test/DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits',
# What hdu is everything in?
'image_hdu': 1,
'badpix_hdu': 2,
'weight_hdu': 3,
'cat_hdu': 2,
# What columns in the catalog have things we need?
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
'sky_col': 'BACKGROUND',
# How large should the postage stamp cutouts of the stars be?
'stamp_size': 31,
},
'psf': {
'model': {
'type': 'GSObjectModel',
'fastfit': True,
'gsobj': 'galsim.Gaussian(sigma=1.0)'
},
'interp': {
'type': 'GPInterp',
'keys': ['u', 'v'],
'kernel': 'RBF(200.0)',
'optimize': False,
}
},
'output': {
'file_name': psf_file
},
}
# using piffify executable
config['verbose'] = 0
with open('gp.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'gp.yaml'])
p.communicate()
piff.read(psf_file)
def test_shapestats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_shapestats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_shapestats.fits')
shape_file = os.path.join('output','test_shapestats.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'ShapeHistograms',
'file_name': shape_file
},
]
},
}
piff.piffify(config, logger)
assert os.path.isfile(shape_file)
# repeat with plotify function
os.remove(shape_file)
piff.plotify(config, logger)
assert os.path.isfile(shape_file)
# Test ShapeHistogramStats directly
psf = piff.read(psf_file)
shapeStats = piff.ShapeHistogramsStats()
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
shapeStats.compute(psf, orig_stars)
# test their characteristics
sigma = 1.3 # (copied from setup())
g1 = 0.23
g2 = -0.17
np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
def test_pickle():
"""Test the reading a file written with python 2 pickling is readable with python 2 or 3.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_pickle.log')
# First, this is the output file written by the above test_single function on python 2.
# Shoudl be trivially readable by python 2, but make sure it is also readable by python 3.
psf = piff.read('input/test_single_py27.piff', logger=logger)
wcs1 = galsim.TanWCS(
galsim.AffineTransform(0.26, 0.05, -0.08, -0.24, galsim.PositionD(1024,1024)),
galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees)
)
wcs2 = galsim.TanWCS(
galsim.AffineTransform(0.25, -0.02, 0.01, 0.24, galsim.PositionD(1024,1024)),
galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees)
)
data1 = fitsio.read('input/test_single_cat1.fits')
data2 = fitsio.read('input/test_single_cat2.fits')
field_center = galsim.CelestialCoord(0 * galsim.degrees, -25 * galsim.degrees)
for chipnum, data, wcs in [(1,data1,wcs1), (2,data2,wcs2)]:
for k in range(len(data)):
x = data['x'][k]
y = data['y'][k]
e1 = data['e1'][k]
e2 = data['e2'][k]
s = data['s'][k]
#print('k,x,y = ',k,x,y)
#print(' true s,e1,e2 = ',s,e1,e2)
image_pos = galsim.PositionD(x,y)
star = piff.Star.makeTarget(x=x, y=y, wcs=wcs, stamp_size=48, pointing=field_center,
chipnum=chipnum)
star = psf.drawStar(star)
#print(' fitted s,e1,e2 = ',star.fit.params)
np.testing.assert_almost_equal(star.fit.params, [s,e1,e2], decimal=6)
# This is a DES Y3 PSF file that Matt Becker reported hadn't been readable with python 3.
# The problem was it had been written with python 2's pickle, which isn't directly
# compatible with python 3. The code has been fixed to make it readable. This unit
# test is just to ensure that it remains so.
# However, it only works if pixmappy is installed, so if not, just bail out.
try:
import pixmappy
except ImportError:
return
fname = os.path.join('input', 'D00240560_r_c01_r2362p01_piff.fits')
psf = piff.PSF.read(fname, logger=logger)
image = psf.draw(x=103.3, y=592.0)
def test_yaml():
# Take DES test image, and test doing a psf run with kNN interpolator
# Now test running it via the config parser
psf_file = os.path.join('output','knn_psf.fits')
config = {
'input' : {
'images' : 'y1_test/DECam_00241238_01.fits.fz',
'cats' : 'y1_test/DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits',
# What hdu is everything in?
'image_hdu': 1,
'badpix_hdu': 2,
'weight_hdu': 3,
'cat_hdu': 2,
# What columns in the catalog have things we need?
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
'sky_col': 'BACKGROUND',
# How large should the postage stamp cutouts of the stars be?
'stamp_size': 31,
},
'psf' : {
'model' : { 'type': 'GSObjectModel',
'fastfit': True,
'gsobj': 'galsim.Gaussian(sigma=1.0)' },
'interp' : { 'type': 'kNNInterp',
'keys': ['u', 'v'],
'n_neighbors': 115,}
},
'output' : { 'file_name' : psf_file },
}
# using piffify executable
config['verbose'] = 0
with open('knn.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'knn.yaml'] )
p.communicate()
psf = piff.read(psf_file)
# by using n_neighbors = 115, when there are only 117 stars in the catalog, we should expect
# that the standard deviation of the interpolated parameters should be small, since almost the
# same set of stars are being averaged in every case.
np.testing.assert_array_less(
np.std([s.fit.params for s in psf.stars], axis=0),
0.01*np.mean([s.fit.params for s in psf.stars], axis=0),
err_msg="Interpolated parameters show too much variation.")
def do_residuals(f, verbose=True, number_plot=0):
c = f.replace('.piff', '.yaml')
piff_name = f.split('/')[-1].split('.piff')[0]
label = 'fitted'
config = piff.read_config(c)
if verbose:
print('psf')
psf = piff.read(f)
# initialize output
directory = '/'.join(c.split('/')[:-1])
config['output']['dir'] = directory
output = piff.Output.process(config['output'])
# select nstars based on stars piece of config
stat = output.stats_list[3] # TODO: this bit is hardcoded
if number_plot != 0:
stat.number_plot = number_plot
stat.indices = np.random.choice(len(psf.stars),
stat.number_plot,
replace=False)
# pass params into output
stat.stars = []
for i, index in enumerate(stat.indices):
star = psf.stars[index]
stat.stars.append(star)
# load their images
if verbose:
print('loading star images')
stat.stars = load_star_images(stat.stars, config)
if verbose:
print('loading model')
stat.models = []
for star in stat.stars:
# draw model star
params = star.fit.params
prof = psf.getProfile(params)
model = psf.drawProfile(star, prof, params, copy_image=True)
stat.models.append(model)
if verbose:
print('writing')
file_name = '{0}/{1}_{2}_{3}'.format(directory, label, piff_name,
os.path.split(stat.file_name)[1])
stat.write(file_name=file_name)
return file_name, stat.stars, stat.models
def test_pickle():
"""Test the reading a file written with python 2 pickling is readable with python 2 or 3.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_pickle.log')
# First, this is the output file written by the above test_single function on python 2.
# Shoudl be trivially readable by python 2, but make sure it is also readable by python 3.
psf = piff.read('input/test_single_py27.piff', logger=logger)
wcs1 = galsim.TanWCS(
galsim.AffineTransform(0.26, 0.05, -0.08, -0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees))
wcs2 = galsim.TanWCS(
galsim.AffineTransform(0.25, -0.02, 0.01, 0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees))
data1 = fitsio.read('input/test_single_cat1.fits')
data2 = fitsio.read('input/test_single_cat2.fits')
field_center = galsim.CelestialCoord(0 * galsim.degrees,
-25 * galsim.degrees)
for chipnum, data, wcs in [(1, data1, wcs1), (2, data2, wcs2)]:
for k in range(len(data)):
x = data['x'][k]
y = data['y'][k]
e1 = data['e1'][k]
e2 = data['e2'][k]
s = data['s'][k]
#print('k,x,y = ',k,x,y)
#print(' true s,e1,e2 = ',s,e1,e2)
image_pos = galsim.PositionD(x, y)
star = piff.Star.makeTarget(x=x,
y=y,
wcs=wcs,
stamp_size=48,
pointing=field_center,
chipnum=chipnum)
star = psf.drawStar(star)
#print(' fitted s,e1,e2 = ',star.fit.params)
np.testing.assert_almost_equal(star.fit.params, [s, e1, e2],
decimal=6)
def fit_psf(directory, config_file_name, print_log, meanify_file_path='', fit_interp_only=False):
do_meanify = meanify_file_path != ''
piff_name = config_file_name
# load config file
config = piff.read_config('{0}/{1}.yaml'.format(directory, config_file_name))
is_optatmo = 'OptAtmo' in config['psf']['type']
# do galsim modules
if 'modules' in config:
galsim.config.ImportModules(config)
# create logger
verbose = config.get('verbose', 3)
if print_log:
logger = piff.setup_logger(verbose=verbose)
else:
if do_meanify:
logger = piff.setup_logger(verbose=verbose, log_file='{0}/{1}_fit_psf_meanify_logger.log'.format(directory, config_file_name))
else:
logger = piff.setup_logger(verbose=verbose, log_file='{0}/{1}_fit_psf_logger.log'.format(directory, config_file_name))
if (do_meanify or fit_interp_only) and is_optatmo:
# load base optics psf
out_path = '{0}/{1}.piff'.format(directory, piff_name)
logger.info('Loading saved PSF at {0}'.format(out_path))
psf = piff.read(out_path)
# load images for train stars
logger.info('loading train stars')
psf.stars = load_star_images(psf.stars, config, logger=logger)
# load test stars and their images
logger.info('loading test stars')
test_stars = read_stars(out_path, logger=logger)
test_stars = load_star_images(test_stars, config, logger=logger)
# make output
config['output']['dir'] = directory
output = piff.Output.process(config['output'], logger=logger)
elif (do_meanify or fit_interp_only) and not is_optatmo:
# welp, not much to do here. shouldn't even have gotten here! :(
logger.warning('Somehow passed the meanify to a non-optatmo argument. This should not happen.')
return
else:
# load stars
stars, wcs, pointing = piff.Input.process(config['input'], logger=logger)
# separate stars
# set seed
np.random.seed(12345)
test_fraction = config.get('test_fraction', 0.2)
test_indx = np.random.choice(len(stars), int(test_fraction * len(stars)), replace=False)
test_stars = []
train_stars = []
# kludgey:
for star_i, star in enumerate(stars):
if star_i in test_indx:
test_stars.append(star)
else:
train_stars.append(star)
# initialize psf
psf = piff.PSF.process(config['psf'], logger=logger)
# piffify
logger.info('Fitting PSF')
psf.fit(train_stars, wcs, pointing, logger=logger)
logger.info('Fitted PSF!')
# fit atmosphere parameters
if is_optatmo:
logger.info('Fitting PSF atmosphere parameters')
logger.info('getting param info for {0} stars'.format(len(psf.stars)))
params = psf.getParamsList(psf.stars)
psf._enable_atmosphere = False
new_stars = []
for star_i, star in zip(range(len(psf.stars)), psf.stars):
if star_i % 100 == 0:
logger.info('Fitting star {0} of {1}'.format(star_i, len(psf.stars)))
try:
model_fitted_star, results = psf.fit_model(star, params=params[star_i], vary_shape=True, vary_optics=False, logger=logger)
new_stars.append(model_fitted_star)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
logger.warning('{0}'.format(str(e)))
logger.warning('Warning! Failed to fit atmosphere model for star {0}. Ignoring star in atmosphere fit'.format(star_i))
psf.stars = new_stars
# save psf
# make sure the output is in the right directory
config['output']['dir'] = directory
output = piff.Output.process(config['output'], logger=logger)
logger.info('Saving PSF')
# save fitted PSF
psf.write(output.file_name, logger=logger)
# and write test stars
write_stars(test_stars, output.file_name, logger=logger)
shape_keys = ['e0', 'e1', 'e2', 'delta1', 'delta2', 'zeta1', 'zeta2']
shape_plot_keys = []
for key in shape_keys:
shape_plot_keys.append(['data_' + key, 'model_' + key, 'd' + key])
if is_optatmo:
interps = config.pop('interps')
interp_keys = interps.keys()
if not (do_meanify or fit_interp_only):
# do noatmo only when we do not have meanify
interp_keys = ['noatmo'] + interp_keys
train_stars = psf.stars
for interp_key in interp_keys:
piff_name = '{0}_{1}'.format(config_file_name, interp_key)
logger.info('Fitting optatmo interpolate for {0}'.format(interp_key))
if interp_key == 'noatmo':
psf.atmo_interp = None
psf._enable_atmosphere = False
passed_test_stars = test_stars
else:
# fit interps
config_interp = interps[interp_key]
if do_meanify:
config_interp['average_fits'] = meanify_file_path
piff_name += '_meanified'
# extract chicut, madcut, snrcut from interp config, if provided
interp_chicut = config_interp.pop('chicut', 0)
interp_madcut = config_interp.pop('madcut', 0)
interp_snrcut = config_interp.pop('snrcut', 0)
# test stars undergo snr cut, but no other cuts
used_interp_stars, passed_test_stars = fit_interp(train_stars, test_stars, config_interp, psf, interp_chicut, interp_madcut, interp_snrcut, logger)
psf.stars = used_interp_stars
# save
out_path = '{0}/{1}.piff'.format(directory, piff_name)
psf.write(out_path, logger=logger)
write_stars(passed_test_stars, out_path, logger=logger)
# evaluate
logger.info('Evaluating {0}'.format(piff_name))
for stars, label in zip([psf.stars, passed_test_stars], ['train', 'test']):
# get shapes
logger.debug('drawing {0} model stars'.format(label))
model_stars = psf.drawStarList(stars)
shapes = measure_star_shape(stars, model_stars, logger=logger)
param_keys = ['atmo_size', 'atmo_g1', 'atmo_g2']
if psf.atmosphere_model == 'vonkarman':
param_keys += ['atmo_L0']
param_keys += ['optics_size', 'optics_g1', 'optics_g2'] + ['z{0:02d}'.format(zi) for zi in range(4, 45)]
if label == 'train':
# if train, plot fitted params
logger.info('Extracting training fit parameters')
params = np.array([star.fit.params for star in stars])
params_var = np.array([star.fit.params_var for star in stars])
for i in range(params.shape[1]):
shapes['{0}_fit'.format(param_keys[i])] = params[:, i]
shapes['{0}_var'.format(param_keys[i])] = params_var[:, i]
logger.info('Getting training fit parameters')
params = psf.getParamsList(stars)
for i in range(params.shape[1]):
shapes[param_keys[i]] = params[:, i]
elif label == 'test':
# if test, plot predicted params
logger.info('Getting test parameters')
params = psf.getParamsList(stars)
for i in range(params.shape[1]):
shapes[param_keys[i]] = params[:, i]
# save shapes
shapes.to_hdf('{0}/shapes_{1}_{2}.h5'.format(directory, label, piff_name), 'data', mode='w')
# plot shapes
fig, axs = plot_2dhist_shapes(shapes, shape_plot_keys, diff_mode=True)
# save
fig.savefig('{0}/plot_2dhist_shapes_{1}_{2}.pdf'.format(directory, label, piff_name))
# plot params
plot_keys = []
plot_keys_i = []
for i in range(params.shape[1]):
plot_keys_i.append(param_keys[i])
if len(plot_keys_i) == 3:
plot_keys.append(plot_keys_i)
plot_keys_i = []
if len(plot_keys_i) > 0:
plot_keys_i += [plot_keys_i[0]] * (3 - len(plot_keys_i))
plot_keys.append(plot_keys_i)
fig, axs = plot_2dhist_shapes(shapes, plot_keys, diff_mode=False)
# save
fig.savefig('{0}/plot_2dhist_params_{1}_{2}.pdf'.format(directory, label, piff_name))
# and repeat for the fit params
if label == 'train':
fig, axs = plot_2dhist_shapes(shapes, [[key + '_fit' for key in kp] for kp in plot_keys], diff_mode=False)
# save
fig.savefig('{0}/plot_2dhist_fit_params_{1}_{2}.pdf'.format(directory, label, piff_name))
# if test, fit the flux and centering
if label == 'test':
logger.info('Fitting the centers and fluxes of {0} test stars'.format(len(stars)))
# fit stars for stats
new_stars = []
for star, param in zip(stars, params):
try:
new_star, res = psf.fit_model(star, param, vary_shape=False, vary_optics=False, logger=logger)
new_stars.append(new_star)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
logger.warning('{0}'.format(str(e)))
logger.warning('Warning! Failed to fit atmosphere model for star {0}. Ignoring star in atmosphere fit'.format(star))
stars = new_stars
# do the output processing
logger.info('Writing Stats Output of {0} stars'.format(label))
for stat in output.stats_list:
stat.compute(psf, stars, logger=logger)
file_name = '{0}/{1}_{2}_{3}'.format(directory, label, piff_name, os.path.split(stat.file_name)[1])
stat.write(file_name=file_name, logger=logger)
else:
logger.info('Evaluating {0}'.format(piff_name))
for stars, label in zip([psf.stars, test_stars], ['train', 'test']):
# get shapes
model_stars = psf.drawStarList(stars)
shapes = measure_star_shape(stars, model_stars, logger=logger)
# save shapes
shapes.to_hdf('{0}/shapes_{1}_{2}.h5'.format(directory, label, piff_name), 'data', mode='w')
# plot shapes
fig, axs = plot_2dhist_shapes(shapes, shape_plot_keys, diff_mode=True)
# save
fig.savefig('{0}/plot_2dhist_shapes_{1}_{2}.pdf'.format(directory, label, piff_name))
logger.info('Writing Stats Output of {0} stars'.format(label))
for stat in output.stats_list:
stat.compute(psf, stars, logger=logger)
file_name = '{0}/{1}_{2}_{3}'.format(directory, label, piff_name, os.path.split(stat.file_name)[1])
stat.write(file_name=file_name, logger=logger)
def test_des_image():
"""Test the whole process with a DES CCD.
"""
import os
import fitsio
image_file = 'input/DECam_00241238_01.fits.fz'
cat_file = 'input/DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits'
orig_image = galsim.fits.read(image_file)
psf_file = os.path.join('output','pixel_des_psf.fits')
if __name__ == '__main__':
# These match what Gary used in fit_des.py
nstars = None
scale = 0.15
size = 31
order = 2
nsigma = 4
else:
# These are faster and good enough for the unit tests.
nstars = 25
scale = 0.26
size = 15
order = 1
nsigma = 1. # This needs to be low to make sure we do test outlier rejection here.
stamp_size = 25
# The configuration dict with the right input fields for the file we're using.
start_sigma = 1.0/2.355 # TODO: Need to make this automatic somehow.
config = {
'input' : {
'nstars': nstars,
'image_file_name' : image_file,
'image_hdu' : 1,
'weight_hdu' : 3,
'badpix_hdu' : 2,
'cat_file_name' : cat_file,
'cat_hdu' : 2,
'x_col' : 'XWIN_IMAGE',
'y_col' : 'YWIN_IMAGE',
'sky_col' : 'BACKGROUND',
'stamp_size' : stamp_size,
'ra' : 'TELRA',
'dec' : 'TELDEC',
'gain' : 'GAINA',
# Test explicitly specifying the wcs (although it is the same here as what is in the
# image anyway).
'wcs' : {
'type': 'Fits',
'file_name': image_file
}
},
'output' : {
'file_name' : psf_file,
},
'psf' : {
'model' : {
'type' : 'PixelGrid',
'scale' : scale,
'size' : size,
'interp' : 'Lanczos(5)',
'start_sigma' : start_sigma,
},
'interp' : {
'type' : 'BasisPolynomial',
'order' : order,
},
'outliers' : {
'type' : 'Chisq',
'nsigma' : nsigma,
'max_remove' : 3
}
},
}
if __name__ == '__main__':
config['verbose'] = 2
else:
config['verbose'] = 0
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if __name__ == '__main__':
# Start by doing things manually:
logger = piff.config.setup_logger(2)
# Largely copied from Gary's fit_des.py, but using the Piff input_handler to
# read the input files.
stars, wcs, pointing = piff.Input.process(config['input'], logger=logger)
if nstars is not None:
stars = stars[:nstars]
# Make model, force PSF centering
model = piff.PixelGrid(scale=scale, size=size, interp=piff.Lanczos(3),
force_model_center=True, start_sigma=start_sigma,
logger=logger)
# Interpolator will be zero-order polynomial.
# Find u, v ranges
interp = piff.BasisPolynomial(order=order, logger=logger)
# Make a psf
psf = piff.SimplePSF(model, interp)
psf.fit(stars, wcs, pointing, logger=logger)
# The difference between the images of the fitted stars and the originals should be
# consistent with noise. Keep track of how many don't meet that goal.
n_bad = 0 # chisq/dof > 2
n_marginal = 0 # chisq/dof > 1.1
n_good = 0 # chisq/dof <= 1.1
# Note: The 2 and 1.1 values here are very arbitrary!
for s in psf.stars:
fitted = psf.drawStar(s)
orig_stamp = orig_image[fitted.image.bounds] - s['sky']
fit_stamp = fitted.image
x0 = int(s['x']+0.5)
y0 = int(s['y']+0.5)
b = galsim.BoundsI(x0-3,x0+3,y0-3,y0+3)
#print('orig center = ',orig_stamp[b].array)
#print('flux = ',orig_stamp.array.sum())
#print('fit center = ',fit_stamp[b].array)
#print('flux = ',fit_stamp.array.sum())
flux = fitted.fit.flux
#print('max diff/flux = ',np.max(np.abs(orig_stamp.array-fit_stamp.array))/flux)
#np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
weight = s.weight # These should be 1/var_pix
resid = fit_stamp - orig_stamp
chisq = np.sum(resid.array**2 * weight.array)
print('chisq = ',chisq)
print('cf. star.chisq, dof = ',s.fit.chisq, s.fit.dof)
assert abs(chisq - s.fit.chisq) < 1.e-3 * chisq
if chisq > 2. * s.fit.dof:
n_bad += 1
elif chisq > 1.1 * s.fit.dof:
n_marginal += 1
else:
n_good += 1
# Check the convenience function that an end user would typically use
offset = s.center_to_offset(s.fit.center)
image = psf.draw(x=s['x'], y=s['y'], stamp_size=stamp_size,
flux=s.fit.flux, offset=offset)
np.testing.assert_almost_equal(image.array, fit_stamp.array, decimal=4)
print('n_good, marginal, bad = ',n_good,n_marginal,n_bad)
# The real counts are 10 and 2. So this says make sure any updates to the code don't make
# things much worse.
assert n_marginal <= 12
assert n_bad <= 3
# Use piffify function
print('start piffify')
piff.piffify(config)
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'], y=stars[0]['y'], stamp_size=stamp_size,
flux=flux, offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
# The first star happens to be a good one, so go ahead and test the arrays directly.
np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
# Test using the piffify executable
with open('pixel_des.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
print('start piffify executable')
p = subprocess.Popen( [piffify_exe, 'pixel_des.yaml'] )
p.communicate()
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'], y=stars[0]['y'], stamp_size=stamp_size,
flux=flux, offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
def test_single_image():
"""Test the whole process with a single image.
Note: This test is based heavily on test_single_image in test_simple.py.
"""
import os
import fitsio
np_rng = np.random.RandomState(1234)
# Make the image
image = galsim.Image(2048, 2048, scale=0.2)
# The (x,y) values will be on a grid 5 x 5 stars with a random sub-pixel offset.
xvals = np.linspace(50., 1950., 5)
yvals = np.linspace(50., 1950., 5)
x_list, y_list = np.meshgrid(xvals, yvals)
x_list = x_list.flatten()
y_list = y_list.flatten()
x_list = x_list + (np_rng.rand(len(x_list)) - 0.5)
y_list = y_list + (np_rng.rand(len(x_list)) - 0.5)
print('x_list = ',x_list)
print('y_list = ',y_list)
# Range of fluxes from 100 to 15000
flux_list = 100. * np.exp(5. * np_rng.rand(len(x_list)))
print('fluxes range from ',np.min(flux_list),np.max(flux_list))
# Draw a Moffat PSF at each location on the image.
# Have the truth values vary quadratically across the image.
beta_fn = lambda x,y: 3.5 - 0.1*(x/1000) + 0.08*(y/1000)**2
fwhm_fn = lambda x,y: 0.9 + 0.05*(x/1000) - 0.03*(y/1000) + 0.02*(x/1000)*(y/1000)
e1_fn = lambda x,y: 0.02 - 0.01*(x/1000)
e2_fn = lambda x,y: -0.03 + 0.02*(x/1000)**2 - 0.01*(y/1000)*2
for x,y,flux in zip(x_list, y_list, flux_list):
beta = beta_fn(x,y)
fwhm = fwhm_fn(x,y)
e1 = e1_fn(x,y)
e2 = e2_fn(x,y)
print(x,y,beta,fwhm,e1,e2)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta, flux=flux).shear(e1=e1, e2=e2)
bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32))
offset = galsim.PositionD( x-int(x)-0.5 , y-int(y)-0.5 )
moffat.drawImage(image=image[bounds], offset=offset, method='no_pixel')
print('drew image')
# Add sky level and noise
sky_level = 1000
noise_sigma = 0.1 # Not much noise to keep this an easy test.
image += sky_level
image.addNoise(galsim.GaussianNoise(sigma=noise_sigma))
# Write out the image to a file
image_file = os.path.join('output','pixel_moffat_image.fits')
image.write(image_file)
print('wrote image')
# Write out the catalog to a file
dtype = [ ('x','f8'), ('y','f8') ]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output','pixel_moffat_cat.fits')
fitsio.write(cat_file, data, clobber=True)
print('wrote catalog')
# Use InputFiles to read these back in
config = { 'image_file_name': image_file,
'cat_file_name': cat_file,
'stamp_size': 32,
'noise' : noise_sigma**2,
'sky' : sky_level,
}
input = piff.InputFiles(config)
assert input.image_file_name == [image_file]
assert input.cat_file_name == [cat_file]
# Check image
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
assert len(input.image_pos) == 1
assert len(input.image_pos[0]) == len(x_list)
np.testing.assert_equal([pos.x for pos in input.image_pos[0]], x_list)
np.testing.assert_equal([pos.y for pos in input.image_pos[0]], y_list)
# Make stars
orig_stars = input.makeStars()
assert len(orig_stars) == len(x_list)
assert orig_stars[0].image.array.shape == (32,32)
# Make a test star, not at the location of any of the model stars to use for each of the
# below tests.
x0 = 1024 # Some random position, not where a star was originally.
y0 = 133
beta = beta_fn(x0,y0)
fwhm = fwhm_fn(x0,y0)
e1 = e1_fn(x0,y0)
e2 = e2_fn(x0,y0)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta).shear(e1=e1, e2=e2)
target_star = piff.Star.makeTarget(x=x0, y=y0, scale=image.scale)
test_im = galsim.ImageD(bounds=target_star.image.bounds, scale=image.scale)
moffat.drawImage(image=test_im, method='no_pixel', use_true_center=False)
print('made test star')
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
order = 2
else:
logger = None
order = 1
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
psf_file = os.path.join('output','pixel_psf.fits')
if __name__ == '__main__':
# Process the star data
model = piff.PixelGrid(0.2, 16, start_sigma=0.9/2.355)
interp = piff.BasisPolynomial(order=order)
pointing = None # wcs is not Celestial here, so pointing needs to be None.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, {0:input.images[0].wcs}, pointing, logger=logger)
# Check that the interpolation is what it should be
print('target.flux = ',target_star.fit.flux)
test_star = psf.drawStar(target_star)
print('flux = ', test_im.array.sum(), test_star.image.array.sum())
print('max diff = ',np.max(np.abs(test_star.image.array-test_im.array)))
np.testing.assert_almost_equal(test_star.image.array/2, test_im.array/2, decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array/2, test_im.array/2, decimal=3)
# Round trip through a file
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.PixelGrid
assert type(psf.interp) is piff.BasisPolynomial
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array/2, test_im.array/2, decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array/2., test_im.array/2., decimal=3)
# Do the whole thing with the config parser
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'x_col' : 'x',
'y_col' : 'y',
'noise' : noise_sigma**2,
'sky' : sky_level,
'stamp_size' : 48 # Bigger than we drew, but should still work.
},
'output' : {
'file_name' : psf_file
},
'psf' : {
'model' : {
'type' : 'PixelGrid',
'scale' : 0.2,
'size' : 16, # Much smaller than the input stamps, but this is plenty here.
'start_sigma' : 0.9/2.355
},
'interp' : {
'type' : 'BasisPolynomial',
'order' : order
},
},
}
if __name__ == '__main__':
config['verbose'] = 2
else:
config['verbose'] = 0
print("Running piffify function")
piff.piffify(config)
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
print("Max abs diff = ",np.max(np.abs(test_star.image.array - test_im.array)))
np.testing.assert_almost_equal(test_star.image.array/2., test_im.array/2., decimal=3)
# Test using the piffify executable
with open('pixel_moffat.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
print("Running piffify executable")
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'pixel_moffat.yaml'] )
p.communicate()
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array/2., test_im.array/2., decimal=3)
# test copy_image property of drawStar and draw
for draw in [psf.drawStar, psf.model.draw]:
target_star_copy = psf.interp.interpolate(piff.Star(target_star.data.copy(), target_star.fit.copy())) # interp is so that when we do psf.model.draw we have fit.params to work with
test_star_copy = draw(target_star_copy, copy_image=True)
test_star_nocopy = draw(target_star_copy, copy_image=False)
# if we modify target_star_copy, then test_star_nocopy should be modified, but not test_star_copy
target_star_copy.image.array[0,0] = 23456
assert test_star_nocopy.image.array[0,0] == target_star_copy.image.array[0,0]
assert test_star_copy.image.array[0,0] != target_star_copy.image.array[0,0]
# however the other pixels SHOULD still be all the same value
assert test_star_nocopy.image.array[1,1] == target_star_copy.image.array[1,1]
assert test_star_copy.image.array[1,1] == target_star_copy.image.array[1,1]
# check that drawing onto an image does not return a copy
image = psf.draw(x=x0, y=y0)
image_reference = psf.draw(x=x0, y=y0, image=image)
image_reference.array[0,0] = 123456
assert image.array[0,0] == image_reference.array[0,0]
def measure_psf_shapes(xlist, ylist, psf_file_name, file_name, use_piff=False):
"""Given x,y positions, a psf solution file, and the wcs, measure shapes and sizes
of the PSF model.
We use the HSM module from GalSim to do this.
Returns e1, e2, size, flag.
"""
print 'Read in PSFEx file: ',psf_file_name
n_psf = len(xlist)
e1_list = [ 999. ] * n_psf
e2_list = [ 999. ] * n_psf
s_list = [ 999. ] * n_psf
flag_list = [ 0 ] * n_psf
try:
if use_piff:
psf = piff.read(psf_file_name)
else:
psf = galsim.des.DES_PSFEx(psf_file_name, file_name)
except Exception as e:
if 'CTYPE' in str(e):
try:
# Workaround for a bug in DES_PSFEx. It tries to read the image file using
# GSFitsWCS, which doesn't work if it's not a normal FITS WCS.
# galsim.fits.read should work correctly in those cases.
psf = galsim.des.DES_PSFEx(psf_file_name)
im = galsim.fits.read(file_name)
psf.wcs = im.wcs
e = None
except Exception as e:
pass
if e is not None:
print 'Caught ',e
flag_list = [ PSFEX_FAILURE ] * n_psf
return e1_list,e2_list,s_list,flag_list
stamp_size = 64
pixel_scale = 0.2
im = galsim.Image(stamp_size, stamp_size, scale=pixel_scale)
for i in range(n_psf):
x = xlist[i]
y = ylist[i]
print 'Measure PSFEx model shape at ',x,y
image_pos = galsim.PositionD(x,y)
#print 'im_pos = ',image_pos
if use_piff:
im = psf.draw(x=x, y=y, image=im)
else:
psf_i = psf.getPSF(image_pos)
im = psf_i.drawImage(image=im, method='no_pixel')
#print 'im = ',im
try:
shape_data = im.FindAdaptiveMom(strict=False)
except:
print ' *** Bad measurement (caught exception). Mask this one.'
flag_list[i] = PSFEX_BAD_MEASUREMENT
continue
#print 'shape_date = ',shape_data
if shape_data.moments_status != 0:
print 'status = ',shape_data.moments_status
print ' *** Bad measurement. Mask this one.'
flag_list[i] = PSFEX_BAD_MEASUREMENT
continue
dx = shape_data.moments_centroid.x - im.trueCenter().x
dy = shape_data.moments_centroid.y - im.trueCenter().y
#print 'centroid = ',shape_data.moments_centroid
#print 'trueCenter = ',im.trueCenter()
#print 'dcentroid = ',dx,dy
if dx**2 + dy**2 > MAX_CENTROID_SHIFT**2:
print ' *** Centroid shifted by ',dx,dy,'. Mask this one.'
flag_list[i] = PSFEX_CENTROID_SHIFT
continue
g1 = shape_data.observed_shape.g1
g2 = shape_data.observed_shape.g2
s = shape_data.moments_sigma * pixel_scale
#print 'g1,g2,s = ',g1,g2,s
e1_list[i] = g1
e2_list[i] = g2
s_list[i] = s
return e1_list,e2_list,s_list,flag_list
def test_draw():
"""Test the various options of the PSF.draw command.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_draw.log')
# Use an existing Piff solution to match as closely as possible how users would actually
# use this function.
psf = piff.read('input/test_single_py27.piff', logger=logger)
# Data that was used to make that file.
wcs = galsim.TanWCS(
galsim.AffineTransform(0.26, 0.05, -0.08, -0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees))
data = fitsio.read('input/test_single_cat1.fits')
field_center = galsim.CelestialCoord(0 * galsim.degrees,
-25 * galsim.degrees)
chipnum = 1
for k in range(len(data)):
x = data['x'][k]
y = data['y'][k]
e1 = data['e1'][k]
e2 = data['e2'][k]
s = data['s'][k]
print('k,x,y = ', k, x, y)
#print(' true s,e1,e2 = ',s,e1,e2)
# First, the same test with this file that is in test_wcs.py:test_pickle()
image_pos = galsim.PositionD(x, y)
star = piff.Star.makeTarget(x=x,
y=y,
wcs=wcs,
stamp_size=48,
pointing=field_center,
chipnum=chipnum)
star = psf.drawStar(star)
#print(' fitted s,e1,e2 = ',star.fit.params)
np.testing.assert_almost_equal(star.fit.params, [s, e1, e2], decimal=6)
# Now use the regular PSF.draw() command. This version is equivalent to the above.
# (It's not equal all the way to machine precision, but pretty close.)
im1 = psf.draw(x, y, chipnum, stamp_size=48)
np.testing.assert_allclose(im1.array,
star.data.image.array,
rtol=1.e-14,
atol=1.e-14)
# The wcs in the image is the wcs of the original image
assert im1.wcs == psf.wcs[1]
# The image is 48 x 48
assert im1.array.shape == (48, 48)
# The bounds are centered close to x,y. Within 0.5 pixel.
np.testing.assert_allclose(im1.bounds.true_center.x, x, atol=0.5)
np.testing.assert_allclose(im1.bounds.true_center.y, y, atol=0.5)
# This version draws the star centered at (x,y). Check the hsm centroid.
hsm = im1.FindAdaptiveMom()
#print('hsm = ',hsm)
np.testing.assert_allclose(hsm.moments_centroid.x, x, atol=0.01)
np.testing.assert_allclose(hsm.moments_centroid.y, y, atol=0.01)
# The total flux should be close to 1.
np.testing.assert_allclose(im1.array.sum(), 1.0, rtol=1.e-3)
# We can center the star at an arbitrary location on the image.
# The default is equivalent to center=(x,y). So check that this is equivalent.
# Also, 48 is the default stamp size, so that can be omitted here.
im2 = psf.draw(x, y, chipnum, center=(x, y))
assert im2.bounds == im1.bounds
np.testing.assert_allclose(im2.array,
im1.array,
rtol=1.e-14,
atol=1.e-14)
# Moving by an integer number of pixels should be very close to the same image
# over a different slice of the array.
im3 = psf.draw(x, y, chipnum, center=(x + 1, y + 3))
assert im3.bounds == im1.bounds
# (Remember -- numpy indexing is y,x!)
# Also, the FFTs will be different in detail, so only match to 1.e-6.
#print('im1 argmax = ',np.unravel_index(np.argmax(im1.array),im1.array.shape))
#print('im3 argmax = ',np.unravel_index(np.argmax(im3.array),im3.array.shape))
np.testing.assert_allclose(im3.array[3:, 1:],
im1.array[:-3, :-1],
rtol=1.e-6,
atol=1.e-6)
hsm = im3.FindAdaptiveMom()
np.testing.assert_allclose(hsm.moments_centroid.x, x + 1, atol=0.01)
np.testing.assert_allclose(hsm.moments_centroid.y, y + 3, atol=0.01)
# Can center at other locations, and the hsm centroids should come out centered pretty
# close to that location.
# (Of course the array will be different here, so can't test that.)
im4 = psf.draw(x, y, chipnum, center=(x + 1.3, y - 0.8))
assert im4.bounds == im1.bounds
hsm = im4.FindAdaptiveMom()
np.testing.assert_allclose(hsm.moments_centroid.x, x + 1.3, atol=0.01)
np.testing.assert_allclose(hsm.moments_centroid.y, y - 0.8, atol=0.01)
# Also allowed is center=True to place in the center of the image.
im5 = psf.draw(x, y, chipnum, center=True)
assert im5.bounds == im1.bounds
assert im5.array.shape == (48, 48)
np.testing.assert_allclose(im5.bounds.true_center.x, x, atol=0.5)
np.testing.assert_allclose(im5.bounds.true_center.y, y, atol=0.5)
np.testing.assert_allclose(im5.array.sum(), 1., rtol=1.e-3)
hsm = im5.FindAdaptiveMom()
center = im5.true_center
np.testing.assert_allclose(hsm.moments_centroid.x, center.x, atol=0.01)
np.testing.assert_allclose(hsm.moments_centroid.y, center.y, atol=0.01)
# Some invalid ways to try to do this. (Must be either True or a tuple.)
np.testing.assert_raises(ValueError,
psf.draw,
x,
y,
chipnum,
center='image')
np.testing.assert_raises(ValueError,
psf.draw,
x,
y,
chipnum,
center=im5.true_center)
# If providing your own image with bounds far away from the star (say centered at 0),
# then center=True works fine to draw in the center of that image.
im6 = im5.copy()
im6.setCenter(0, 0)
psf.draw(x, y, chipnum, center=True, image=im6)
assert im6.bounds.center == galsim.PositionI(0, 0)
np.testing.assert_allclose(im6.array.sum(), 1., rtol=1.e-3)
hsm = im6.FindAdaptiveMom()
center = im6.true_center
np.testing.assert_allclose(hsm.moments_centroid.x, center.x, atol=0.01)
np.testing.assert_allclose(hsm.moments_centroid.y, center.y, atol=0.01)
np.testing.assert_allclose(im6.array,
im5.array,
rtol=1.e-14,
atol=1.e-14)
# Check non-even stamp size. Also, not unit flux while we're at it.
im7 = psf.draw(x,
y,
chipnum,
center=(x + 1.3, y - 0.8),
stamp_size=43,
flux=23.7)
assert im7.array.shape == (43, 43)
np.testing.assert_allclose(im7.bounds.true_center.x, x, atol=0.5)
np.testing.assert_allclose(im7.bounds.true_center.y, y, atol=0.5)
np.testing.assert_allclose(im7.array.sum(), 23.7, rtol=1.e-3)
hsm = im7.FindAdaptiveMom()
np.testing.assert_allclose(hsm.moments_centroid.x, x + 1.3, atol=0.01)
np.testing.assert_allclose(hsm.moments_centroid.y, y - 0.8, atol=0.01)
# Can't do mixed even/odd shape with stamp_size, but it will respect a provided image.
im8 = galsim.Image(43, 44)
im8.setCenter(
x, y
) # It will respect the given bounds, so put it near the right place.
psf.draw(x,
y,
chipnum,
center=(x + 1.3, y - 0.8),
image=im8,
flux=23.7)
assert im8.array.shape == (44, 43)
np.testing.assert_allclose(im8.array.sum(), 23.7, rtol=1.e-3)
hsm = im8.FindAdaptiveMom()
np.testing.assert_allclose(hsm.moments_centroid.x, x + 1.3, atol=0.01)
np.testing.assert_allclose(hsm.moments_centroid.y, y - 0.8, atol=0.01)
# The offset parameter can add an additional to whatever center is used.
# Here center=None, so this is equivalent to im4 above.
im9 = psf.draw(x, y, chipnum, offset=(1.3, -0.8))
assert im9.bounds == im1.bounds
hsm = im9.FindAdaptiveMom()
np.testing.assert_allclose(im9.array,
im4.array,
rtol=1.e-14,
atol=1.e-14)
# With both, they are effectively added together. Not sure if there would be a likely
# use for this, but it's allowed. (The above with default center is used in unit
# tests a number of times, so that version at least is useful if only for us.
# I'm hard pressed to imaging end users wanting to specify things this way though.)
im10 = psf.draw(x,
y,
chipnum,
center=(x + 0.8, y - 0.3),
offset=(0.5, -0.5))
assert im10.bounds == im1.bounds
np.testing.assert_allclose(im10.array,
im4.array,
rtol=1.e-14,
atol=1.e-14)
def test_reserve():
"""Test the reserve_frac option.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_single_reserve.log')
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
true_params = [sigma, g1, g2]
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y in zip(x_list, y_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
psf.drawImage(image[bounds],
center=galsim.PositionD(x, y),
method='no_pixel')
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('output', 'test_simple_reserve_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output', 'test_simple_reserve_cat.fits')
fitsio.write(cat_file, data, clobber=True)
psf_file = os.path.join('output', 'test_simple_reserve_psf.fits')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'reserve_frac': 0.2,
'stamp_size': 32,
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
},
'output': {
'file_name': psf_file
},
}
piff.piffify(config, logger)
psf = piff.read(psf_file)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
print('chisq = ', psf.chisq)
print('dof = ', psf.dof)
nreserve = len([s for s in psf.stars if s.is_reserve])
ntot = len(psf.stars)
print('reserve = %s/%s' % (nreserve, ntot))
assert nreserve == 2
assert ntot == 10
print('dof =? ', (32 * 32 - 6) * (ntot - nreserve))
assert psf.dof == (32 * 32 - 6) * (ntot - nreserve)
for star in psf.stars:
# Fits should be good for both reserve and non-reserve stars
np.testing.assert_almost_equal(star.fit.params, true_params, decimal=4)
def test_yaml():
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_gp.log')
# Take DES test image, and test doing a psf run with GP interpolator
# Use config parser:
psf_file = os.path.join('output', 'gp_psf.fits')
config = {
'input': {
# These can be regular strings
'image_file_name': 'input/DECam_00241238_01.fits.fz',
# Or any GalSim str value type. e.g. FormattedStr
'cat_file_name': {
'type': 'FormattedStr',
'format':
'%s/DECam_%08d_%02d_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits',
'items': [
'input', # dir
241238, # expnum
1 # chipnum
]
},
# What hdu is everything in?
'image_hdu': 1,
'badpix_hdu': 2,
'weight_hdu': 3,
'cat_hdu': 2,
# What columns in the catalog have things we need?
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
'sky_col': 'BACKGROUND',
# How large should the postage stamp cutouts of the stars be?
'stamp_size': 21,
},
'psf': {
'model': {
'type': 'GSObjectModel',
'fastfit': True,
'gsobj': 'galsim.Gaussian(sigma=1.0)'
},
'interp': {
'type': 'GPInterp',
'keys': ['u', 'v'],
'optimizer': 'none',
'kernel': 'RBF(200.0)'
}
},
'output': {
'file_name': psf_file
},
}
piff.piffify(config, logger)
psf = piff.read(psf_file)
assert type(psf.model) is piff.GSObjectModel
assert type(psf.interp) is piff.GPInterp
print('nstars = ', len(psf.stars))
target = psf.stars[17]
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
target.fit.params,
decimal=3)
# This should also work if the target doesn't have a fit yet.
print('interpolate ', piff.Star(target.data, None))
test_star = psf.interp.interpolate(piff.Star(target.data, None))
np.testing.assert_almost_equal(test_star.fit.params,
target.fit.params,
decimal=3)
def test_des_image():
"""Test the whole process with a DES CCD.
"""
import os
import fitsio
image_file = 'y1_test/DECam_00241238_01.fits.fz'
cat_file = 'y1_test/DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits'
orig_image = galsim.fits.read(image_file)
psf_file = os.path.join('output', 'pixel_des_psf.fits')
if __name__ == '__main__':
# These match what Gary used in fit_des.py
nstars = None
scale = 0.15
size = 41
else:
# These are faster and good enough for the unit tests.
nstars = 25
scale = 0.2
size = 21
stamp_size = 51
# The configuration dict with the right input fields for the file we're using.
start_sigma = 1.0 / 2.355 # TODO: Need to make this automatic somehow.
config = {
'input': {
'images': image_file,
'image_hdu': 1,
'weight_hdu': 3,
'badpix_hdu': 2,
'cats': cat_file,
'cat_hdu': 2,
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'sky_col': 'BACKGROUND',
'stamp_size': stamp_size,
'ra': 'TELRA',
'dec': 'TELDEC',
'gain': 'GAINA',
},
'output': {
'file_name': psf_file,
},
'psf': {
'model': {
'type': 'PixelGrid',
'scale': scale,
'size': size,
'start_sigma': start_sigma,
},
'interp': {
'type': 'BasisPolynomial',
'order': 2,
},
},
}
if __name__ == '__main__': config['verbose'] = 3
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if True:
# Start by doing things manually:
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
# Largely copied from Gary's fit_des.py, but using the Piff input_handler to
# read the input files.
stars, wcs, pointing = piff.Input.process(config['input'],
logger=logger)
if nstars is not None:
stars = stars[:nstars]
# Make model, force PSF centering
model = piff.PixelGrid(scale=scale,
size=size,
interp=piff.Lanczos(3),
force_model_center=True,
start_sigma=start_sigma,
logger=logger)
# Interpolator will be zero-order polynomial.
# Find u, v ranges
interp = piff.BasisPolynomial(order=2, logger=logger)
# Make a psf
psf = piff.SimplePSF(model, interp)
psf.fit(stars, wcs, pointing, logger=logger)
# The difference between the images of the fitted stars and the originals should be
# consistent with noise. Keep track of how many don't meet that goal.
n_bad = 0 # chisq/dof > 2
n_marginal = 0 # chisq/dof > 1.1
n_good = 0 # chisq/dof <= 1.1
# Note: The 2 and 1.1 values here are very arbitrary!
for s in psf.stars:
fitted = psf.drawStar(s)
orig_stamp = orig_image[fitted.image.bounds] - s['sky']
fit_stamp = fitted.image
x0 = int(s['x'] + 0.5)
y0 = int(s['y'] + 0.5)
b = galsim.BoundsI(x0 - 3, x0 + 3, y0 - 3, y0 + 3)
#print('orig center = ',orig_stamp[b].array)
#print('flux = ',orig_stamp.array.sum())
#print('fit center = ',fit_stamp[b].array)
#print('flux = ',fit_stamp.array.sum())
flux = fitted.fit.flux
#print('max diff/flux = ',np.max(np.abs(orig_stamp.array-fit_stamp.array))/flux)
#np.testing.assert_almost_equal(fit_stamp.array/flux, orig_stamp.array/flux, decimal=2)
weight = s.weight # These should be 1/var_pix
resid = fit_stamp - orig_stamp
chisq = np.sum(resid.array**2 * weight.array)
print('chisq = ', chisq)
print('cf. star.chisq, dof = ', s.fit.chisq, s.fit.dof)
assert abs(chisq - s.fit.chisq) < 1.e-3 * chisq
if chisq > 2. * s.fit.dof:
n_bad += 1
elif chisq > 1.1 * s.fit.dof:
n_marginal += 1
else:
n_good += 1
# Check the convenience function that an end user would typically use
offset = s.center_to_offset(s.fit.center)
image = psf.draw(x=s['x'],
y=s['y'],
stamp_size=stamp_size,
flux=s.fit.flux,
offset=offset)
np.testing.assert_almost_equal(image.array,
fit_stamp.array,
decimal=4)
print('n_good, marginal, bad = ', n_good, n_marginal, n_bad)
# The real counts are 10 and 2. So this says make sure any updates to the code don't make
# things much worse.
assert n_marginal <= 12
assert n_bad <= 3
# Use piffify function
if __name__ == '__main__':
print('start piffify')
piff.piffify(config)
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'],
y=stars[0]['y'],
stamp_size=stamp_size,
flux=flux,
offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
# The first star happens to be a good one, so go ahead and test the arrays directly.
np.testing.assert_almost_equal(fit_stamp.array / flux,
orig_stamp.array / flux,
decimal=2)
# Test using the piffify executable
config['verbose'] = 0
with open('pixel_des.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
print('start piffify executable')
p = subprocess.Popen([piffify_exe, 'pixel_des.yaml'])
p.communicate()
print('read stars')
stars, wcs, pointing = piff.Input.process(config['input'])
print('read psf')
psf = piff.read(psf_file)
stars = [psf.model.initialize(s) for s in stars]
flux = stars[0].fit.flux
offset = stars[0].center_to_offset(stars[0].fit.center)
fit_stamp = psf.draw(x=stars[0]['x'],
y=stars[0]['y'],
stamp_size=stamp_size,
flux=flux,
offset=offset)
orig_stamp = orig_image[stars[0].image.bounds] - stars[0]['sky']
np.testing.assert_almost_equal(fit_stamp.array / flux,
orig_stamp.array / flux,
decimal=2)
def test_single_image():
"""Test the whole process with a single image.
Note: This test is based heavily on test_single_image in test_simple.py.
"""
import os
import fitsio
np_rng = np.random.RandomState(1234)
# Make the image
image = galsim.Image(2048, 2048, scale=0.2)
# The (x,y) values will be on a grid 5 x 5 stars with a random sub-pixel offset.
xvals = np.linspace(50., 1950., 5)
yvals = np.linspace(50., 1950., 5)
x_list, y_list = np.meshgrid(xvals, yvals)
x_list = x_list.flatten()
y_list = y_list.flatten()
x_list = x_list + (np_rng.rand(len(x_list)) - 0.5)
y_list = y_list + (np_rng.rand(len(x_list)) - 0.5)
print('x_list = ', x_list)
print('y_list = ', y_list)
# Range of fluxes from 100 to 15000
flux_list = 100. * np.exp(5. * np_rng.rand(len(x_list)))
print('fluxes range from ', np.min(flux_list), np.max(flux_list))
# Draw a Moffat PSF at each location on the image.
# Have the truth values vary quadratically across the image.
beta_fn = lambda x, y: 3.5 - 0.1 * (x / 1000) + 0.08 * (y / 1000)**2
fwhm_fn = lambda x, y: 0.9 + 0.05 * (x / 1000) - 0.03 * (
y / 1000) + 0.02 * (x / 1000) * (y / 1000)
e1_fn = lambda x, y: 0.02 - 0.01 * (x / 1000)
e2_fn = lambda x, y: -0.03 + 0.02 * (x / 1000)**2 - 0.01 * (y / 1000) * 2
for x, y, flux in zip(x_list, y_list, flux_list):
beta = beta_fn(x, y)
fwhm = fwhm_fn(x, y)
e1 = e1_fn(x, y)
e2 = e2_fn(x, y)
print(x, y, beta, fwhm, e1, e2)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta, flux=flux).shear(e1=e1,
e2=e2)
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
moffat.drawImage(image=image[bounds], offset=offset, method='no_pixel')
print('drew image')
# Write out the image to a file
image_file = os.path.join('data', 'pixel_moffat_image.fits')
image.write(image_file)
print('wrote image')
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('data', 'pixel_moffat_cat.fits')
fitsio.write(cat_file, data, clobber=True)
print('wrote catalog')
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file, stamp_size=32)
assert input.image_files == [image_file]
assert input.cat_files == [cat_file]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Make stars
orig_stars = input.makeStars()
assert len(orig_stars) == len(x_list)
assert orig_stars[0].image.array.shape == (32, 32)
# Make a test star, not at the location of any of the model stars to use for each of the
# below tests.
x0 = 1024 # Some random position, not where a star was originally.
y0 = 133
beta = beta_fn(x0, y0)
fwhm = fwhm_fn(x0, y0)
e1 = e1_fn(x0, y0)
e2 = e2_fn(x0, y0)
moffat = galsim.Moffat(fwhm=fwhm, beta=beta).shear(e1=e1, e2=e2)
target_star = piff.Star.makeTarget(x=x0, y=y0, scale=image.scale)
test_im = galsim.ImageD(bounds=target_star.image.bounds, scale=image.scale)
moffat.drawImage(image=test_im, method='no_pixel', use_true_center=False)
print('made test star')
# These tests are slow, and it's really just doing the same thing three times, so
# only do the first one when running via nosetests.
if True:
# Process the star data
model = piff.PixelGrid(0.2, 16, start_sigma=0.9 / 2.355)
interp = piff.BasisPolynomial(order=2)
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
pointing = None # wcs is not Celestial here, so pointing needs to be None.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, {0: input.images[0].wcs}, pointing, logger=logger)
# Check that the interpolation is what it should be
print('target.flux = ', target_star.fit.flux)
test_star = psf.drawStar(target_star)
#print('test_im center = ',test_im[b].array)
#print('flux = ',test_im.array.sum())
#print('interp_im center = ',test_star.image[b].array)
#print('flux = ',test_star.image.array.sum())
#print('max diff = ',np.max(np.abs(test_star.image.array-test_im.array)))
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array, test_im.array, decimal=3)
# Round trip through a file
psf_file = os.path.join('output', 'pixel_psf.fits')
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.PixelGrid
assert type(psf.interp) is piff.BasisPolynomial
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Check the convenience function that an end user would typically use
image = psf.draw(x=x0, y=y0)
np.testing.assert_almost_equal(image.array, test_im.array, decimal=3)
# Do the whole thing with the config parser
config = {
'input': {
'images': image_file,
'cats': cat_file,
'x_col': 'x',
'y_col': 'y',
'stamp_size': 48 # Bigger than we drew, but should still work.
},
'output': {
'file_name': psf_file
},
'psf': {
'model': {
'type': 'PixelGrid',
'scale': 0.2,
'size':
16, # Much smaller than the input stamps, but this is plenty here.
'start_sigma': 0.9 / 2.355
},
'interp': {
'type': 'BasisPolynomial',
'order': 2
},
},
}
if __name__ == '__main__':
print("Running piffify function")
piff.piffify(config)
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
# Test using the piffify executable
config['verbose'] = 0
with open('pixel_moffat.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
if __name__ == '__main__':
print("Running piffify executable")
if os.path.exists(psf_file):
os.remove(psf_file)
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'pixel_moffat.yaml'])
p.communicate()
psf = piff.read(psf_file)
test_star = psf.drawStar(target_star)
np.testing.assert_almost_equal(test_star.image.array,
test_im.array,
decimal=3)
def test_yaml():
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_gp.log')
# Take DES test image, and test doing a psf run with GP interpolator
# Use config parser:
for gp_piff in ['GPInterp', 'GPInterp2pcf']:
psf_file = os.path.join('output','gp_psf.fits')
config = {
'input' : {
# These can be regular strings
'image_file_name' : 'input/DECam_00241238_01.fits.fz',
# Or any GalSim str value type. e.g. FormattedStr
'cat_file_name' : {
'type': 'FormattedStr',
'format': '%s/DECam_%08d_%02d_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits',
'items': [
'input', # dir
241238, # expnum
1 # chipnum
]
},
# What hdu is everything in?
'image_hdu' : 1,
'badpix_hdu' : 2,
'weight_hdu' : 3,
'cat_hdu' : 2,
# What columns in the catalog have things we need?
'x_col' : 'XWIN_IMAGE',
'y_col' : 'YWIN_IMAGE',
'ra' : 'TELRA',
'dec' : 'TELDEC',
'gain' : 'GAINA',
'sky_col' : 'BACKGROUND',
# How large should the postage stamp cutouts of the stars be?
'stamp_size' : 21,
},
'psf' : {
'model' : { 'type' : 'GSObjectModel',
'fastfit' : True,
'gsobj' : 'galsim.Gaussian(sigma=1.0)' },
'interp' : { 'type' : gp_piff,
'keys' : ['u', 'v'],
'kernel' : 'RBF(200.0)',
'optimize' : False,}
},
'output' : { 'file_name' : psf_file },
}
if __name__ != '__main__':
config['input']['nstars'] = 25
piff.piffify(config, logger)
psf = piff.read(psf_file)
assert type(psf.model) is piff.GSObjectModel
assert type(psf.interp) is piff.GPInterp or type(psf.interp) is piff.GPInterp2pcf
print('nstars = ',len(psf.stars))
target = psf.stars[17]
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, target.fit.params, decimal=3)
# This should also work if the target doesn't have a fit yet.
print('interpolate ',piff.Star(target.data,None))
test_star = psf.interp.interpolate(piff.Star(target.data,None))
np.testing.assert_almost_equal(test_star.fit.params, target.fit.params, decimal=3)
def zernike(directory, config_file_name, piff_name, do_interp):
config = piff.read_config('{0}/{1}.yaml'.format(directory, config_file_name))
logger = piff.setup_logger(verbose=3)
# load base optics psf
out_path = '{0}/{1}.piff'.format(directory, piff_name)
psf = piff.read(out_path)
# load images for train stars
psf.stars = load_star_images(psf.stars, config, logger=logger)
stars = psf.stars
params = psf.getParamsList(stars)
# if do_interp, draw star models and fit radial profile
if do_interp:
# draw model stars
model_stars = psf.drawStarList(stars)
# fit radial piece
radial_agg = collect_radial_profiles(stars, model_stars)
interpfunc = interp1d(radial_agg['r'].values, radial_agg['dI'].values)
radial_agg.to_hdf('{0}/radial_{1}_{2}.h5'.format(directory, 'train', piff_name), 'data')
fig = Figure(figsize = (10, 5))
ax = fig.add_subplot(1, 1, 1)
ax.plot(radial_agg['r'], radial_agg['dI'])
ax.set_xlabel('r')
ax.set_ylabel('Residual radial image')
canvas = FigureCanvasAgg(fig)
# Do this after we've set the canvas to use Agg to avoid warning.
fig.set_tight_layout(True)
plot_path = '{0}/radial_{1}_{2}.pdf'.format(directory, 'train', piff_name)
logger.info('saving plot to {0}'.format(plot_path))
canvas.print_figure(plot_path, dpi=100)
# do the fits of the stars
logger.info('Fitting {0} stars'.format(len(stars)))
model_fitted_stars = []
for star_i, star in zip(range(len(stars)), stars):
if (star_i + 1) % int(max([len(stars) * 0.05, 1])) == 0:
logger.info('doing {0} out of {1}:'.format(star_i + 1, len(stars)))
try:
model_fitted_star, results = fit_with_radial(psf, star, interpfunc, vary_shape=True, vary_optics=True)
model_fitted_stars.append(model_fitted_star)
if (star_i + 1) % int(max([len(stars) * 0.05, 1])) == 0:
logger.debug(lmfit.fit_report(results, min_correl=0.5))
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
logger.warning('{0}'.format(str(e)))
logger.warning('Warning! Failed to fit atmosphere model for star {0}. Ignoring star in atmosphere fit'.format(star_i))
stars = model_fitted_stars
logger.info('Drawing final model stars')
drawn_stars = [drawProfile(psf, star, psf.getProfile(star.fit.params), star.fit.params, use_fit=True, copy_image=True, interpfunc=interpfunc) for star in stars]
else:
# else just do regular zernike fit
logger.info('Fitting {0} stars'.format(len(stars)))
model_fitted_stars = []
for star_i, star in zip(range(len(stars)), stars):
if (star_i + 1) % int(max([len(stars) * 0.05, 1])) == 0:
logger.info('doing {0} out of {1}:'.format(star_i + 1, len(stars)))
try:
if (star_i + 1) % int(max([len(stars) * 0.05, 1])) == 0:
model_fitted_star, results = psf.fit_model(star, params=params[star_i], vary_shape=True, vary_optics=True, mode='pixel', logger=logger)
else:
model_fitted_star, results = psf.fit_model(star, params=params[star_i], vary_shape=True, vary_optics=True, mode='pixel')
model_fitted_stars.append(model_fitted_star)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
logger.warning('{0}'.format(str(e)))
logger.warning('Warning! Failed to fit atmosphere model for star {0}. Ignoring star in atmosphere fit'.format(star_i))
stars = model_fitted_stars
logger.info('Drawing final model stars')
drawn_stars = [psf.drawProfile(star, psf.getProfile(star.fit.params), star.fit.params, copy_image=True, use_fit=True) for star in stars]
logger.info('Measuring star shapes')
shapes = measure_star_shape(stars, drawn_stars, logger=logger)
logger.info('Adding fitted params and params_var')
shape_keys = ['e0', 'e1', 'e2', 'delta1', 'delta2', 'zeta1', 'zeta2']
shape_plot_keys = []
for key in shape_keys:
shape_plot_keys.append(['data_' + key, 'model_' + key, 'd' + key])
param_keys = ['atmo_size', 'atmo_g1', 'atmo_g2'] + ['optics_size', 'optics_g1', 'optics_g2'] + ['z{0:02d}'.format(zi) for zi in range(4, 45)]
logger.info('Extracting training fit parameters')
params = np.array([star.fit.params for star in stars])
params_var = np.array([star.fit.params_var for star in stars])
for i in range(params.shape[1]):
shapes['{0}_fit'.format(param_keys[i])] = params[:, i]
shapes['{0}_var'.format(param_keys[i])] = params_var[:, i]
shapes['chisq'] = np.array([star.fit.chisq for star in stars])
shapes['dof'] = np.array([star.fit.dof for star in stars])
logger.info('saving shapes')
shapes.to_hdf('{0}/zernikeshapes_{1}_{2}_zernike{3}.h5'.format(directory, 'train', piff_name, ['_reg', '_interp'][do_interp]), 'data')
logger.info('saving stars')
fits_path = '{0}/zernikestars_{1}_{2}_zernike{3}.fits'.format(directory, 'train', piff_name, ['_reg', '_interp'][do_interp])
with fitsio.FITS(fits_path, 'rw', clobber=True) as f:
piff.Star.write(stars, f, extname='zernike_stars')
logger.info('making 2d plots')
# plot shapes
fig, axs = plot_2dhist_shapes(shapes, shape_plot_keys, diff_mode=True)
# save
fig.savefig('{0}/zernike{3}_shapes_{1}_{2}.pdf'.format(directory, 'train', piff_name, ['_reg', '_interp'][do_interp]))
# plot params
plot_keys = []
plot_keys_i = []
for i in range(params.shape[1]):
plot_keys_i.append(param_keys[i])
if len(plot_keys_i) == 3:
plot_keys.append(plot_keys_i)
plot_keys_i = []
if len(plot_keys_i) > 0:
plot_keys_i += [plot_keys_i[0]] * (3 - len(plot_keys_i))
plot_keys.append(plot_keys_i)
fig, axs = plot_2dhist_shapes(shapes, [[key + '_fit' for key in kp] for kp in plot_keys], diff_mode=False)
fig.savefig('{0}/zernike{3}_fit_params_{1}_{2}.pdf'.format(directory, 'train', piff_name, ['_reg', '_interp'][do_interp]))
nstars = min([20, len(stars)])
indices = np.random.choice(len(stars), nstars, replace=False)
logger.info('saving {0} star images'.format(nstars))
fig = Figure(figsize = (4 * 4, 3 * nstars))
for i, indx in enumerate(indices):
axs = [ fig.add_subplot(nstars, 4, i * 4 + j + 1) for j in range(4)]
# select a star
star = stars[indx]
# draw the model star
params = star.fit.params
prof = psf.getProfile(params)
if do_interp:
star_drawn = drawProfile(psf, star, psf.getProfile(star.fit.params), star.fit.params, use_fit=True, copy_image=True, interpfunc=interpfunc)
else:
star_drawn = psf.drawProfile(star, prof, params, use_fit=True, copy_image=True)
# make plot
draw_stars(star, star_drawn, fig, axs)
canvas = FigureCanvasAgg(fig)
# Do this after we've set the canvas to use Agg to avoid warning.
fig.set_tight_layout(True)
# save files based on what is listed
plot_path = '{0}/zernike{3}_stars_{1}_{2}.pdf'.format(directory, 'train', piff_name, ['_reg', '_interp'][do_interp])
logger.info('saving plot to {0}'.format(plot_path))
canvas.print_figure(plot_path, dpi=100)
def test_parallel():
# Run the same test as test_single, but using nproc
wcs1 = galsim.TanWCS(
galsim.AffineTransform(0.26, 0.05, -0.08, -0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees))
wcs2 = galsim.TanWCS(
galsim.AffineTransform(0.25, -0.02, 0.01, 0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees))
field_center = galsim.CelestialCoord(0 * galsim.degrees,
-25 * galsim.degrees)
if __name__ == '__main__':
nstars = 20 # per ccd
else:
nstars = 6 # per ccd
rng = np.random.RandomState(1234)
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra1, dec1 = wcs1.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data1 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im1 = drawImage(2048, 2048, wcs1, x, y, e1, e2, s)
im1.write('output/test_parallel_im1.fits')
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra2, dec2 = wcs2.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
# Same functions of u,v, but using the positions on chip 2
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data2 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im2 = drawImage(2048, 2048, wcs2, x, y, e1, e2, s)
im2.write('output/test_parallel_im2.fits')
ra12 = np.concatenate([ra1, ra2])
dec12 = np.concatenate([dec1, dec2])
data12 = np.array(list(zip(ra12, dec12)),
dtype=[('ra', float), ('dec', float)])
fitsio.write('output/test_parallel.fits', data12, clobber=True)
# im3 is blank. Will give errors trying to measure PSF from it.
im3 = galsim.Image(2048, 2048, wcs=wcs2)
im3.write('output/test_parallel_im3.fits')
psf_file = os.path.join('output', 'test_single.fits')
config = {
'input': {
# A third way to input these same file names. Use GalSim config values and
# explicitly specify the number of images to read
'nimages': 2,
'image_file_name': {
'type': 'FormattedStr',
'format': '%s/test_parallel_im%d.fits',
'items': ['output', '$image_num+1'],
},
'cat_file_name': 'output/test_parallel.fits',
'chipnum': '$image_num+1',
'ra_col': 'ra',
'dec_col': 'dec',
'ra_units': 'rad',
'dec_units': 'rad',
'nproc': -1,
},
'psf': {
'type': 'SingleChip',
'model': {
'type': 'Moffat',
'beta': 2.5,
},
'interp': {
'type': 'Polynomial',
'order': 2,
},
'nproc': 2,
},
'output': {
'file_name': psf_file,
},
}
with CaptureLog(level=2) as cl:
piff.piffify(config, logger=cl.logger)
psf = piff.read(psf_file)
for chipnum, data, wcs in [(1, data1, wcs1), (2, data2, wcs2)]:
for k in range(nstars):
x = data['x'][k]
y = data['y'][k]
e1 = data['e1'][k]
e2 = data['e2'][k]
s = data['s'][k]
image_pos = galsim.PositionD(x, y)
star = piff.Star.makeTarget(x=x,
y=y,
wcs=wcs,
stamp_size=48,
pointing=field_center,
chipnum=chipnum)
star = psf.drawStar(star)
np.testing.assert_almost_equal(star.fit.params, [s, e1, e2],
decimal=6)
# Finally, check that the logger properly captures the subprocess logs
with CaptureLog(level=2) as cl:
psf = piff.process(config, cl.logger)
#print('with nproc=2, log = ',cl.output)
assert "Processing catalog 1" in cl.output
assert "Processing catalog 2" in cl.output
assert "Building solution for chip 1" in cl.output
assert "Building solution for chip 2" in cl.output
# Check that errors in the solution get properly reported.
config['input']['nimages'] = 3
with CaptureLog(level=2) as cl:
psf = piff.process(config, cl.logger)
assert "Removed 6 stars in initialize" in cl.output
assert "No stars. Cannot find PSF model." in cl.output
assert "Solutions failed for chipnums: [3]" in cl.output
# Check that errors in the multiprocessing input get properly reported.
config['input']['ra_col'] = 'invalid'
with CaptureLog(level=2) as cl:
with np.testing.assert_raises(ValueError):
psf = piff.process(config, cl.logger)
assert "ra_col = invalid is not a column" in cl.output
# With nproc=1, the error is raised directly.
config['input']['nproc'] = 1
config['verbose'] = 0
with np.testing.assert_raises(ValueError):
psf = piff.process(config)
# But just the input error. Not the one in fitting.
config['psf']['nproc'] = 1
config['input']['ra_col'] = 'ra'
config['verbose'] = 1
with CaptureLog(level=1) as cl:
psf = piff.process(config, logger=cl.logger)
assert "No stars. Cannot find PSF model." in cl.output
assert "Ignoring this failure and continuing on." in cl.output
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
flag_list = [0, 0, 12, 0, 0, 1, 0, 0, 0, 0]
use_list = [1, 1, 1, 1, 1, 0, 1, 1, 0, 1]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y, flag, use in zip(x_list, y_list, flag_list, use_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag:
print('corrupting star at ', x, y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('data', 'simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2'), ('use', 'i2')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
data['use'] = use_list
cat_file = os.path.join('data', 'simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file)
assert input.image_files == [image_file]
assert input.cat_files == [cat_file]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Repeat, using flag and use columns this time.
input = piff.InputFiles(image_file,
cat_file,
flag_col='flag',
use_col='use',
stamp_size=48)
assert input.flag_col == 'flag'
assert input.use_col == 'use'
input.readImages()
input.readStarCatalogs()
assert len(input.cats[0]) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48, 48)
# Process the star data
# can only compare to truth if include_pixel=False
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars]
interp.solve(fitted_stars)
print('mean = ', interp.mean)
# Check that the interpolation is what it should be
target = piff.Star.makeTarget(x=1024,
y=123) # Any position would work here.
true_params = [sigma, g1, g2]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Now test running it via the config parser
psf_file = os.path.join('output', 'simple_psf.fits')
config = {
'input': {
'images': image_file,
'cats': cat_file,
'flag_col': 'flag',
'use_col': 'use',
'stamp_size': 48
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
},
'output': {
'file_name': psf_file
},
}
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(verbose=0)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Round trip to a file
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test using the piffify executable
os.remove(psf_file)
config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test that we can make rho statistics
min_sep = 1
max_sep = 100
bin_size = 0.1
stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size)
stats.compute(psf, orig_stars)
rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5]
for rho in rhos:
# Test the range of separations
radius = np.exp(rho.logr)
# last bin can be one bigger than max_sep
np.testing.assert_array_less(radius,
np.exp(np.log(max_sep) + bin_size))
np.testing.assert_array_less(min_sep, radius)
np.testing.assert_array_almost_equal(np.diff(rho.logr),
bin_size,
decimal=5)
# Test that the max absolute value of each rho isn't crazy
np.testing.assert_array_less(np.abs(rho.xip), 1)
# # Check that each rho isn't precisely zero. This means the sum of abs > 0
np.testing.assert_array_less(0, np.sum(np.abs(rho.xip)))
# Test the plotting and writing
rho_psf_file = os.path.join('output', 'simple_psf_rhostats.pdf')
stats.write(rho_psf_file)
# Test that we can make summary shape statistics, using HSM
shapeStats = piff.ShapeHistogramsStats()
shapeStats.compute(psf, orig_stars)
# test their characteristics
np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
shape_psf_file = os.path.join('output', 'simple_psf_shapestats.pdf')
shapeStats.write(shape_psf_file)
# Test that we can use the config parser for both RhoStats and ShapeHistogramsStats
config['output']['stats'] = [
{
'type': 'ShapeHistograms',
'file_name': shape_psf_file
},
{
'type': 'Rho',
'file_name': rho_psf_file
},
{
'type': 'TwoDHist',
'file_name': os.path.join('output',
'simple_psf_twodhiststats.pdf'),
'number_bins_u': 3,
'number_bins_v': 3,
},
{
'type': 'TwoDHist',
'file_name': os.path.join('output',
'simple_psf_twodhiststats_std.pdf'),
'reducing_function': 'np.std',
'number_bins_u': 3,
'number_bins_v': 3,
},
]
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
piff.piffify(config, logger)
# Test using the piffify executable
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
def test_depr_select():
# A bunch of keys used to be allowed in input, but now belong in select.
# Check that the old way still works, but gives a warning.
# This is the input from test_stars in test_input.py
dir = 'input'
image_file = 'test_input_image_00.fits'
cat_file = 'test_input_cat_00.fits'
psf_file = os.path.join('output','test_depr_select.fits')
config = {
'input' : {
'dir' : dir,
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'weight_hdu' : 1,
'sky_col' : 'sky',
'gain_col' : 'gain',
'use_partial' : True,
'nstars': 15, # Just to make the test faster
},
'select': {
'max_snr' : 200,
'min_snr' : 20,
'hsm_size_reject' : 20,
'max_edge_frac': 0.25,
'stamp_center_size': 10,
'max_mask_pixels' : 20,
'reserve_frac' : 0.1,
'seed': 1234,
},
'psf': {
'model' : {'type': 'Gaussian'},
'interp' : {'type': 'Mean'},
},
'output' : { 'file_name' : psf_file },
}
logger = piff.config.setup_logger(log_file='output/test_depr_select.log')
# This is the new API
print('config = ',config)
piff.piffify(config, logger)
psf1 = piff.read(psf_file)
im1 = psf1.draw(x=23, y=34, stamp_size=16)
print('len1 = ',len(psf1.stars))
# This is the old API
config['input'].update(config['select'])
del config['select']
config = galsim.config.CleanConfig(config)
print('config = ',config)
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert "WARNING: Items [" in cl.output
assert "] should now be in the 'select' field of the config file." in cl.output
psf2 = piff.read(psf_file)
print('len2 = ',len(psf2.stars))
assert len(psf1.stars) == len(psf2.stars)
im2 = psf2.draw(x=23, y=34, stamp_size=16)
np.testing.assert_allclose(im2.array, im1.array)
# Also ok for some items to be in select, but erroneously put some in input.
config['input']['min_snr'] = config['select'].pop('min_snr')
config['input']['max_snr'] = config['select'].pop('max_snr')
config = galsim.config.CleanConfig(config)
print('config = ',config)
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert "WARNING: Items ['max_snr', 'min_snr'] should now be in the 'select' field of the config file." in cl.output
psf3 = piff.read(psf_file)
print('len3 = ',len(psf3.stars))
assert len(psf1.stars) == len(psf3.stars)
im3 = psf3.draw(x=23, y=34, stamp_size=16)
np.testing.assert_allclose(im3.array, im1.array)
ncolor = len(colors)
nx = len(xs)
ny = len(ys)
src = info['src_info'][n]
band = src['band']
image_path = src['image_path']
piff_path = src['piff_path']
piff_info = src['piff_info'] # ccdnum, expnum, nstar, desdm_flags, fwhm_cen,
# exp_star_t_mean, exp_star_t_std, star_t_mean, star_t_std
ccdnum = piff_info['ccdnum']
print(n, band, piff_path)
psf = piff.read(piff_path)
print(piff_info)
print('chisq = ', psf.chisq)
print('dof = ', psf.dof)
print('nremoved = ', psf.nremoved)
print('nused = ', len(psf.stars))
print('colors = ', [s['GI_COLOR'] for s in psf.stars])
allT = np.zeros((ncolor, nx, ny), dtype=float)
allg1 = np.zeros((ncolor, nx, ny), dtype=float)
allg2 = np.zeros((ncolor, nx, ny), dtype=float)
for icolor, color in enumerate(colors):
print('color = ', color)
for ix, x in enumerate(xs):
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ]
y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ]
flag_list = [ 0, 0, 12, 0, 0, 1, 0, 0, 0, 0 ]
use_list = [ 1, 1, 1, 1, 1, 0, 1, 1, 0, 1 ]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x,y,flag,use in zip(x_list, y_list, flag_list, use_list):
bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32))
offset = galsim.PositionD( x-int(x)-0.5 , y-int(y)-0.5 )
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag:
print('corrupting star at ',x,y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
# Write out the image to a file
image_file = os.path.join('data','simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [ ('x','f8'), ('y','f8'), ('flag','i2'), ('use','i2') ]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
data['use'] = use_list
cat_file = os.path.join('data','simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
input = piff.InputFiles(image_file, cat_file)
assert input.image_files == [ image_file ]
assert input.cat_files == [ cat_file ]
assert input.x_col == 'x'
assert input.y_col == 'y'
# Check image
input.readImages()
assert len(input.images) == 1
np.testing.assert_equal(input.images[0].array, image.array)
# Check catalog
input.readStarCatalogs()
assert len(input.cats) == 1
np.testing.assert_equal(input.cats[0]['x'], x_list)
np.testing.assert_equal(input.cats[0]['y'], y_list)
# Repeat, using flag and use columns this time.
input = piff.InputFiles(image_file, cat_file, flag_col='flag', use_col='use', stamp_size=48)
assert input.flag_col == 'flag'
assert input.use_col == 'use'
input.readImages()
input.readStarCatalogs()
assert len(input.cats[0]) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48,48)
# Process the star data
model = piff.Gaussian()
interp = piff.Mean()
fitted_stars = [ model.fit(star) for star in orig_stars ]
interp.solve(fitted_stars)
print('mean = ',interp.mean)
# Check that the interpolation is what it should be
target = piff.Star.makeTarget(x=1024, y=123) # Any position would work here.
true_params = [ sigma, g1, g2 ]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Now test running it via the config parser
psf_file = os.path.join('output','simple_psf.fits')
config = {
'input' : {
'images' : image_file,
'cats' : cat_file,
'flag_col' : 'flag',
'use_col' : 'use',
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian' },
'interp' : { 'type' : 'Mean' },
},
'output' : { 'file_name' : psf_file },
}
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=3)
else:
logger = piff.config.setup_logger(verbose=1)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Round trip to a file
psf.write(psf_file, logger)
psf = piff.read(psf_file, logger)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Test using the piffify executable
os.remove(psf_file)
with open('simple.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'simple.yaml'] )
p.communicate()
psf = piff.read(psf_file)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=5)
# Test that we can make rho statistics
min_sep = 1
max_sep = 100
bin_size = 0.1
stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size)
stats.compute(psf, orig_stars)
rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5]
for rho in rhos:
# Test the range of separations
radius = np.exp(rho.logr)
# last bin can be one bigger than max_sep
np.testing.assert_array_less(radius, np.exp(np.log(max_sep) + bin_size))
np.testing.assert_array_less(min_sep, radius)
np.testing.assert_array_almost_equal(np.diff(rho.logr), bin_size, decimal=5)
# Test that the max absolute value of each rho isn't crazy
np.testing.assert_array_less(np.abs(rho.xip), 1)
# Check that each rho isn't precisely zero. This means the sum of abs > 0
np.testing.assert_array_less(0, np.sum(np.abs(rho.xip)))
# Test the plotting and writing
rho_psf_file = os.path.join('output','simple_psf_rhostats.png')
stats.write(rho_psf_file)
# Test that we can make summary shape statistics, using HSM
shapeStats = piff.ShapeHistogramsStats()
shapeStats.compute(psf, orig_stars)
# test their characteristics
np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
shape_psf_file = os.path.join('output','simple_psf_shapestats.png')
shapeStats.write(shape_psf_file)
# Test that we can use the config parser for both RhoStats and ShapeHistogramsStats
config['output']['stats'] = [
{
'type': 'ShapeHistograms',
'file_name': shape_psf_file
},
{
'type': 'Rho',
'file_name': rho_psf_file
},
]
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
piff.piffify(config)
# Test using the piffify executable
os.remove(psf_file)
os.remove(rho_psf_file)
os.remove(shape_psf_file)
with open('simple.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'simple.yaml'] )
p.communicate()
def test_bad_hsm():
"""Test that stats don't break when all stars end up being flagged with hsm errors.
"""
image_file = os.path.join('input','DECam_00241238_01.fits.fz')
cat_file = os.path.join('input',
'DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits')
psf_file = os.path.join('output','bad_hsm.fits')
twodhist_file = os.path.join('output','bad_hsm_twod.pdf')
whisker_file = os.path.join('output','bad_hsm_whisk.pdf')
rho_file = os.path.join('output','bad_hsm_rho.pdf')
shape_file = os.path.join('output','bad_hsm_shape.pdf')
star_file = os.path.join('output','bad_hsm_star.pdf')
hsm_file = os.path.join('output','bad_hsm_hsm.fits')
sizemag_file = os.path.join('output','bad_hsm_sizemag.png')
stamp_size = 25
# The configuration dict with the right input fields for the file we're using.
config = {
'input' : {
'nstars': 8,
'image_file_name' : image_file,
'image_hdu' : 1,
'weight_hdu' : 3,
'badpix_hdu' : 2,
'cat_file_name' : cat_file,
'cat_hdu' : 2,
# These next two are intentionally backwards. The PixelGrid will find some kind
# of solution, but it will be complex garbage, and hsm will fail for them.
'x_col' : 'YWIN_IMAGE',
'y_col' : 'XWIN_IMAGE',
'sky_col' : 'BACKGROUND',
'stamp_size' : stamp_size,
'ra' : 'TELRA',
'dec' : 'TELDEC',
'gain' : 'GAINA',
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'TwoDHist',
'file_name': twodhist_file,
},
{
'type': 'Whisker',
'file_name': whisker_file,
},
{ # Note: stats doesn't have to be a list.
'type': 'Rho',
'file_name': rho_file
},
{
'type': 'ShapeHist',
'file_name': shape_file,
},
{
'type': 'Star',
'file_name': star_file,
},
{
'type': 'SizeMag',
'file_name': sizemag_file,
},
{
'type': 'HSMCatalog',
'file_name': hsm_file,
},
],
},
'psf' : {
'model' : {
'type' : 'PixelGrid',
'scale' : 0.3,
'size' : 10,
},
'interp' : { 'type' : 'Mean' },
'outliers' : {
'type' : 'Chisq',
'nsigma' : 0.05 # This will throw out all but 1, which adds an additional
# test of Star stats when nstars < nplot
}
},
}
if __name__ == '__main__':
logger = piff.config.setup_logger(1)
else:
config['verbose'] = 0
logger = None
for f in [twodhist_file, rho_file, shape_file, star_file, hsm_file]:
if os.path.exists(f):
os.remove(f)
piff.piffify(config, logger=logger)
# Confirm that all but one star was rejected, since that was part of the intent of this test.
psf = piff.read(psf_file)
print('stars = ',psf.stars)
print('nremoved = ',psf.nremoved)
assert len(psf.stars) == 1
assert psf.nremoved == 7 # There were 8 to start.
for f in [twodhist_file, rho_file, shape_file, star_file, sizemag_file, hsm_file]:
assert os.path.exists(f)
# Check hsm file with bad measurements
# The one star that was left still fails hsm measurement here.
data = fitsio.read(hsm_file)
for col in ['ra', 'dec', 'x', 'y', 'u', 'v',
'T_data', 'g1_data', 'g2_data',
'T_model', 'g1_model', 'g2_model',
'flux', 'reserve', 'flag_truth', 'flag_model']:
assert len(data[col]) == 1
print('flag_truth = ',data['flag_truth'])
print('flag_model = ',data['flag_model'])
np.testing.assert_array_equal(data['flag_truth'], 7)
np.testing.assert_array_equal(data['flag_model'], 7)
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_single_image.log')
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
flag_list = [1, 1, 13, 1, 1, 4, 1, 1, 0, 1]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y, flag in zip(x_list, y_list, flag_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag & 4:
print('corrupting star at ', x, y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('output', 'simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
cat_file = os.path.join('output', 'simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
config = {'image_file_name': image_file, 'cat_file_name': cat_file}
input = piff.InputFiles(config, logger=logger)
assert input.image_file_name == [image_file]
assert input.cat_file_name == [cat_file]
# Check image
assert input.nimages == 1
image1, _, image_pos, _, _, _ = input.getRawImageData(0)
np.testing.assert_equal(image1.array, image.array)
# Check catalog
np.testing.assert_equal([pos.x for pos in image_pos], x_list)
np.testing.assert_equal([pos.y for pos in image_pos], y_list)
# Repeat, using flag columns this time.
config = {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': '1',
'skip_flag': '4',
'stamp_size': 48
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, _, image_pos, _, _, _ = input.getRawImageData(0)
assert len(image_pos) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48, 48)
# Process the star data
# can only compare to truth if include_pixel=False
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars]
interp.solve(fitted_stars)
print('mean = ', interp.mean)
# Check that the interpolation is what it should be
# Any position would work here.
chipnum = 0
x = 1024
y = 123
orig_wcs = input.getWCS()[chipnum]
orig_pointing = input.getPointing()
image_pos = galsim.PositionD(x, y)
world_pos = piff.StarData.calculateFieldPos(image_pos, orig_wcs,
orig_pointing)
u, v = world_pos.x, world_pos.y
stamp_size = config['stamp_size']
target = piff.Star.makeTarget(x=x,
y=y,
u=u,
v=v,
wcs=orig_wcs,
stamp_size=stamp_size,
pointing=orig_pointing)
true_params = [sigma, g1, g2]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Check default values of options
psf = piff.SimplePSF(model, interp)
assert psf.chisq_thresh == 0.1
assert psf.max_iter == 30
assert psf.outliers == None
assert psf.extra_interp_properties == []
# Now test running it via the config parser
psf_file = os.path.join('output', 'simple_psf.fits')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': 1,
'skip_flag': 4,
'stamp_size': stamp_size
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
'max_iter': 10,
'chisq_thresh': 0.2,
},
'output': {
'file_name': psf_file
},
}
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
interp = piff.Mean()
psf = piff.SimplePSF(model, interp, max_iter=10, chisq_thresh=0.2)
assert psf.chisq_thresh == 0.2
assert psf.max_iter == 10
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# test that drawStar and drawStarList work
test_star = psf.drawStar(target)
test_star_list = psf.drawStarList([target])[0]
np.testing.assert_equal(test_star.fit.params, test_star_list.fit.params)
np.testing.assert_equal(test_star.image.array, test_star_list.image.array)
# test copy_image property of drawStar and draw
for draw in [psf.drawStar, psf.model.draw]:
target_star_copy = psf.interp.interpolate(
piff.Star(target.data.copy(), target.fit.copy()))
# interp is so that when we do psf.model.draw we have fit.params to work with
test_star_copy = draw(target_star_copy, copy_image=True)
test_star_nocopy = draw(target_star_copy, copy_image=False)
# if we modify target_star_copy, then test_star_nocopy should be modified,
# but not test_star_copy
target_star_copy.image.array[0, 0] = 23456
assert test_star_nocopy.image.array[
0, 0] == target_star_copy.image.array[0, 0]
assert test_star_copy.image.array[0,
0] != target_star_copy.image.array[0,
0]
# however the other pixels SHOULD still be all the same value
assert test_star_nocopy.image.array[
1, 1] == target_star_copy.image.array[1, 1]
assert test_star_copy.image.array[1,
1] == target_star_copy.image.array[1,
1]
# test that draw works
test_image = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center)
# this image should be the same values as test_star
assert test_image == test_star.image
# test that draw does not copy the image
image_ref = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center,
image=test_image)
image_ref.array[0, 0] = 123456789
assert test_image.array[0, 0] == image_ref.array[0, 0]
assert test_star.image.array[0, 0] != test_image.array[0, 0]
assert test_star.image.array[1, 1] == test_image.array[1, 1]
# Round trip to a file
psf.write(psf_file, logger)
psf2 = piff.read(psf_file, logger)
assert type(psf2.model) is piff.Gaussian
assert type(psf2.interp) is piff.Mean
assert psf2.chisq == psf.chisq
assert psf2.last_delta_chisq == psf.last_delta_chisq
assert psf2.chisq_thresh == psf.chisq_thresh
assert psf2.max_iter == psf.max_iter
assert psf2.dof == psf.dof
assert psf2.nremoved == psf.nremoved
test_star = psf2.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf3 = piff.read(psf_file)
assert type(psf3.model) is piff.Gaussian
assert type(psf3.interp) is piff.Mean
assert psf3.chisq == psf.chisq
assert psf3.last_delta_chisq == psf.last_delta_chisq
assert psf3.chisq_thresh == psf.chisq_thresh
assert psf3.max_iter == psf.max_iter
assert psf3.dof == psf.dof
assert psf3.nremoved == psf.nremoved
test_star = psf3.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test using the piffify executable
os.remove(psf_file)
# This would be simpler as a direct assignment, but this once, test the way you would set
# this from the command line, which would call parse_variables.
piff.config.parse_variables(config, ['verbose=0'], logger=logger)
#config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
config2 = piff.config.read_config('simple.yaml')
assert config == config2
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
psf4 = piff.read(psf_file)
assert type(psf4.model) is piff.Gaussian
assert type(psf4.interp) is piff.Mean
assert psf4.chisq == psf.chisq
assert psf4.last_delta_chisq == psf.last_delta_chisq
assert psf4.chisq_thresh == psf.chisq_thresh
assert psf4.max_iter == psf.max_iter
assert psf4.dof == psf.dof
assert psf4.nremoved == psf.nremoved
test_star = psf4.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# With very low max_iter, we hit the warning about non-convergence
config['psf']['max_iter'] = 1
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert 'PSF fit did not converge' in cl.output
def test_shapestats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_shapestats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_shapestats.fits')
shape_file = os.path.join('output','test_shapestats.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'ShapeHist',
'file_name': shape_file
},
]
},
}
piff.piffify(config, logger)
assert os.path.isfile(shape_file)
# repeat with plotify function
os.remove(shape_file)
piff.plotify(config, logger)
assert os.path.isfile(shape_file)
# Test ShapeHistStats directly
psf = piff.read(psf_file)
shapeStats = piff.ShapeHistStats(nbins=5) # default is sqrt(nstars)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
with np.testing.assert_raises(RuntimeError):
shapeStats.write() # Cannot write before compute
shapeStats.compute(psf, orig_stars)
shapeStats.plot(histtype='bar', log=True) # can supply additional args for matplotlib
# test their characteristics
sigma = 1.3 # (copied from setup())
T = 2*sigma**2
g1 = 0.23
g2 = -0.17
np.testing.assert_array_almost_equal(T, shapeStats.T, decimal=4)
np.testing.assert_array_almost_equal(T, shapeStats.T_model, decimal=3)
np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4)
np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3)
np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4)
np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3)
def test_load_images():
"""Test the load_images function
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_load_image2.log')
# Same setup as test_single_image, but without flags
image = galsim.Image(2048, 2048, scale=0.26)
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y in zip(x_list, y_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
psf.drawImage(image[bounds],
center=galsim.PositionD(x, y),
method='no_pixel')
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
image_file = os.path.join('output', 'test_load_images_im.fits')
image.write(image_file)
dtype = [('x', 'f8'), ('y', 'f8')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output', 'test_load_images_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Make star data
config = {'image_file_name': image_file, 'cat_file_name': cat_file}
orig_stars, wcs, pointing = piff.Input.process(config, logger)
# Fit these with a simple Mean, Gaussian
model = piff.Gaussian()
interp = piff.Mean()
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
psf_file = os.path.join('output', 'test_load_images_psf.fits')
psf.write(psf_file, logger)
# Read this file back in. It has the star data, but the images are blank.
psf2 = piff.read(psf_file, logger)
assert len(psf2.stars) == 10
for star in psf2.stars:
np.testing.assert_array_equal(star.image.array, 0.)
loaded_stars = piff.Star.load_images(psf2.stars, image_file)
for star, orig in zip(loaded_stars, psf.stars):
np.testing.assert_array_equal(star.image.array, orig.image.array)
# Can optionally supply sky to subtract
loaded_stars = piff.Star.load_images(psf2.stars, image_file, sky=10)
for star, orig in zip(loaded_stars, psf.stars):
np.testing.assert_array_equal(star.image.array, orig.image.array - 10)
from piff.optatmo_psf import poly, poly_full
test_mode = False
out_dir = '/nfs/slac/g/ki/ki18/cpd/Projects/piff_des/analytics'
# In[310]:
config = piff.read_config(
'/u/ki/cpd/ki19/piff_test/y3/mar_mesh_configs_fix_sph/00233466/Science-20121120s1-v20i2_limited/2018.03.29/config.yaml'
)
# In[311]:
# create an OptAtmoPSF for drawing
psf_fit = piff.read(
'/u/ki/cpd/ki19/piff_test/y3/mar_mesh_configs_fix_sph/00233466/Science-20121120s1-v20i2_limited/2018.03.29/psf.piff'
)
# In[312]:
# load up some stars
# do import modules so we can import pixmappy wcs
galsim.config.ImportModules(config)
# only load one ccd
config['input']['image_file_name'] = config['input'][
'image_file_name'].replace('*', '10')
# only load 10 stars in the ccd
config['input']['nstars'] = 10
if test_mode:
config['input']['stamp_size'] = 15
def test_rhostats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_rhostats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_rhostats.fits')
rho_file = os.path.join('output','test_rhostats.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : { # Note: stats doesn't have to be a list.
'type': 'Rho',
'file_name': rho_file
}
},
}
piff.piffify(config, logger)
assert os.path.isfile(rho_file)
# repeat with plotify function
os.remove(rho_file)
piff.plotify(config, logger)
assert os.path.isfile(rho_file)
# Test rho statistics directly.
min_sep = 1
max_sep = 100
bin_size = 0.1
psf = piff.read(psf_file)
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size)
stats.compute(psf, orig_stars)
rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5]
for rho in rhos:
# Test the range of separations
radius = np.exp(rho.logr)
np.testing.assert_array_less(radius, max_sep)
np.testing.assert_array_less(min_sep, radius)
# bin_size is reduced slightly to get integer number of bins
assert rho.bin_size < bin_size
assert np.isclose(rho.bin_size, bin_size, rtol=0.1)
np.testing.assert_array_almost_equal(np.diff(rho.logr), rho.bin_size, decimal=5)
# Test that the max absolute value of each rho isn't crazy
np.testing.assert_array_less(np.abs(rho.xip), 1)
# # Check that each rho isn't precisely zero. This means the sum of abs > 0
np.testing.assert_array_less(0, np.sum(np.abs(rho.xip)))
# Test using the piffify executable
os.remove(rho_file)
config['verbose'] = 0
with open('rho.yaml','w') as f:
f.write(yaml.dump(config, default_flow_style=False))
piffify_exe = get_script_name('piffify')
p = subprocess.Popen( [piffify_exe, 'rho.yaml'] )
p.communicate()
assert os.path.isfile(rho_file)
# Test using the plotify executable
os.remove(rho_file)
plotify_exe = get_script_name('plotify')
p = subprocess.Popen( [plotify_exe, 'rho.yaml'] )
p.communicate()
assert os.path.isfile(rho_file)
# test running plotify with dir in config, with no logger, and with a modules specification.
# (all to improve test coverage)
config['output']['dir'] = '.'
config['modules'] = [ 'custom_wcs' ]
os.remove(rho_file)
piff.plotify(config)
assert os.path.isfile(rho_file)
def __init__(self, file_name):
self.file_name = file_name
self._piff = piff.read(
os.path.expanduser(os.path.expandvars(file_name)))
self._did_fit = False
def test_starstats_config():
"""Test running stats through a config file.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_starstats_config.log')
image_file = os.path.join('output','test_stats_image.fits')
cat_file = os.path.join('output','test_stats_cat.fits')
psf_file = os.path.join('output','test_starstats.fits')
star_file = os.path.join('output', 'test_starstats.pdf')
star_noadjust_file = os.path.join('output', 'test_starstats_noadjust.pdf')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 48
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False },
'interp' : { 'type' : 'Mean' },
},
'output' : {
'file_name' : psf_file,
'stats' : [
{
'type': 'Star',
'file_name': star_file,
'number_plot': 5,
'adjust_stars': True,
}
]
}
}
piff.piffify(config, logger)
assert os.path.isfile(star_file)
# repeat with plotify function
os.remove(star_file)
piff.plotify(config, logger)
assert os.path.isfile(star_file)
# check default number_plot
psf = piff.read(psf_file)
starStats = piff.StarStats()
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
starStats.compute(psf, orig_stars)
assert starStats.number_plot == len(starStats.stars)
assert starStats.number_plot == len(starStats.models)
assert starStats.number_plot == len(starStats.indices)
np.testing.assert_array_equal(starStats.stars[2].image.array,
orig_stars[starStats.indices[2]].image.array)
# check number_plot = 6
starStats = piff.StarStats(number_plot=6)
starStats.compute(psf, orig_stars)
assert len(starStats.stars) == 6
# check number_plot >> len(stars)
starStats = piff.StarStats(number_plot=1000000)
starStats.compute(psf, orig_stars)
assert len(starStats.stars) == len(orig_stars)
# if use all stars, no randomness
np.testing.assert_array_equal(starStats.stars[3].image.array, orig_stars[3].image.array)
np.testing.assert_array_equal(starStats.indices, np.arange(len(orig_stars)))
# check number_plot = 0
starStats = piff.StarStats(number_plot=0)
starStats.compute(psf, orig_stars)
assert len(starStats.stars) == len(orig_stars)
# if use all stars, no randomness
np.testing.assert_array_equal(starStats.stars[3].image.array, orig_stars[3].image.array)
np.testing.assert_array_equal(starStats.indices, np.arange(len(orig_stars)))
# rerun with adjust stars and see if it did the right thing
# first with starstats == False
starStats = piff.StarStats(number_plot=0, adjust_stars=False)
starStats.compute(psf, orig_stars, logger=logger)
fluxs_noadjust = np.array([s.fit.flux for s in starStats.stars])
ds_noadjust = np.array([s.fit.center for s in starStats.stars])
# check that fluxes 1
np.testing.assert_array_equal(fluxs_noadjust, 1)
# check that ds are 0
np.testing.assert_array_equal(ds_noadjust, 0)
# now with starstats == True
starStats = piff.StarStats(number_plot=0, adjust_stars=True)
starStats.compute(psf, orig_stars, logger=logger)
fluxs_adjust = np.array([s.fit.flux for s in starStats.stars])
ds_adjust = np.array([s.fit.center for s in starStats.stars])
# copy the right values from setup()
dx = 0.31
dy = -0.32
flux = 123.45
# compare fluxes
np.testing.assert_allclose(fluxs_adjust, flux, rtol=1e-4)
# compare dx and dy, keeping in mind that ds_adjust is dx/y * 0.26 (scale)
dx_adjust = ds_adjust[:, 0] / 0.26
dy_adjust = ds_adjust[:, 1] / 0.26
np.testing.assert_allclose(dx_adjust, dx, rtol=1e-4)
np.testing.assert_allclose(dy_adjust, dy, rtol=1e-4)
# do once with adjust_stars = False to graphically demonstrate
config['output']['stats'][0]['file_name'] = star_noadjust_file
config['output']['stats'][0]['adjust_stars'] = False
piff.plotify(config, logger)
assert os.path.isfile(star_noadjust_file)
def measure_psf_shapes(xlist, ylist, psf_file_name, file_name, use_piff=False):
"""Given x,y positions, a psf solution file, and the wcs, measure shapes and sizes
of the PSF model.
We use the HSM module from GalSim to do this.
Returns e1, e2, size, flag.
"""
print 'Read in PSFEx file: ', psf_file_name
n_psf = len(xlist)
e1_list = [999.] * n_psf
e2_list = [999.] * n_psf
s_list = [999.] * n_psf
flag_list = [0] * n_psf
try:
if use_piff:
psf = piff.read(psf_file_name)
else:
psf = galsim.des.DES_PSFEx(psf_file_name, file_name)
except Exception as e:
if 'CTYPE' in str(e):
try:
# Workaround for a bug in DES_PSFEx. It tries to read the image file using
# GSFitsWCS, which doesn't work if it's not a normal FITS WCS.
# galsim.fits.read should work correctly in those cases.
psf = galsim.des.DES_PSFEx(psf_file_name)
im = galsim.fits.read(file_name)
psf.wcs = im.wcs
e = None
except Exception as e:
pass
if e is not None:
print 'Caught ', e
flag_list = [PSFEX_FAILURE] * n_psf
return e1_list, e2_list, s_list, flag_list
stamp_size = 64
pixel_scale = 0.2
im = galsim.Image(stamp_size, stamp_size, scale=pixel_scale)
for i in range(n_psf):
x = xlist[i]
y = ylist[i]
print 'Measure PSFEx model shape at ', x, y
image_pos = galsim.PositionD(x, y)
#print 'im_pos = ',image_pos
if use_piff:
im = psf.draw(x=x, y=y, image=im)
else:
psf_i = psf.getPSF(image_pos)
im = psf_i.drawImage(image=im, method='no_pixel')
#print 'im = ',im
try:
shape_data = im.FindAdaptiveMom(strict=False)
except:
print ' *** Bad measurement (caught exception). Mask this one.'
flag_list[i] = PSFEX_BAD_MEASUREMENT
continue
#print 'shape_date = ',shape_data
if shape_data.moments_status != 0:
print 'status = ', shape_data.moments_status
print ' *** Bad measurement. Mask this one.'
flag_list[i] = PSFEX_BAD_MEASUREMENT
continue
dx = shape_data.moments_centroid.x - im.trueCenter().x
dy = shape_data.moments_centroid.y - im.trueCenter().y
#print 'centroid = ',shape_data.moments_centroid
#print 'trueCenter = ',im.trueCenter()
#print 'dcentroid = ',dx,dy
if dx**2 + dy**2 > MAX_CENTROID_SHIFT**2:
print ' *** Centroid shifted by ', dx, dy, '. Mask this one.'
flag_list[i] = PSFEX_CENTROID_SHIFT
continue
g1 = shape_data.observed_shape.g1
g2 = shape_data.observed_shape.g2
s = shape_data.moments_sigma * pixel_scale
#print 'g1,g2,s = ',g1,g2,s
e1_list[i] = g1
e2_list[i] = g2
s_list[i] = s
return e1_list, e2_list, s_list, flag_list
