def test_weight():
"""Test the weight map and bad pixel masks
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file=os.path.join('output', 'test_input_weight.log'))
# If no weight or badpix is specified, the weights are all equal.
config = {
'image_file_name': 'input/test_input_image_00.fits',
'cat_file_name': 'input/test_input_cat_00.fits',
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
image, weight, image_pos, _, _, _ = input.getRawImageData(0)
assert len(image_pos) == 100
assert image.array.shape == (1024, 1024)
assert weight.array.shape == (1024, 1024)
np.testing.assert_array_equal(weight.array, 1.0)
# The default weight and badpix masks that GalSim makes don't do any masking, so this
# is the almost the same as above, but the weight value is 1/sky.
config = {
'image_file_name': 'input/test_input_image_00.fits',
'cat_file_name': 'input/test_input_cat_00.fits',
'weight_hdu': 1,
'badpix_hdu': 2,
'sky_col':
'sky', # Used to determine what the value of weight should be
'gain_col': 'gain',
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
image, weight, image_pos, sky, gain, _ = input.getRawImageData(0)
assert len(image_pos) == 100
assert image.array.shape == (1024, 1024)
assert weight.array.shape == (1024, 1024)
sky = sky[0]
gain = gain[0]
read_noise = 10
expected_noise = sky / gain + read_noise**2 / gain**2
np.testing.assert_almost_equal(weight.array, expected_noise**-1)
# Can set the noise by hand
config = {
'image_file_name': 'input/test_input_image_00.fits',
'cat_file_name': 'input/test_input_cat_00.fits',
'noise': 32,
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, weight, _, _, _, _ = input.getRawImageData(0)
assert weight.array.shape == (1024, 1024)
np.testing.assert_almost_equal(weight.array, 32.**-1)
# Some old versions of fitsio had a bug where the badpix mask could be offset by 32768.
# We move them back to 0
config = {
'image_file_name': 'input/test_input_image_00.fits',
'cat_file_name': 'input/test_input_cat_00.fits',
'weight_hdu': 1,
'badpix_hdu': 5,
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, weight, _, _, _, _ = input.getRawImageData(0)
assert weight.array.shape == (1024, 1024)
np.testing.assert_almost_equal(weight.array, expected_noise**-1)
config['badpix_hdu'] = 6
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, weight, _, _, _, _ = input.getRawImageData(0)
assert weight.array.shape == (1024, 1024)
np.testing.assert_almost_equal(weight.array, expected_noise**-1)
# Various ways to get all weight values == 0 (which will emit a logger message, but isn't
# an error).
config['weight_hdu'] = 1
config['badpix_hdu'] = 7 # badpix > 0
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, weight, _, _, _, _ = input.getRawImageData(0)
assert weight.array.shape == (1024, 1024)
np.testing.assert_almost_equal(weight.array, 0.)
config['weight_hdu'] = 3 # wt = 0
config['badpix_hdu'] = 2
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, weight, _, _, _, _ = input.getRawImageData(0)
assert weight.array.shape == (1024, 1024)
np.testing.assert_almost_equal(weight.array, 0.)
config['weight_hdu'] = 8 # Even cols are = 0
config['badpix_hdu'] = 9 # Odd cols are > 0
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, weight, _, _, _, _ = input.getRawImageData(0)
assert weight.array.shape == (1024, 1024)
np.testing.assert_almost_equal(weight.array, 0.)
# Negative valued weights are invalid
config['weight_hdu'] = 4
input = piff.InputFiles(config)
with CaptureLog() as cl:
_, weight, _, _, _, _ = input.getRawImageData(0, logger=cl.logger)
assert 'Warning: weight map has invalid negative-valued pixels.' in cl.output
def test_single_image():
"""Test the simple case of one image and one catalog.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(
log_file='output/test_single_image.log')
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars. Include some flagged and not used locations.
x_list = [
123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39,
1033.29, 1409.31
]
y_list = [
345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19,
1409.20, 123.11
]
flag_list = [1, 1, 13, 1, 1, 4, 1, 1, 0, 1]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x, y, flag in zip(x_list, y_list, flag_list):
bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31),
int(y + 32))
offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5)
psf.drawImage(image=image[bounds], method='no_pixel', offset=offset)
# corrupt the ones that are marked as flagged
if flag & 4:
print('corrupting star at ', x, y)
ar = image[bounds].array
im_max = np.max(ar) * 0.2
ar[ar > im_max] = im_max
image.addNoise(
galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('output', 'simple_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2')]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
data['flag'] = flag_list
cat_file = os.path.join('output', 'simple_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Use InputFiles to read these back in
config = {'image_file_name': image_file, 'cat_file_name': cat_file}
input = piff.InputFiles(config, logger=logger)
assert input.image_file_name == [image_file]
assert input.cat_file_name == [cat_file]
# Check image
assert input.nimages == 1
image1, _, image_pos, _, _, _ = input.getRawImageData(0)
np.testing.assert_equal(image1.array, image.array)
# Check catalog
np.testing.assert_equal([pos.x for pos in image_pos], x_list)
np.testing.assert_equal([pos.y for pos in image_pos], y_list)
# Repeat, using flag columns this time.
config = {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': '1',
'skip_flag': '4',
'stamp_size': 48
}
input = piff.InputFiles(config, logger=logger)
assert input.nimages == 1
_, _, image_pos, _, _, _ = input.getRawImageData(0)
assert len(image_pos) == 7
# Make star data
orig_stars = input.makeStars()
assert len(orig_stars) == 7
assert orig_stars[0].image.array.shape == (48, 48)
# Process the star data
# can only compare to truth if include_pixel=False
model = piff.Gaussian(fastfit=True, include_pixel=False)
interp = piff.Mean()
fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars]
interp.solve(fitted_stars)
print('mean = ', interp.mean)
# Check that the interpolation is what it should be
# Any position would work here.
chipnum = 0
x = 1024
y = 123
orig_wcs = input.getWCS()[chipnum]
orig_pointing = input.getPointing()
image_pos = galsim.PositionD(x, y)
world_pos = piff.StarData.calculateFieldPos(image_pos, orig_wcs,
orig_pointing)
u, v = world_pos.x, world_pos.y
stamp_size = config['stamp_size']
target = piff.Star.makeTarget(x=x,
y=y,
u=u,
v=v,
wcs=orig_wcs,
stamp_size=stamp_size,
pointing=orig_pointing)
true_params = [sigma, g1, g2]
test_star = interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Check default values of options
psf = piff.SimplePSF(model, interp)
assert psf.chisq_thresh == 0.1
assert psf.max_iter == 30
assert psf.outliers == None
assert psf.extra_interp_properties == []
# Now test running it via the config parser
psf_file = os.path.join('output', 'simple_psf.fits')
config = {
'input': {
'image_file_name': image_file,
'cat_file_name': cat_file,
'flag_col': 'flag',
'use_flag': 1,
'skip_flag': 4,
'stamp_size': stamp_size
},
'psf': {
'model': {
'type': 'Gaussian',
'fastfit': True,
'include_pixel': False
},
'interp': {
'type': 'Mean'
},
'max_iter': 10,
'chisq_thresh': 0.2,
},
'output': {
'file_name': psf_file
},
}
orig_stars, wcs, pointing = piff.Input.process(config['input'], logger)
# Use a SimplePSF to process the stars data this time.
interp = piff.Mean()
psf = piff.SimplePSF(model, interp, max_iter=10, chisq_thresh=0.2)
assert psf.chisq_thresh == 0.2
assert psf.max_iter == 10
psf.fit(orig_stars, wcs, pointing, logger=logger)
test_star = psf.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# test that drawStar and drawStarList work
test_star = psf.drawStar(target)
test_star_list = psf.drawStarList([target])[0]
np.testing.assert_equal(test_star.fit.params, test_star_list.fit.params)
np.testing.assert_equal(test_star.image.array, test_star_list.image.array)
# test copy_image property of drawStar and draw
for draw in [psf.drawStar, psf.model.draw]:
target_star_copy = psf.interp.interpolate(
piff.Star(target.data.copy(), target.fit.copy()))
# interp is so that when we do psf.model.draw we have fit.params to work with
test_star_copy = draw(target_star_copy, copy_image=True)
test_star_nocopy = draw(target_star_copy, copy_image=False)
# if we modify target_star_copy, then test_star_nocopy should be modified,
# but not test_star_copy
target_star_copy.image.array[0, 0] = 23456
assert test_star_nocopy.image.array[
0, 0] == target_star_copy.image.array[0, 0]
assert test_star_copy.image.array[0,
0] != target_star_copy.image.array[0,
0]
# however the other pixels SHOULD still be all the same value
assert test_star_nocopy.image.array[
1, 1] == target_star_copy.image.array[1, 1]
assert test_star_copy.image.array[1,
1] == target_star_copy.image.array[1,
1]
# test that draw works
test_image = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center)
# this image should be the same values as test_star
assert test_image == test_star.image
# test that draw does not copy the image
image_ref = psf.draw(x=target['x'],
y=target['y'],
stamp_size=config['input']['stamp_size'],
flux=target.fit.flux,
offset=target.fit.center,
image=test_image)
image_ref.array[0, 0] = 123456789
assert test_image.array[0, 0] == image_ref.array[0, 0]
assert test_star.image.array[0, 0] != test_image.array[0, 0]
assert test_star.image.array[1, 1] == test_image.array[1, 1]
# Round trip to a file
psf.write(psf_file, logger)
psf2 = piff.read(psf_file, logger)
assert type(psf2.model) is piff.Gaussian
assert type(psf2.interp) is piff.Mean
assert psf2.chisq == psf.chisq
assert psf2.last_delta_chisq == psf.last_delta_chisq
assert psf2.chisq_thresh == psf.chisq_thresh
assert psf2.max_iter == psf.max_iter
assert psf2.dof == psf.dof
assert psf2.nremoved == psf.nremoved
test_star = psf2.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Do the whole thing with the config parser
os.remove(psf_file)
piff.piffify(config, logger)
psf3 = piff.read(psf_file)
assert type(psf3.model) is piff.Gaussian
assert type(psf3.interp) is piff.Mean
assert psf3.chisq == psf.chisq
assert psf3.last_delta_chisq == psf.last_delta_chisq
assert psf3.chisq_thresh == psf.chisq_thresh
assert psf3.max_iter == psf.max_iter
assert psf3.dof == psf.dof
assert psf3.nremoved == psf.nremoved
test_star = psf3.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# Test using the piffify executable
os.remove(psf_file)
# This would be simpler as a direct assignment, but this once, test the way you would set
# this from the command line, which would call parse_variables.
piff.config.parse_variables(config, ['verbose=0'], logger=logger)
#config['verbose'] = 0
with open('simple.yaml', 'w') as f:
f.write(yaml.dump(config, default_flow_style=False))
config2 = piff.config.read_config('simple.yaml')
assert config == config2
piffify_exe = get_script_name('piffify')
p = subprocess.Popen([piffify_exe, 'simple.yaml'])
p.communicate()
psf4 = piff.read(psf_file)
assert type(psf4.model) is piff.Gaussian
assert type(psf4.interp) is piff.Mean
assert psf4.chisq == psf.chisq
assert psf4.last_delta_chisq == psf.last_delta_chisq
assert psf4.chisq_thresh == psf.chisq_thresh
assert psf4.max_iter == psf.max_iter
assert psf4.dof == psf.dof
assert psf4.nremoved == psf.nremoved
test_star = psf4.interp.interpolate(target)
np.testing.assert_almost_equal(test_star.fit.params,
true_params,
decimal=4)
# With very low max_iter, we hit the warning about non-convergence
config['psf']['max_iter'] = 1
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert 'PSF fit did not converge' in cl.output
def test_sizemag():
"""Test the SizeMag selection algorithm.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
config = piff.config.read_config('sizemag.yaml')
config['select'] = {'type': 'SizeMag'}
objects, _, _ = piff.Input.process(config['input'], logger=logger)
stars = piff.Select.process(config['select'], objects, logger=logger)
# This finds more stars than the simple SmallBright selector found.
print('nstars = ', len(stars))
assert len(stars) == 140
# A few of these have lower CLASS_STAR values, but still most are > 0.95
class_star = np.array([s['CLASS_STAR'] for s in stars])
print('class_star = ', class_star)
print('min class_star = ', np.min(class_star))
print('N class_star > 0.95 = ', np.sum(class_star > 0.95))
assert np.sum(class_star > 0.95) == 140
# This goes a bit fainter than the SmallBright selector did (which is kind of the point).
mag_auto = np.array([s['MAG_AUTO'] for s in stars])
print('mag_auto = ', mag_auto)
print('max mag_auto = ', np.max(mag_auto))
assert np.max(mag_auto) > 16
assert np.max(mag_auto) < 17
# Sizes are all pretty similar (but not exactly by construction anymore)
sizes = [s.hsm[3] for s in stars]
print('mean size = ', np.mean(sizes))
print('median size = ', np.median(sizes))
print('min/max size = ', np.min(sizes), np.max(sizes))
print('spread = ', (np.max(sizes) - np.min(sizes)) / np.median(sizes))
assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.1
# Try some different parameter values.
config['select'] = {
'type': 'SizeMag',
'initial_select': {
'type': 'Properties',
'where': '(CLASS_STAR > 0.9) & (MAG_AUTO < 16)',
},
'purity': 0,
'num_iter': 2,
'fit_order': 0,
}
stars = piff.Select.process(config['select'], objects, logger=logger)
# Somewhat fewer, but still works reasonably ok.
print('nstars = ', len(stars))
assert len(stars) == 135
class_star = np.array([s['CLASS_STAR'] for s in stars])
print('class_star = ', class_star)
print('min class_star = ', np.min(class_star))
print('N class_star > 0.95 = ', np.sum(class_star > 0.95))
assert np.sum(class_star > 0.95) == 135
sizes = [s.hsm[3] for s in stars]
print('mean size = ', np.mean(sizes))
print('median size = ', np.median(sizes))
print('min/max size = ', np.min(sizes), np.max(sizes))
print('spread = ', (np.max(sizes) - np.min(sizes)) / np.median(sizes))
assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.1
# Check that it doesn't crap out with bad initial objects.
# Here the initial selection is pretty much all galaxies, not stars.
config['select'] = {
'type': 'SizeMag',
'initial_select': {
'type': 'Properties',
'where': '(CLASS_STAR < 0.2) & (MAG_AUTO > 15)',
},
}
stars = piff.Select.process(config['select'], objects, logger=logger)
sizes = [s.hsm[3] for s in stars]
print('mean size = ', np.mean(sizes))
print('median size = ', np.median(sizes))
print('min/max size = ', np.min(sizes), np.max(sizes))
# It doesn't fail, but it (obviously) doesn't find a good stellar locus either.
assert np.max(sizes) > 1.5 # Real stellar locus is at about T = 0.5
# Make sure it doesn't crap out if all the input objects are stars
config['input'] = {
'dir': 'input',
'image_file_name': 'DECam_00241238_01.fits.fz',
'image_hdu': 1,
'badpix_hdu': 2,
'weight_hdu': 3,
'cat_file_name':
'DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits',
'cat_hdu': 2,
'x_col': 'XWIN_IMAGE',
'y_col': 'YWIN_IMAGE',
'sky_col': 'BACKGROUND',
'stamp_size': 25,
}
config['select'] = {
'type': 'SizeMag',
'initial_select': {
# Use all the input objects
}
}
objects, _, _ = piff.Input.process(config['input'], logger=logger)
stars = piff.Select.process(config['select'], objects, logger=logger)
# Check pathologically few input objects.
select = piff.SizeMagSelect(config['select'])
with np.testing.assert_raises(RuntimeError):
stars = select.selectStars(objects[:0], logger=logger)
with CaptureLog() as cl:
stars = select.selectStars(objects[:1], logger=cl.logger)
print(cl.output)
assert "Too few candidate stars (1) to use fit_order=2." in cl.output
print("len(stars) = ", len(stars))
assert len(stars) == 0
with CaptureLog() as cl:
stars = select.selectStars(objects[:5], logger=cl.logger)
print(cl.output)
# (Note: one was clipped, so only N=4 when the error message triggered.)
assert "Too few candidate stars (4) to use fit_order=2." in cl.output
print("len(stars) = ", len(stars))
assert len(stars) == 0
# With fit_order=0, 1 input star is still tricky, but it should go through.
config['select'] = {
'type': 'SizeMag',
'fit_order': 0,
'initial_select': {}
}
select = piff.SizeMagSelect(config['select'])
stars = select.selectStars(objects[:1], logger=logger)
print("len(stars) = ", len(stars))
assert len(stars) == 1
# With fit_order=1, 3 input stars is the minimum.
config['select'] = {
'type': 'SizeMag',
'fit_order': 1,
'initial_select': {}
}
select = piff.SizeMagSelect(config['select'])
with CaptureLog(3) as cl:
stars = select.selectStars(objects[:2], logger=cl.logger)
print(cl.output)
print("len(stars) = ", len(stars))
assert len(stars) == 0
with CaptureLog(3) as cl:
stars = select.selectStars(objects[:3], logger=cl.logger)
print(cl.output)
print("len(stars) = ", len(stars))
assert len(stars) == 3
# Error to have other parameters
config['select'] = {
'type': 'SizeMag',
'order': 3,
}
with np.testing.assert_raises(ValueError):
select = piff.SmallBrightSelect(config['select'])
# But ok to have parameters that the base class will handle.
config['select'] = {
'type': 'SizeMag',
'purity': 0,
'hsm_size_reject': 4,
'min_snr': 50,
}
piff.Select.process(config['select'], objects, logger=logger)
def test_smallbright():
"""Test the SmallBright selection algorithm.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(2)
else:
logger = None
config = piff.config.read_config('sizemag.yaml')
config['select'] = {'type': 'SmallBright'}
objects, _, _ = piff.Input.process(config['input'], logger=logger)
stars = piff.Select.process(config['select'], objects, logger=logger)
# This does a pretty decent job actually. Finds 88 stars.
print('nstars = ', len(stars))
assert len(stars) == 88
# They are all ones that CLASS_STAR also identified as stars.
class_star = np.array([s['CLASS_STAR'] for s in stars])
print('class_star = ', class_star)
print('min class_star = ', np.min(class_star))
assert np.all(class_star > 0.95)
mag_auto = np.array([s['MAG_AUTO'] for s in stars])
print('mag_auto = ', mag_auto)
print('max mag_auto = ', np.max(mag_auto))
assert np.max(mag_auto) < 16
# Sizes are all pretty similar (by construction of the algorithm)
sizes = [s.hsm[3] for s in stars]
print('mean size = ', np.mean(sizes))
print('median size = ', np.median(sizes))
print('min/max size = ', np.min(sizes), np.max(sizes))
assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.1
# Try some different parameter values.
config['select'] = {
'type': 'SmallBright',
'bright_fraction': 0.1,
'small_fraction': 0.5,
'locus_fraction': 0.8,
'max_spread': 0.05,
}
stars = piff.Select.process(config['select'], objects, logger=logger)
# Fewer stars since limited to brighter subset
# Different systems give slightly different results here. So give a range.
print('nstars = ', len(stars))
assert 30 < len(stars) < 40
# But still finds all high confidence stars
class_star = np.array([s['CLASS_STAR'] for s in stars])
print('class_star = ', class_star)
print('min class_star = ', np.min(class_star))
assert np.all(class_star > 0.95)
# And sizes are now restricted to max_spread = 0.05
sizes = [s.hsm[3] for s in stars]
print('mean size = ', np.mean(sizes))
print('median size = ', np.median(sizes))
print('min/max size = ', np.min(sizes), np.max(sizes))
assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.05
# Error to have other parameters
config['select'] = {
'type': 'SmallBright',
'bright_frac': 0.1,
'small_frac': 0.5,
}
with np.testing.assert_raises(ValueError):
piff.Select.process(config['select'], objects, logger=logger)
# But ok to have parameters that the base class will handle.
config['select'] = {
'type': 'SmallBright',
'locus_fraction': 0.8,
'hsm_size_reject': 4,
'min_snr': 50,
}
piff.Select.process(config['select'], objects, logger=logger)
# Set up a pathological input that hits the code where niter=10.
# Normal inputs rarely need more than 3 iterations, so this is really a backstop
# against very strange behavior. I don't know if it's possible for the code to
# go into an infinite loop, where having a maxiter would be critical, but it's
# at least conceivable for it to be slowly converging, so might as well keep it.
config['select'] = {
'type': 'SmallBright',
'bright_fraction': 1.,
'small_fraction': 0.1,
'locus_fraction': 1.,
'max_spread': 1.,
}
# If sizes are spaced such that the small ones are consistently farther apart than bigger
# ones, then the median/iqr iteration will keep slowly shifting to a larger size.
# Having T go as 1/sqrt(i+1) seems to work to make this happen.
for i in range(len(objects)):
objects[i]._hsm = (1., 0., 0., 1. / (i + 1.)**0.25, 0., 0., 0)
select = piff.SmallBrightSelect(config['select'])
with CaptureLog() as cl:
stars = select.selectStars(objects, logger=cl.logger)
assert "Max iter = 10 reached." in cl.output
print("len(stars) = ", len(stars))
# These would be pathalogical values to set (as were the above values), but the
# code handles them gracefully by changing 0 into the minimum plausible value.
# However, it will end up not being able to find a stellar locus, so len(stars) = 0
config['select'] = {
'type': 'SmallBright',
'bright_fraction': 0.,
'small_fraction': 0.,
'locus_fraction': 0.,
}
select = piff.SmallBrightSelect(config['select'])
with CaptureLog() as cl:
stars = select.selectStars(objects, logger=cl.logger)
assert "Failed to find bright/small stellar locus" in cl.output
print("len(stars) = ", len(stars))
assert len(stars) == 0
# With this arrangement, it does better. (Since the problem above is that all the fluxes
# are the same, so the "brightest" 2 are also the biggest 2 and have a large iqr.)
config['select'] = {
'type': 'SmallBright',
'bright_fraction': 1.,
'small_fraction': 0.,
'locus_fraction': 0.,
}
select = piff.SmallBrightSelect(config['select'])
with CaptureLog() as cl:
stars = select.selectStars(objects, logger=cl.logger)
print("len(stars) = ", len(stars))
assert len(stars) == 134
# Check pathologically few input objects.
config['select'] = {
'type': 'SmallBright',
}
select = piff.SmallBrightSelect(config['select'])
with CaptureLog() as cl:
stars = select.selectStars(objects[:0], logger=cl.logger)
print(cl.output)
assert "No input objects" in cl.output
print("len(stars) = ", len(stars))
assert len(stars) == 0
with CaptureLog() as cl:
stars = select.selectStars(objects[:1], logger=cl.logger)
print(cl.output)
assert "Only 1 input object." in cl.output
print("len(stars) = ", len(stars))
assert len(stars) == 0
with CaptureLog() as cl:
stars = select.selectStars(objects[:2], logger=cl.logger)
print(cl.output)
assert "Failed to find bright/small stellar locus" in cl.output
print("len(stars) = ", len(stars))
assert len(stars) == 0
def test_parallel():
# Run the same test as test_single, but using nproc
wcs1 = galsim.TanWCS(
galsim.AffineTransform(0.26, 0.05, -0.08, -0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees))
wcs2 = galsim.TanWCS(
galsim.AffineTransform(0.25, -0.02, 0.01, 0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees))
field_center = galsim.CelestialCoord(0 * galsim.degrees,
-25 * galsim.degrees)
if __name__ == '__main__':
nstars = 20 # per ccd
else:
nstars = 6 # per ccd
rng = np.random.RandomState(1234)
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra1, dec1 = wcs1.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data1 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im1 = drawImage(2048, 2048, wcs1, x, y, e1, e2, s)
im1.write('output/test_parallel_im1.fits')
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra2, dec2 = wcs2.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
# Same functions of u,v, but using the positions on chip 2
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data2 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im2 = drawImage(2048, 2048, wcs2, x, y, e1, e2, s)
im2.write('output/test_parallel_im2.fits')
ra12 = np.concatenate([ra1, ra2])
dec12 = np.concatenate([dec1, dec2])
data12 = np.array(list(zip(ra12, dec12)),
dtype=[('ra', float), ('dec', float)])
fitsio.write('output/test_parallel.fits', data12, clobber=True)
# im3 is blank. Will give errors trying to measure PSF from it.
im3 = galsim.Image(2048, 2048, wcs=wcs2)
im3.write('output/test_parallel_im3.fits')
psf_file = os.path.join('output', 'test_single.fits')
config = {
'input': {
# A third way to input these same file names. Use GalSim config values and
# explicitly specify the number of images to read
'nimages': 2,
'image_file_name': {
'type': 'FormattedStr',
'format': '%s/test_parallel_im%d.fits',
'items': ['output', '$image_num+1'],
},
'cat_file_name': 'output/test_parallel.fits',
'chipnum': '$image_num+1',
'ra_col': 'ra',
'dec_col': 'dec',
'ra_units': 'rad',
'dec_units': 'rad',
'nproc': -1,
},
'psf': {
'type': 'SingleChip',
'model': {
'type': 'Moffat',
'beta': 2.5,
},
'interp': {
'type': 'Polynomial',
'order': 2,
},
'nproc': 2,
},
'output': {
'file_name': psf_file,
},
}
with CaptureLog(level=2) as cl:
piff.piffify(config, logger=cl.logger)
psf = piff.read(psf_file)
for chipnum, data, wcs in [(1, data1, wcs1), (2, data2, wcs2)]:
for k in range(nstars):
x = data['x'][k]
y = data['y'][k]
e1 = data['e1'][k]
e2 = data['e2'][k]
s = data['s'][k]
image_pos = galsim.PositionD(x, y)
star = piff.Star.makeTarget(x=x,
y=y,
wcs=wcs,
stamp_size=48,
pointing=field_center,
chipnum=chipnum)
star = psf.drawStar(star)
np.testing.assert_almost_equal(star.fit.params, [s, e1, e2],
decimal=6)
# Finally, check that the logger properly captures the subprocess logs
with CaptureLog(level=2) as cl:
psf = piff.process(config, cl.logger)
#print('with nproc=2, log = ',cl.output)
assert "Processing catalog 1" in cl.output
assert "Processing catalog 2" in cl.output
assert "Building solution for chip 1" in cl.output
assert "Building solution for chip 2" in cl.output
# Check that errors in the solution get properly reported.
config['input']['nimages'] = 3
with CaptureLog(level=2) as cl:
psf = piff.process(config, cl.logger)
assert "Removed 6 stars in initialize" in cl.output
assert "No stars. Cannot find PSF model." in cl.output
assert "Solutions failed for chipnums: [3]" in cl.output
# Check that errors in the multiprocessing input get properly reported.
config['input']['ra_col'] = 'invalid'
with CaptureLog(level=2) as cl:
with np.testing.assert_raises(ValueError):
psf = piff.process(config, cl.logger)
assert "ra_col = invalid is not a column" in cl.output
# With nproc=1, the error is raised directly.
config['input']['nproc'] = 1
config['verbose'] = 0
with np.testing.assert_raises(ValueError):
psf = piff.process(config)
# But just the input error. Not the one in fitting.
config['psf']['nproc'] = 1
config['input']['ra_col'] = 'ra'
config['verbose'] = 1
with CaptureLog(level=1) as cl:
psf = piff.process(config, logger=cl.logger)
assert "No stars. Cannot find PSF model." in cl.output
assert "Ignoring this failure and continuing on." in cl.output
def test_single():
"""Same as test_focal, but using the SingleCCD PSF type, which does a separate fit on each CCD.
"""
wcs1 = galsim.TanWCS(
galsim.AffineTransform(0.26, 0.05, -0.08, -0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees))
wcs2 = galsim.TanWCS(
galsim.AffineTransform(0.25, -0.02, 0.01, 0.24,
galsim.PositionD(1024, 1024)),
galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees))
field_center = galsim.CelestialCoord(0 * galsim.degrees,
-25 * galsim.degrees)
if __name__ == '__main__':
nstars = 20 # per ccd
logger = piff.config.setup_logger(verbose=2)
else:
nstars = 6 # per ccd
logger = piff.config.setup_logger(log_file='output/test_single.log')
rng = np.random.RandomState(1234)
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra1, dec1 = wcs1.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data1 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im1 = drawImage(2048, 2048, wcs1, x, y, e1, e2, s)
im1.write('output/test_single_im1.fits')
fitsio.write('output/test_single_cat1.fits', data1, clobber=True)
x = rng.random_sample(nstars) * 2000 + 24
y = rng.random_sample(nstars) * 2000 + 24
ra2, dec2 = wcs2.toWorld(x, y, units='rad')
u, v = field_center.project_rad(ra1, dec1, projection='gnomonic')
# Same functions of u,v, but using the positions on chip 2
e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2
e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2
s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v
data2 = np.array(list(zip(x, y, e1, e2, s)),
dtype=[('x', float), ('y', float), ('e1', float),
('e2', float), ('s', float)])
im2 = drawImage(2048, 2048, wcs2, x, y, e1, e2, s)
im2.write('output/test_single_im2.fits')
fitsio.write('output/test_single_cat2.fits', data2, clobber=True)
ra12 = np.concatenate([ra1, ra2])
dec12 = np.concatenate([dec1, dec2])
data12 = np.array(list(zip(ra12, dec12)),
dtype=[('ra', float), ('dec', float)])
fitsio.write('output/test_single_cat12.fits', data12, clobber=True)
# Try to fit with the right model (Moffat) and interpolant (2nd order polyomial)
# Should work very well, since no noise.
config = {
'input': {
# A third way to input these same file names. Use GalSim config values and
# explicitly specify the number of images to read
'nimages': 2,
'image_file_name': {
'type': 'FormattedStr',
'format': '%s/test_single_im%d.fits',
'items': ['output', '$image_num+1']
},
'cat_file_name': {
'type': 'FormattedStr',
'format': '%s/test_single_cat%d.fits',
'items': ['output', '$image_num+1']
},
# Use chipnum = 1,2 rather than the default 0,1.
'chipnum': '$image_num+1',
'x_col': 'x',
'y_col': 'y',
'ra': 0.,
'dec': -25.,
},
'psf': {
'type': 'SingleChip',
'model': {
'type': 'Moffat',
'beta': 2.5
},
'interp': {
'type': 'Polynomial',
'order': 2
}
},
}
if __name__ != '__main__':
config['verbose'] = 0
with CaptureLog(level=2) as cl:
psf = piff.process(config, cl.logger)
#print('without nproc, log = ',cl.output)
assert "Building solution for chip 1" in cl.output
assert "Building solution for chip 2" in cl.output
for chipnum, data, wcs in [(1, data1, wcs1), (2, data2, wcs2)]:
for k in range(nstars):
x = data['x'][k]
y = data['y'][k]
e1 = data['e1'][k]
e2 = data['e2'][k]
s = data['s'][k]
#print('k,x,y = ',k,x,y)
#print(' true s,e1,e2 = ',s,e1,e2)
image_pos = galsim.PositionD(x, y)
star = piff.Star.makeTarget(x=x,
y=y,
wcs=wcs,
stamp_size=48,
pointing=field_center,
chipnum=chipnum)
star = psf.drawStar(star)
#print(' fitted s,e1,e2 = ',star.fit.params)
np.testing.assert_almost_equal(star.fit.params, [s, e1, e2],
decimal=6)
# Chipnum is required as a property to use SingleCCDPSF
star1 = piff.Star.makeTarget(x=x,
y=y,
wcs=wcs,
stamp_size=48,
pointing=field_center)
with np.testing.assert_raises(ValueError):
psf.drawStar(star1)
star2 = piff.Star(star1.data,
star.fit) # If has a fit, it hits a different error
with np.testing.assert_raises(ValueError):
psf.drawStar(star2)
def test_depr_select():
# A bunch of keys used to be allowed in input, but now belong in select.
# Check that the old way still works, but gives a warning.
# This is the input from test_stars in test_input.py
dir = 'input'
image_file = 'test_input_image_00.fits'
cat_file = 'test_input_cat_00.fits'
psf_file = os.path.join('output','test_depr_select.fits')
config = {
'input' : {
'dir' : dir,
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'weight_hdu' : 1,
'sky_col' : 'sky',
'gain_col' : 'gain',
'use_partial' : True,
'nstars': 15, # Just to make the test faster
},
'select': {
'max_snr' : 200,
'min_snr' : 20,
'hsm_size_reject' : 20,
'max_edge_frac': 0.25,
'stamp_center_size': 10,
'max_mask_pixels' : 20,
'reserve_frac' : 0.1,
'seed': 1234,
},
'psf': {
'model' : {'type': 'Gaussian'},
'interp' : {'type': 'Mean'},
},
'output' : { 'file_name' : psf_file },
}
logger = piff.config.setup_logger(log_file='output/test_depr_select.log')
# This is the new API
print('config = ',config)
piff.piffify(config, logger)
psf1 = piff.read(psf_file)
im1 = psf1.draw(x=23, y=34, stamp_size=16)
print('len1 = ',len(psf1.stars))
# This is the old API
config['input'].update(config['select'])
del config['select']
config = galsim.config.CleanConfig(config)
print('config = ',config)
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert "WARNING: Items [" in cl.output
assert "] should now be in the 'select' field of the config file." in cl.output
psf2 = piff.read(psf_file)
print('len2 = ',len(psf2.stars))
assert len(psf1.stars) == len(psf2.stars)
im2 = psf2.draw(x=23, y=34, stamp_size=16)
np.testing.assert_allclose(im2.array, im1.array)
# Also ok for some items to be in select, but erroneously put some in input.
config['input']['min_snr'] = config['select'].pop('min_snr')
config['input']['max_snr'] = config['select'].pop('max_snr')
config = galsim.config.CleanConfig(config)
print('config = ',config)
with CaptureLog(level=1) as cl:
piff.piffify(config, cl.logger)
assert "WARNING: Items ['max_snr', 'min_snr'] should now be in the 'select' field of the config file." in cl.output
psf3 = piff.read(psf_file)
print('len3 = ',len(psf3.stars))
assert len(psf1.stars) == len(psf3.stars)
im3 = psf3.draw(x=23, y=34, stamp_size=16)
np.testing.assert_allclose(im3.array, im1.array)
def test_load_images():
"""Test the load_images function
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_load_image2.log')
# Same setup as test_single_image, but without flags
image = galsim.Image(2048, 2048, scale=0.26)
x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ]
y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ]
sigma = 1.3
g1 = 0.23
g2 = -0.17
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x,y in zip(x_list, y_list):
bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32))
psf.drawImage(image[bounds], center=galsim.PositionD(x,y), method='no_pixel')
image.addNoise(galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
sky = 10.
image += sky
image_file = os.path.join('output','test_load_images_im.fits')
image.write(image_file)
dtype = [ ('x','f8'), ('y','f8') ]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output','test_load_images_cat.fits')
fitsio.write(cat_file, data, clobber=True)
# Make star data
config = { 'image_file_name' : image_file,
'cat_file_name': cat_file,
'sky': 10
}
orig_stars, wcs, pointing = piff.Input.process(config, logger)
# Fit these with a simple Mean, Gaussian
model = piff.Gaussian()
interp = piff.Mean()
psf = piff.SimplePSF(model, interp)
psf.fit(orig_stars, wcs, pointing, logger=logger)
psf_file = os.path.join('output','test_load_images_psf.fits')
psf.write(psf_file, logger)
# Read this file back in. It has the star data, but the images are blank.
psf2 = piff.read(psf_file, logger)
assert len(psf2.stars) == 10
for star in psf2.stars:
np.testing.assert_array_equal(star.image.array, 0.)
# First the old API:
with CaptureLog() as cl:
loaded_stars = piff.Star.load_images(psf2.stars, image_file, logger=cl.logger)
assert "WARNING: The Star.load_images function is deprecated." in cl.output
for star, orig in zip(loaded_stars, psf.stars):
np.testing.assert_array_equal(star.image.array, orig.image.array)
# Can optionally supply a different sky to subtract
# (This is not allowed by the new API, but it probably doesn't make much sense.)
loaded_stars = piff.Star.load_images(psf2.stars, image_file, sky=2*sky)
for star, orig in zip(loaded_stars, psf.stars):
np.testing.assert_array_equal(star.image.array, orig.image.array - sky)
# Now the new API:
psf2 = piff.read(psf_file, logger) # Reset stars to not have images in them.
loaded_stars = piff.InputFiles(config).load_images(psf2.stars)
for star, orig in zip(loaded_stars, psf.stars):
np.testing.assert_array_equal(star.image.array, orig.image.array)
def test_reserve():
"""Test the reserve_frac option.
"""
if __name__ == '__main__':
logger = piff.config.setup_logger(verbose=2)
else:
logger = piff.config.setup_logger(log_file='output/test_single_reserve.log')
# Make the image
image = galsim.Image(2048, 2048, scale=0.26)
# Where to put the stars.
x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ]
y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ]
# Draw a Gaussian PSF at each location on the image.
sigma = 1.3
g1 = 0.23
g2 = -0.17
true_params = [ sigma, g1, g2 ]
psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2)
for x,y in zip(x_list, y_list):
bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32))
psf.drawImage(image[bounds], center=galsim.PositionD(x,y), method='no_pixel')
image.addNoise(galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6))
# Write out the image to a file
image_file = os.path.join('output','test_simple_reserve_image.fits')
image.write(image_file)
# Write out the catalog to a file
dtype = [ ('x','f8'), ('y','f8') ]
data = np.empty(len(x_list), dtype=dtype)
data['x'] = x_list
data['y'] = y_list
cat_file = os.path.join('output','test_simple_reserve_cat.fits')
fitsio.write(cat_file, data, clobber=True)
psf_file = os.path.join('output','test_simple_reserve_psf.fits')
config = {
'input' : {
'image_file_name' : image_file,
'cat_file_name' : cat_file,
'stamp_size' : 32,
},
'select' : {
'reserve_frac' : 0.2,
},
'psf' : {
'model' : { 'type' : 'Gaussian',
'fastfit': True,
'include_pixel': False},
'interp' : { 'type' : 'Mean' },
},
'output' : { 'file_name' : psf_file },
}
piff.piffify(config, logger)
psf = piff.read(psf_file)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
print('chisq = ',psf.chisq)
print('dof = ',psf.dof)
nreserve = len([s for s in psf.stars if s.is_reserve])
ntot = len(psf.stars)
print('reserve = %s/%s'%(nreserve,ntot))
assert nreserve == 2
assert ntot == 10
print('dof =? ',(32*32 - 3) * (ntot-nreserve))
assert psf.dof == (32*32 - 3) * (ntot-nreserve)
for star in psf.stars:
# Fits should be good for both reserve and non-reserve stars
np.testing.assert_almost_equal(star.fit.params, true_params, decimal=4)
# Check the old API of putting reserve_frac in input
config['input']['reserve_frac'] = 0.2
del config['select']
with CaptureLog() as cl:
piff.piffify(config, cl.logger)
assert "WARNING: Items ['reserve_frac'] should now be in the 'select' field" in cl.output
psf = piff.read(psf_file)
assert type(psf.model) is piff.Gaussian
assert type(psf.interp) is piff.Mean
nreserve = len([s for s in psf.stars if s.is_reserve])
ntot = len(psf.stars)
assert nreserve == 2
assert ntot == 10