def test_gaussian(): gaussian = piff.Gaussian(include_pixel=False) print('test gaussian') star = make_empty_star() # test gaussian alone sigma = 1 g1 = -0.1 g2 = 0.05 model = piff.Optical(r0=0, sigma=sigma, template='des') star = model.draw(star) # insert assert statement about sigma np.testing.assert_almost_equal(gaussian.fit(star).fit.params[0], sigma, 5) # gaussian and shear model = piff.Optical(r0=0, sigma=sigma, g1=g1, g2=g2, template='des') star = model.draw(star) params = gaussian.fit(star).fit.params np.testing.assert_almost_equal(params[0], sigma, 5) np.testing.assert_almost_equal(params[1], g1, 5) np.testing.assert_almost_equal(params[2], g2, 5) # now gaussian, shear, aberration, r0 star = make_empty_star(params=[0.5, 0.8, -0.7, 0.5, -0.2, 0.9, -1, 2.0]) model = piff.Optical(r0=0.1, sigma=sigma, g1=g1, g2=g2, template='des') star = model.draw(star)
def generate_starlist(n_samples=500): # create n_samples images from the 63 ccds and pixel coordinates np_rng = np.random.RandomState(1234) icens = np_rng.randint(100, 2048, n_samples) jcens = np_rng.randint(100, 4096, n_samples) ccdnums = np_rng.randint(1, 63, n_samples) icenter = 1000 jcenter = 2000 # throw out any icens and jcens that are within 400 pixels of the center conds = (np.abs(icens - icenter) > 400) | (np.abs(jcens - jcenter) > 400) icens = icens[conds] jcens = jcens[conds] ccdnums = ccdnums[conds] sigmas, g1s, g2s = psf_model(icens, jcens, icenter, jcenter) # throw in a 2d polynomial function for sigma g1 and g2 # all sigma > 0, all g1 < 0, and g2 straddles. star_list = [make_star(icen, jcen, ccdnum, sigma, g1, g2) for icen, jcen, ccdnum, sigma, g1, g2 in zip(icens, jcens, ccdnums, sigmas, g1s, g2s)] # load up model and draw the stars model = piff.Gaussian(fastfit=True) star_list = [model.draw(star) for star in star_list] star_list = [model.initialize(star) for star in star_list] star_list = [model.fit(star) for star in star_list] return star_list, model
def test_Gaussian(): """This is about the simplest possible model I could think of. It just uses the HSM adaptive moments routine to measure the moments, and then it models the PSF as a Gaussian. """ # Here is the true PSF sigma = 1.3 g1 = 0.23 g2 = -0.17 psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) # Draw the PSF onto an image. Let's go ahead and give it a non-trivial WCS. wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29) image = galsim.Image(64,64, wcs=wcs) # This is only going to come out right if we (unphysically) don't convolve by the pixel. psf.drawImage(image, method='no_pixel') # Make a StarData instance for this image stardata = piff.StarData(image, image.true_center) star = piff.Star(stardata, None) # Fit the model from the image model = piff.Gaussian(include_pixel=False) star = model.initialize(star) fit = model.fit(star).fit print('True sigma = ',sigma,', model sigma = ',fit.params[0]) print('True g1 = ',g1,', model g1 = ',fit.params[1]) print('True g2 = ',g2,', model g2 = ',fit.params[2]) # This test is pretty accurate, since we didn't add any noise and didn't convolve by # the pixel, so the image is very accurately a sheared Gaussian. true_params = [ sigma, g1, g2 ] np.testing.assert_almost_equal(fit.params[0], sigma, decimal=7) np.testing.assert_almost_equal(fit.params[1], g1, decimal=7) np.testing.assert_almost_equal(fit.params[2], g2, decimal=7) np.testing.assert_almost_equal(fit.params, true_params, decimal=7) # Now test running it via the config parser config = { 'model' : { 'type' : 'Gaussian', 'include_pixel': False } } if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger(log_file='output/test_Gaussian.log') model = piff.Model.process(config['model'], logger) fit = model.fit(star).fit # Same tests. np.testing.assert_almost_equal(fit.params[0], sigma, decimal=7) np.testing.assert_almost_equal(fit.params[1], g1, decimal=7) np.testing.assert_almost_equal(fit.params[2], g2, decimal=7) np.testing.assert_almost_equal(fit.params, true_params, decimal=7)
def test_twodstats(): """Make sure we can execute and print a readout of the plot """ if __name__ == '__main__': logger = piff.config.setup_logger(2) else: logger = None model = piff.Gaussian(fastfit=True) interp = piff.Polynomial(order=1) # should find that order=1 is better # create background model stars, true_model = generate_starlist(100) psf = piff.SimplePSF(model, interp) psf.fit(stars, None, None) # check the coeffs of sigma and g2, which are actually linear fits # skip g1 since it is actually a 2d parabola # factor of 0.263 is to account for going from pixel xy to wcs uv np.testing.assert_almost_equal(psf.interp.coeffs[0].flatten(), np.array([0.4, 0, 1. / (0.263 * 2048), 0]), decimal=4) np.testing.assert_almost_equal(psf.interp.coeffs[2].flatten(), np.array([-0.1 * 1000 / 2048, 0, 0.1 / (0.263 * 2048), 0]), decimal=4) stats = piff.TwoDHistStats(number_bins_u=5, number_bins_v=5, reducing_function='np.mean') stats.compute(psf, stars, logger=logger) # check the twodhists # get the average value in the bin u_i = 3 v_i = 3 icen = stats.twodhists['u'][v_i, u_i] / 0.263 jcen = stats.twodhists['v'][v_i, u_i] / 0.263 print('icen = ',icen) print('jcen = ',jcen) icenter = 1000 jcenter = 2000 # the average value in the bin should match up with the model for the average coordinates sigma, g1, g2 = psf_model(icen, jcen, icenter, jcenter) sigma_average = stats.twodhists['T'][v_i, u_i] g1_average = stats.twodhists['g1'][v_i, u_i] g2_average = stats.twodhists['g2'][v_i, u_i] # assert equal to 4th decimal print('sigma, g1, g2 = ',[sigma,g1,g2]) print('av sigma, g1, g2 = ',[sigma_average,g1_average,g2_average]) np.testing.assert_almost_equal([sigma, g1, g2], [sigma_average, g1_average, g2_average], decimal=2) # Test the plotting and writing twodstats_file = os.path.join('output','twodstats.pdf') stats.write(twodstats_file) # repeat for whisker stats = piff.WhiskerStats(number_bins_u=21, number_bins_v=21, reducing_function='np.mean') stats.compute(psf, stars) # Test the plotting and writing twodstats_file = os.path.join('output','whiskerstats.pdf') stats.write(twodstats_file)
def test_shearing(): print('test shearing') # make sure if we put in common mode ellipticities that things change star = make_empty_star() g1 = 0 g2 = 0.05 model = piff.Optical(r0=0.1, g1=g1, g2=g2, template='des') star = model.draw(star) gaussian = piff.Gaussian(include_pixel=False) star_gaussian = gaussian.fit(star) np.testing.assert_almost_equal(star_gaussian.fit.params[1], g1, 5) np.testing.assert_almost_equal(star_gaussian.fit.params[2], g2, 5)
def test_extra_interp(): # Test that specifying extra_interp_properties works properly # TODO: This is a very bare bones test of the interface. There is basically no test of # this functionality at all yet. TBD! sigma = 1.3 g1 = 0.23 g2 = -0.17 psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29) image = galsim.Image(64, 64, wcs=wcs) psf.drawImage(image, method='no_pixel') # use g-i color as an extra property for interpolation. props = dict(gi_color=0.3) print('props = ', props) star = piff.Star(piff.StarData(image, image.true_center, properties=props), None) model = piff.Gaussian(fastfit=True, include_pixel=False) interp = piff.Mean() psf = piff.SimplePSF(model, interp, extra_interp_properties=['gi_color']) assert psf.extra_interp_properties == ['gi_color'] # Note: Mean doesn't actually do anything useful with the extra properties, so this # isn't really testing anything other than that the code doesn't completely break. pointing = galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees) psf.fit([star], wcs={0: wcs}, pointing=pointing) # Not much of a check here. Just check that it actually draws something with flux ~= 1 im = psf.draw(x=5, y=7, gi_color=0.3) np.testing.assert_allclose(im.array.sum(), 1.0, rtol=1.e-3) # Check missing or extra properties with np.testing.assert_raises(TypeError): psf.draw(x=5, y=7) with np.testing.assert_raises(TypeError): psf.draw(x=5, y=7, gi_color=0.3, ri_color=3) # No stars raises an error. (This can happen in practice if all stars are excluded on input.) with np.testing.assert_raises(RuntimeError): psf.fit([], wcs={0: wcs}, pointing=pointing) # Also for SingleChipPSf psf2 = piff.SingleChipPSF(psf, extra_interp_properties=['gi_color']) assert psf2.extra_interp_properties == ['gi_color'] with np.testing.assert_raises(TypeError): psf2.draw(x=5, y=7, chipnum=0)
def test_gaussian(): gaussian = piff.Gaussian(include_pixel=False) print('test gaussian') star = make_empty_star() # test gaussian alone sigma = 1.7 g1 = -0.1 g2 = 0.05 model = piff.Optical(r0=0, sigma=sigma, template='des') star = model.draw(star) # insert assert statement about sigma # We don't expect sigma to match. But it should be the biggest component, so close-ish np.testing.assert_allclose(gaussian.fit(star).fit.params[0], sigma, rtol=1.e-2) # omitting atm params is equivalent kwargs = piff.optical_model.optical_templates['des'].copy() del kwargs['r0'] model2 = piff.Optical(sigma=sigma, **kwargs) star2 = model2.draw(star) assert star.image == star2.image # gaussian and shear model = piff.Optical(r0=0, sigma=sigma, g1=g1, g2=g2, template='des') star = model.draw(star) params = gaussian.fit(star).fit.params np.testing.assert_allclose(gaussian.fit(star).fit.params[0], sigma, rtol=1.e-2) # Shears are much closer, since the optical part is round. np.testing.assert_allclose(params[1], g1, rtol=1.e-4) np.testing.assert_allclose(params[2], g2, rtol=1.e-4) # now gaussian, shear, aberration, r0 star = make_empty_star(params=[0.5, 0.8, -0.7, 0.5, -0.2, 0.9, -1, 2.0]) model = piff.Optical(r0=0.1, sigma=sigma, g1=g1, g2=g2, template='des') star = model.draw(star)
def test_load_images(): """Test the load_images function """ if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger( log_file='output/test_load_image2.log') # Same setup as test_single_image, but without flags image = galsim.Image(2048, 2048, scale=0.26) x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ] y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ] sigma = 1.3 g1 = 0.23 g2 = -0.17 psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) for x, y in zip(x_list, y_list): bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31), int(y + 32)) psf.drawImage(image[bounds], center=galsim.PositionD(x, y), method='no_pixel') image.addNoise( galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6)) image_file = os.path.join('output', 'test_load_images_im.fits') image.write(image_file) dtype = [('x', 'f8'), ('y', 'f8')] data = np.empty(len(x_list), dtype=dtype) data['x'] = x_list data['y'] = y_list cat_file = os.path.join('output', 'test_load_images_cat.fits') fitsio.write(cat_file, data, clobber=True) # Make star data config = {'image_file_name': image_file, 'cat_file_name': cat_file} orig_stars, wcs, pointing = piff.Input.process(config, logger) # Fit these with a simple Mean, Gaussian model = piff.Gaussian() interp = piff.Mean() psf = piff.SimplePSF(model, interp) psf.fit(orig_stars, wcs, pointing, logger=logger) psf_file = os.path.join('output', 'test_load_images_psf.fits') psf.write(psf_file, logger) # Read this file back in. It has the star data, but the images are blank. psf2 = piff.read(psf_file, logger) assert len(psf2.stars) == 10 for star in psf2.stars: np.testing.assert_array_equal(star.image.array, 0.) loaded_stars = piff.Star.load_images(psf2.stars, image_file) for star, orig in zip(loaded_stars, psf.stars): np.testing.assert_array_equal(star.image.array, orig.image.array) # Can optionally supply sky to subtract loaded_stars = piff.Star.load_images(psf2.stars, image_file, sky=10) for star, orig in zip(loaded_stars, psf.stars): np.testing.assert_array_equal(star.image.array, orig.image.array - 10)
def test_single_image(): """Test the simple case of one image and one catalog. """ if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger( log_file='output/test_single_image.log') # Make the image image = galsim.Image(2048, 2048, scale=0.26) # Where to put the stars. Include some flagged and not used locations. x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ] y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ] flag_list = [1, 1, 13, 1, 1, 4, 1, 1, 0, 1] # Draw a Gaussian PSF at each location on the image. sigma = 1.3 g1 = 0.23 g2 = -0.17 psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) for x, y, flag in zip(x_list, y_list, flag_list): bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31), int(y + 32)) offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5) psf.drawImage(image=image[bounds], method='no_pixel', offset=offset) # corrupt the ones that are marked as flagged if flag & 4: print('corrupting star at ', x, y) ar = image[bounds].array im_max = np.max(ar) * 0.2 ar[ar > im_max] = im_max image.addNoise( galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6)) # Write out the image to a file image_file = os.path.join('output', 'simple_image.fits') image.write(image_file) # Write out the catalog to a file dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2')] data = np.empty(len(x_list), dtype=dtype) data['x'] = x_list data['y'] = y_list data['flag'] = flag_list cat_file = os.path.join('output', 'simple_cat.fits') fitsio.write(cat_file, data, clobber=True) # Use InputFiles to read these back in config = {'image_file_name': image_file, 'cat_file_name': cat_file} input = piff.InputFiles(config, logger=logger) assert input.image_file_name == [image_file] assert input.cat_file_name == [cat_file] # Check image assert input.nimages == 1 image1, _, image_pos, _, _, _ = input.getRawImageData(0) np.testing.assert_equal(image1.array, image.array) # Check catalog np.testing.assert_equal([pos.x for pos in image_pos], x_list) np.testing.assert_equal([pos.y for pos in image_pos], y_list) # Repeat, using flag columns this time. config = { 'image_file_name': image_file, 'cat_file_name': cat_file, 'flag_col': 'flag', 'use_flag': '1', 'skip_flag': '4', 'stamp_size': 48 } input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, _, image_pos, _, _, _ = input.getRawImageData(0) assert len(image_pos) == 7 # Make star data orig_stars = input.makeStars() assert len(orig_stars) == 7 assert orig_stars[0].image.array.shape == (48, 48) # Process the star data # can only compare to truth if include_pixel=False model = piff.Gaussian(fastfit=True, include_pixel=False) interp = piff.Mean() fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars] interp.solve(fitted_stars) print('mean = ', interp.mean) # Check that the interpolation is what it should be # Any position would work here. chipnum = 0 x = 1024 y = 123 orig_wcs = input.getWCS()[chipnum] orig_pointing = input.getPointing() image_pos = galsim.PositionD(x, y) world_pos = piff.StarData.calculateFieldPos(image_pos, orig_wcs, orig_pointing) u, v = world_pos.x, world_pos.y stamp_size = config['stamp_size'] target = piff.Star.makeTarget(x=x, y=y, u=u, v=v, wcs=orig_wcs, stamp_size=stamp_size, pointing=orig_pointing) true_params = [sigma, g1, g2] test_star = interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Check default values of options psf = piff.SimplePSF(model, interp) assert psf.chisq_thresh == 0.1 assert psf.max_iter == 30 assert psf.outliers == None assert psf.extra_interp_properties == [] # Now test running it via the config parser psf_file = os.path.join('output', 'simple_psf.fits') config = { 'input': { 'image_file_name': image_file, 'cat_file_name': cat_file, 'flag_col': 'flag', 'use_flag': 1, 'skip_flag': 4, 'stamp_size': stamp_size }, 'psf': { 'model': { 'type': 'Gaussian', 'fastfit': True, 'include_pixel': False }, 'interp': { 'type': 'Mean' }, 'max_iter': 10, 'chisq_thresh': 0.2, }, 'output': { 'file_name': psf_file }, } orig_stars, wcs, pointing = piff.Input.process(config['input'], logger) # Use a SimplePSF to process the stars data this time. interp = piff.Mean() psf = piff.SimplePSF(model, interp, max_iter=10, chisq_thresh=0.2) assert psf.chisq_thresh == 0.2 assert psf.max_iter == 10 psf.fit(orig_stars, wcs, pointing, logger=logger) test_star = psf.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # test that drawStar and drawStarList work test_star = psf.drawStar(target) test_star_list = psf.drawStarList([target])[0] np.testing.assert_equal(test_star.fit.params, test_star_list.fit.params) np.testing.assert_equal(test_star.image.array, test_star_list.image.array) # test copy_image property of drawStar and draw for draw in [psf.drawStar, psf.model.draw]: target_star_copy = psf.interp.interpolate( piff.Star(target.data.copy(), target.fit.copy())) # interp is so that when we do psf.model.draw we have fit.params to work with test_star_copy = draw(target_star_copy, copy_image=True) test_star_nocopy = draw(target_star_copy, copy_image=False) # if we modify target_star_copy, then test_star_nocopy should be modified, # but not test_star_copy target_star_copy.image.array[0, 0] = 23456 assert test_star_nocopy.image.array[ 0, 0] == target_star_copy.image.array[0, 0] assert test_star_copy.image.array[0, 0] != target_star_copy.image.array[0, 0] # however the other pixels SHOULD still be all the same value assert test_star_nocopy.image.array[ 1, 1] == target_star_copy.image.array[1, 1] assert test_star_copy.image.array[1, 1] == target_star_copy.image.array[1, 1] # test that draw works test_image = psf.draw(x=target['x'], y=target['y'], stamp_size=config['input']['stamp_size'], flux=target.fit.flux, offset=target.fit.center) # this image should be the same values as test_star assert test_image == test_star.image # test that draw does not copy the image image_ref = psf.draw(x=target['x'], y=target['y'], stamp_size=config['input']['stamp_size'], flux=target.fit.flux, offset=target.fit.center, image=test_image) image_ref.array[0, 0] = 123456789 assert test_image.array[0, 0] == image_ref.array[0, 0] assert test_star.image.array[0, 0] != test_image.array[0, 0] assert test_star.image.array[1, 1] == test_image.array[1, 1] # Round trip to a file psf.write(psf_file, logger) psf2 = piff.read(psf_file, logger) assert type(psf2.model) is piff.Gaussian assert type(psf2.interp) is piff.Mean assert psf2.chisq == psf.chisq assert psf2.last_delta_chisq == psf.last_delta_chisq assert psf2.chisq_thresh == psf.chisq_thresh assert psf2.max_iter == psf.max_iter assert psf2.dof == psf.dof assert psf2.nremoved == psf.nremoved test_star = psf2.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Do the whole thing with the config parser os.remove(psf_file) piff.piffify(config, logger) psf3 = piff.read(psf_file) assert type(psf3.model) is piff.Gaussian assert type(psf3.interp) is piff.Mean assert psf3.chisq == psf.chisq assert psf3.last_delta_chisq == psf.last_delta_chisq assert psf3.chisq_thresh == psf.chisq_thresh assert psf3.max_iter == psf.max_iter assert psf3.dof == psf.dof assert psf3.nremoved == psf.nremoved test_star = psf3.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Test using the piffify executable os.remove(psf_file) # This would be simpler as a direct assignment, but this once, test the way you would set # this from the command line, which would call parse_variables. piff.config.parse_variables(config, ['verbose=0'], logger=logger) #config['verbose'] = 0 with open('simple.yaml', 'w') as f: f.write(yaml.dump(config, default_flow_style=False)) config2 = piff.config.read_config('simple.yaml') assert config == config2 piffify_exe = get_script_name('piffify') p = subprocess.Popen([piffify_exe, 'simple.yaml']) p.communicate() psf4 = piff.read(psf_file) assert type(psf4.model) is piff.Gaussian assert type(psf4.interp) is piff.Mean assert psf4.chisq == psf.chisq assert psf4.last_delta_chisq == psf.last_delta_chisq assert psf4.chisq_thresh == psf.chisq_thresh assert psf4.max_iter == psf.max_iter assert psf4.dof == psf.dof assert psf4.nremoved == psf.nremoved test_star = psf4.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # With very low max_iter, we hit the warning about non-convergence config['psf']['max_iter'] = 1 with CaptureLog(level=1) as cl: piff.piffify(config, cl.logger) assert 'PSF fit did not converge' in cl.output
def test_var(): """Check that the variance estimate in params_var is sane. """ # Here is the true PSF scale = 1.3 g1 = 0.23 g2 = -0.17 du = 0.1 dv = 0.4 flux = 500 wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29) noise = 0.2 gsobjs = [ galsim.Gaussian(sigma=1.0), galsim.Kolmogorov(half_light_radius=1.0), galsim.Moffat(half_light_radius=1.0, beta=3.0), galsim.Moffat(half_light_radius=1.0, beta=2.5, trunc=3.0) ] # Mix of centered = True/False, # fastfit = True/False, # include_pixel = True/False models = [ piff.Gaussian(fastfit=False, include_pixel=False, centered=False), piff.Kolmogorov(fastfit=True, include_pixel=True, centered=False), piff.Moffat(fastfit=False, beta=4.8, include_pixel=True, centered=True), piff.Moffat(fastfit=True, beta=2.5, trunc=3.0, include_pixel=False, centered=True) ] names = ['Gaussian', 'Kolmogorov', 'Moffat3', 'Moffat2.5'] for gsobj, model, name in zip(gsobjs, models, names): print() print("gsobj = ", gsobj) print() psf = gsobj.dilate(scale).shear(g1=g1, g2=g2).shift(du, dv).withFlux(flux) image = psf.drawImage(nx=64, ny=64, wcs=wcs, method='no_pixel') weight = image.copy() weight.fill(1 / noise**2) # Save this one without noise. image1 = image.copy() image1.addNoise(galsim.GaussianNoise(sigma=noise)) # Make a StarData instance for this image stardata = piff.StarData(image, image.true_center, weight) star = piff.Star(stardata, None) star = model.initialize(star) fit = model.fit(star).fit file_name = 'input/test_%s_var.npz' % name print(file_name) if not os.path.isfile(file_name): num_runs = 1000 all_params = [] for i in range(num_runs): image1 = image.copy() image1.addNoise(galsim.GaussianNoise(sigma=noise)) sd = piff.StarData(image1, image1.true_center, weight) s = piff.Star(sd, None) try: s = model.initialize(s) s = model.fit(s) except RuntimeError as e: # Occasionally hsm fails. print('Caught ', e) continue print(s.fit.params) all_params.append(s.fit.params) var = np.var(all_params, axis=0) np.savez(file_name, var=var) var = np.load(file_name)['var'] print('params = ', fit.params) print('empirical var = ', var) print('piff estimate = ', fit.params_var) print('ratio = ', fit.params_var / var) print('max ratio = ', np.max(fit.params_var / var)) print('min ratio = ', np.min(fit.params_var / var)) print('mean ratio = ', np.mean(fit.params_var / var)) # Note: The fastfit=False estimates are better -- typically better than 10% # The fastfit=True estimates are much rougher. Especially size. Need rtol=0.3. np.testing.assert_allclose(fit.params_var, var, rtol=0.3)
def test_direct(): """ Simple test for directly instantiated Gaussian, Kolmogorov, and Moffat without going through GSObjectModel explicitly. """ # Here is the true PSF scale = 1.3 g1 = 0.23 g2 = -0.17 du = 0.1 dv = 0.4 gsobjs = [ galsim.Gaussian(sigma=1.0), galsim.Kolmogorov(half_light_radius=1.0), galsim.Moffat(half_light_radius=1.0, beta=3.0), galsim.Moffat(half_light_radius=1.0, beta=2.5, trunc=3.0) ] models = [ piff.Gaussian(fastfit=True, include_pixel=False), piff.Kolmogorov(fastfit=True, include_pixel=False), piff.Moffat(fastfit=True, beta=3.0, include_pixel=False), piff.Moffat(fastfit=True, beta=2.5, trunc=3.0, include_pixel=False) ] for gsobj, model in zip(gsobjs, models): print() print("gsobj = ", gsobj) print() psf = gsobj.dilate(scale).shear(g1=g1, g2=g2).shift(du, dv) # Draw the PSF onto an image. Let's go ahead and give it a non-trivial WCS. wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29) image = galsim.Image(64, 64, wcs=wcs) # This is only going to come out right if we (unphysically) don't convolve by the pixel. psf.drawImage(image, method='no_pixel') # Make a StarData instance for this image stardata = piff.StarData(image, image.true_center) star = piff.Star(stardata, None) star = model.initialize(star) # First try fastfit. print('Fast fit') fit = model.fit(star).fit print('True scale = ', scale, ', model scale = ', fit.params[0]) print('True g1 = ', g1, ', model g1 = ', fit.params[1]) print('True g2 = ', g2, ', model g2 = ', fit.params[2]) print('True du = ', du, ', model du = ', fit.center[0]) print('True dv = ', dv, ', model dv = ', fit.center[1]) # This test is fairly accurate, since we didn't add any noise and didn't convolve by # the pixel, so the image is very accurately a sheared GSObject. # These tests are more strict above. The truncated Moffat included here but not there # doesn't work quite as well. np.testing.assert_allclose(fit.params[0], scale, rtol=1e-4) np.testing.assert_allclose(fit.params[1], g1, rtol=0, atol=1e-5) np.testing.assert_allclose(fit.params[2], g2, rtol=0, atol=1e-5) np.testing.assert_allclose(fit.center[0], du, rtol=0, atol=1e-5) np.testing.assert_allclose(fit.center[1], dv, rtol=0, atol=1e-5) # Also need to test ability to serialize outfile = os.path.join('output', 'gsobject_direct_test.fits') with fitsio.FITS(outfile, 'rw', clobber=True) as f: model.write(f, 'psf_model') with fitsio.FITS(outfile, 'r') as f: roundtrip_model = piff.GSObjectModel.read(f, 'psf_model') assert model.__dict__ == roundtrip_model.__dict__ # repeat with fastfit=False models = [ piff.Gaussian(fastfit=False, include_pixel=False), piff.Kolmogorov(fastfit=False, include_pixel=False), piff.Moffat(fastfit=False, beta=3.0, include_pixel=False), piff.Moffat(fastfit=False, beta=2.5, trunc=3.0, include_pixel=False) ] for gsobj, model in zip(gsobjs, models): print() print("gsobj = ", gsobj) print() psf = gsobj.dilate(scale).shear(g1=g1, g2=g2).shift(du, dv) # Draw the PSF onto an image. Let's go ahead and give it a non-trivial WCS. wcs = galsim.JacobianWCS(0.26, 0.05, -0.08, -0.29) image = galsim.Image(64, 64, wcs=wcs) # This is only going to come out right if we (unphysically) don't convolve by the pixel. psf.drawImage(image, method='no_pixel') # Make a StarData instance for this image stardata = piff.StarData(image, image.true_center) star = piff.Star(stardata, None) star = model.initialize(star) print('Slow fit') fit = model.fit(star).fit print('True scale = ', scale, ', model scale = ', fit.params[0]) print('True g1 = ', g1, ', model g1 = ', fit.params[1]) print('True g2 = ', g2, ', model g2 = ', fit.params[2]) print('True du = ', du, ', model du = ', fit.center[0]) print('True dv = ', dv, ', model dv = ', fit.center[1]) # This test is fairly accurate, since we didn't add any noise and didn't convolve by # the pixel, so the image is very accurately a sheared GSObject. np.testing.assert_allclose(fit.params[0], scale, rtol=1e-5) np.testing.assert_allclose(fit.params[1], g1, rtol=0, atol=1e-5) np.testing.assert_allclose(fit.params[2], g2, rtol=0, atol=1e-5) np.testing.assert_allclose(fit.center[0], du, rtol=0, atol=1e-5) np.testing.assert_allclose(fit.center[1], dv, rtol=0, atol=1e-5) # Also need to test ability to serialize outfile = os.path.join('output', 'gsobject_direct_test.fits') with fitsio.FITS(outfile, 'rw', clobber=True) as f: model.write(f, 'psf_model') with fitsio.FITS(outfile, 'r') as f: roundtrip_model = piff.GSObjectModel.read(f, 'psf_model') assert model.__dict__ == roundtrip_model.__dict__
def test_twodstats(): """Make sure we can execute and print a readout of the plot """ if __name__ == '__main__': logger = piff.config.setup_logger(2) else: logger = None model = piff.Gaussian(fastfit=True) interp = piff.Polynomial(order=1) # should find that order=1 is better # create background model stars, true_model = generate_starlist(100) psf = piff.SimplePSF(model, interp) psf.fit(stars, None, None) stars = psf.stars # These have the right fit parameters # check the coeffs of sigma and g2, which are actually linear fits # skip g1 since it is actually a 2d parabola # factor of 0.263 is to account for going from pixel xy to wcs uv np.testing.assert_almost_equal(psf.interp.coeffs[0].flatten(), np.array([0.4, 0, 1. / (0.263 * 2048), 0]), decimal=4) np.testing.assert_almost_equal( psf.interp.coeffs[2].flatten(), np.array([-0.1 * 1000 / 2048, 0, 0.1 / (0.263 * 2048), 0]), decimal=4) stats = piff.TwoDHistStats(nbins_u=5, nbins_v=5) # implicitly np.median stats.compute(psf, stars, logger=logger) # check the twodhists # get the average value in the bin u_i = 3 v_i = 3 icen = stats.twodhists['u'][v_i, u_i] / 0.263 jcen = stats.twodhists['v'][v_i, u_i] / 0.263 print('icen = ', icen) print('jcen = ', jcen) icenter = 1000 jcenter = 2000 # the average value in the bin should match up with the model for the average coordinates sigma, g1, g2 = psf_model(icen, jcen, icenter, jcenter) sigma_average = stats.twodhists['T'][v_i, u_i] g1_average = stats.twodhists['g1'][v_i, u_i] g2_average = stats.twodhists['g2'][v_i, u_i] # assert equal to 4th decimal print('sigma, g1, g2 = ', [sigma, g1, g2]) print('av sigma, g1, g2 = ', [sigma_average, g1_average, g2_average]) np.testing.assert_almost_equal([sigma, g1, g2], [sigma_average, g1_average, g2_average], decimal=2) # Test the plotting and writing twodstats_file = os.path.join('output', 'twodstats.pdf') stats.write(twodstats_file) with np.testing.assert_raises(ValueError): stats.write() # If not given in constructor, must give file name here. # repeat for whisker stats = piff.WhiskerStats(nbins_u=21, nbins_v=21, reducing_function='np.mean') stats.compute(psf, stars) # Test the plotting and writing whisker_file = os.path.join('output', 'whiskerstats.pdf') stats.write(whisker_file) with np.testing.assert_raises(ValueError): stats.write() # With large number of bins, many will have no objects. This is ok. # Also, can use other np functions like max, std, instead to get different stats # Not sure when these would be useful, but they are allowed. # And, check usage where file_name is given in init. twodstats_file2 = os.path.join('output', 'twodstats.pdf') stats2 = piff.TwoDHistStats(nbins_u=50, nbins_v=50, reducing_function='np.std', file_name=twodstats_file2) with np.testing.assert_raises(RuntimeError): stats2.write() # Cannot write before compute stats2.compute(psf, stars, logger=logger) stats2.write() whisker_file2 = os.path.join('output', 'whiskerstats.pdf') stats2 = piff.WhiskerStats(nbins_u=100, nbins_v=100, reducing_function='np.max', file_name=whisker_file2) with np.testing.assert_raises(RuntimeError): stats2.write() # Cannot write before compute stats2.compute(psf, stars) stats2.write()
def test_single_image(): """Test the simple case of one image and one catalog. """ # Make the image image = galsim.Image(2048, 2048, scale=0.26) # Where to put the stars. Include some flagged and not used locations. x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ] y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ] flag_list = [0, 0, 12, 0, 0, 1, 0, 0, 0, 0] use_list = [1, 1, 1, 1, 1, 0, 1, 1, 0, 1] # Draw a Gaussian PSF at each location on the image. sigma = 1.3 g1 = 0.23 g2 = -0.17 psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) for x, y, flag, use in zip(x_list, y_list, flag_list, use_list): bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31), int(y + 32)) offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5) psf.drawImage(image=image[bounds], method='no_pixel', offset=offset) # corrupt the ones that are marked as flagged if flag: print('corrupting star at ', x, y) ar = image[bounds].array im_max = np.max(ar) * 0.2 ar[ar > im_max] = im_max image.addNoise( galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6)) # Write out the image to a file image_file = os.path.join('data', 'simple_image.fits') image.write(image_file) # Write out the catalog to a file dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2'), ('use', 'i2')] data = np.empty(len(x_list), dtype=dtype) data['x'] = x_list data['y'] = y_list data['flag'] = flag_list data['use'] = use_list cat_file = os.path.join('data', 'simple_cat.fits') fitsio.write(cat_file, data, clobber=True) # Use InputFiles to read these back in input = piff.InputFiles(image_file, cat_file) assert input.image_files == [image_file] assert input.cat_files == [cat_file] assert input.x_col == 'x' assert input.y_col == 'y' # Check image input.readImages() assert len(input.images) == 1 np.testing.assert_equal(input.images[0].array, image.array) # Check catalog input.readStarCatalogs() assert len(input.cats) == 1 np.testing.assert_equal(input.cats[0]['x'], x_list) np.testing.assert_equal(input.cats[0]['y'], y_list) # Repeat, using flag and use columns this time. input = piff.InputFiles(image_file, cat_file, flag_col='flag', use_col='use', stamp_size=48) assert input.flag_col == 'flag' assert input.use_col == 'use' input.readImages() input.readStarCatalogs() assert len(input.cats[0]) == 7 # Make star data orig_stars = input.makeStars() assert len(orig_stars) == 7 assert orig_stars[0].image.array.shape == (48, 48) # Process the star data # can only compare to truth if include_pixel=False model = piff.Gaussian(fastfit=True, include_pixel=False) interp = piff.Mean() fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars] interp.solve(fitted_stars) print('mean = ', interp.mean) # Check that the interpolation is what it should be target = piff.Star.makeTarget(x=1024, y=123) # Any position would work here. true_params = [sigma, g1, g2] test_star = interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Now test running it via the config parser psf_file = os.path.join('output', 'simple_psf.fits') config = { 'input': { 'images': image_file, 'cats': cat_file, 'flag_col': 'flag', 'use_col': 'use', 'stamp_size': 48 }, 'psf': { 'model': { 'type': 'Gaussian', 'fastfit': True, 'include_pixel': False }, 'interp': { 'type': 'Mean' }, }, 'output': { 'file_name': psf_file }, } if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger(verbose=0) orig_stars, wcs, pointing = piff.Input.process(config['input'], logger) # Use a SimplePSF to process the stars data this time. psf = piff.SimplePSF(model, interp) psf.fit(orig_stars, wcs, pointing, logger=logger) test_star = psf.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Round trip to a file psf.write(psf_file, logger) psf = piff.read(psf_file, logger) assert type(psf.model) is piff.Gaussian assert type(psf.interp) is piff.Mean test_star = psf.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Do the whole thing with the config parser os.remove(psf_file) piff.piffify(config, logger) psf = piff.read(psf_file) test_star = psf.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Test using the piffify executable os.remove(psf_file) config['verbose'] = 0 with open('simple.yaml', 'w') as f: f.write(yaml.dump(config, default_flow_style=False)) piffify_exe = get_script_name('piffify') p = subprocess.Popen([piffify_exe, 'simple.yaml']) p.communicate() psf = piff.read(psf_file) test_star = psf.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Test that we can make rho statistics min_sep = 1 max_sep = 100 bin_size = 0.1 stats = piff.RhoStats(min_sep=min_sep, max_sep=max_sep, bin_size=bin_size) stats.compute(psf, orig_stars) rhos = [stats.rho1, stats.rho2, stats.rho3, stats.rho4, stats.rho5] for rho in rhos: # Test the range of separations radius = np.exp(rho.logr) # last bin can be one bigger than max_sep np.testing.assert_array_less(radius, np.exp(np.log(max_sep) + bin_size)) np.testing.assert_array_less(min_sep, radius) np.testing.assert_array_almost_equal(np.diff(rho.logr), bin_size, decimal=5) # Test that the max absolute value of each rho isn't crazy np.testing.assert_array_less(np.abs(rho.xip), 1) # # Check that each rho isn't precisely zero. This means the sum of abs > 0 np.testing.assert_array_less(0, np.sum(np.abs(rho.xip))) # Test the plotting and writing rho_psf_file = os.path.join('output', 'simple_psf_rhostats.pdf') stats.write(rho_psf_file) # Test that we can make summary shape statistics, using HSM shapeStats = piff.ShapeHistogramsStats() shapeStats.compute(psf, orig_stars) # test their characteristics np.testing.assert_array_almost_equal(sigma, shapeStats.T, decimal=4) np.testing.assert_array_almost_equal(sigma, shapeStats.T_model, decimal=3) np.testing.assert_array_almost_equal(g1, shapeStats.g1, decimal=4) np.testing.assert_array_almost_equal(g1, shapeStats.g1_model, decimal=3) np.testing.assert_array_almost_equal(g2, shapeStats.g2, decimal=4) np.testing.assert_array_almost_equal(g2, shapeStats.g2_model, decimal=3) shape_psf_file = os.path.join('output', 'simple_psf_shapestats.pdf') shapeStats.write(shape_psf_file) # Test that we can use the config parser for both RhoStats and ShapeHistogramsStats config['output']['stats'] = [ { 'type': 'ShapeHistograms', 'file_name': shape_psf_file }, { 'type': 'Rho', 'file_name': rho_psf_file }, { 'type': 'TwoDHist', 'file_name': os.path.join('output', 'simple_psf_twodhiststats.pdf'), 'number_bins_u': 3, 'number_bins_v': 3, }, { 'type': 'TwoDHist', 'file_name': os.path.join('output', 'simple_psf_twodhiststats_std.pdf'), 'reducing_function': 'np.std', 'number_bins_u': 3, 'number_bins_v': 3, }, ] os.remove(psf_file) os.remove(rho_psf_file) os.remove(shape_psf_file) piff.piffify(config, logger) # Test using the piffify executable os.remove(psf_file) os.remove(rho_psf_file) os.remove(shape_psf_file) config['verbose'] = 0 with open('simple.yaml', 'w') as f: f.write(yaml.dump(config, default_flow_style=False)) p = subprocess.Popen([piffify_exe, 'simple.yaml']) p.communicate()
def test_chisq(): """Test the Chisq outlier class """ if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger( log_file='output/test_single_image.log') # Make the image image = galsim.Image(512, 512, scale=0.26) nstars = 1000 # enough that there could be some overlaps. Also, some over the edge. rng = np.random.RandomState(1234) x = rng.random_sample(nstars) * 512 y = rng.random_sample(nstars) * 512 sigma = np.ones_like(x) * 0.4 # Most have this sigma g1 = np.ones_like(x) * 0.023 # Most are pretty round. g2 = np.ones_like(x) * 0.012 flux = np.exp(rng.normal(size=nstars)) print('flux range = ', np.min(flux), np.max(flux), np.median(flux), np.mean(flux)) # Make a few intentionally wrong. g1[35] = 0.29 g1[188] = -0.15 sigma[239] = 0.2 g2[347] = -0.15 sigma[551] = 1.3 g2[809] = 0.05 g1[922] = -0.03 # Draw a Gaussian PSF at each location on the image. for i in range(nstars): psf = galsim.Gaussian(sigma=sigma[i]).shear(g1=g1[i], g2=g2[i]) stamp = psf.drawImage(scale=0.26, center=galsim.PositionD(x[i], y[i])) b = stamp.bounds & image.bounds image[b] += stamp[b] noise = 0.02 image.addNoise( galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=noise)) image_file = os.path.join('output', 'test_chisq_im.fits') image.write(image_file) # Write out the catalog to a file dtype = [('x', 'f8'), ('y', 'f8')] data = np.empty(len(x), dtype=dtype) data['x'] = x data['y'] = y cat_file = os.path.join('output', 'test_chisq_cat.fits') fitsio.write(cat_file, data, clobber=True) # Read the catalog in as stars. config = { 'image_file_name': image_file, 'cat_file_name': cat_file, 'noise': noise**2, # Variance here is sigma^2 'stamp_size': 15, 'use_partial': True, } input = piff.InputFiles(config, logger=logger) stars = input.makeStars() # Skip the solve step. Just give it the right answer and see what it finds for outliers model = piff.Gaussian() interp = piff.Mean() interp.mean = np.array([0.4, 0.023, 0.012]) psf = piff.SimplePSF(model, interp) stars = psf.interpolateStarList(stars) stars = [psf.model.reflux(s, logger=logger) for s in stars] outliers1 = piff.ChisqOutliers(nsigma=5) stars1, nremoved1 = outliers1.removeOutliers(stars, logger=logger) print('nremoved1 = ', nremoved1) assert len(stars1) == len(stars) - nremoved1 # This is what nsigma=5 means in terms of probability outliers2 = piff.ChisqOutliers(prob=5.733e-7) stars2, nremoved2 = outliers2.removeOutliers(stars, logger=logger) print('nremoved2 = ', nremoved2) assert len(stars2) == len(stars) - nremoved2 assert nremoved1 == nremoved2 # The following is nearly equivalent for this particular data set. # For dof=222 (what most of these have, this probability converts to # thresh = 455.40143379 # or ndof = 2.0513578 # But note that when using the above prop or nsigma, the code uses a tailored threshold # different for each star's particular dof, which varies (since some are off the edge). outliers3 = piff.ChisqOutliers(thresh=455.401) stars3, nremoved3 = outliers3.removeOutliers(stars, logger=logger) print('nremoved3 = ', nremoved3) assert len(stars3) == len(stars) - nremoved3 outliers4 = piff.ChisqOutliers(ndof=2.05136) stars4, nremoved4 = outliers4.removeOutliers(stars, logger=logger) print('nremoved4 = ', nremoved4) assert len(stars4) == len(stars) - nremoved4 assert nremoved3 == nremoved4 # Regression tests. If these change, make sure we understand why. assert nremoved1 == nremoved2 == 58 assert nremoved3 == nremoved4 == 16 # Much less, since edge objects aren't being removed # nearly as often as when they have a custom thresh. # Can't provide multiple thresh specifications np.testing.assert_raises(TypeError, piff.ChisqOutliers, nsigma=5, prob=1.e-3) np.testing.assert_raises(TypeError, piff.ChisqOutliers, nsigma=5, thresh=100) np.testing.assert_raises(TypeError, piff.ChisqOutliers, nsigma=5, ndof=3) np.testing.assert_raises(TypeError, piff.ChisqOutliers, prob=1.e-3, thresh=100) np.testing.assert_raises(TypeError, piff.ChisqOutliers, prob=1.e-3, ndof=3) np.testing.assert_raises(TypeError, piff.ChisqOutliers, thresh=100, ndof=3) # Need to specifiy it somehow. np.testing.assert_raises(TypeError, piff.ChisqOutliers)