def test_weight(): """Test the weight map and bad pixel masks """ if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger( log_file=os.path.join('output', 'test_input_weight.log')) # If no weight or badpix is specified, the weights are all equal. config = { 'image_file_name': 'input/test_input_image_00.fits', 'cat_file_name': 'input/test_input_cat_00.fits', } input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 image, weight, image_pos, _, _, _ = input.getRawImageData(0) assert len(image_pos) == 100 assert image.array.shape == (1024, 1024) assert weight.array.shape == (1024, 1024) np.testing.assert_array_equal(weight.array, 1.0) # The default weight and badpix masks that GalSim makes don't do any masking, so this # is the almost the same as above, but the weight value is 1/sky. config = { 'image_file_name': 'input/test_input_image_00.fits', 'cat_file_name': 'input/test_input_cat_00.fits', 'weight_hdu': 1, 'badpix_hdu': 2, 'sky_col': 'sky', # Used to determine what the value of weight should be 'gain_col': 'gain', } input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 image, weight, image_pos, sky, gain, _ = input.getRawImageData(0) assert len(image_pos) == 100 assert image.array.shape == (1024, 1024) assert weight.array.shape == (1024, 1024) sky = sky[0] gain = gain[0] read_noise = 10 expected_noise = sky / gain + read_noise**2 / gain**2 np.testing.assert_almost_equal(weight.array, expected_noise**-1) # Can set the noise by hand config = { 'image_file_name': 'input/test_input_image_00.fits', 'cat_file_name': 'input/test_input_cat_00.fits', 'noise': 32, } input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, weight, _, _, _, _ = input.getRawImageData(0) assert weight.array.shape == (1024, 1024) np.testing.assert_almost_equal(weight.array, 32.**-1) # Some old versions of fitsio had a bug where the badpix mask could be offset by 32768. # We move them back to 0 config = { 'image_file_name': 'input/test_input_image_00.fits', 'cat_file_name': 'input/test_input_cat_00.fits', 'weight_hdu': 1, 'badpix_hdu': 5, } input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, weight, _, _, _, _ = input.getRawImageData(0) assert weight.array.shape == (1024, 1024) np.testing.assert_almost_equal(weight.array, expected_noise**-1) config['badpix_hdu'] = 6 input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, weight, _, _, _, _ = input.getRawImageData(0) assert weight.array.shape == (1024, 1024) np.testing.assert_almost_equal(weight.array, expected_noise**-1) # Various ways to get all weight values == 0 (which will emit a logger message, but isn't # an error). config['weight_hdu'] = 1 config['badpix_hdu'] = 7 # badpix > 0 input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, weight, _, _, _, _ = input.getRawImageData(0) assert weight.array.shape == (1024, 1024) np.testing.assert_almost_equal(weight.array, 0.) config['weight_hdu'] = 3 # wt = 0 config['badpix_hdu'] = 2 input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, weight, _, _, _, _ = input.getRawImageData(0) assert weight.array.shape == (1024, 1024) np.testing.assert_almost_equal(weight.array, 0.) config['weight_hdu'] = 8 # Even cols are = 0 config['badpix_hdu'] = 9 # Odd cols are > 0 input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, weight, _, _, _, _ = input.getRawImageData(0) assert weight.array.shape == (1024, 1024) np.testing.assert_almost_equal(weight.array, 0.) # Negative valued weights are invalid config['weight_hdu'] = 4 input = piff.InputFiles(config) with CaptureLog() as cl: _, weight, _, _, _, _ = input.getRawImageData(0, logger=cl.logger) assert 'Warning: weight map has invalid negative-valued pixels.' in cl.output
def test_single_image(): """Test the simple case of one image and one catalog. """ if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger( log_file='output/test_single_image.log') # Make the image image = galsim.Image(2048, 2048, scale=0.26) # Where to put the stars. Include some flagged and not used locations. x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ] y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ] flag_list = [1, 1, 13, 1, 1, 4, 1, 1, 0, 1] # Draw a Gaussian PSF at each location on the image. sigma = 1.3 g1 = 0.23 g2 = -0.17 psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) for x, y, flag in zip(x_list, y_list, flag_list): bounds = galsim.BoundsI(int(x - 31), int(x + 32), int(y - 31), int(y + 32)) offset = galsim.PositionD(x - int(x) - 0.5, y - int(y) - 0.5) psf.drawImage(image=image[bounds], method='no_pixel', offset=offset) # corrupt the ones that are marked as flagged if flag & 4: print('corrupting star at ', x, y) ar = image[bounds].array im_max = np.max(ar) * 0.2 ar[ar > im_max] = im_max image.addNoise( galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6)) # Write out the image to a file image_file = os.path.join('output', 'simple_image.fits') image.write(image_file) # Write out the catalog to a file dtype = [('x', 'f8'), ('y', 'f8'), ('flag', 'i2')] data = np.empty(len(x_list), dtype=dtype) data['x'] = x_list data['y'] = y_list data['flag'] = flag_list cat_file = os.path.join('output', 'simple_cat.fits') fitsio.write(cat_file, data, clobber=True) # Use InputFiles to read these back in config = {'image_file_name': image_file, 'cat_file_name': cat_file} input = piff.InputFiles(config, logger=logger) assert input.image_file_name == [image_file] assert input.cat_file_name == [cat_file] # Check image assert input.nimages == 1 image1, _, image_pos, _, _, _ = input.getRawImageData(0) np.testing.assert_equal(image1.array, image.array) # Check catalog np.testing.assert_equal([pos.x for pos in image_pos], x_list) np.testing.assert_equal([pos.y for pos in image_pos], y_list) # Repeat, using flag columns this time. config = { 'image_file_name': image_file, 'cat_file_name': cat_file, 'flag_col': 'flag', 'use_flag': '1', 'skip_flag': '4', 'stamp_size': 48 } input = piff.InputFiles(config, logger=logger) assert input.nimages == 1 _, _, image_pos, _, _, _ = input.getRawImageData(0) assert len(image_pos) == 7 # Make star data orig_stars = input.makeStars() assert len(orig_stars) == 7 assert orig_stars[0].image.array.shape == (48, 48) # Process the star data # can only compare to truth if include_pixel=False model = piff.Gaussian(fastfit=True, include_pixel=False) interp = piff.Mean() fitted_stars = [model.fit(model.initialize(star)) for star in orig_stars] interp.solve(fitted_stars) print('mean = ', interp.mean) # Check that the interpolation is what it should be # Any position would work here. chipnum = 0 x = 1024 y = 123 orig_wcs = input.getWCS()[chipnum] orig_pointing = input.getPointing() image_pos = galsim.PositionD(x, y) world_pos = piff.StarData.calculateFieldPos(image_pos, orig_wcs, orig_pointing) u, v = world_pos.x, world_pos.y stamp_size = config['stamp_size'] target = piff.Star.makeTarget(x=x, y=y, u=u, v=v, wcs=orig_wcs, stamp_size=stamp_size, pointing=orig_pointing) true_params = [sigma, g1, g2] test_star = interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Check default values of options psf = piff.SimplePSF(model, interp) assert psf.chisq_thresh == 0.1 assert psf.max_iter == 30 assert psf.outliers == None assert psf.extra_interp_properties == [] # Now test running it via the config parser psf_file = os.path.join('output', 'simple_psf.fits') config = { 'input': { 'image_file_name': image_file, 'cat_file_name': cat_file, 'flag_col': 'flag', 'use_flag': 1, 'skip_flag': 4, 'stamp_size': stamp_size }, 'psf': { 'model': { 'type': 'Gaussian', 'fastfit': True, 'include_pixel': False }, 'interp': { 'type': 'Mean' }, 'max_iter': 10, 'chisq_thresh': 0.2, }, 'output': { 'file_name': psf_file }, } orig_stars, wcs, pointing = piff.Input.process(config['input'], logger) # Use a SimplePSF to process the stars data this time. interp = piff.Mean() psf = piff.SimplePSF(model, interp, max_iter=10, chisq_thresh=0.2) assert psf.chisq_thresh == 0.2 assert psf.max_iter == 10 psf.fit(orig_stars, wcs, pointing, logger=logger) test_star = psf.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # test that drawStar and drawStarList work test_star = psf.drawStar(target) test_star_list = psf.drawStarList([target])[0] np.testing.assert_equal(test_star.fit.params, test_star_list.fit.params) np.testing.assert_equal(test_star.image.array, test_star_list.image.array) # test copy_image property of drawStar and draw for draw in [psf.drawStar, psf.model.draw]: target_star_copy = psf.interp.interpolate( piff.Star(target.data.copy(), target.fit.copy())) # interp is so that when we do psf.model.draw we have fit.params to work with test_star_copy = draw(target_star_copy, copy_image=True) test_star_nocopy = draw(target_star_copy, copy_image=False) # if we modify target_star_copy, then test_star_nocopy should be modified, # but not test_star_copy target_star_copy.image.array[0, 0] = 23456 assert test_star_nocopy.image.array[ 0, 0] == target_star_copy.image.array[0, 0] assert test_star_copy.image.array[0, 0] != target_star_copy.image.array[0, 0] # however the other pixels SHOULD still be all the same value assert test_star_nocopy.image.array[ 1, 1] == target_star_copy.image.array[1, 1] assert test_star_copy.image.array[1, 1] == target_star_copy.image.array[1, 1] # test that draw works test_image = psf.draw(x=target['x'], y=target['y'], stamp_size=config['input']['stamp_size'], flux=target.fit.flux, offset=target.fit.center) # this image should be the same values as test_star assert test_image == test_star.image # test that draw does not copy the image image_ref = psf.draw(x=target['x'], y=target['y'], stamp_size=config['input']['stamp_size'], flux=target.fit.flux, offset=target.fit.center, image=test_image) image_ref.array[0, 0] = 123456789 assert test_image.array[0, 0] == image_ref.array[0, 0] assert test_star.image.array[0, 0] != test_image.array[0, 0] assert test_star.image.array[1, 1] == test_image.array[1, 1] # Round trip to a file psf.write(psf_file, logger) psf2 = piff.read(psf_file, logger) assert type(psf2.model) is piff.Gaussian assert type(psf2.interp) is piff.Mean assert psf2.chisq == psf.chisq assert psf2.last_delta_chisq == psf.last_delta_chisq assert psf2.chisq_thresh == psf.chisq_thresh assert psf2.max_iter == psf.max_iter assert psf2.dof == psf.dof assert psf2.nremoved == psf.nremoved test_star = psf2.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Do the whole thing with the config parser os.remove(psf_file) piff.piffify(config, logger) psf3 = piff.read(psf_file) assert type(psf3.model) is piff.Gaussian assert type(psf3.interp) is piff.Mean assert psf3.chisq == psf.chisq assert psf3.last_delta_chisq == psf.last_delta_chisq assert psf3.chisq_thresh == psf.chisq_thresh assert psf3.max_iter == psf.max_iter assert psf3.dof == psf.dof assert psf3.nremoved == psf.nremoved test_star = psf3.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # Test using the piffify executable os.remove(psf_file) # This would be simpler as a direct assignment, but this once, test the way you would set # this from the command line, which would call parse_variables. piff.config.parse_variables(config, ['verbose=0'], logger=logger) #config['verbose'] = 0 with open('simple.yaml', 'w') as f: f.write(yaml.dump(config, default_flow_style=False)) config2 = piff.config.read_config('simple.yaml') assert config == config2 piffify_exe = get_script_name('piffify') p = subprocess.Popen([piffify_exe, 'simple.yaml']) p.communicate() psf4 = piff.read(psf_file) assert type(psf4.model) is piff.Gaussian assert type(psf4.interp) is piff.Mean assert psf4.chisq == psf.chisq assert psf4.last_delta_chisq == psf.last_delta_chisq assert psf4.chisq_thresh == psf.chisq_thresh assert psf4.max_iter == psf.max_iter assert psf4.dof == psf.dof assert psf4.nremoved == psf.nremoved test_star = psf4.interp.interpolate(target) np.testing.assert_almost_equal(test_star.fit.params, true_params, decimal=4) # With very low max_iter, we hit the warning about non-convergence config['psf']['max_iter'] = 1 with CaptureLog(level=1) as cl: piff.piffify(config, cl.logger) assert 'PSF fit did not converge' in cl.output
def test_sizemag(): """Test the SizeMag selection algorithm. """ if __name__ == '__main__': logger = piff.config.setup_logger(2) else: logger = None config = piff.config.read_config('sizemag.yaml') config['select'] = {'type': 'SizeMag'} objects, _, _ = piff.Input.process(config['input'], logger=logger) stars = piff.Select.process(config['select'], objects, logger=logger) # This finds more stars than the simple SmallBright selector found. print('nstars = ', len(stars)) assert len(stars) == 140 # A few of these have lower CLASS_STAR values, but still most are > 0.95 class_star = np.array([s['CLASS_STAR'] for s in stars]) print('class_star = ', class_star) print('min class_star = ', np.min(class_star)) print('N class_star > 0.95 = ', np.sum(class_star > 0.95)) assert np.sum(class_star > 0.95) == 140 # This goes a bit fainter than the SmallBright selector did (which is kind of the point). mag_auto = np.array([s['MAG_AUTO'] for s in stars]) print('mag_auto = ', mag_auto) print('max mag_auto = ', np.max(mag_auto)) assert np.max(mag_auto) > 16 assert np.max(mag_auto) < 17 # Sizes are all pretty similar (but not exactly by construction anymore) sizes = [s.hsm[3] for s in stars] print('mean size = ', np.mean(sizes)) print('median size = ', np.median(sizes)) print('min/max size = ', np.min(sizes), np.max(sizes)) print('spread = ', (np.max(sizes) - np.min(sizes)) / np.median(sizes)) assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.1 # Try some different parameter values. config['select'] = { 'type': 'SizeMag', 'initial_select': { 'type': 'Properties', 'where': '(CLASS_STAR > 0.9) & (MAG_AUTO < 16)', }, 'purity': 0, 'num_iter': 2, 'fit_order': 0, } stars = piff.Select.process(config['select'], objects, logger=logger) # Somewhat fewer, but still works reasonably ok. print('nstars = ', len(stars)) assert len(stars) == 135 class_star = np.array([s['CLASS_STAR'] for s in stars]) print('class_star = ', class_star) print('min class_star = ', np.min(class_star)) print('N class_star > 0.95 = ', np.sum(class_star > 0.95)) assert np.sum(class_star > 0.95) == 135 sizes = [s.hsm[3] for s in stars] print('mean size = ', np.mean(sizes)) print('median size = ', np.median(sizes)) print('min/max size = ', np.min(sizes), np.max(sizes)) print('spread = ', (np.max(sizes) - np.min(sizes)) / np.median(sizes)) assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.1 # Check that it doesn't crap out with bad initial objects. # Here the initial selection is pretty much all galaxies, not stars. config['select'] = { 'type': 'SizeMag', 'initial_select': { 'type': 'Properties', 'where': '(CLASS_STAR < 0.2) & (MAG_AUTO > 15)', }, } stars = piff.Select.process(config['select'], objects, logger=logger) sizes = [s.hsm[3] for s in stars] print('mean size = ', np.mean(sizes)) print('median size = ', np.median(sizes)) print('min/max size = ', np.min(sizes), np.max(sizes)) # It doesn't fail, but it (obviously) doesn't find a good stellar locus either. assert np.max(sizes) > 1.5 # Real stellar locus is at about T = 0.5 # Make sure it doesn't crap out if all the input objects are stars config['input'] = { 'dir': 'input', 'image_file_name': 'DECam_00241238_01.fits.fz', 'image_hdu': 1, 'badpix_hdu': 2, 'weight_hdu': 3, 'cat_file_name': 'DECam_00241238_01_psfcat_tb_maxmag_17.0_magcut_3.0_findstars.fits', 'cat_hdu': 2, 'x_col': 'XWIN_IMAGE', 'y_col': 'YWIN_IMAGE', 'sky_col': 'BACKGROUND', 'stamp_size': 25, } config['select'] = { 'type': 'SizeMag', 'initial_select': { # Use all the input objects } } objects, _, _ = piff.Input.process(config['input'], logger=logger) stars = piff.Select.process(config['select'], objects, logger=logger) # Check pathologically few input objects. select = piff.SizeMagSelect(config['select']) with np.testing.assert_raises(RuntimeError): stars = select.selectStars(objects[:0], logger=logger) with CaptureLog() as cl: stars = select.selectStars(objects[:1], logger=cl.logger) print(cl.output) assert "Too few candidate stars (1) to use fit_order=2." in cl.output print("len(stars) = ", len(stars)) assert len(stars) == 0 with CaptureLog() as cl: stars = select.selectStars(objects[:5], logger=cl.logger) print(cl.output) # (Note: one was clipped, so only N=4 when the error message triggered.) assert "Too few candidate stars (4) to use fit_order=2." in cl.output print("len(stars) = ", len(stars)) assert len(stars) == 0 # With fit_order=0, 1 input star is still tricky, but it should go through. config['select'] = { 'type': 'SizeMag', 'fit_order': 0, 'initial_select': {} } select = piff.SizeMagSelect(config['select']) stars = select.selectStars(objects[:1], logger=logger) print("len(stars) = ", len(stars)) assert len(stars) == 1 # With fit_order=1, 3 input stars is the minimum. config['select'] = { 'type': 'SizeMag', 'fit_order': 1, 'initial_select': {} } select = piff.SizeMagSelect(config['select']) with CaptureLog(3) as cl: stars = select.selectStars(objects[:2], logger=cl.logger) print(cl.output) print("len(stars) = ", len(stars)) assert len(stars) == 0 with CaptureLog(3) as cl: stars = select.selectStars(objects[:3], logger=cl.logger) print(cl.output) print("len(stars) = ", len(stars)) assert len(stars) == 3 # Error to have other parameters config['select'] = { 'type': 'SizeMag', 'order': 3, } with np.testing.assert_raises(ValueError): select = piff.SmallBrightSelect(config['select']) # But ok to have parameters that the base class will handle. config['select'] = { 'type': 'SizeMag', 'purity': 0, 'hsm_size_reject': 4, 'min_snr': 50, } piff.Select.process(config['select'], objects, logger=logger)
def test_smallbright(): """Test the SmallBright selection algorithm. """ if __name__ == '__main__': logger = piff.config.setup_logger(2) else: logger = None config = piff.config.read_config('sizemag.yaml') config['select'] = {'type': 'SmallBright'} objects, _, _ = piff.Input.process(config['input'], logger=logger) stars = piff.Select.process(config['select'], objects, logger=logger) # This does a pretty decent job actually. Finds 88 stars. print('nstars = ', len(stars)) assert len(stars) == 88 # They are all ones that CLASS_STAR also identified as stars. class_star = np.array([s['CLASS_STAR'] for s in stars]) print('class_star = ', class_star) print('min class_star = ', np.min(class_star)) assert np.all(class_star > 0.95) mag_auto = np.array([s['MAG_AUTO'] for s in stars]) print('mag_auto = ', mag_auto) print('max mag_auto = ', np.max(mag_auto)) assert np.max(mag_auto) < 16 # Sizes are all pretty similar (by construction of the algorithm) sizes = [s.hsm[3] for s in stars] print('mean size = ', np.mean(sizes)) print('median size = ', np.median(sizes)) print('min/max size = ', np.min(sizes), np.max(sizes)) assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.1 # Try some different parameter values. config['select'] = { 'type': 'SmallBright', 'bright_fraction': 0.1, 'small_fraction': 0.5, 'locus_fraction': 0.8, 'max_spread': 0.05, } stars = piff.Select.process(config['select'], objects, logger=logger) # Fewer stars since limited to brighter subset # Different systems give slightly different results here. So give a range. print('nstars = ', len(stars)) assert 30 < len(stars) < 40 # But still finds all high confidence stars class_star = np.array([s['CLASS_STAR'] for s in stars]) print('class_star = ', class_star) print('min class_star = ', np.min(class_star)) assert np.all(class_star > 0.95) # And sizes are now restricted to max_spread = 0.05 sizes = [s.hsm[3] for s in stars] print('mean size = ', np.mean(sizes)) print('median size = ', np.median(sizes)) print('min/max size = ', np.min(sizes), np.max(sizes)) assert (np.max(sizes) - np.min(sizes)) / np.median(sizes) < 0.05 # Error to have other parameters config['select'] = { 'type': 'SmallBright', 'bright_frac': 0.1, 'small_frac': 0.5, } with np.testing.assert_raises(ValueError): piff.Select.process(config['select'], objects, logger=logger) # But ok to have parameters that the base class will handle. config['select'] = { 'type': 'SmallBright', 'locus_fraction': 0.8, 'hsm_size_reject': 4, 'min_snr': 50, } piff.Select.process(config['select'], objects, logger=logger) # Set up a pathological input that hits the code where niter=10. # Normal inputs rarely need more than 3 iterations, so this is really a backstop # against very strange behavior. I don't know if it's possible for the code to # go into an infinite loop, where having a maxiter would be critical, but it's # at least conceivable for it to be slowly converging, so might as well keep it. config['select'] = { 'type': 'SmallBright', 'bright_fraction': 1., 'small_fraction': 0.1, 'locus_fraction': 1., 'max_spread': 1., } # If sizes are spaced such that the small ones are consistently farther apart than bigger # ones, then the median/iqr iteration will keep slowly shifting to a larger size. # Having T go as 1/sqrt(i+1) seems to work to make this happen. for i in range(len(objects)): objects[i]._hsm = (1., 0., 0., 1. / (i + 1.)**0.25, 0., 0., 0) select = piff.SmallBrightSelect(config['select']) with CaptureLog() as cl: stars = select.selectStars(objects, logger=cl.logger) assert "Max iter = 10 reached." in cl.output print("len(stars) = ", len(stars)) # These would be pathalogical values to set (as were the above values), but the # code handles them gracefully by changing 0 into the minimum plausible value. # However, it will end up not being able to find a stellar locus, so len(stars) = 0 config['select'] = { 'type': 'SmallBright', 'bright_fraction': 0., 'small_fraction': 0., 'locus_fraction': 0., } select = piff.SmallBrightSelect(config['select']) with CaptureLog() as cl: stars = select.selectStars(objects, logger=cl.logger) assert "Failed to find bright/small stellar locus" in cl.output print("len(stars) = ", len(stars)) assert len(stars) == 0 # With this arrangement, it does better. (Since the problem above is that all the fluxes # are the same, so the "brightest" 2 are also the biggest 2 and have a large iqr.) config['select'] = { 'type': 'SmallBright', 'bright_fraction': 1., 'small_fraction': 0., 'locus_fraction': 0., } select = piff.SmallBrightSelect(config['select']) with CaptureLog() as cl: stars = select.selectStars(objects, logger=cl.logger) print("len(stars) = ", len(stars)) assert len(stars) == 134 # Check pathologically few input objects. config['select'] = { 'type': 'SmallBright', } select = piff.SmallBrightSelect(config['select']) with CaptureLog() as cl: stars = select.selectStars(objects[:0], logger=cl.logger) print(cl.output) assert "No input objects" in cl.output print("len(stars) = ", len(stars)) assert len(stars) == 0 with CaptureLog() as cl: stars = select.selectStars(objects[:1], logger=cl.logger) print(cl.output) assert "Only 1 input object." in cl.output print("len(stars) = ", len(stars)) assert len(stars) == 0 with CaptureLog() as cl: stars = select.selectStars(objects[:2], logger=cl.logger) print(cl.output) assert "Failed to find bright/small stellar locus" in cl.output print("len(stars) = ", len(stars)) assert len(stars) == 0
def test_parallel(): # Run the same test as test_single, but using nproc wcs1 = galsim.TanWCS( galsim.AffineTransform(0.26, 0.05, -0.08, -0.24, galsim.PositionD(1024, 1024)), galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees)) wcs2 = galsim.TanWCS( galsim.AffineTransform(0.25, -0.02, 0.01, 0.24, galsim.PositionD(1024, 1024)), galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees)) field_center = galsim.CelestialCoord(0 * galsim.degrees, -25 * galsim.degrees) if __name__ == '__main__': nstars = 20 # per ccd else: nstars = 6 # per ccd rng = np.random.RandomState(1234) x = rng.random_sample(nstars) * 2000 + 24 y = rng.random_sample(nstars) * 2000 + 24 ra1, dec1 = wcs1.toWorld(x, y, units='rad') u, v = field_center.project_rad(ra1, dec1, projection='gnomonic') e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2 e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2 s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v data1 = np.array(list(zip(x, y, e1, e2, s)), dtype=[('x', float), ('y', float), ('e1', float), ('e2', float), ('s', float)]) im1 = drawImage(2048, 2048, wcs1, x, y, e1, e2, s) im1.write('output/test_parallel_im1.fits') x = rng.random_sample(nstars) * 2000 + 24 y = rng.random_sample(nstars) * 2000 + 24 ra2, dec2 = wcs2.toWorld(x, y, units='rad') u, v = field_center.project_rad(ra1, dec1, projection='gnomonic') # Same functions of u,v, but using the positions on chip 2 e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2 e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2 s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v data2 = np.array(list(zip(x, y, e1, e2, s)), dtype=[('x', float), ('y', float), ('e1', float), ('e2', float), ('s', float)]) im2 = drawImage(2048, 2048, wcs2, x, y, e1, e2, s) im2.write('output/test_parallel_im2.fits') ra12 = np.concatenate([ra1, ra2]) dec12 = np.concatenate([dec1, dec2]) data12 = np.array(list(zip(ra12, dec12)), dtype=[('ra', float), ('dec', float)]) fitsio.write('output/test_parallel.fits', data12, clobber=True) # im3 is blank. Will give errors trying to measure PSF from it. im3 = galsim.Image(2048, 2048, wcs=wcs2) im3.write('output/test_parallel_im3.fits') psf_file = os.path.join('output', 'test_single.fits') config = { 'input': { # A third way to input these same file names. Use GalSim config values and # explicitly specify the number of images to read 'nimages': 2, 'image_file_name': { 'type': 'FormattedStr', 'format': '%s/test_parallel_im%d.fits', 'items': ['output', '$image_num+1'], }, 'cat_file_name': 'output/test_parallel.fits', 'chipnum': '$image_num+1', 'ra_col': 'ra', 'dec_col': 'dec', 'ra_units': 'rad', 'dec_units': 'rad', 'nproc': -1, }, 'psf': { 'type': 'SingleChip', 'model': { 'type': 'Moffat', 'beta': 2.5, }, 'interp': { 'type': 'Polynomial', 'order': 2, }, 'nproc': 2, }, 'output': { 'file_name': psf_file, }, } with CaptureLog(level=2) as cl: piff.piffify(config, logger=cl.logger) psf = piff.read(psf_file) for chipnum, data, wcs in [(1, data1, wcs1), (2, data2, wcs2)]: for k in range(nstars): x = data['x'][k] y = data['y'][k] e1 = data['e1'][k] e2 = data['e2'][k] s = data['s'][k] image_pos = galsim.PositionD(x, y) star = piff.Star.makeTarget(x=x, y=y, wcs=wcs, stamp_size=48, pointing=field_center, chipnum=chipnum) star = psf.drawStar(star) np.testing.assert_almost_equal(star.fit.params, [s, e1, e2], decimal=6) # Finally, check that the logger properly captures the subprocess logs with CaptureLog(level=2) as cl: psf = piff.process(config, cl.logger) #print('with nproc=2, log = ',cl.output) assert "Processing catalog 1" in cl.output assert "Processing catalog 2" in cl.output assert "Building solution for chip 1" in cl.output assert "Building solution for chip 2" in cl.output # Check that errors in the solution get properly reported. config['input']['nimages'] = 3 with CaptureLog(level=2) as cl: psf = piff.process(config, cl.logger) assert "Removed 6 stars in initialize" in cl.output assert "No stars. Cannot find PSF model." in cl.output assert "Solutions failed for chipnums: [3]" in cl.output # Check that errors in the multiprocessing input get properly reported. config['input']['ra_col'] = 'invalid' with CaptureLog(level=2) as cl: with np.testing.assert_raises(ValueError): psf = piff.process(config, cl.logger) assert "ra_col = invalid is not a column" in cl.output # With nproc=1, the error is raised directly. config['input']['nproc'] = 1 config['verbose'] = 0 with np.testing.assert_raises(ValueError): psf = piff.process(config) # But just the input error. Not the one in fitting. config['psf']['nproc'] = 1 config['input']['ra_col'] = 'ra' config['verbose'] = 1 with CaptureLog(level=1) as cl: psf = piff.process(config, logger=cl.logger) assert "No stars. Cannot find PSF model." in cl.output assert "Ignoring this failure and continuing on." in cl.output
def test_single(): """Same as test_focal, but using the SingleCCD PSF type, which does a separate fit on each CCD. """ wcs1 = galsim.TanWCS( galsim.AffineTransform(0.26, 0.05, -0.08, -0.24, galsim.PositionD(1024, 1024)), galsim.CelestialCoord(-5 * galsim.arcmin, -25 * galsim.degrees)) wcs2 = galsim.TanWCS( galsim.AffineTransform(0.25, -0.02, 0.01, 0.24, galsim.PositionD(1024, 1024)), galsim.CelestialCoord(5 * galsim.arcmin, -25 * galsim.degrees)) field_center = galsim.CelestialCoord(0 * galsim.degrees, -25 * galsim.degrees) if __name__ == '__main__': nstars = 20 # per ccd logger = piff.config.setup_logger(verbose=2) else: nstars = 6 # per ccd logger = piff.config.setup_logger(log_file='output/test_single.log') rng = np.random.RandomState(1234) x = rng.random_sample(nstars) * 2000 + 24 y = rng.random_sample(nstars) * 2000 + 24 ra1, dec1 = wcs1.toWorld(x, y, units='rad') u, v = field_center.project_rad(ra1, dec1, projection='gnomonic') e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2 e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2 s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v data1 = np.array(list(zip(x, y, e1, e2, s)), dtype=[('x', float), ('y', float), ('e1', float), ('e2', float), ('s', float)]) im1 = drawImage(2048, 2048, wcs1, x, y, e1, e2, s) im1.write('output/test_single_im1.fits') fitsio.write('output/test_single_cat1.fits', data1, clobber=True) x = rng.random_sample(nstars) * 2000 + 24 y = rng.random_sample(nstars) * 2000 + 24 ra2, dec2 = wcs2.toWorld(x, y, units='rad') u, v = field_center.project_rad(ra1, dec1, projection='gnomonic') # Same functions of u,v, but using the positions on chip 2 e1 = 0.02 + 2.e-5 * u - 3.e-9 * u**2 + 2.e-9 * v**2 e2 = -0.04 - 3.e-5 * v + 1.e-9 * u * v + 3.e-9 * v**2 s = 0.3 + 8.e-9 * (u**2 + v**2) - 1.e-9 * u * v data2 = np.array(list(zip(x, y, e1, e2, s)), dtype=[('x', float), ('y', float), ('e1', float), ('e2', float), ('s', float)]) im2 = drawImage(2048, 2048, wcs2, x, y, e1, e2, s) im2.write('output/test_single_im2.fits') fitsio.write('output/test_single_cat2.fits', data2, clobber=True) ra12 = np.concatenate([ra1, ra2]) dec12 = np.concatenate([dec1, dec2]) data12 = np.array(list(zip(ra12, dec12)), dtype=[('ra', float), ('dec', float)]) fitsio.write('output/test_single_cat12.fits', data12, clobber=True) # Try to fit with the right model (Moffat) and interpolant (2nd order polyomial) # Should work very well, since no noise. config = { 'input': { # A third way to input these same file names. Use GalSim config values and # explicitly specify the number of images to read 'nimages': 2, 'image_file_name': { 'type': 'FormattedStr', 'format': '%s/test_single_im%d.fits', 'items': ['output', '$image_num+1'] }, 'cat_file_name': { 'type': 'FormattedStr', 'format': '%s/test_single_cat%d.fits', 'items': ['output', '$image_num+1'] }, # Use chipnum = 1,2 rather than the default 0,1. 'chipnum': '$image_num+1', 'x_col': 'x', 'y_col': 'y', 'ra': 0., 'dec': -25., }, 'psf': { 'type': 'SingleChip', 'model': { 'type': 'Moffat', 'beta': 2.5 }, 'interp': { 'type': 'Polynomial', 'order': 2 } }, } if __name__ != '__main__': config['verbose'] = 0 with CaptureLog(level=2) as cl: psf = piff.process(config, cl.logger) #print('without nproc, log = ',cl.output) assert "Building solution for chip 1" in cl.output assert "Building solution for chip 2" in cl.output for chipnum, data, wcs in [(1, data1, wcs1), (2, data2, wcs2)]: for k in range(nstars): x = data['x'][k] y = data['y'][k] e1 = data['e1'][k] e2 = data['e2'][k] s = data['s'][k] #print('k,x,y = ',k,x,y) #print(' true s,e1,e2 = ',s,e1,e2) image_pos = galsim.PositionD(x, y) star = piff.Star.makeTarget(x=x, y=y, wcs=wcs, stamp_size=48, pointing=field_center, chipnum=chipnum) star = psf.drawStar(star) #print(' fitted s,e1,e2 = ',star.fit.params) np.testing.assert_almost_equal(star.fit.params, [s, e1, e2], decimal=6) # Chipnum is required as a property to use SingleCCDPSF star1 = piff.Star.makeTarget(x=x, y=y, wcs=wcs, stamp_size=48, pointing=field_center) with np.testing.assert_raises(ValueError): psf.drawStar(star1) star2 = piff.Star(star1.data, star.fit) # If has a fit, it hits a different error with np.testing.assert_raises(ValueError): psf.drawStar(star2)
def test_depr_select(): # A bunch of keys used to be allowed in input, but now belong in select. # Check that the old way still works, but gives a warning. # This is the input from test_stars in test_input.py dir = 'input' image_file = 'test_input_image_00.fits' cat_file = 'test_input_cat_00.fits' psf_file = os.path.join('output','test_depr_select.fits') config = { 'input' : { 'dir' : dir, 'image_file_name' : image_file, 'cat_file_name' : cat_file, 'weight_hdu' : 1, 'sky_col' : 'sky', 'gain_col' : 'gain', 'use_partial' : True, 'nstars': 15, # Just to make the test faster }, 'select': { 'max_snr' : 200, 'min_snr' : 20, 'hsm_size_reject' : 20, 'max_edge_frac': 0.25, 'stamp_center_size': 10, 'max_mask_pixels' : 20, 'reserve_frac' : 0.1, 'seed': 1234, }, 'psf': { 'model' : {'type': 'Gaussian'}, 'interp' : {'type': 'Mean'}, }, 'output' : { 'file_name' : psf_file }, } logger = piff.config.setup_logger(log_file='output/test_depr_select.log') # This is the new API print('config = ',config) piff.piffify(config, logger) psf1 = piff.read(psf_file) im1 = psf1.draw(x=23, y=34, stamp_size=16) print('len1 = ',len(psf1.stars)) # This is the old API config['input'].update(config['select']) del config['select'] config = galsim.config.CleanConfig(config) print('config = ',config) with CaptureLog(level=1) as cl: piff.piffify(config, cl.logger) assert "WARNING: Items [" in cl.output assert "] should now be in the 'select' field of the config file." in cl.output psf2 = piff.read(psf_file) print('len2 = ',len(psf2.stars)) assert len(psf1.stars) == len(psf2.stars) im2 = psf2.draw(x=23, y=34, stamp_size=16) np.testing.assert_allclose(im2.array, im1.array) # Also ok for some items to be in select, but erroneously put some in input. config['input']['min_snr'] = config['select'].pop('min_snr') config['input']['max_snr'] = config['select'].pop('max_snr') config = galsim.config.CleanConfig(config) print('config = ',config) with CaptureLog(level=1) as cl: piff.piffify(config, cl.logger) assert "WARNING: Items ['max_snr', 'min_snr'] should now be in the 'select' field of the config file." in cl.output psf3 = piff.read(psf_file) print('len3 = ',len(psf3.stars)) assert len(psf1.stars) == len(psf3.stars) im3 = psf3.draw(x=23, y=34, stamp_size=16) np.testing.assert_allclose(im3.array, im1.array)
def test_load_images(): """Test the load_images function """ if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger(log_file='output/test_load_image2.log') # Same setup as test_single_image, but without flags image = galsim.Image(2048, 2048, scale=0.26) x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ] y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ] sigma = 1.3 g1 = 0.23 g2 = -0.17 psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) for x,y in zip(x_list, y_list): bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32)) psf.drawImage(image[bounds], center=galsim.PositionD(x,y), method='no_pixel') image.addNoise(galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6)) sky = 10. image += sky image_file = os.path.join('output','test_load_images_im.fits') image.write(image_file) dtype = [ ('x','f8'), ('y','f8') ] data = np.empty(len(x_list), dtype=dtype) data['x'] = x_list data['y'] = y_list cat_file = os.path.join('output','test_load_images_cat.fits') fitsio.write(cat_file, data, clobber=True) # Make star data config = { 'image_file_name' : image_file, 'cat_file_name': cat_file, 'sky': 10 } orig_stars, wcs, pointing = piff.Input.process(config, logger) # Fit these with a simple Mean, Gaussian model = piff.Gaussian() interp = piff.Mean() psf = piff.SimplePSF(model, interp) psf.fit(orig_stars, wcs, pointing, logger=logger) psf_file = os.path.join('output','test_load_images_psf.fits') psf.write(psf_file, logger) # Read this file back in. It has the star data, but the images are blank. psf2 = piff.read(psf_file, logger) assert len(psf2.stars) == 10 for star in psf2.stars: np.testing.assert_array_equal(star.image.array, 0.) # First the old API: with CaptureLog() as cl: loaded_stars = piff.Star.load_images(psf2.stars, image_file, logger=cl.logger) assert "WARNING: The Star.load_images function is deprecated." in cl.output for star, orig in zip(loaded_stars, psf.stars): np.testing.assert_array_equal(star.image.array, orig.image.array) # Can optionally supply a different sky to subtract # (This is not allowed by the new API, but it probably doesn't make much sense.) loaded_stars = piff.Star.load_images(psf2.stars, image_file, sky=2*sky) for star, orig in zip(loaded_stars, psf.stars): np.testing.assert_array_equal(star.image.array, orig.image.array - sky) # Now the new API: psf2 = piff.read(psf_file, logger) # Reset stars to not have images in them. loaded_stars = piff.InputFiles(config).load_images(psf2.stars) for star, orig in zip(loaded_stars, psf.stars): np.testing.assert_array_equal(star.image.array, orig.image.array)
def test_reserve(): """Test the reserve_frac option. """ if __name__ == '__main__': logger = piff.config.setup_logger(verbose=2) else: logger = piff.config.setup_logger(log_file='output/test_single_reserve.log') # Make the image image = galsim.Image(2048, 2048, scale=0.26) # Where to put the stars. x_list = [ 123.12, 345.98, 567.25, 1094.94, 924.15, 1532.74, 1743.11, 888.39, 1033.29, 1409.31 ] y_list = [ 345.43, 567.45, 1094.32, 924.29, 1532.92, 1743.83, 888.83, 1033.19, 1409.20, 123.11 ] # Draw a Gaussian PSF at each location on the image. sigma = 1.3 g1 = 0.23 g2 = -0.17 true_params = [ sigma, g1, g2 ] psf = galsim.Gaussian(sigma=sigma).shear(g1=g1, g2=g2) for x,y in zip(x_list, y_list): bounds = galsim.BoundsI(int(x-31), int(x+32), int(y-31), int(y+32)) psf.drawImage(image[bounds], center=galsim.PositionD(x,y), method='no_pixel') image.addNoise(galsim.GaussianNoise(rng=galsim.BaseDeviate(1234), sigma=1e-6)) # Write out the image to a file image_file = os.path.join('output','test_simple_reserve_image.fits') image.write(image_file) # Write out the catalog to a file dtype = [ ('x','f8'), ('y','f8') ] data = np.empty(len(x_list), dtype=dtype) data['x'] = x_list data['y'] = y_list cat_file = os.path.join('output','test_simple_reserve_cat.fits') fitsio.write(cat_file, data, clobber=True) psf_file = os.path.join('output','test_simple_reserve_psf.fits') config = { 'input' : { 'image_file_name' : image_file, 'cat_file_name' : cat_file, 'stamp_size' : 32, }, 'select' : { 'reserve_frac' : 0.2, }, 'psf' : { 'model' : { 'type' : 'Gaussian', 'fastfit': True, 'include_pixel': False}, 'interp' : { 'type' : 'Mean' }, }, 'output' : { 'file_name' : psf_file }, } piff.piffify(config, logger) psf = piff.read(psf_file) assert type(psf.model) is piff.Gaussian assert type(psf.interp) is piff.Mean print('chisq = ',psf.chisq) print('dof = ',psf.dof) nreserve = len([s for s in psf.stars if s.is_reserve]) ntot = len(psf.stars) print('reserve = %s/%s'%(nreserve,ntot)) assert nreserve == 2 assert ntot == 10 print('dof =? ',(32*32 - 3) * (ntot-nreserve)) assert psf.dof == (32*32 - 3) * (ntot-nreserve) for star in psf.stars: # Fits should be good for both reserve and non-reserve stars np.testing.assert_almost_equal(star.fit.params, true_params, decimal=4) # Check the old API of putting reserve_frac in input config['input']['reserve_frac'] = 0.2 del config['select'] with CaptureLog() as cl: piff.piffify(config, cl.logger) assert "WARNING: Items ['reserve_frac'] should now be in the 'select' field" in cl.output psf = piff.read(psf_file) assert type(psf.model) is piff.Gaussian assert type(psf.interp) is piff.Mean nreserve = len([s for s in psf.stars if s.is_reserve]) ntot = len(psf.stars) assert nreserve == 2 assert ntot == 10