def test_subpixel_conv_diff(): import matplotlib.pyplot as plt from thesis_lib.testdata.generators import read_or_generate_image sp_img, sp_tab = read_or_generate_image('scopesim_grid_16_perturb2_mag18_24_subpixel') conv_img, conv_tab = read_or_generate_image('scopesim_grid_16_perturb2_mag18_24') plt.figure() plt.imshow(sp_img - conv_img) plt.show()
def benchmark_parameter_constraints(posbound, fluxbound, groupsize, seed=0): def recipe(): return scopesim_groups(N1d=1, border=300, group_size=groupsize, group_radius=12, jitter=13, magnitude=lambda N: np.random.normal(21, 2, N), custom_subpixel_psf=epsf, seed=seed) cfg.bounds = { 'x_0': (posbound, posbound), 'y_0': (posbound, posbound), 'flux_0': (fluxbound, fluxbound) } img, tab = read_or_generate_image(f'1x{groupsize}_groups_seed{seed}', cfg, recipe) session = Session(cfg, image=img, input_table=tab) session.fitter = TRFLSQFitter() session.epsf = epsf session.determine_psf_parameters() n_calls, time = timeit.Timer(lambda: session.do_astrometry()).autorange() return {'runtime': time / n_calls}
def objective(cutout_size: int, fitshape_half: int, sigma: float, iters: int): try: config = Config() config.use_catalogue_positions = True config.photometry_iterations = 1 config.oversampling = 2 config.smoothing = util.make_gauss_kernel(sigma) config.fitshape = fitshape_half*2+1 config.cutout_size = cutout_size config.epsfbuilder_iters=iters image, input_table = generators.read_or_generate_image(image_recipe, image_name, config.image_folder) result = run_photometry(image, input_table, image_name, config) result_table = util.match_observation_to_source(input_table, result.result_table) loss = np.sqrt(np.sum(result_table['offset']**2)) return loss except Exception as ex: import traceback print('\033[93m##############\033[0m') print(f'error in objective({cutout_size}, {fitshape_half}, {sigma}, {iters})') print('\033[93m##############\033[0m') error = ''.join(traceback.format_exception(type(ex), ex, ex.__traceback__)) print(error) raise ex
def session_grid(config_for_tests): config_for_tests.photometry_iterations = 1 config_for_tests.use_catalogue_positions = True config_for_tests.max_epsf_stars = 40 image, table = read_or_generate_image('testgrid', config_for_tests, lambda: convolved_grid(N1d=10)) session = astrometry_wrapper.Session(config_for_tests, image, table) # TODO set this? session.image_name # TODO add method to let it be picked up automatically? Maybe container for image, name, table? return session
def do_photometry(seed): rng = np.random.default_rng(seed=seed) def recipe(): return scopesim_grid(N1d=7, border=100, perturbation=7, magnitude=lambda N: rng.uniform(15, 26, N), custom_subpixel_psf=anisocado_psf, seed=seed) img_noisefloor, tab_noisefloor = read_or_generate_image( f'grid7_pert7_seed{seed}', recipe=recipe, force_generate=False) guess_table = tab_noisefloor.copy() guess_table['x_0'] = guess_table['x'].copy() guess_table['y_0'] = guess_table['y'].copy() guess_table['x_orig'] = guess_table['x'].copy() guess_table['y_orig'] = guess_table['y'].copy() guess_table['flux_0'] = guess_table['f'] guess_table['x_0'] += rng.uniform(-0.1, 0.1, len(guess_table)) guess_table['y_0'] += rng.uniform(-0.1, 0.1, len(guess_table)) fit_stages = [ FitStage(10, 1e-10, 1e-11, np.inf, all_individual), # first stage: get flux approximately right FitStage(60, 0.5, 0.5, 10_000, all_individual), FitStage(130, 0.1, 0.1, 5_000, all_individual), #FitStage(10, 0.1, 0.1, 1, all_individual) #FitStage(10, 0.3, 0.3, 1, all_individual), # optimize position, keep flux constant #FitStage(10, 0.2, 0.2, 5000, all_individual), #FitStage(30, 0.05, 0.05, 100, all_individual) ] photometry = IncrementalFitPhotometry(SExtractorBackground(), anisocado_psf, max_group_size=1, group_extension_radius=50, fit_stages=fit_stages, use_noise=True) result_table = photometry.do_photometry(img_noisefloor, guess_table) result_table['id'] += seed * 10000 return result_table, photometry.residual(result_table)
def diy(): """ Examples of how to generate images yourself :return: """ # generate Image for immediate use with python, won't use existing file: # see also available recipes in testdata_generators image, input_table = scopesim_grid() # choose args as needed # generate different versions of same image for statistics for i in range(10): recipe = lambda: scopesim_grid(N1d=16, perturbation=2., magnitude=lambda N: np.random.uniform( 18, 24, N)) fname = f'scopesim_grid_16_perturb2_mag18_24_{i}' image, input_table = read_or_generate_image(recipe, fname, 'test_images')
def do_photometry_nocrowd(seed): rng = np.random.default_rng(seed=seed) def recipe(): return scopesim_grid(N1d=8, border=50, perturbation=2, magnitude=lambda N: rng.uniform(18, 24, N), custom_subpixel_psf=anisocado_psf, seed=seed) img_grid_nocrowd, tab_grid_nowcrowd = read_or_generate_image( f'grid8_pert2_seed{seed}', recipe=recipe, force_generate=False) #img_grid, tab_grid = read_or_generate_image('scopesim_grid_16_perturb2_mag18_24_subpixel') guess_table = prepare_table(tab_grid_nowcrowd) photometry = grid_photometry() result_table = photometry.do_photometry(img_grid_nocrowd, guess_table) return result_table
def test_session(session_single): # equivalent way of session_single.image = 'testsingle' image, input_table = read_or_generate_image('testsingle') session_single.image = image session_single.input_table = input_table session_single.find_stars() session_single.select_epsfstars_auto() session_single.make_epsf() # Here we could e.g. change starfinder and re_run find_stars() # TODO # session.cull_detections() # session.select_epsfstars_qof() session_single.make_epsf() session_single.do_astrometry() assert session_single.tables.result_table assert len(session_single.tables.result_table) == 1
def do_photometry_nosat(seed): rng = np.random.default_rng(seed=seed) def recipe(): rng = np.random.default_rng(seed) return scopesim_grid(N1d=16, border=50, perturbation=2, magnitude=lambda N: rng.uniform(19.5, 24, N), custom_subpixel_psf=anisocado_psf, seed=seed) img_grid_nosat, tab_grid_nosat = read_or_generate_image( f'grid16_pert2_mag195_24_seed{seed}', recipe=recipe, force_generate=False) guess_table = prepare_table(tab_grid_nosat) photometry = grid_photometry() result_table = photometry.do_photometry(img_grid_nosat, guess_table) return result_table
def view_objective(cutout_size: int, fitshape_half: int, sigma: float, iters: int): config = Config() config.use_catalogue_positions = True config.photometry_iterations = 1 config.oversampling = 2 config.smoothing = util.make_gauss_kernel(sigma) config.fitshape = fitshape_half * 2 + 1 config.cutout_size = cutout_size config.epsfbuilder_iters = iters image, input_table = generators.read_or_generate_image( image_recipe, image_name, config.image_folder) result = run_photometry(image, input_table, image_name, config) result_table = util.match_observation_to_source(input_table, result.result_table) plot_xy_deviation(result_table) plot_deviation_vs_magnitude(result_table) plot_image_with_source_and_measured(image, input_table, result_table) return result, result_table
magnitude=lambda N: np.random.normal(22, 2, N)), 'gausscluster_N2000_mag22_lowpass': lambda: gaussian_cluster(2000, magnitude=lambda N: np.random.normal(22, 2, N), psf_transform=lowpass()), } if __name__ == '__main__': # download scopesim stuff to temporary location old_dir = os.path.abspath(os.getcwd()) with tempfile.TemporaryDirectory() as dir: os.chdir(dir) download(ask=False) for fname, recipe in benchmark_images.items(): read_or_generate_image(recipe, fname, os.path.join(old_dir, 'test_images')) def diy(): """ Examples of how to generate images yourself :return: """ # generate Image for immediate use with python, won't use existing file: # see also available recipes in testdata_generators image, input_table = scopesim_grid() # choose args as needed # generate different versions of same image for statistics for i in range(10): recipe = lambda: scopesim_grid(N1d=16,
def session_multi(config_for_tests) -> astrometry_wrapper.Session: config_for_tests.photometry_iterations = 1 image, table = read_or_generate_image('testmulti', config_for_tests, lambda: multi_source_testimage()) session = astrometry_wrapper.Session(config_for_tests, image, table) return session
#FitStage(30, 0.1, 0.1, 5_000, all_simultaneous) ] photometry_grid = IncrementalFitPhotometry(SExtractorBackground(), anisocado_psf, max_group_size=8, group_extension_radius=100, fit_stages=fit_stages_grid, use_noise=True) with open(out_dir / 'processing_params_synthetic_grid.txt', 'w') as f: f.write(pformat(photometry_grid.__dict__)) return photometry_grid # %% img_grid, tab_grid = read_or_generate_image( 'scopesim_grid_16_perturb2_mag18_24_subpixel') guess_table = prepare_table(tab_grid) photometry = grid_photometry() grid_result = cached(lambda: photometry.do_photometry(img_grid, guess_table), cache_dir / 'synthetic_grid', rerun=False) [(np.min(np.abs(i)), np.mean(np.abs(i)), np.max(np.abs(i))) for i in calc_devation(grid_result)] # %% #visualize_grouper(img_grid, tab_grid, 4, 90) # %% plot_residual(photometry, img_grid, grid_result)
guess_table['y_0'] += rng.uniform(-0.5, 0.5, len(guess_table)) guess_table.sort(fluxguessname, reverse=True) return guess_table # %% [markdown] # # Tight group # %% rng = np.random.default_rng(seed=10) def recipe(): return scopesim_groups(N1d=1, border=500, group_size=8, group_radius=6, jitter=8, magnitude=lambda N: rng.uniform(20.5, 21.5, size=N), custom_subpixel_psf=model, seed=11) img_tight, table = read_or_generate_image('tight8group', recipe=recipe) guess_table_tight = prepare_table(table) fit_stages_tight = [FitStage(10, 0.001, 0.001, np.inf, all_individual), # first stage: flux is wildly off, get decent guess FitStage(10, 2., 2., 50_000, all_individual), FitStage(10, 1., 1., 20_000, brightest_simultaneous(3)), FitStage(10, 1., 1., 20_000, brightest_simultaneous(5)), FitStage(10, 0.5, 0.5, np.inf, all_simultaneous), FitStage(50, 0.2, 0.2, 10_000, all_simultaneous) ] # %% photometry_tight = IncrementalFitPhotometry(MMMBackground(), model,
anisocado_psf = make_anisocado_model() rng = np.random.default_rng(seed=12) def recipe(): return scopesim_grid(N1d=7, border=100, perturbation=7, magnitude=lambda N: rng.uniform(20, 24, N), custom_subpixel_psf=anisocado_psf, seed=11) img_grid, tab_grid = read_or_generate_image('grid7_pert7', recipe=recipe, force_generate=False) guess_table = tab_grid.copy() guess_table['x_0'] = guess_table['x'].copy() guess_table['y_0'] = guess_table['y'].copy() guess_table['x_orig'] = guess_table['x'].copy() guess_table['y_orig'] = guess_table['y'].copy() guess_table['flux_0'] = guess_table['f'] guess_table['x_0'] += rng.uniform(-0.02, 0.02, len(guess_table)) guess_table['y_0'] += rng.uniform(-0.02, 0.02, len(guess_table)) fit_stages = [ FitStage(10, 1e-10, 1e-11, np.inf, all_individual), # first stage: get flux approximately right
# offsets += [np.nan] * len(seen_indices - set(lookup_tree.indices)) # offsets += [np.nan] * abs(len(input_table)-len(res_table)) # offsets = np.array(offsets) # offsets -= np.nanmean(offsets) # offsets[np.isnan(offsets)] = 50. # # return np.sqrt(np.sum(np.array(offsets)**2)) # # return starfinder_objective from thesis_lib.standalone_analysis.parameter_tuning import make_starfinder_objective if __name__ == '__main__': name = 'gausscluster_N2000_mag22' recipe = thesis_lib.testdata.definitions.benchmark_images[name] img, input_table = generators.read_or_generate_image(recipe, name) starfinder_obj = make_starfinder_objective(recipe, name) starfinder_dims = [ Real(-6., 3, name='threshold'), Real(2., 8, name='fwhm'), Real(2., 5.5, name='sigma_radius'), Real(-15., 0., name='roundlo'), Real(0., 15., name='roundhi'), Real(-15, 0., name='sharplo'), Real(0., 10., name='sharphi') ] starfinder_optimizer = skopt.Optimizer(dimensions=starfinder_dims, n_jobs=mp.cpu_count(), random_state=1,
import matplotlib.pyplot as plt from astropy.convolution import Gaussian2DKernel import numpy as np from astropy.modeling.models import Gaussian2D from photutils.psf import * from astropy.nddata import NDData cutout_size = 30 σ = 8 from thesis_lib.testdata.generators import read_or_generate_image, convolved_grid img, input_table = read_or_generate_image( lambda: convolved_grid(8, perturbation=8, kernel=Gaussian2DKernel( x_stddev=σ, x_size=201, y_size=201)), 'prepare_model_test') y, x = np.mgrid[-2 * σ:2 * σ + 0.1:0.5, -2 * σ:2 * σ + .1:0.5] epsf_data = Gaussian2D(x_stddev=σ, y_stddev=σ)(x, y) epsf_ana_pre = EPSFModel(epsf_data, flux=1, normalize=False, oversampling=2) epsf_ana = prepare_psf_model(epsf_ana_pre.copy(), renormalize_psf=False, fluxname='flux') stars = extract_stars(NDData(img), input_table, size=cutout_size) epsf_fit_pre, _ = EPSFBuilder(oversampling=2, maxiters=3, progress_bar=True)(stars) epsf_fit = prepare_psf_model(epsf_fit_pre.copy(),
from thesis_lib.testdata.generators import read_or_generate_image from thesis_lib.experimental.saturation_model import SaturationModel, read_scopesim_linearity from thesis_lib.config import Config from thesis_lib import scopesim_helper from thesis_lib import util # - scopesim_helper.download() # takeaway: # - appyling saturation after epsf fitting does not help, as the epsf is kind of adapted to it already # - inverting before all EPSF photometry stuff seems more usefull # - χ^2 is bad for sorting candidates, as it picks faint ones. need to compensate for brightness somehow... img, input_table = read_or_generate_image('gausscluster_N2000_mag22_subpixel') saturation_model = SaturationModel( read_scopesim_linearity(Config.instance().scopesim_working_dir / 'inst_pkgs/MICADO/FPA_linearity.dat')) img = saturation_model.inverse_eval(img) mean, median, std = sigma_clipped_stats(img) finder = DAOStarFinder(threshold=median - 5 * std, fwhm=3.5, sigma_radius=2.7) image_no_background = img - median all_stars = finder(img) stars_tbl = all_stars.copy() stars_tbl.rename_columns(['xcentroid', 'ycentroid'], ['x', 'y'])
def session_single(config_for_tests) -> astrometry_wrapper.Session: image, table = read_or_generate_image('testsingle', config_for_tests, lambda: one_source_testimage()) session = astrometry_wrapper.Session(config_for_tests, image, table) return session
def recipe(): return scopesim_groups(N1d=13, border=70, group_size=1, group_radius=1, jitter=15, magnitude=lambda N: [20.5] * N, custom_subpixel_psf=epsf, seed=10) cfg.fithshape = 199 cfg.bounds = {'x_0': (0.3, 0.3), 'y_0': (0.3, 0.3), 'flux_0': (10000, 10000)} cfg.niters = 1 img, tab = read_or_generate_image(f'169x1_group', cfg, recipe) trialsession = Session(cfg, image=img, input_table=tab) trialsession.fitter = TRFLSQFitter() trialsession.epsf = epsf trialsession.determine_psf_parameters() trialsession.do_astrometry() # %% plot_image_with_source_and_measured(img, tab, trialsession.tables.result_table) pass # %% plot_xy_deviation(trialsession.tables.result_table) pass # %% [markdown]
import matplotlib.pyplot as plt #download() epsf = make_anisocado_model() cfg = config.Config() cfg.use_catalogue_positions = True cfg.separation_factor = 20 cfg.photometry_iterations = 1 cfg.perturb_catalogue_guess = 0.01 def recipe(): return scopesim_groups(N1d=1, border=500, group_size=10, group_radius=10, jitter=12, magnitude=lambda N: [20.5]*N, custom_subpixel_psf=epsf, seed=10) cfg.fithshape=91 cfg.bounds = {'x_0': (0.2, 0.2), 'y_0': (0.2, 0.2), 'flux_0': (10000, 10000)} img, tab = read_or_generate_image(f'1x10_group', cfg, recipe) trialsession = Session(cfg, image=img, input_table=tab) trialsession.fitter = TRFLSQFitter() trialsession.epsf = epsf trialsession.determine_psf_parameters() trialsession.do_astrometry_mine() plot_image_with_source_and_measured(img, tab, trialsession.tables.result_table) plot_xy_deviation(trialsession.tables.result_table) plt.show()
from thesis_lib.testdata.generators import read_or_generate_image from thesis_lib.testdata.definitions import benchmark_images for key in benchmark_images: read_or_generate_image(key)
(result.x[col] - x0_low) / (x0_high - x0_low) * img_points, (result.x[row] - x1_low) / (x1_high - x1_low) * img_points, 'r*') ua.set_xticks(xticks) ua.set_yticks(yticks) ua.set_xticklabels(xtick_labels) ua.set_yticklabels(ytick_labels) ua.set_xlabel(dimensions[col]) ua.set_ylabel(dimensions[row]) #fig.colorbar(im, ax=ax[j, i]) result_filename = 'daofind_opt_full_redo.pkl' image_name = 'gausscluster_N2000_mag22' image_recipe = benchmark_images[image_name] img, ref_table = read_or_generate_image(image_recipe, image_name) # for lowpass: # roundlo = -32.6 # roundhi = 3.91 # sharplo = -21.70 # sharphi = 17.73 # threshold, fwhm, sigma_radius = [-0.5961880021208964, 2.820694454041728, 3.0288937031114287] # for no lowpass: # roundlo, roundhi, sharplo, sharphi = -10, 8.5, -11, 2.4 # threshold, fwhm, sigma_radius = -2.942323484472957, 2.24827752258762, 4.367898600244883 # threshold, fwhm, sigma_radius = -0.9985482815217563, 2.790086326795894, 4.015963504846637 #-11.288660990445317, 1.000361717764853, -2.5904444426411564, 1.4934012105230134
if __name__ == '__main__': model = make_anisocado_model() rng = np.random.default_rng(seed=11) def recipe(): return scopesim_groups(N1d=1, border=500, group_size=10, group_radius=12, jitter=12, magnitude=lambda N: [20.5] * N, custom_subpixel_psf=model, seed=10) img, table = read_or_generate_image('group_image_10', recipe=recipe) guess_table = table.copy() guess_table.rename_columns(['x', 'y', 'f'], [xname, yname, fluxname]) guess_table.sort(fluxname, reverse=True) guess_table[xname] += rng.uniform(-0.01, 0.01, len(guess_table)) guess_table[yname] += rng.uniform(-0.01, 0.01, len(guess_table)) guess_table[fluxname] *= 1e15 objective, x0, bounds = make_objective_function(model, guess_table, img, to_optimize=set(), posbound=0.05) # t=timeit.Timer(lambda: least_squares(objective, x0, bounds=bounds)).repeat(repeat=5, number=1) # print(t)