def test_subpixel_conv_diff():
import matplotlib.pyplot as plt
from thesis_lib.testdata.generators import read_or_generate_image
sp_img, sp_tab = read_or_generate_image('scopesim_grid_16_perturb2_mag18_24_subpixel')
conv_img, conv_tab = read_or_generate_image('scopesim_grid_16_perturb2_mag18_24')
plt.figure()
plt.imshow(sp_img - conv_img)
plt.show()
def benchmark_parameter_constraints(posbound, fluxbound, groupsize, seed=0):
def recipe():
return scopesim_groups(N1d=1,
border=300,
group_size=groupsize,
group_radius=12,
jitter=13,
magnitude=lambda N: np.random.normal(21, 2, N),
custom_subpixel_psf=epsf,
seed=seed)
cfg.bounds = {
'x_0': (posbound, posbound),
'y_0': (posbound, posbound),
'flux_0': (fluxbound, fluxbound)
}
img, tab = read_or_generate_image(f'1x{groupsize}_groups_seed{seed}', cfg,
recipe)
session = Session(cfg, image=img, input_table=tab)
session.fitter = TRFLSQFitter()
session.epsf = epsf
session.determine_psf_parameters()
n_calls, time = timeit.Timer(lambda: session.do_astrometry()).autorange()
return {'runtime': time / n_calls}
def objective(cutout_size: int, fitshape_half: int, sigma: float, iters: int):
try:
config = Config()
config.use_catalogue_positions = True
config.photometry_iterations = 1
config.oversampling = 2
config.smoothing = util.make_gauss_kernel(sigma)
config.fitshape = fitshape_half*2+1
config.cutout_size = cutout_size
config.epsfbuilder_iters=iters
image, input_table = generators.read_or_generate_image(image_recipe, image_name, config.image_folder)
result = run_photometry(image, input_table, image_name, config)
result_table = util.match_observation_to_source(input_table, result.result_table)
loss = np.sqrt(np.sum(result_table['offset']**2))
return loss
except Exception as ex:
import traceback
print('\033[93m##############\033[0m')
print(f'error in objective({cutout_size}, {fitshape_half}, {sigma}, {iters})')
print('\033[93m##############\033[0m')
error = ''.join(traceback.format_exception(type(ex), ex, ex.__traceback__))
print(error)
raise ex
def session_grid(config_for_tests):
config_for_tests.photometry_iterations = 1
config_for_tests.use_catalogue_positions = True
config_for_tests.max_epsf_stars = 40
image, table = read_or_generate_image('testgrid', config_for_tests,
lambda: convolved_grid(N1d=10))
session = astrometry_wrapper.Session(config_for_tests, image, table)
# TODO set this? session.image_name
# TODO add method to let it be picked up automatically? Maybe container for image, name, table?
return session
def do_photometry(seed):
rng = np.random.default_rng(seed=seed)
def recipe():
return scopesim_grid(N1d=7,
border=100,
perturbation=7,
magnitude=lambda N: rng.uniform(15, 26, N),
custom_subpixel_psf=anisocado_psf,
seed=seed)
img_noisefloor, tab_noisefloor = read_or_generate_image(
f'grid7_pert7_seed{seed}', recipe=recipe, force_generate=False)
guess_table = tab_noisefloor.copy()
guess_table['x_0'] = guess_table['x'].copy()
guess_table['y_0'] = guess_table['y'].copy()
guess_table['x_orig'] = guess_table['x'].copy()
guess_table['y_orig'] = guess_table['y'].copy()
guess_table['flux_0'] = guess_table['f']
guess_table['x_0'] += rng.uniform(-0.1, 0.1, len(guess_table))
guess_table['y_0'] += rng.uniform(-0.1, 0.1, len(guess_table))
fit_stages = [
FitStage(10, 1e-10, 1e-11, np.inf,
all_individual), # first stage: get flux approximately right
FitStage(60, 0.5, 0.5, 10_000, all_individual),
FitStage(130, 0.1, 0.1, 5_000, all_individual),
#FitStage(10, 0.1, 0.1, 1, all_individual)
#FitStage(10, 0.3, 0.3, 1, all_individual), # optimize position, keep flux constant
#FitStage(10, 0.2, 0.2, 5000, all_individual),
#FitStage(30, 0.05, 0.05, 100, all_individual)
]
photometry = IncrementalFitPhotometry(SExtractorBackground(),
anisocado_psf,
max_group_size=1,
group_extension_radius=50,
fit_stages=fit_stages,
use_noise=True)
result_table = photometry.do_photometry(img_noisefloor, guess_table)
result_table['id'] += seed * 10000
return result_table, photometry.residual(result_table)
def diy():
"""
Examples of how to generate images yourself
:return:
"""
# generate Image for immediate use with python, won't use existing file:
# see also available recipes in testdata_generators
image, input_table = scopesim_grid() # choose args as needed
# generate different versions of same image for statistics
for i in range(10):
recipe = lambda: scopesim_grid(N1d=16,
perturbation=2.,
magnitude=lambda N: np.random.uniform(
18, 24, N))
fname = f'scopesim_grid_16_perturb2_mag18_24_{i}'
image, input_table = read_or_generate_image(recipe, fname,
'test_images')
def do_photometry_nocrowd(seed):
rng = np.random.default_rng(seed=seed)
def recipe():
return scopesim_grid(N1d=8,
border=50,
perturbation=2,
magnitude=lambda N: rng.uniform(18, 24, N),
custom_subpixel_psf=anisocado_psf,
seed=seed)
img_grid_nocrowd, tab_grid_nowcrowd = read_or_generate_image(
f'grid8_pert2_seed{seed}', recipe=recipe, force_generate=False)
#img_grid, tab_grid = read_or_generate_image('scopesim_grid_16_perturb2_mag18_24_subpixel')
guess_table = prepare_table(tab_grid_nowcrowd)
photometry = grid_photometry()
result_table = photometry.do_photometry(img_grid_nocrowd, guess_table)
return result_table
def test_session(session_single):
# equivalent way of
session_single.image = 'testsingle'
image, input_table = read_or_generate_image('testsingle')
session_single.image = image
session_single.input_table = input_table
session_single.find_stars()
session_single.select_epsfstars_auto()
session_single.make_epsf()
# Here we could e.g. change starfinder and re_run find_stars()
# TODO
# session.cull_detections()
# session.select_epsfstars_qof()
session_single.make_epsf()
session_single.do_astrometry()
assert session_single.tables.result_table
assert len(session_single.tables.result_table) == 1
def do_photometry_nosat(seed):
rng = np.random.default_rng(seed=seed)
def recipe():
rng = np.random.default_rng(seed)
return scopesim_grid(N1d=16,
border=50,
perturbation=2,
magnitude=lambda N: rng.uniform(19.5, 24, N),
custom_subpixel_psf=anisocado_psf,
seed=seed)
img_grid_nosat, tab_grid_nosat = read_or_generate_image(
f'grid16_pert2_mag195_24_seed{seed}',
recipe=recipe,
force_generate=False)
guess_table = prepare_table(tab_grid_nosat)
photometry = grid_photometry()
result_table = photometry.do_photometry(img_grid_nosat, guess_table)
return result_table
def view_objective(cutout_size: int, fitshape_half: int, sigma: float,
iters: int):
config = Config()
config.use_catalogue_positions = True
config.photometry_iterations = 1
config.oversampling = 2
config.smoothing = util.make_gauss_kernel(sigma)
config.fitshape = fitshape_half * 2 + 1
config.cutout_size = cutout_size
config.epsfbuilder_iters = iters
image, input_table = generators.read_or_generate_image(
image_recipe, image_name, config.image_folder)
result = run_photometry(image, input_table, image_name, config)
result_table = util.match_observation_to_source(input_table,
result.result_table)
plot_xy_deviation(result_table)
plot_deviation_vs_magnitude(result_table)
plot_image_with_source_and_measured(image, input_table, result_table)
return result, result_table
magnitude=lambda N: np.random.normal(22, 2, N)),
'gausscluster_N2000_mag22_lowpass':
lambda: gaussian_cluster(2000,
magnitude=lambda N: np.random.normal(22, 2, N),
psf_transform=lowpass()),
}
if __name__ == '__main__':
# download scopesim stuff to temporary location
old_dir = os.path.abspath(os.getcwd())
with tempfile.TemporaryDirectory() as dir:
os.chdir(dir)
download(ask=False)
for fname, recipe in benchmark_images.items():
read_or_generate_image(recipe, fname,
os.path.join(old_dir, 'test_images'))
def diy():
"""
Examples of how to generate images yourself
:return:
"""
# generate Image for immediate use with python, won't use existing file:
# see also available recipes in testdata_generators
image, input_table = scopesim_grid() # choose args as needed
# generate different versions of same image for statistics
for i in range(10):
recipe = lambda: scopesim_grid(N1d=16,
def session_multi(config_for_tests) -> astrometry_wrapper.Session:
config_for_tests.photometry_iterations = 1
image, table = read_or_generate_image('testmulti', config_for_tests,
lambda: multi_source_testimage())
session = astrometry_wrapper.Session(config_for_tests, image, table)
return session
#FitStage(30, 0.1, 0.1, 5_000, all_simultaneous)
]
photometry_grid = IncrementalFitPhotometry(SExtractorBackground(),
anisocado_psf,
max_group_size=8,
group_extension_radius=100,
fit_stages=fit_stages_grid,
use_noise=True)
with open(out_dir / 'processing_params_synthetic_grid.txt', 'w') as f:
f.write(pformat(photometry_grid.__dict__))
return photometry_grid
# %%
img_grid, tab_grid = read_or_generate_image(
'scopesim_grid_16_perturb2_mag18_24_subpixel')
guess_table = prepare_table(tab_grid)
photometry = grid_photometry()
grid_result = cached(lambda: photometry.do_photometry(img_grid, guess_table),
cache_dir / 'synthetic_grid',
rerun=False)
[(np.min(np.abs(i)), np.mean(np.abs(i)), np.max(np.abs(i)))
for i in calc_devation(grid_result)]
# %%
#visualize_grouper(img_grid, tab_grid, 4, 90)
# %%
plot_residual(photometry, img_grid, grid_result)
guess_table['y_0'] += rng.uniform(-0.5, 0.5, len(guess_table))
guess_table.sort(fluxguessname, reverse=True)
return guess_table
# %% [markdown]
# # Tight group
# %%
rng = np.random.default_rng(seed=10)
def recipe():
return scopesim_groups(N1d=1, border=500, group_size=8, group_radius=6, jitter=8,
magnitude=lambda N: rng.uniform(20.5, 21.5, size=N),
custom_subpixel_psf=model, seed=11)
img_tight, table = read_or_generate_image('tight8group', recipe=recipe)
guess_table_tight = prepare_table(table)
fit_stages_tight = [FitStage(10, 0.001, 0.001, np.inf, all_individual), # first stage: flux is wildly off, get decent guess
FitStage(10, 2., 2., 50_000, all_individual),
FitStage(10, 1., 1., 20_000, brightest_simultaneous(3)),
FitStage(10, 1., 1., 20_000, brightest_simultaneous(5)),
FitStage(10, 0.5, 0.5, np.inf, all_simultaneous),
FitStage(50, 0.2, 0.2, 10_000, all_simultaneous)
]
# %%
photometry_tight = IncrementalFitPhotometry(MMMBackground(),
model,
anisocado_psf = make_anisocado_model()
rng = np.random.default_rng(seed=12)
def recipe():
return scopesim_grid(N1d=7,
border=100,
perturbation=7,
magnitude=lambda N: rng.uniform(20, 24, N),
custom_subpixel_psf=anisocado_psf,
seed=11)
img_grid, tab_grid = read_or_generate_image('grid7_pert7',
recipe=recipe,
force_generate=False)
guess_table = tab_grid.copy()
guess_table['x_0'] = guess_table['x'].copy()
guess_table['y_0'] = guess_table['y'].copy()
guess_table['x_orig'] = guess_table['x'].copy()
guess_table['y_orig'] = guess_table['y'].copy()
guess_table['flux_0'] = guess_table['f']
guess_table['x_0'] += rng.uniform(-0.02, 0.02, len(guess_table))
guess_table['y_0'] += rng.uniform(-0.02, 0.02, len(guess_table))
fit_stages = [
FitStage(10, 1e-10, 1e-11, np.inf,
all_individual), # first stage: get flux approximately right
# offsets += [np.nan] * len(seen_indices - set(lookup_tree.indices))
# offsets += [np.nan] * abs(len(input_table)-len(res_table))
# offsets = np.array(offsets)
# offsets -= np.nanmean(offsets)
# offsets[np.isnan(offsets)] = 50.
#
# return np.sqrt(np.sum(np.array(offsets)**2))
#
# return starfinder_objective
from thesis_lib.standalone_analysis.parameter_tuning import make_starfinder_objective
if __name__ == '__main__':
name = 'gausscluster_N2000_mag22'
recipe = thesis_lib.testdata.definitions.benchmark_images[name]
img, input_table = generators.read_or_generate_image(recipe, name)
starfinder_obj = make_starfinder_objective(recipe, name)
starfinder_dims = [
Real(-6., 3, name='threshold'),
Real(2., 8, name='fwhm'),
Real(2., 5.5, name='sigma_radius'),
Real(-15., 0., name='roundlo'),
Real(0., 15., name='roundhi'),
Real(-15, 0., name='sharplo'),
Real(0., 10., name='sharphi')
]
starfinder_optimizer = skopt.Optimizer(dimensions=starfinder_dims,
n_jobs=mp.cpu_count(),
random_state=1,
import matplotlib.pyplot as plt
from astropy.convolution import Gaussian2DKernel
import numpy as np
from astropy.modeling.models import Gaussian2D
from photutils.psf import *
from astropy.nddata import NDData
cutout_size = 30
σ = 8
from thesis_lib.testdata.generators import read_or_generate_image, convolved_grid
img, input_table = read_or_generate_image(
lambda: convolved_grid(8,
perturbation=8,
kernel=Gaussian2DKernel(
x_stddev=σ, x_size=201, y_size=201)),
'prepare_model_test')
y, x = np.mgrid[-2 * σ:2 * σ + 0.1:0.5, -2 * σ:2 * σ + .1:0.5]
epsf_data = Gaussian2D(x_stddev=σ, y_stddev=σ)(x, y)
epsf_ana_pre = EPSFModel(epsf_data, flux=1, normalize=False, oversampling=2)
epsf_ana = prepare_psf_model(epsf_ana_pre.copy(),
renormalize_psf=False,
fluxname='flux')
stars = extract_stars(NDData(img), input_table, size=cutout_size)
epsf_fit_pre, _ = EPSFBuilder(oversampling=2, maxiters=3,
progress_bar=True)(stars)
epsf_fit = prepare_psf_model(epsf_fit_pre.copy(),
from thesis_lib.testdata.generators import read_or_generate_image
from thesis_lib.experimental.saturation_model import SaturationModel, read_scopesim_linearity
from thesis_lib.config import Config
from thesis_lib import scopesim_helper
from thesis_lib import util
# -
scopesim_helper.download()
# takeaway:
# - appyling saturation after epsf fitting does not help, as the epsf is kind of adapted to it already
# - inverting before all EPSF photometry stuff seems more usefull
# - χ^2 is bad for sorting candidates, as it picks faint ones. need to compensate for brightness somehow...
img, input_table = read_or_generate_image('gausscluster_N2000_mag22_subpixel')
saturation_model = SaturationModel(
read_scopesim_linearity(Config.instance().scopesim_working_dir /
'inst_pkgs/MICADO/FPA_linearity.dat'))
img = saturation_model.inverse_eval(img)
mean, median, std = sigma_clipped_stats(img)
finder = DAOStarFinder(threshold=median - 5 * std, fwhm=3.5, sigma_radius=2.7)
image_no_background = img - median
all_stars = finder(img)
stars_tbl = all_stars.copy()
stars_tbl.rename_columns(['xcentroid', 'ycentroid'], ['x', 'y'])
def session_single(config_for_tests) -> astrometry_wrapper.Session:
image, table = read_or_generate_image('testsingle', config_for_tests,
lambda: one_source_testimage())
session = astrometry_wrapper.Session(config_for_tests, image, table)
return session
def recipe():
return scopesim_groups(N1d=13,
border=70,
group_size=1,
group_radius=1,
jitter=15,
magnitude=lambda N: [20.5] * N,
custom_subpixel_psf=epsf,
seed=10)
cfg.fithshape = 199
cfg.bounds = {'x_0': (0.3, 0.3), 'y_0': (0.3, 0.3), 'flux_0': (10000, 10000)}
cfg.niters = 1
img, tab = read_or_generate_image(f'169x1_group', cfg, recipe)
trialsession = Session(cfg, image=img, input_table=tab)
trialsession.fitter = TRFLSQFitter()
trialsession.epsf = epsf
trialsession.determine_psf_parameters()
trialsession.do_astrometry()
# %%
plot_image_with_source_and_measured(img, tab, trialsession.tables.result_table)
pass
# %%
plot_xy_deviation(trialsession.tables.result_table)
pass
# %% [markdown]
import matplotlib.pyplot as plt
#download()
epsf = make_anisocado_model()
cfg = config.Config()
cfg.use_catalogue_positions = True
cfg.separation_factor = 20
cfg.photometry_iterations = 1
cfg.perturb_catalogue_guess = 0.01
def recipe():
return scopesim_groups(N1d=1, border=500, group_size=10, group_radius=10, jitter=12,
magnitude=lambda N: [20.5]*N,
custom_subpixel_psf=epsf, seed=10)
cfg.fithshape=91
cfg.bounds = {'x_0': (0.2, 0.2), 'y_0': (0.2, 0.2), 'flux_0': (10000, 10000)}
img, tab = read_or_generate_image(f'1x10_group', cfg, recipe)
trialsession = Session(cfg, image=img, input_table=tab)
trialsession.fitter = TRFLSQFitter()
trialsession.epsf = epsf
trialsession.determine_psf_parameters()
trialsession.do_astrometry_mine()
plot_image_with_source_and_measured(img, tab, trialsession.tables.result_table)
plot_xy_deviation(trialsession.tables.result_table)
plt.show()
from thesis_lib.testdata.generators import read_or_generate_image
from thesis_lib.testdata.definitions import benchmark_images
for key in benchmark_images:
read_or_generate_image(key)
(result.x[col] - x0_low) / (x0_high - x0_low) * img_points,
(result.x[row] - x1_low) / (x1_high - x1_low) * img_points,
'r*')
ua.set_xticks(xticks)
ua.set_yticks(yticks)
ua.set_xticklabels(xtick_labels)
ua.set_yticklabels(ytick_labels)
ua.set_xlabel(dimensions[col])
ua.set_ylabel(dimensions[row])
#fig.colorbar(im, ax=ax[j, i])
result_filename = 'daofind_opt_full_redo.pkl'
image_name = 'gausscluster_N2000_mag22'
image_recipe = benchmark_images[image_name]
img, ref_table = read_or_generate_image(image_recipe, image_name)
# for lowpass:
# roundlo = -32.6
# roundhi = 3.91
# sharplo = -21.70
# sharphi = 17.73
# threshold, fwhm, sigma_radius = [-0.5961880021208964, 2.820694454041728, 3.0288937031114287]
# for no lowpass:
# roundlo, roundhi, sharplo, sharphi = -10, 8.5, -11, 2.4
# threshold, fwhm, sigma_radius = -2.942323484472957, 2.24827752258762, 4.367898600244883
# threshold, fwhm, sigma_radius = -0.9985482815217563, 2.790086326795894, 4.015963504846637
#-11.288660990445317, 1.000361717764853, -2.5904444426411564, 1.4934012105230134
if __name__ == '__main__':
model = make_anisocado_model()
rng = np.random.default_rng(seed=11)
def recipe():
return scopesim_groups(N1d=1,
border=500,
group_size=10,
group_radius=12,
jitter=12,
magnitude=lambda N: [20.5] * N,
custom_subpixel_psf=model,
seed=10)
img, table = read_or_generate_image('group_image_10', recipe=recipe)
guess_table = table.copy()
guess_table.rename_columns(['x', 'y', 'f'], [xname, yname, fluxname])
guess_table.sort(fluxname, reverse=True)
guess_table[xname] += rng.uniform(-0.01, 0.01, len(guess_table))
guess_table[yname] += rng.uniform(-0.01, 0.01, len(guess_table))
guess_table[fluxname] *= 1e15
objective, x0, bounds = make_objective_function(model,
guess_table,
img,
to_optimize=set(),
posbound=0.05)
# t=timeit.Timer(lambda: least_squares(objective, x0, bounds=bounds)).repeat(repeat=5, number=1)
# print(t)
