def phot(
image,
syntax,
df,
fwhm,
LOG=None,
):
import sys
import numpy as np
import os
import logging
from autophot.packages.aperture import ap_phot
from autophot.packages.functions import mag
logger = logging.getLogger(__name__)
try:
ap_dict = {}
ap_dict['inf_ap'] = syntax['inf_ap_size'] * fwhm
ap_dict['ap'] = syntax['ap_size'] * fwhm
positions = list(zip(np.array(df['x_pix']), np.array(df['y_pix'])))
for key, val in ap_dict.items():
try:
ap, bkg = ap_phot(positions,
image,
radius=val,
r_in=val + syntax['r_in_size'] * fwhm,
r_out=val + syntax['r_out_size'] * fwhm)
df['flux_' + str(key)] = ap
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.info(exc_type, fname, exc_tb.tb_lineno, e)
pass
df['mag_inst'] = mag(df.flux_ap / syntax['exp_time'], 0)
if syntax['save_dataframe']:
df.to_csv(str(syntax['fname_dataframe']) + '.csv')
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.info(exc_type, fname, exc_tb.tb_lineno, e)
return df
def rm_bkg(source,
syntax,
xc=0,
yc=0):
import logging
import warnings
import numpy as np
from astropy.stats import SigmaClip
from photutils import Background2D, MedianBackground,MADStdBackgroundRMS, BkgIDWInterpolator
from autophot.packages.aperture import ap_phot
from astropy.modeling import models, fitting
from photutils import CircularAperture
from photutils import SExtractorBackground
logger = logging.getLogger(__name__)
# try:
# source_size = syntax['image_radius']
# except:
source_size = 2*syntax['fwhm']
try:
if syntax['psf_bkg_surface'] and not syntax['psf_bkg_poly']:
sigma_clip = SigmaClip(sigma=syntax['lim_SNR'])
bkg_estimator = MADStdBackgroundRMS()
positions = [source.shape[0]/2,source.shape[1]/2]
aperture = CircularAperture(positions, r=source_size)
masks = aperture.to_mask(method='center')
mask_array = masks.to_image(shape=((source.shape[0], source.shape[1])))
bkg = Background2D(source,
box_size = (3, 3),
mask = mask_array,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=SExtractorBackground(),
interpolator= BkgIDWInterpolator(),
# edge_method = 'pad'
)
surface = bkg.background
# bkg_median = np.nanmedian(bkg.background_median)
source_bkg_free = source - surface
elif syntax['psf_bkg_poly']:
surface_function_init = models.Polynomial2D(degree=syntax['psf_bkg_poly_degree'])
fit_surface = fitting.LevMarLSQFitter()
x = np.arange(0,source.shape[0])
y = np.arange(0,source.shape[0])
xx,yy= np.meshgrid(x,y)
positions = [source.shape[0]/2,source.shape[1]/2]
aperture = CircularAperture(positions, r=source_size)
masks = aperture.to_mask(method='center')
mask_array = masks.to_image(shape=((source.shape[0], source.shape[1])))
source[mask_array.astype(bool)] == np.nan
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
surface_fit = fit_surface(surface_function_init, xx, yy, source)
surface = surface_fit(xx,yy)
# bkg_median = np.nanmedian(surface)
source_bkg_free = source - surface
elif syntax['psf_bkg_local']:
pos = list(zip([xc],[yc]))
ap,bkg = ap_phot(pos,
source,
radius = syntax['ap_size'] * syntax['fwhm'],
r_in = syntax['r_in_size'] * syntax['fwhm'],
r_out = syntax['r_out_size'] * syntax['fwhm'])
surface = np.ones(source.shape) * bkg
source_bkg_free = source - surface
except Exception as e:
logger.exception(e)
return source_bkg_free,surface
def subtract_bkg(source, syntax, xc=0, yc=0):
import logging
import warnings
import numpy as np
from astropy.stats import SigmaClip
from photutils import Background2D, MedianBackground
from autophot.packages.aperture import ap_phot
from astropy.modeling import models, fitting
logger = logging.getLogger(__name__)
try:
if syntax['psf_bkg_surface']:
sigma_clip = SigmaClip(sigma=syntax['Lim_SNR'])
bkg_estimator = MedianBackground()
bkg = Background2D(source, (3, 3),
filter_size=(5, 5),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
surface = bkg.background
bkg_median = np.nanmedian(bkg.background_median)
source_bkg_free = source - surface
if syntax['psf_bkg_poly']:
surface_function_init = models.Polynomial2D(
degree=syntax['psf_bkg_poly_degree'])
fit_surface = fitting.LevMarLSQFitter()
x = np.arange(0, source.shape[0])
y = np.arange(0, source.shape[0])
xx, yy = np.meshgrid(x, y)
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
surface_fit = fit_surface(surface_function_init, xx, yy,
source)
surface = surface_fit(xx, yy)
bkg_median = np.nanmedian(surface)
source_bkg_free = source - surface
if syntax['psf_bkg_local']:
pos = list(zip([xc], [yc]))
ap, bkg = ap_phot(pos,
source,
radius=syntax['ap_size'] * syntax['fwhm'],
r_in=syntax['r_in_size'] * syntax['fwhm'],
r_out=syntax['r_out_size'] * syntax['fwhm'])
source_bkg_free = source - (np.ones(source.shape) * bkg)
bkg_median = bkg[0]
except Exception as e:
logger.exception(e)
return source_bkg_free, bkg_median
def build_r_table(base_image,selected_sources,syntax,fwhm):
'''
Build tables of residuals from bright isolated sources given in selected_sources dataframe
Function will function selected function to these sources and normialise there residaul array to build a residual image
which will then be used to make a PSF for the image
'''
import numpy as np
from astropy.stats import sigma_clipped_stats
from photutils import DAOStarFinder
import pandas as pd
import lmfit
import logging
from autophot.packages.functions import pix_dist,gauss_sigma2fwhm
from autophot.packages.uncertain import SNR
from autophot.packages.aperture import ap_phot
from autophot.packages.functions import gauss_2d,gauss_fwhm2sigma
from autophot.packages.functions import moffat_2d,moffat_fwhm
if syntax['use_moffat']:
fitting_model = moffat_2d
fitting_model_fwhm = moffat_fwhm
else:
fitting_model = gauss_2d
fitting_model_fwhm = gauss_sigma2fwhm
try:
logger = logging.getLogger(__name__)
image = base_image.copy()
# Only fit to a small image with radius ~the fwhm
fitting_radius = int(np.ceil(fwhm))
# for matchinf each source residual image with will regrid the image for shifting later
regriding_size = int(syntax['regrid_size'])
# m = regriding_size
if regriding_size % 2 > 0:
logger.info('regrid size must be even adding 1')
regriding_size += 1
# FWHM/sigma fits
fwhm_fit = []
# what sources will be used
construction_sources = []
# Residual Table in extended format
residual_table = np.zeros((int(2 * syntax['scale'] * regriding_size), int(2 * syntax['scale']*regriding_size)))
# if syntax['remove_sat']:
# len_with_sat = len(selected_sources)
# selected_sources = selected_sources[selected_sources['flux_ap']+selected_sources['median']<= syntax['sat_lvl']]
# print('%d saturdated PSF stars removed' % (len_with_sat-len(selected_sources)))
selected_sources['dist'] = pix_dist(syntax['target_x_pix'],selected_sources.x_pix,
syntax['target_y_pix'],selected_sources.y_pix)
selected_sources_mask = sigma_clip(selected_sources['median'], sigma=3, maxiters=5,masked=True)
selected_sources = selected_sources[~selected_sources_mask.mask]
if syntax['use_local_stars_for_PSF']:
'''
Use local stars given by 'use_acrmin' parameter
'''
selected_sources_test = selected_sources[selected_sources['dist'] <= syntax['local_radius']]
selected_sources = selected_sources_test
flux_idx = [i for i in selected_sources.flux_ap.sort_values(ascending = False).index]
sources_used = 1
n = 0
failsafe = 0
psf_mag = []
image_radius_lst = []
sources_dict = {}
while sources_used <= syntax['psf_source_no']:
if failsafe>25:
logger.info('PSF - Failed to build psf')
residual_table=None
fwhm_fit = fwhm
if n >= len(flux_idx):
if sources_used >= syntax['min_psf_source_no']:
logger.info('Using worst case scenario number of sources')
break
logger.info('PSF - Ran out of sources')
residual_table=None
fwhm_fit = fwhm
break
try:
idx = flux_idx[n]
n+=1
# Inital guess at where psf source is is
psf_image = image[int(selected_sources.y_pix[idx]-syntax['scale']): int(selected_sources.y_pix[idx] + syntax['scale']),
int(selected_sources.x_pix[idx]-syntax['scale']): int(selected_sources.x_pix[idx] + syntax['scale'])]
if len(psf_image) == 0:
logger.info('PSF image ERROR')
continue
try:
if np.min(psf_image) == np.nan:
continue
except:
continue
mean, median, std = sigma_clipped_stats(psf_image, sigma = syntax['source_sigma_close_up'], maxiters = syntax['iters'])
daofind = DAOStarFinder(fwhm=np.floor(fwhm),
threshold = syntax['lim_SNR']*std,
roundlo = -1.0, roundhi = 1.0,
sharplo = 0.2, sharphi = 1.0)
sources = daofind(psf_image - median)
if sources is None:
sources = []
if len(sources) > 1:
dist = [list(pix_dist(
sources['xcentroid'][i],
sources['xcentroid'],
sources['ycentroid'][i],
sources['ycentroid']) for i in range(len(sources)))]
dist = np.array(list(set(np.array(dist).flatten())))
if all(dist < 2):
pass
else:
continue
psf_image = image[int(selected_sources.y_pix[idx]-syntax['scale']): int(selected_sources.y_pix[idx]+syntax['scale']),
int(selected_sources.x_pix[idx]-syntax['scale']): int(selected_sources.x_pix[idx]+syntax['scale'])]
psf_image_bkg_free,bkg_surface = rm_bkg(psf_image,syntax,psf_image.shape[0]/2,psf_image.shape[0]/2)
x = np.arange(0,2*syntax['scale'])
xx,yy= np.meshgrid(x,x)
pars = lmfit.Parameters()
pars.add('A',value = np.nanmax(psf_image_bkg_free),min=0)
pars.add('x0',value = psf_image_bkg_free.shape[1]/2,min = 0, max =psf_image_bkg_free.shape[1] )
pars.add('y0',value = psf_image_bkg_free.shape[0]/2,min = 0, max =psf_image_bkg_free.shape[0])
pars.add('sky',value = np.nanmedian(psf_image_bkg_free))
if syntax['use_moffat']:
pars.add('alpha',value = syntax['image_params']['alpha'],
min = 0,
vary = syntax['fit_PSF_FWHM'] )
pars.add('beta',value = syntax['image_params']['beta'],
min = 0,
vary = syntax['vary_moff_beta'] or syntax['fit_PSF_FWHM'] )
else:
pars.add('sigma', value = syntax['image_params']['sigma'],
min = 0,
max = gauss_fwhm2sigma(syntax['max_fit_fwhm']),
vary = syntax['vary_moff_beta'] or syntax['fit_PSF_FWHM']
)
if syntax['use_moffat']:
def residual(p):
p = p.valuesdict()
return (psf_image_bkg_free - moffat_2d((xx,yy),p['x0'],p['y0'],p['sky'],p['A'],dict(alpha=p['alpha'],beta=p['beta'])).reshape(psf_image_bkg_free.shape)).flatten()
else:
def residual(p):
p = p.valuesdict()
return (psf_image_bkg_free - gauss_2d((xx,yy),p['x0'],p['y0'],p['sky'],p['A'],dict(sigma=p['sigma'])).reshape(psf_image_bkg_free.shape)).flatten()
mini = lmfit.Minimizer(residual, pars,nan_policy = 'omit')
result = mini.minimize(method = 'least_squares')
xc = result.params['x0'].value
yc = result.params['y0'].value
ap_range = np.arange(0.1,syntax['scale']/fwhm,1/25)
ap_sum = []
for nm in ap_range:
ap,bkg = ap_phot( [(xc,yc)] ,
psf_image_bkg_free,
radius = nm * fwhm,
r_in = syntax['r_in_size'] * fwhm,
r_out = syntax['r_out_size'] * fwhm)
ap_sum.append(ap)
ap_sum = ap_sum/np.nanmax(ap_sum)
radius = ap_range[np.argmax(ap_sum>=syntax['norm_count_sum'])]
image_radius = radius * fwhm
image_radius_lst.append(image_radius)
'''
refit only focusing on highest SNR area given by fitting radius
'''
# global pixel coorindates base on bn gaussian fit
xc_global = xc - syntax['scale'] + int(selected_sources.x_pix[idx])
yc_global = yc - syntax['scale'] + int(selected_sources.y_pix[idx])
# recenter image absed on location of best fit x and y
psf_image = image[int(yc_global-syntax['scale']): int(yc_global + syntax['scale']),
int(xc_global-syntax['scale']): int(xc_global + syntax['scale'])]
psf_image_bkg_free,bkg_median = rm_bkg(psf_image,syntax,psf_image.shape[0]/2,psf_image.shape[0]/2)
psf_image_slice = psf_image_bkg_free[int(psf_image_bkg_free.shape[0]/2 - fitting_radius):int(psf_image_bkg_free.shape[0]/2 + fitting_radius) ,
int(psf_image_bkg_free.shape[0]/2 - fitting_radius):int(psf_image_bkg_free.shape[0]/2 + fitting_radius) ]
x_slice = np.arange(0,2*fitting_radius)
xx_sl,yy_sl= np.meshgrid(x_slice,x_slice)
pars = lmfit.Parameters()
pars.add('A',value = np.nanmean(psf_image_slice),min = 0,max = np.nanmax(psf_image_slice) )
pars.add('x0',value = psf_image_slice.shape[1]/2,min = 0,max = psf_image_slice.shape[1])
pars.add('y0',value = psf_image_slice.shape[0]/2,min = 0,max = psf_image_slice.shape[0] )
if syntax['use_moffat']:
pars.add('alpha',value = syntax['image_params']['alpha'],
min = 0,
vary = syntax['fit_PSF_FWHM'] )
pars.add('beta',value = syntax['image_params']['beta'],
min = 0,
vary = syntax['vary_moff_beta'] or syntax['fit_PSF_FWHM'] )
else:
pars.add('sigma', value = syntax['image_params']['sigma'],
min = 0,
max = gauss_fwhm2sigma(syntax['max_fit_fwhm']),
vary = syntax['fit_PSF_FWHM']
)
if syntax['use_moffat']:
def residual(p):
p = p.valuesdict()
return (psf_image_slice - moffat_2d((xx_sl,yy_sl),p['x0'],p['y0'],0,p['A'],dict(alpha=p['alpha'],beta=p['beta'])).reshape(psf_image_slice .shape)).flatten()
else:
def residual(p):
p = p.valuesdict()
return (psf_image_slice - gauss_2d((xx_sl,yy_sl),p['x0'],p['y0'],0,p['A'],dict(sigma=p['sigma'])).reshape(psf_image_slice.shape)).flatten()
mini = lmfit.Minimizer(residual, pars,nan_policy = 'omit')
result = mini.minimize(method = 'least_squares')
# print(result.params)
positions = list(zip([xc_global ],[yc_global ]))
psf_counts,psf_bkg = ap_phot(positions,
image,
radius = syntax['ap_size'] * fwhm,
r_in = syntax['r_in_size'] * fwhm,
r_out = syntax['r_out_size'] * fwhm)
if syntax['use_moffat']:
PSF_FWHM = fitting_model_fwhm(dict(alpha=result.params['alpha'],beta=result.params['beta']))
else:
PSF_FWHM = fitting_model_fwhm(dict(sigma=result.params['sigma']))
PSF_SNR = SNR(psf_counts,psf_bkg,syntax['exp_time'],0,syntax['ap_size']*fwhm,syntax['gain'],0)[0]
if np.isnan(PSF_SNR) or np.isnan(PSF_FWHM):
logger.debug('PSF Contruction source fitting error')
continue
if PSF_SNR < syntax['construction_SNR'] and syntax['exp_time'] > 1:
logger.debug('PSF constuction source too low: %s' % int(PSF_SNR))
continue
else:
logger.info('SNR: %d FWHM: %.3f' % (PSF_SNR,PSF_FWHM))
# print('\rPSF source %d / %d :: SNR: %d' % (int(PSF_SNR)),end = '')
pass
# print(result.params)
xc = result.params['x0'].value
yc = result.params['y0'].value
H = result.params['A'].value
H_err = result.params['A'].stderr
xc_correction = xc - fitting_radius + syntax['scale']
yc_correction = yc - fitting_radius + syntax['scale']
if syntax['use_moffat']:
residual = psf_image_bkg_free - moffat_2d((xx,yy),xc_correction,yc_correction,
0,H,
dict(alpha=result.params['alpha'],beta=result.params['beta'])).reshape(psf_image_bkg_free.shape)
PSF_FWHM = fitting_model_fwhm(dict(alpha=result.params['alpha'],beta=result.params['beta']))
else:
residual = psf_image_bkg_free - gauss_2d((xx,yy),xc_correction,yc_correction,
0,H,
dict(sigma=result.params['sigma'])).reshape(psf_image_bkg_free.shape)
PSF_FWHM = fitting_model_fwhm(dict(sigma=result.params['sigma']))
residual /= H
psf_mag.append(-2.5*np.log10(H))
residual_regrid = np.repeat(np.repeat(residual, regriding_size, axis=0), regriding_size, axis=1)
x_roll = scale_roll(fitting_radius,xc,regriding_size)
y_roll = scale_roll(fitting_radius,yc,regriding_size)
residual_roll = np.roll(np.roll(residual_regrid,y_roll,axis=0),x_roll,axis = 1)
residual_table += residual_roll
sources_dict['PSF_%d'%sources_used] = {}
# print(H_err)
sources_dict['PSF_%d'%sources_used]['x_pix'] = xc_global
sources_dict['PSF_%d'%sources_used]['y_pix'] = yc_global
sources_dict['PSF_%d'%sources_used]['H_psf'] = float(H/syntax['exp_time'])
# sources_dict['PSF_%d'%sources_used]['H_psf_err'] = float(H_err/syntax['exp_time'])
sources_dict['PSF_%d'%sources_used]['fwhm'] = PSF_FWHM
sources_dict['PSF_%d'%sources_used]['x_best'] = xc_correction
sources_dict['PSF_%d'%sources_used]['y_best'] = yc_correction
sources_dict['PSF_%d'%sources_used]['close_up'] = psf_image_bkg_free
sources_dict['PSF_%d'%sources_used]['residual'] = residual
sources_dict['PSF_%d'%sources_used]['regrid'] = residual_regrid
sources_dict['PSF_%d'%sources_used]['roll'] = residual_roll
sources_dict['PSF_%d'%sources_used]['x_roll'] =x_roll
sources_dict['PSF_%d'%sources_used]['y_roll'] =y_roll
logger.debug('Residual table updated: %d / %d ' % (sources_used,syntax['psf_source_no']))
print('\rResidual table updated: %d / %d ' % (sources_used,syntax['psf_source_no']) ,end = '')
sources_used +=1
fwhm_fit.append(PSF_FWHM)
except Exception as e:
# logger.exception(e)
logger.error('** Fitting error - trying another source**')
failsafe+=1
n+=1
continue
print(' ')
if sources_used < syntax['min_psf_source_no']:
logger.warning('BUILDING PSF: Not enough useable sources found')
return None,None,construction_sources.append([np.nan]*5),syntax
logger.debug('PSF Successful')
#
# Get average of residual table
residual_table/= sources_used
# regrid residual table to psf size
residual_table = rebin(residual_table,( int(2*syntax['scale']),int(2*syntax['scale'])))
# construction_sources = pd.DataFrame(construction_sources)
construction_sources = pd.DataFrame.from_dict(sources_dict, orient='index',
columns=['x_pix','y_pix','H_psf','H_psf_err','fwhm','x_best','y_best'])
construction_sources.reset_index(inplace = True)
if syntax['plots_PSF_residual']:
from autophot.packages.create_plots import plot_PSF_model_steps
plot_PSF_model_steps(sources_dict,syntax,image)
if syntax['plots_PSF_sources']:
from autophot.packages.create_plots import plot_PSF_construction_grid
plot_PSF_construction_grid(construction_sources,image,syntax)
image_radius_lst = np.array(image_radius_lst)
syntax['image_radius'] = image_radius_lst.mean()
logger.info('Image_radius [pix] : %.3f +/- %.3f' % (image_radius_lst.mean(), image_radius_lst.std()))
except Exception as e:
logger.exception('BUILDING PSF: ',e)
raise Exception
return residual_table,fwhm_fit,construction_sources,syntax