def findSkyLevel(self):
Psky = fm.array(
self.data_sci['array'].copy().values /
(self.data_sci['integtime'].copy().values)[:, None]).mean('ch')
date = self.data_sci['date'].copy().values
t_sci = np.array([s[:-4] for s in date], 'datetime64[us]')
flag1 = t_sci < t_sci[-1]
#flag2 = data_sci['bufpos'] == 'ON'
flag = flag1 #& flag2
model_sky = models.Polynomial1D(2)
model_sky.c0 = 0.0
model_sky.c1 = -1.0
model_sky.c2 = 1.0
pfit = fitting.LinearLSQFitter()
opfit = fitting.FittingWithOutlierRemoval(pfit,
sigma_clip,
niter=15,
sigma=2.0)
#fitted_sky = pfit(model_sky, t_sci[flag], Psky[flag])
fitted_sky, filtered_data = opfit(model_sky, t_sci[flag] - t_sci[0],
Psky[flag])
#opfit = fitting.FittingWithOutlierRemoval(pfit, sigma_clip,niter=3, sigma=3.0)
#fitted_sky,filtered_data = opfit(model_sky,t_sci[flag][~filtered_data],Psky[flag][~filtered_data])
#print(filtered_data)
#print(fitted_sky.c1,fitted_sky.c0)
self.t_sci = t_sci
self.fitted_sky = fitted_sky
self.Psky = Psky
self.flag = flag
self.filtered_data = filtered_data
def fit_meanphot_vs_varphot_levmar(meanphot,
varphot,
nfr=5,
itersigma=4.0,
niter=5,
mode='linear'):
# does not currently work, using fit_meanphot_vs_varphot()
if mode == 'linear':
fit = fitting.LinearLSQFitter()
elif mode == 'LevMar':
fit = fitting.LevMarLSQFitter()
# initialize the outlier removal fitter
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=niter,
sigma=itersigma)
# initialize a linear model
line_init = models.Linear1D()
# fit the data with the fitter
sigmask, fitted_line = or_fit(line_init, meanphot, varphot)
slope = fitted_line.slope.value
g1_iter = get_g1_from_slope_T_N(slope, N=nfr)
if mode == 'LevMar':
cov_diag = np.diag(or_fit.fit_info['param_cov'])
print('cov diag:')
print(cov_diag)
return fitted_line, sigmask, g1_iter
def fit_star2d_outlier_removal(x,
y,
z,
sigma=3.0,
niter=50,
guess=None,
bounds=None):
"""Star2D parameters: amplitude, x_mean,y_mean,stddev,saturation"""
gg_init = Star2D()
if guess is not None:
for ip, p in enumerate(gg_init.param_names):
getattr(gg_init, p).value = guess[ip]
if bounds is not None:
for ip, p in enumerate(gg_init.param_names):
getattr(gg_init, p).min = bounds[0][ip]
getattr(gg_init, p).max = bounds[1][ip]
gg_init.saturation.fixed = True
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=niter,
sigma=sigma)
# get fitted model and filtered data
filtered_data, or_fitted_model = or_fit(gg_init, x, y, z)
if parameters.VERBOSE: print or_fitted_model
return or_fitted_model
def fit_poly1d_outlier_removal(x, y, order=2, sigma=3.0, niter=3):
gg_init = models.Polynomial1D(order)
gg_init.c0.min = np.min(y)
gg_init.c0.max = 2 * np.max(y)
gg_init.c1 = 0
gg_init.c2 = 0
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=niter,
sigma=sigma)
# get fitted model and filtered data
filtered_data, or_fitted_model = or_fit(gg_init, x, y)
'''
import matplotlib.pyplot as plt
plt.figure(figsize=(8,5))
plt.plot(x, y, 'gx', label="original data")
plt.plot(x, filtered_data, 'r+', label="filtered data")
plt.plot(x, or_fitted_model(x), 'r--',
label="model fitted w/ filtered data")
plt.legend(loc=2, numpoints=1)
plt.show()
'''
return or_fitted_model
def guide_star_seeing(subframe):
# subframe = subframe - np.median(subframe)
subframe = subframe - np.percentile(subframe,5)
sub_frame_l = int(np.shape(subframe)[0])
y, x = np.mgrid[:sub_frame_l, :sub_frame_l]
# Fit with constant, bounds, tied x and y sigmas and outlier rejection:
gaussian_init = models.Const2D(0.0) + models.Gaussian2D(subframe[int(sub_frame_l/2),int(sub_frame_l/2)],int(sub_frame_l/2),int(sub_frame_l/2),8/2.355,8/2.355,0)
gaussian_init.x_stddev_1.min = 1.0/2.355
gaussian_init.x_stddev_1.max = 20.0/2.355
gaussian_init.y_stddev_1.min = 1.0/2.355
gaussian_init.y_stddev_1.max = 20.0/2.355
gaussian_init.y_stddev_1.tied = tie_sigma
gaussian_init.theta_1.fixed = True
fit_gauss = fitting.FittingWithOutlierRemoval(fitting.LevMarLSQFitter(),sigma_clip,niter=3,sigma=3.0)
# gaussian, mask = fit_gauss(gaussian_init, x, y, subframe)
gain = 8.21 #e per ADU
read_noise = 2.43 #ADU
weights = gain / np.sqrt(np.absolute(subframe)*gain + (read_noise*gain)**2) #1/sigma for each pixel
gaussian, mask = fit_gauss(gaussian_init, x, y, subframe, weights)
fwhm_x = 2.355*gaussian.x_stddev_1.value
fwhm_y = 2.355*gaussian.y_stddev_1.value
x_seeing = fwhm_x * 0.579
y_seeing = fwhm_y * 0.579
return(x_seeing,y_seeing)
def color_corrections(aij_stars, aij_mags, apass_index, apass_color,
apass_R_mags, good_match):
#Changing from huber to astropy fit
# Create empty list for the corrections (slope and intercept)
corrections = []
#Create empy list for the error in the corrections
fit_error = []
all_Rminusr_error = []
#loop over all images
for idx in range(aij_mags.shape[1]):
#create BminusV list
BminusV = []
#Create Rminusr list
Rminusr = []
#loop over the apass_index and the placement in the apass_index
for aij_star, el in enumerate(apass_index):
#check if the aij star has a corresponding apass match
if good_match[aij_star]:
#Make sure the aij stars magnitude in the image isn't friggin huge
if aij_stars[aij_star].magnitude[idx] < 100:
#Add the color of that star (according to apass) to the bminusv list
BminusV.append(apass_color[el])
#add the difference between aijs magnitude and apass transformed magnitude to the Rminusr list
Rminusr.append(apass_R_mags[el] -
aij_stars[aij_star].magnitude[idx])
#No idea what this is meant to do
#if Rminusr[-1] < 20 and idx == 0:
#print(aij_star)
Rminusr_new = np.array([d for d in Rminusr if d != 'masked'])
BminusV_new = np.array(
[b for b, d in zip(BminusV, Rminusr) if d != 'masked'])
#Astropy Fit
g_init = models.Polynomial1D(1)
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=3,
sigma=3.0)
# get fitted model and filtered data
filtered_data, or_fitted_model = or_fit(g_init, BminusV_new,
Rminusr_new)
fitted_model = fit(g_init, BminusV_new, Rminusr_new)
#print(fitted_model)
#append corrections data to list
fit_list = [or_fitted_model.c1.value, or_fitted_model.c0.value]
corrections.append(fit_list)
plt.ylim(20, 21)
plt.title(idx)
plt.plot(BminusV_new, Rminusr_new, 'o')
plt.plot(BminusV_new,
or_fitted_model(BminusV_new),
'g--',
label="model fitted w/ filtered data")
plt.show()
print(or_fitted_model.c1.value)
return corrections, BminusV
def fit_airy_2d(data, x=None, y=None, sigma_factor=0):
"""Fit an AiryDisk2D model to the data.
Parameters
----------
data: 2D float array
should be a 2d array, the initial center is used to estimate
the fit center
x: float (optional)
xcenter location
y: float (optional)
ycenter location
sigma_factor: float (optional)
If sigma_factor > 0 then clipping will be performed
on the data during the model fit
Returns
-------
The the fit model for Airy2D function
"""
delta = int(len(data) / 2) # guess the center
ldata = len(data)
if x is None:
x = delta
if y is None:
y = delta
fixed_pars = {"x_0": True, "y_0": True} # hold these constant
yy, xx = np.mgrid[:ldata, :ldata]
# Initialize the fitter
fitter = fitting.LevMarLSQFitter()
if sigma_factor > 0:
fit = fitting.FittingWithOutlierRemoval(fitter,
sigma_clip,
niter=3,
sigma=sigma_factor)
else:
fit = fitter
# AiryDisk2D(amplitude, x_0, y_0, radius) + constant
model = (models.AiryDisk2D(
np.max(data), x_0=x, y_0=y, radius=delta, fixed=fixed_pars) +
models.Polynomial2D(c0_0=data.min(), degree=0))
with warnings.catch_warnings():
# Ignore model warnings for new_plot_window
warnings.simplefilter('ignore')
results = fit(model, xx, yy, data)
if sigma_factor > 0:
return results[1]
else:
return results
def fit_gaussian_2d(data, sigma=3., theta=0., sigma_factor=0):
"""center the data by fitting a 2d gaussian to the region.
Parameters
----------
data: 2D float array
should be a 2d array, the initial center is used to estimate
the fit center
sigma: float (optional)
The sigma value for the starting gaussian model
theta: float(optional)
The theta value for the starting gaussian model
sigma_factor: float (optional)
If sigma_factor > 0 then clipping will be performed
on the data during the model fit
Returns
-------
The full gaussian fit model, from which the center can be extracted
"""
# use a smaller bounding box so that we are only fitting the local data
delta = int(len(data) / 2) # guess the center
amp = data.max() - data.min() # guess the amplitude
ldata = len(data)
yy, xx = np.mgrid[:ldata, :ldata]
# Initialize the fitter
fitter = fitting.LevMarLSQFitter()
if sigma_factor > 0:
fit = fitting.FittingWithOutlierRemoval(fitter,
sigma_clip,
niter=3,
sigma=sigma_factor)
else:
fit = fitter
# Gaussian2D(amp,xmean,ymean,xstd,ystd,theta) + a constant
model = (models.Gaussian2D(amp, delta, delta, sigma, sigma, theta) +
models.Polynomial2D(c0_0=data.min(), degree=0))
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
results = fit(model, xx, yy, data)
# previous yield amp, ycenter, xcenter, sigma, offset
# if sigma clipping is used, results is a tuple of data, model
if sigma_factor > 0:
return results[1]
else:
return results
def fit_poly2d_outlier_removal(x, y, z, order=2, sigma=3.0, niter=30):
gg_init = models.Polynomial2D(order)
gg_init.c0_0.min = np.min(z)
gg_init.c0_0.max = 2 * np.max(z)
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=niter,
sigma=sigma)
# get fitted model and filtered data
filtered_data, or_fitted_model = or_fit(gg_init, x, y, z)
return or_fitted_model
def astropy_clipping_fit(x, row, variance):
gauss_model = models.Gaussian1D() + models.Const1D()
fit = fitting.LevMarLSQFitter()
clipping_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=30,
sigma=15.0)
# initialize fitters
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=3,
sigma=6.0)
g_init = models.Gaussian1D(
amplitude=np.max(row), mean=len(x) / 2,
stddev=1.0) + models.Const1D(amplitude=np.min(row))
filtered_data, or_fitted_model = or_fit(g_init,
x,
row,
weights=1.0 / variance)
return or_fitted_model.mean_0.value
def fit_meanphot_vs_varphot(meanphot, varphot, nfr=5, itersigma=4.0, niter=5):
fit = fitting.LinearLSQFitter()
# initialize the outlier removal fitter
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=niter,
sigma=itersigma)
# initialize a linear model
line_init = models.Linear1D()
# fit the data with the fitter
sigmask, fitted_line = or_fit(line_init, meanphot, varphot)
slope = fitted_line.slope.value
g1_iter = get_g1_from_slope_T_N(slope, N=nfr)
return fitted_line, sigmask, g1_iter
def fit_poly_n(data, deg=1, sigma_factor=0):
"""Fit a Polynomial 1D model to the data.
Parameters
----------
data: float array
should be a 1d or 2d array
deg: int
The degree of polynomial to fit
sigma_factor: float (optional)
If sigma_factor > 0 then clipping will be performed
on the data during the model fit
Returns
-------
The the polynomial fit model for the function
"""
if len(data) < deg + 1:
raise ValueError("fit_poly_n: Need more data for fit")
# define the model
poly = models.Polynomial1D(deg)
# set the axis range for fitting
ax = np.arange(len(data))
# define the fitter
fitter = fitting.LinearLSQFitter()
if sigma_factor > 0:
fit = fitting.FittingWithOutlierRemoval(fitter,
sigma_clip,
sigma=sigma_factor,
niter=3)
else:
fit = fitter
try:
result = fit(poly, ax, data)
except ValueError:
result = None
if sigma_factor > 0:
result = result[1]
return result
def fit_line_sigma_clip(x,
y,
yunc=None,
slope=1,
intercept=0,
niter=5,
sigma=3):
'''
INPUT
* x - array of x values
* y - array of y values
OPTIONAL INPUT
* yunc - uncertainty in y values
* slope - intial guess for slope
* intercept - intial guess for intercept
* niter - number of sigma clipping iterations; default is 3
* sigma - sigma to use in clipping; default is 3
RETURNS
* Linear1D model, with slope and intercept
* you can get fitted y values with fitted_line(x)
* mask - array indicating the points that were cut in sigma clipping process
REFERENCES
https://docs.astropy.org/en/stable/modeling/example-fitting-line.html
'''
if yunc is None:
yunc = np.ones(len(y))
# initialize a linear fitter
fit = fitting.LinearLSQFitter()
# initialize the outlier removal fitter
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=niter,
sigma=sigma)
# initialize a linear model
line_init = models.Linear1D(slope=slope, intercept=intercept)
# fit the data with the fitter
fitted_line, mask = or_fit(line_init, x, y, weights=1.0 / yunc)
return fitted_line, mask
def plot(filenames, Flat = True):
time_table, counts_unique_table = tab(filenames)
counts_unique = np.squeeze(np.array([counts_unique_table[k] for k in counts_unique_table.keys()]))
time = np.squeeze(np.array([time_table[k] for k in time_table.keys()]))
if Flat == True:
plt.figure()
fit = fitting.LinearLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit, sigma_clip, niter=2, sigma=2.5)
line_init = models.Linear1D()
fitted_line, mask = or_fit(line_init, time, counts_unique)
filtered_data = np.ma.masked_array(counts_unique, mask=mask)
plt.plot(time, counts_unique, 'ko', fillstyle='none')
plt.plot(time, filtered_data, 'ko')
plt.plot(time, fitted_line(time), 'k-')
plt.xlabel('Time (s)')
plt.ylabel('Counts')
plt.title('Atik T=-1 Flat')
plt.savefig('Atik_T=-1_Flat.pdf')
else:
time = sorted(np.squeeze(np.array([time_table[k] for k in time_table.keys()])))
fig1,ax1 = plt.subplots()
counts_norm = counts_unique/max(counts_unique)
ax1.errorbar(time, counts_norm,
yerr = np.std(counts_norm, dtype=np.float64),
fmt = ' ',
marker = 'o',
markersize = 5,
elinewidth=1)
ax1.set_xlabel('Time (s)')
ax1.set_ylabel('Counts')
ax1.legend(['Block1', 'Block2', 'Block3', 'Block4', 'Block5', 'Block6',
'Block7', 'Block8', 'Block9', 'Block10', 'Block11',
'Block12', 'Block13', 'Block14', 'Block14', 'Block16'])
slope, intercept, r_value, p_value, std_err = linregress(time, counts_norm)
## plt.title('Atik T=3 Dark - Normalized counts')
## plt.savefig('Atik_T=3_Dark.pdf')
return(intercept, plt.show())
def G1(wave_range, flux_range, A0, eline0, sigma0, plot=0):
def G_model(x, A0=1, eline0=1, sigma0=300):
l_0 = eline0
model0 = A0 * np.exp(-0.5 * (x - l_0)**2 / (sigma0**2))
return model0
def G_deriv(x, A0=1, eline0=1, sigma0=300):
# Jacobian of G_model
l_0 = eline0
y0 = (x - l_0) / (sigma0)
model0 = A0 * np.exp(-0.5 * y0**2)
d_A0 = np.exp(-0.5 * y0**2)
d_sigma0 = A0 * d_A0 * (x - l_0)**2 / sigma0**3
d_l0 = A0 * d_A0 * (x - l_0) / sigma0**2
return [d_A0, d_l0, d_sigma0]
# initialize fitters
from astropy.modeling import models, fitting
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=1,
sigma=3.0)
GaussModel = custom_model(G_model, fit_deriv=G_deriv)
model = GaussModel()
or_fitted_model, mask = or_fit(model, wave_range, flux_range)
#or_fitted_model = fit(model, wave_range,flux_range)
A0_best, sigma_best, eline_best = mask.A0, mask.sigma0, mask.eline0
if plot == 1:
plt.plot(wave_range, flux_range, 'k+')
plt.plot(wave_range,
G_model(wave_range, A0_best, eline_best, sigma_best),
"b-",
label='best_fit')
plt.legend()
return [A0_best, eline_best, sigma_best, or_fitted_model] #best_vals
def fitB4(coord_iso):
u = coord_iso[0]
v = coord_iso[1]
r = np.sqrt(u**2 + v**2)
E=np.zeros(len(u))
for i in range(0,len(u)):
if np.sign(u[i])>0 and np.sign(v[i])>0 : E[i]=np.arctan(v[i]/u[i])
elif np.sign(u[i])>0 and np.sign(v[i])<0 : E[i]=2*np.pi+np.arctan(v[i]/u[i])
else : E[i]=np.pi+np.arctan(v[i]/u[i])
Es = np.linspace(0,2*np.pi,num=100)
dic = {'frequency': True, 'phase': True}
g_init = (models.Sine1D(amplitude=0.1, frequency=1.5/np.pi, fixed=dic)
+models.Sine1D(amplitude=0.1, frequency=2/np.pi, fixed=dic)
+models.Sine1D(amplitude=0.1, frequency=1.5/np.pi,
phase=0.25, fixed=dic)
+models.Sine1D(amplitude=0.1, frequency=2/np.pi,
phase=0.25, fixed=dic)
+models.Const1D(amplitude=1.0,fixed={'amplitude':True}))
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit, sigma_clip, niter=3, sigma=3.0)
filtered_data, or_fitted_model = or_fit(g_init, E, r)
fitted_model = fit(g_init, E, r)
plt.figure(figsize=(8,5))
plt.plot(E, r, 'gx', label="original data")
plt.plot(E, filtered_data, 'r+', label="filtered data")
plt.plot(Es, fitted_model(Es), 'g-',
label="model fitted w/ original data")
plt.plot(Es, or_fitted_model(Es), 'r--',
label="model fitted w/ filtered data")
plt.legend(loc=2, numpoints=1)
return or_fitted_model[3].amplitude.value
def estimate_peak_width(data, min=2, max=8):
"""
Estimates the FWHM of the spectral features (arc lines) by fitting
Gaussians to the brightest peaks.
Parameters
----------
data: ndarray
1D data array (will be modified)
min: int
minimum plausible peak width
max: int
maximum plausible peak width (not inclusive)
Returns
-------
float: estimate of FWHM of features
"""
all_widths = []
for fwidth in range(min, max + 1): # plausible range of widths
data_copy = data.copy() # We'll be editing the data
widths = []
for i in range(15): # 15 brightest peaks, should ensure we get real ones
index = 2 * fwidth + np.argmax(data_copy[2 * fwidth:-2 * fwidth - 1])
data_to_fit = data_copy[index - 2 * fwidth:index + 2 * fwidth + 1]
m_init = models.Gaussian1D(stddev=0.42466 * fwidth) + models.Const1D(np.min(data_to_fit))
m_init.mean_0.bounds = [-1, 1]
m_init.amplitude_1.fixed = True
fit_it = fitting.FittingWithOutlierRemoval(fitting.LevMarLSQFitter(),
sigma_clip, sigma=3)
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
m_final, _ = fit_it(m_init, np.arange(-2 * fwidth, 2 * fwidth + 1),
data_to_fit)
# Quick'n'dirty logic to remove "peaks" at edges of CCDs
if m_final.amplitude_1 != 0 and m_final.stddev_0 < fwidth:
widths.append(m_final.stddev_0 / 0.42466)
# Set data to zero so no peak is found here
data_copy[index - 2 * fwidth:index + 2 * fwidth + 1] = 0.
all_widths.append(sigma_clip(widths).mean())
return sigma_clip(all_widths).mean()
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None):
"""
Adds an illumination mask to each AD object
Parameters
----------
suffix: str
suffix to be added to output files
illum_mask: str/None
name of illumination mask mask (None -> use default)
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
for ad, illum in zip(
*gt.make_lists(adinputs, illum_mask, force_ad=True)):
if ad.phu.get(timestamp_key):
log.warning(
'No changes will be made to {}, since it has '
'already been processed by addIllumMaskToDQ'.format(
ad.filename))
continue
ad_detsec = ad.detector_section()
no_bridges = all(detsec.y1 > 1600 and detsec.y2 < 2900
for detsec in ad_detsec)
has_48rows = (all(detsec.y2 == 4224 for detsec in ad_detsec)
and 'Hamamatsu' in ad.detector_name(pretty=True))
if illum:
log.fullinfo("Using {} as illumination mask".format(
illum.filename))
final_illum = gt.clip_auxiliary_data(ad,
aux=illum,
aux_type='bpm',
return_dtype=DQ.datatype)
for ext, illum_ext in zip(ad, final_illum):
if illum_ext is not None:
# Ensure we're only adding the unilluminated bit
iext = np.where(illum_ext.data > 0, DQ.unilluminated,
0).astype(DQ.datatype)
ext.mask = iext if ext.mask is None else ext.mask | iext
elif not no_bridges: # i.e. there are bridges.
# Default operation for GMOS full-frame LS
# The 95% cut should ensure that we're sampling something
# bright (even for an arc)
# The max is intended to handle R150 data, where many of
# the extensions are unilluminated
row_medians = np.max(np.array(
[np.percentile(ext.data, 95, axis=1) for ext in ad]),
axis=0)
rows = np.arange(len(row_medians))
m_init = models.Polynomial1D(degree=3)
fit_it = fitting.FittingWithOutlierRemoval(
fitting.LinearLSQFitter(),
outlier_func=sigma_clip,
sigma_upper=1,
sigma_lower=3)
m_final, _ = fit_it(m_init, rows, row_medians)
model_fit = m_final(rows)
# Find points which are significantly below the smooth illumination fit
# First ensure we don't worry about single rows
row_mask = at.boxcar(model_fit - row_medians > 0.1 * model_fit,
operation=np.logical_and,
size=1)
row_mask = at.boxcar(row_mask, operation=np.logical_or, size=3)
for ext in ad:
ext.mask |= (row_mask * DQ.unilluminated).astype(
DQ.datatype)[:, np.newaxis]
if has_48rows:
actual_rows = 48 // ad.detector_y_bin()
for ext in ad:
ext.mask[:actual_rows] |= DQ.unilluminated
# Timestamp and update filename
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=suffix, strip=True)
return adinputs
def fit_mex_hat_1d(data, sigma_factor=0, center_at=None, weighted=False):
"""Fit a 1D Mexican Hat function to the data.
Parameters
----------
data: 2D float array
should be a 2d array, the initial center is used to estimate
the fit center
sigma_factor: float (optional)
If sigma_factor > 0 then clipping will be performed
on the data during the model fit
center_at: float or None
None by default, set to value to use as center
weighted: bool
if weighted is True, then weight the values by basic
uncertainty hueristic
Returns
-------
The the fit model for mexican hat 1D function
"""
ldata = len(data)
if center_at:
x0 = 0
else:
x0 = int(ldata / 2.)
# assumes negligable background
if weighted:
z = np.nan_to_num(1. / np.sqrt(data)) # use as weight
x = np.arange(ldata)
fixed_pars = {"x_0": True}
# Initialize the fitter
fitter = fitting.LevMarLSQFitter()
if sigma_factor > 0:
fit = fitting.FittingWithOutlierRemoval(fitter,
sigma_clip,
niter=3,
sigma=sigma_factor)
else:
fit = fitter
# Mexican Hat 1D + constant
model = (models.MexicanHat1D(
amplitude=np.max(data), x_0=x0, sigma=2., fixed=fixed_pars) +
models.Polynomial1D(c0=data.min(), degree=0))
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
if weighted:
results = fit(model, x, data, weights=z)
else:
results = fit(model, x, data)
# previous yield amp, ycenter, xcenter, sigma, offset
if sigma_factor > 0:
return results[1]
else:
return results
def fit_moffat_1d(data,
gamma=2.,
alpha=1.,
sigma_factor=0.,
center_at=None,
weighted=False):
"""Fit a 1D moffat profile to the data and return the fit.
Parameters
----------
data: 2D data array
The input sigma to use
gamma: float (optional)
The input gamma to use
alpha: float (optional)
The input alpha to use
sigma_factor: float (optional)
If sigma>0 then sigma clipping of the data is performed
at that level
center_at: float or None
None by default, set to value to use as center
weighted: bool
if weighted is True, then weight the values by basic
uncertainty hueristic
Returns
-------
The fitted 1D moffat model for the data
"""
# data is assumed to already be chunked to a reasonable size
ldata = len(data)
# guesstimate mean
if center_at:
x0 = 0
else:
x0 = int(ldata / 2.)
# assumes negligable background
if weighted:
z = np.nan_to_num(1. / np.sqrt(data)) # use as weight
x = np.arange(ldata)
# Initialize the fitter
fitter = fitting.LevMarLSQFitter()
if sigma_factor > 0:
fit = fitting.FittingWithOutlierRemoval(fitter,
sigma_clip,
niter=3,
sigma=sigma_factor)
else:
fit = fitter
# Moffat1D + constant
model = (models.Moffat1D(
amplitude=max(data), x_0=x0, gamma=gamma, alpha=alpha) +
models.Polynomial1D(c0=data.min(), degree=0))
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
if weighted:
results = fit(model, x, data, weights=z)
else:
results = fit(model, x, data)
# previous yield amp, ycenter, xcenter, sigma, offset
# if sigma clipping is used, results is a tuple of data, model
if sigma_factor > 0:
return results[1]
else:
return results
def rb_iter_contfit(wave, flux, error, **kwargs):
'''
Iterative continuum fitter using Legendre polynomials
Input: -
wavelength array
flux array
error array
optional input:
maxiter :- maximum iteration [25 default]
order :- polynomial order of fit [4 default]
output: -
fit_final : Final fitted continuum array
resid_final : residual error array
fit_error : error on the fit [standard deviation of the residual]
Written by: Rongmon Bordoloi
Tested on Python 3.7 Sep 4 2019
--------------------------
example :
from IGM import rb_iter_contfit as r
out= r.rb_iter_contfit(wave,flux,error,order=5)
out[0] = fitted continuum
'''
if 'maxiter' in kwargs:
maxiter = kwargs['maxiter']
else:
maxiter = 25
if 'order' in kwargs:
order = kwargs['order']
else:
order = 4
#Initialize a mask
mask = np.ones((np.size(wave), ))
# Looking for chip gaps
chip_gap = np.where(((error == 0) & (flux == 0)) | (error - flux == 0))
if (np.size(chip_gap) > 0):
#error[chip_gap]=1;
#flux[chip_gap]=1;
mask[chip_gap] = 0
# Now get rid of negative error values
qq = np.where(error <= 0)
if (np.size(qq) > 0):
error[qq] = np.median(error)
#Do a sanity check to avoid bad flux values
q = np.where(flux <= 0)
if (np.size(q) > 0):
flux[q] = error[q]
w = 1 / error**2. # Weight
index = 0 #Initialize the counter
outside_chip_gap = np.where(((error != 0) & (flux != 0))
| (error - flux != 0))
med_err = np.median(error[outside_chip_gap])
med_flux = np.median(flux[outside_chip_gap])
# Now take out any possible emission or absorption features
#%Flux values less than the median error are 1sigma within zero, so should be masked
#%Try to exclude emission. anything above 2sigma over the median flux should be masked
bd = np.where((flux < med_err)) #| (flux > (med_flux+3*med_err)))
nbd = len(bd[0])
if (nbd > 0):
mask[bd] = 0
mask = np.array(mask)
qq = np.where(mask == 1)
#Here I'm cutting out any chip gap or any other absorption feature. These are permanently excluded from the fit
flux_new = flux[qq]
wave_new = wave[qq]
weights = np.array(w[qq])
# Fit the data using Legedre Polynomials
g_init = models.Legendre1D(order)
# initialize fitters
fit = fitting.LevMarLSQFitter()
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
new_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=maxiter,
sigma=3.0)
filtered_fit, filtered_data = new_fit(g_init, wave_new,
flux_new) #,weights=weights)
''''
while (index < maxiter):
print("Index="+np.str(index))
index=index+1
oldmask=mask
# Fit the data using Legedre Polynomials
g_init = models.Legendre1D(order)
# initialize fitters
fit = fitting.LevMarLSQFitter()
new_fit = fitting.FittingWithOutlierRemoval(fit, sigma_clip,niter=maxiter, sigma=3.0)
filtered_data, new_fit = new_fit(g_init, wave_new,flux_new,weights=weights)
#fit_g = fitting.LevMarLSQFitter()
#new_fit = fit_g(g_init, wave_new,flux_new) # Learn how to use the weights....
cont=new_fit(wave_new)
resid=flux_new/np.double(cont)
#mederr=np.sqrt(scipy.stats.moment(resid,2)) #;;Use the median error to set "sigma" for the clipping. If you median is skewed, i think you're pretty much hosed
mederr=np.std(resid)
# Do some sigma clipping here
median_val=np.median(resid)
kappa=3.; # 3 sigma clipping
bd = np.where( (resid > (median_val+kappa*mederr)) | (resid < (median_val - mederr*kappa)))
gd= np.where( (resid <= (median_val+kappa*mederr)) & (resid >= (median_val - mederr*kappa)))
print np.size(gd)
print np.size(bd)
if (np.size(gd) > 0):
mask[gd] = 1 # Allowing previously rejected points to reenter
if (np.size(bd) > 0):
mask[bd] = 0
if (index >0):
diff=oldmask-mask
qq = np.where(diff != 0.)
if (np.size(qq)==0):
print index
print "No more Clipping"
index=maxiter # ;;mask and oldmask are identical if all elements match
# Updating everything
qq=np.where(mask==1)
flux_new=flux[qq]
wave_new=wave[qq]
weights=w[qq]
'''
#pdb.set_trace()
fit_final = filtered_fit(wave)
resid_final = flux / fit_final
fit_error = np.std(
resid_final) #np.sqrt(scipy.stats.moment(resid_final,2))
return fit_final, resid_final, fit_error
# y_initvalues = np.ones_like(binsX_TRGB) * 1.0
# y_upperBounds = np.ones_like(binsX_TRGB) * 1.1
# y_lowerBounds = np.ones_like(binsX_TRGB) * 0.9
y_initvalues = 0.31 * (binsX_TRGB - 2.1)**2 - 3.9
y_upperBounds = 0.25 * (binsX_TRGB - 2)**2 - 3.0
y_lowerBounds = 0.25 * (binsX_TRGB - 2)**2 - 5.0
gauss_mean = np.zeros((numX_TRGB))
gauss_amplitude = np.zeros((numX_TRGB))
gauss_stddev = np.zeros((numX_TRGB))
p_degree = 1 # max polynom degree in the approximating function
fit = fitting.LevMarLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=10,
sigma=3.0)
def find_nearest(array, value):
array = np.asarray(array)
idx = (np.abs(array - value)).argmin()
return array[idx]
for i in range(numX_TRGB):
# print(i,'of',numX_TRGB)
# g_init -- Gauss model with initial values
sliceTRGB = regionTRGB[:, i] / np.max(regionTRGB[:, i])
y_bounds = (y_lowerBounds[i], y_upperBounds[i])
mapPeak = np.max(sliceTRGB)
def plot_stars_fit_plane(img_file, vmin=None, vmax=None, fignum=None):
img, hdr = fits.getdata(img_file, header=True)
if 'BINFAC' in hdr:
# FLI CAMERA
ps = 0.04 # arcsecs / pix
binfac = hdr['BINFAC']
ps = ps * binfac
else:
ps = 0.12
_in = open(img_file.replace('.fits', '_fwhm.pickle'), 'rb')
sources = pickle.load(_in)
min_fwhm = pickle.load(_in)
maj_fwhm = pickle.load(_in)
elon = pickle.load(_in)
phi = pickle.load(_in)
_in.close()
# Only use the brightest sources for calculating the mean. This is just for printing.
min_fwhm_mean, min_fwhm_med, min_fwhm_std = sigma_clipped_stats(min_fwhm)
maj_fwhm_mean, maj_fwhm_med, maj_fwhm_std = sigma_clipped_stats(maj_fwhm)
elon_mean, elon_med, elon_std = sigma_clipped_stats(elon)
phi_mean, phi_med, phi_std = sigma_clipped_stats(phi)
print(' Number of sources = ', len(sources))
print(' Minor fwhm = {0:.2f} +/- {1:.2f} arcsec'.format(
min_fwhm_med * ps, min_fwhm_std * ps))
print(' Major fwhm = {0:.2f} +/- {1:.2f} arcsec'.format(
maj_fwhm_med * ps, maj_fwhm_std * ps))
print(' Elongation = {0:.2f} +/- {1:.2f}'.format(
elon_med, elon_std))
print(' Pos. Angle = {0:.2f} +/- {1:.2f} rad'.format(
phi_med, phi_std))
x = sources['xcentroid']
y = sources['ycentroid']
z = min_fwhm * ps
def plot_plane(xdat, ydat, zdat, title, vmin, vmax, image=False):
plt.clf()
plt.subplots_adjust(left=0.13)
if image == False:
plt.scatter(xdat,
ydat,
c=zdat,
s=50,
edgecolor='none',
vmin=vmin,
vmax=vmax,
cmap='plasma')
else:
plt.imshow(zdat,
vmin=vmin,
vmax=vmax,
origin='lower',
cmap='plasma')
plt.xlabel('X (pixels)')
plt.ylabel('Y (pixels)')
plt.title(title)
plt.colorbar(label='FWHM (")')
plt.axis('equal')
##########
# Fit the data to a plane.
##########
if (len(sources) > 10):
p_init = models.Polynomial2D(degree=1)
fitter1 = fitting.LevMarLSQFitter()
fitter2 = fitting.FittingWithOutlierRemoval(fitter1,
sigma_clip,
niter=3,
sigma=3)
z_good, fit2_model = fitter2(p_init, x, y, z)
n_tot = len(z_good)
n_good = z_good.count()
n_bad = n_tot - n_good
print('Keeping {0:d} of {1:d}, {2:d} outliers'.format(
n_good, n_tot, n_bad))
y_grid, x_grid = np.mgrid[:int(y.max()), :int(x.max())]
plane2 = fit2_model(x_grid, y_grid)
else:
z_good = z
plane2 = None
##########
# Plotting
##########
if vmin is None:
vmin = z_good.min()
if vmax is None:
vmax = z_good.max()
if plane2 is not None:
plt.figure(1, figsize=(8, 6))
plt.clf()
plot_plane(x_grid, y_grid, plane2, 'Plane 2', vmin, vmax, image=True)
if fignum is None:
plt.figure(figsize=(8, 6))
else:
plt.figure(fignum, figsize=(8, 6))
plt.clf()
plot_plane(x, y, z, img_file, vmin, vmax, image=False)
plt.plot(x[z_good.mask == True],
y[z_good.mask == True],
'kx',
ms=10,
linestyle='none')
print('Maximum Tilt Range in Model Plane is: {0:.2f}" - {1:.2f}"'.format(
plane2.min(), plane2.max()))
print(
'Maximum Tilt Delta in Model Plane is: {0:.2f}"'.format(plane2.max() -
plane2.min()))
return
def fit_profile(self, bgdist = None, fitdist = None, fitfunc=None, verbose=False, beamwidth=None, bgdegree = 1):
"""
Fit a model to the filament's master profile
Parameters
------
self: An instance of the radfil_class
fitdist: number-like or tuple-like with a length of 2
The radial distance (in units of pc) out to which you'd like to fit your profile.
When the input has a length of 2, data points with distances between the two values will be
used in the fitting. The negative direction is always to the left of the spline direction,
which always runs from smaller axis-0 indices to larger axis-0 indices.
bgdist: tuple-like, with a shape (2,)
The radial distance range that defines the data points to be used in background subtraction; if None no background is fit
fitfunc: string
Options include "Gaussian" or "Plummer"
bgdegree: integer (default = 1)
The order of the polynomial used in background subtraction (options are 1 or 0). Active only when wrap = False.
beamwidth: float or int
If not inputed into the make_fil_spine method, beamwidth needs to be provided to calculate deconvolved FWHM of Gaussian/Plummer Fits
If not provided, deconvolved FWHM values will be set to nan
Attributes
------
xbg, ybg: 1D numpy.ndarray (list-like)
Data used for background subtraction.
xfit, yfit: 1D numpy.ndarray (list-like)
Data used in fitting.
bgfit: astropy.modeling.functional_models (1st-order) or float (0th-order)
The background removal information.
profilefit: astropy.modeling.functional_models
The fitting results.
param_cov: the covariance matrix of the parameters
"""
#Check to make sure user entered valid function
if isinstance(fitfunc, str):
if (fitfunc.lower() == 'plummer') or (fitfunc.lower() == 'gaussian'):
self.fitfunc = fitfunc.lower()
fitfunc_style = self.fitfunc.capitalize()
else:
raise ValueError("Reset fitfunc; You have not entered a valid function. Input 'Gaussian' or 'Plummer'")
else:
raise ValueError("Set a fitfunc; You have not entered a valid function. Input 'Gaussian' or 'Plummer'")
#Check whether beamwidth already exists, or whether they have inputed one here to compute deconvolved FWHM
if (hasattr(self,'beamwith')==False) & (type(beamwidth)!=None):
if isinstance(beamwidth, numbers.Number):
if (self.header is not None):
self.beamwidth = beamwidth * u.arcsec
else:
self.beamwidth = beamwidth * u.pix
else:
self.beamwidth = None
# Mask for bg removal
## take only bgdist, which should be a 2-tuple or 2-list
if np.asarray(bgdist).shape == (2,):
self.bgdist = np.sort(bgdist)
## below can be merged... ##########
if self.wrap:
maskbg = ((self.masterx >= self.bgdist[0])&\
(self.masterx < self.bgdist[1])&\
np.isfinite(self.mastery))
else:
maskbg = ((abs(self.masterx) >= self.bgdist[0])&\
(abs(self.masterx) < self.bgdist[1])&\
np.isfinite(self.mastery))
if sum(maskbg) == 0.:
raise ValueError("Reset bgdist; there is no data to fit for the background.")
else:
self.bgdist = None
warnings.warn("No background removal will be performed.")
# Mask for fitting
## Anything inside `fitdist` pc is used in fitting.
if isinstance(fitdist, numbers.Number):
self.fitdist = fitdist
mask = ((self.masterx >= (-self.fitdist))&\
(self.masterx < self.fitdist)&\
np.isfinite(self.mastery))
if sum(mask) == 0.:
raise ValueError("Reset fitdist; there is no data inside fitdist.")
elif np.asarray(fitdist).shape == (2,):
self.fitdist = np.sort(fitdist)
mask = ((self.masterx >= self.fitdist[0])&\
(self.masterx < self.fitdist[1])&\
np.isfinite(self.mastery))
if sum(mask) == 0.:
raise ValueError("Reset fitdist; there is no data inside fitdist.")
## Fit all data if no fitdist
else:
self.fitdist = None
## Just fool-proof
mask = (np.isfinite(self.masterx)&\
np.isfinite(self.mastery))
if sum(mask) == 0.:
raise ValueError("Reset fitdist; there is no data inside fitdist.")
# Fit for the background, and remove
## If bgdist (yes, background removal.)
if np.asarray(self.bgdist).shape == (2,):
## In the case where the profile is wrapped, simply take the mean in the background.
## This is because that a linear fit (with a slope) with only one side is not definite.
if self.wrap:
xbg, ybg = self.masterx, self.mastery
xbg, ybg = xbg[maskbg], ybg[maskbg]
self.xbg, self.ybg = xbg, ybg
self.bgfit = models.Polynomial1D(degree = 0,
c0 = np.median(self.ybg)) ### No fitting!
self.ybg_filtered = None ## no filtering during background removal
## Remove bg without fitting (or essentially a constant fit).
xfit, yfit = self.masterx[mask], self.mastery[mask]
yfit = yfit - self.bgfit(xfit) ##########
## pass nobs to fitter; masked
if self.binning:
self.nobsfit = self.masternobs[mask]
else:
self.nobsfit = None
print "The profile is wrapped. Use the 0th order polynomial in BG subtraction."
## A first-order bg removal is carried out only when the profile is not wrapped.
else:
## Fit bg
xbg, ybg = self.masterx, self.mastery
xbg, ybg = xbg[maskbg], ybg[maskbg]
self.xbg, self.ybg = xbg, ybg
bg_init = models.Polynomial1D(degree = bgdegree) ##########
fit_bg = fitting.LinearLSQFitter()
## outlier removal; use sigma clipping, set to 3 sigmas
fit_bg_or = fitting.FittingWithOutlierRemoval(fit_bg, sigma_clip,
niter=10, sigma=3.)
bg = fit_bg(bg_init, self.xbg, self.ybg)
data_or, bg_or = fit_bg_or(bg_init, self.xbg, self.ybg)
self.bgfit = bg_or.copy()
self.ybg_filtered = data_or ## a masked array returned by the outlier removal
## Remove bg and prepare for fitting
xfit, yfit = self.masterx[mask], self.mastery[mask]
yfit = yfit - self.bgfit(xfit)
## pass nobs to fitter; masked
if self.binning:
self.nobsfit = self.masternobs[mask]
else:
self.nobsfit = None
## If no bgdist
else:
self.bgfit = None
self.xbg, self.ybg = None, None
self.ybg_filtered = None
## Set up fitting without bg removal.
xfit, yfit = self.masterx[mask], self.mastery[mask]
## pass nobs to fitter; masked
if self.binning:
self.nobsfit = self.masternobs[mask]
else:
self.nobsfit = None
self.xfit, self.yfit = xfit, yfit
# Fit Model
## Gaussian model
if self.fitfunc == "gaussian":
g_init = models.Gaussian1D(amplitude = .8*np.max(self.yfit),
mean = 0.,
stddev=np.std(self.xfit),
fixed = {'mean': True},
bounds = {'amplitude': (0., np.inf),
'stddev': (0., np.inf)})
fit_g = fitting.LevMarLSQFitter()
if self.binning:
g = fit_g(g_init, self.xfit, self.yfit, weights = self.nobsfit)
else:
g = fit_g(g_init, self.xfit, self.yfit)
self.profilefit = g.copy()
self.param_cov=fit_g.fit_info['param_cov'] #store covariance matrix of the parameters
print '==== Gaussian ===='
print 'amplitude: %.3E'%self.profilefit.parameters[0]
print 'width: %.3f'%self.profilefit.parameters[2]
## Plummer-like model
elif self.fitfunc == "plummer":
g_init = Plummer1D(amplitude = .8*np.max(self.yfit),
powerIndex=2.,
flatteningRadius = np.std(self.xfit))
fit_g = fitting.LevMarLSQFitter()
if self.binning:
g = fit_g(g_init, self.xfit, self.yfit, weights = self.nobsfit)
else:
g = fit_g(g_init, self.xfit, self.yfit)
self.profilefit = g.copy()
self.param_cov=fit_g.fit_info['param_cov'] #store covariance matrix of the parameters
self.profilefit.parameters[2] = abs(self.profilefit.parameters[2]) #Make sure R_flat always positive
print '==== Plummer-like ===='
print 'amplitude: %.3E'%self.profilefit.parameters[0]
print 'p: %.3f'%self.profilefit.parameters[1]
print 'R_flat: %.3f'%self.profilefit.parameters[2]
else:
raise ValueError("Reset fitfunc; no valid function entered. Options include 'Gaussian' or 'Plummer'")
### Plot background fit if bgdist is not none ###
if self.bgdist is not None:
fig, ax = plt.subplots(figsize = (8, 8.), ncols = 1, nrows = 2)
axis = ax[0]
#Adjust axes limits
xlim=np.max(np.absolute([np.nanpercentile(self.xall[np.isfinite(self.yall)],1),np.nanpercentile(self.xall[np.isfinite(self.yall)],99)]))
if not self.wrap:
axis.set_xlim(-xlim,+xlim)
else:
axis.set_xlim(0., +xlim)
axis.set_ylim(np.nanpercentile(self.yall,0)-np.abs(0.5*np.nanpercentile(self.yall,0)),np.nanpercentile(self.yall,99.9)+np.abs(0.25*np.nanpercentile(self.yall,99.9)))
axis.plot(self.xall, self.yall, 'k.', markersize = 1., alpha = .1)
##########
if self.binning:
plotbinx, plotbiny = np.ravel(zip(self.bins[:-1], self.bins[1:])), np.ravel(zip(self.mastery, self.mastery))
axis.plot(plotbinx, plotbiny,
'r-')
# Plot the range
plot_bgdist = self.bgdist.copy()
plot_bgdist[~np.isfinite(plot_bgdist)] = np.asarray(axis.get_xlim())[~np.isfinite(plot_bgdist)]
axis.fill_between(plot_bgdist, *axis.get_ylim(),
facecolor = (0., 1., 0., .05),
edgecolor = 'g',
linestyle = '--',
linewidth = 1.)
axis.fill_between(-plot_bgdist, *axis.get_ylim(),
facecolor = (0., 1., 0., .05),
edgecolor = 'g',
linestyle = '--',
linewidth = 1.)
axis.plot(np.linspace(axis.get_xlim()[0],axis.get_xlim()[1],200), self.bgfit(np.linspace(axis.get_xlim()[0],axis.get_xlim()[1],200)),'g-', lw=3)
axis.set_xticklabels([])
axis.tick_params(labelsize=14)
xplot = self.xall
yplot = self.yall - self.bgfit(xplot)
#Add labels#
if self.bgfit.degree == 1:
axis.text(0.03, 0.95,"y=({:.2E})x+({:.2E})".format(self.bgfit.parameters[1],self.bgfit.parameters[0]),ha='left',va='top', fontsize=14, fontweight='bold',transform=axis.transAxes)#,bbox={'facecolor':'white', 'edgecolor':'none', 'alpha':1.0, 'pad':1})
elif self.bgfit.degree == 0:
axis.text(0.03, 0.95,"y=({:.2E})".format(self.bgfit.c0.value),ha='left',va='top', fontsize=14, fontweight='bold',transform=axis.transAxes)
else:
warnings.warn("Labeling BG functions of higher degrees during plotting are not supported yet.")
axis.text(0.97, 0.95,"Background\nFit", ha='right',va='top', fontsize=20, fontweight='bold',color='green',transform=axis.transAxes)#,bbox={'facecolor':'white', 'edgecolor':'none', 'alpha':1.0, 'pad':1})
axis=ax[1]
else:
fig, ax = plt.subplots(figsize = (8, 4.), ncols = 1, nrows = 1)
axis = ax
xplot=self.xall
yplot=self.yall
## Plot model
#Adjust axis limit based on percentiles of data
xlim=np.max(np.absolute([np.nanpercentile(self.xall[np.isfinite(self.yall)],1),np.nanpercentile(self.xall[np.isfinite(self.yall)],99)]))
if not self.wrap:
axis.set_xlim(-xlim,+xlim)
else:
axis.set_xlim(0., +xlim)
axis.set_ylim(np.nanpercentile(yplot,0)-np.abs(0.5*np.nanpercentile(yplot,0)),np.nanpercentile(yplot,99.9)+np.abs(0.25*np.nanpercentile(yplot,99.9)))
axis.plot(xplot, yplot, 'k.', markersize = 1., alpha = .1)
if self.binning:
if self.bgdist is not None:
plotbinx, plotbiny = np.ravel(zip(self.bins[:-1], self.bins[1:])), np.ravel(zip(self.mastery-self.bgfit(self.masterx), self.mastery-self.bgfit(self.masterx)))
else:
plotbinx, plotbiny = np.ravel(zip(self.bins[:-1], self.bins[1:])), np.ravel(zip(self.mastery, self.mastery))
axis.plot(plotbinx, plotbiny,
'r-')
# Plot the range
if self.fitdist is not None:
## symmetric fitting range
if isinstance(self.fitdist, numbers.Number):
axis.fill_between([-self.fitdist, self.fitdist], *axis.get_ylim(),
facecolor = (0., 0., 1., .05),
edgecolor = 'b',
linestyle = '--',
linewidth = 1.)
## asymmetric fitting range
elif np.asarray(self.fitdist).shape == (2,):
plot_fitdist = self.fitdist.copy()
plot_fitdist[~np.isfinite(plot_fitdist)] = np.asarray(axis.get_xlim())[~np.isfinite(plot_fitdist)]
axis.fill_between(plot_fitdist, *axis.get_ylim(),
facecolor = (0., 0., 1., .05),
edgecolor = 'b',
linestyle = '--',
linewidth = 1.)
## no fitting range; all data are used
else:
axis.fill_between(axis.get_xlim(), *axis.get_ylim(),
facecolor = (0., 0., 1., .05),
edgecolor = 'b',
linestyle = '--',
linewidth = 1.)
# Plot the predicted curve
axis.plot(np.linspace(axis.get_xlim()[0],axis.get_xlim()[1],200), self.profilefit(np.linspace(axis.get_xlim()[0],axis.get_xlim()[1],200)), 'b-', lw = 3., alpha = .6)
axis.text(0.03, 0.95,"{}={:.2E}\n{}={:.2f}\n{}={:.2f}".format(self.profilefit.param_names[0],self.profilefit.parameters[0],self.profilefit.param_names[1],self.profilefit.parameters[1],self.profilefit.param_names[2],self.profilefit.parameters[2]),ha='left',va='top', fontsize=14, fontweight='bold',transform=axis.transAxes)#,bbox={'facecolor':'white', 'edgecolor':'none', 'alpha':1.0, 'pad':1})
axis.text(0.97, 0.95,"{}\nFit".format(fitfunc_style), ha='right',va='top', fontsize=20, color='blue',fontweight='bold',transform=axis.transAxes)#,bbox={'facecolor':'white', 'edgecolor':'none', 'alpha':1.0, 'pad':1})
axis.tick_params(labelsize=14)
#add axis info
fig.tight_layout()
fig.subplots_adjust(hspace=0)
fig.text(0.5, -0.05, "Radial Distance ({})".format(str(self.imgscale.unit)),fontsize=25,ha='center')
fig.text(-0.05, 0.5, "Profile Height",fontsize=25,va='center',rotation=90)
# Return a dictionary to store the key setup Parameters
params = {'bgdist': self.bgdist,
'fitdist': self.fitdist,
'fitfunc': self.fitfunc}
self._params['fit_profile'] = params
# Return a dictionary to store the results
## All the fits are `astropy.model` objects.
results = {'bgfit': self.bgfit,
'profilefit': self.profilefit,
'xbg': self.xbg,
'ybg': self.ybg,
'xfit': self.xfit,
'yfit': self.yfit}
self._results['fit_profile'] = results
if self.fitfunc == "gaussian":
FWHM = 2.*np.sqrt(2.*np.log(2.))*self.profilefit.parameters[2]
if self.beamwidth is not None:
if (self.beamwidth.unit == u.arcsec) and (self.imgscale_ang is not None):
beamwidth_phys = (self.beamwidth/self.imgscale_ang).decompose()*self.imgscale.value
print 'Physical Size of the Beam:', beamwidth_phys*self.imgscale.unit
if np.isfinite(np.sqrt(FWHM**2.-beamwidth_phys**2.)):
FWHM_deconv = np.sqrt(FWHM**2.-beamwidth_phys**2.).value
else:
FWHM_deconv = np.nan
warnings.warn("The Gaussian width is not resolved.")
elif (self.beamwidth.unit == u.pix):
beamwidth_phys = self.beamwidth.value
print 'Beamwidth in the Pixel Unit:', self.beamwidth
if np.isfinite(np.sqrt(FWHM**2.-beamwidth_phys**2.)):
FWHM_deconv = np.sqrt(FWHM**2.-beamwidth_phys**2.).value
else:
FWHM_deconv = np.nan
warnings.warn("The width is not resolved.")
else:
FWHM_deconv = np.nan
warnings.warn("A beamwidth is not found. Deconvolved FWHMs cannot be derived.")
else:
FWHM_deconv = np.nan
warnings.warn("A beamwidth is not found. Deconvolved FWHMs cannot be derived.")
if self.fitfunc == "plummer":
FWHM = 2.*self.profilefit.parameters[2]*np.sqrt(2.**(2./(self.profilefit.parameters[1]-1.)) - 1.)
if self.beamwidth is not None:
if (self.beamwidth.unit == u.arcsec) and (self.imgscale_ang is not None):
beamwidth_phys = (self.beamwidth/self.imgscale_ang).decompose()*self.imgscale.value
print 'Physical Size of the Beam:', beamwidth_phys*self.imgscale.unit
if np.isfinite(np.sqrt(FWHM**2.-beamwidth_phys**2.)):
FWHM_deconv = np.sqrt(FWHM**2.-beamwidth_phys**2.).value
else:
FWHM_deconv = np.nan
warnings.warn("The width is not resolved.")
elif (self.beamwidth.unit == u.pix):
beamwidth_phys = self.beamwidth.value
print 'Beamwidth in the Pixel Unit:', self.beamwidth
if np.isfinite(np.sqrt(FWHM**2.-beamwidth_phys**2.)):
FWHM_deconv = np.sqrt(FWHM**2.-beamwidth_phys**2.).value
else:
FWHM_deconv = np.nan
warnings.warn("The width is not resolved.")
else:
FWHM_deconv = np.nan
warnings.warn("A beamwidth is not found. Deconvolved FWHMs cannot be derived.")
else:
FWHM_deconv = np.nan
warnings.warn("A beamwidth is not found. Deconvolved FWHMs cannot be derived.")
self.FWHM, self.FWHM_deconv = FWHM, FWHM_deconv
self._results['FWHM'] = FWHM
self._results['FWHM_deconv'] = FWHM_deconv
return self
def calculate_transform_coefficients(input_mag,
catalog_mag,
color,
input_mag_error=None,
catalog_mag_error=None,
faintest_mag=None,
order=1,
sigma=2.0,
gain=None,
verbose=False,
extended_output=False):
"""
Calculate linear transform coefficients from input magnitudes to catalog
magnitudes.
Parameters
----------
input_mag : numpy array or astropy Table column
Input magnitudes; for example, instrumental magnitudes.
catalog_mag : numpy array or astropy Table column
Catalog (or reference) magnitudes; the magnitudes to which the
input_mag will eventually be transformed.
color : numpy array or astropy Table column
Colors to use in determining transform coefficients.
input_mag_error : numpy array or astropy Table column, optional
Error in input magnitudes. Default is zero.
catalog_mag_error : numpy array or astropy Table column, optional
Error in catalog magnitudes. Default is zero.
faintest_mag_for_transform : float, optional
If this is not ``None``, the magnitude of the faintest catalog stars
to use in computing transform coefficients.
order : int, optional
Order of the polynomial fit to use in correcting for color.
sigma : float, optional
Value of sigma to use to reject outliers while fitting using
sigma clipping.
gain : float, optional
If not ``None``, adjust the instrumental magnitude by
-2.5 * log10(gain), i.e. gain correct the magnitude.
verbose : bool, optional
If ``True``, print some diagnostic information.
extended_output : bool, optional
If ``True``, return additional information.
Returns
-------
filtered_data : `~numpy.ma.core.MaskedArray`
The data, with the mask set ``True`` for the data that was *omitted*
from the fit.
model : `astropy.modeling.FittableModel`
Entries in the model are the coefficients in the fit made to the
data. Since the model is always a polynomial, these are terms in
a polynomial in the order of ascending power. In other words, the
coefficient ``ci`` is the coefficient of the term ``x**i``.
If ``extended_output=True``, then also return:
fit_input : tuple
A tuple of color, magnitude for only the stars brighter than
``faintest_mag_for_transform``. These are input to the sigma-clipping
fitter.
used_in_fit : tuple
A tuple of color, magnitude for only the stars brighter than
``faintest_mag_for_transform`` that were not sigma-cliped out.
Notes
-----
This function has some pretty serious limitations right now:
+ Errors in the independent variable are ignored.
+ Outliers are rejected using a modified loss function (Huber loss) that
cannot be modified.
+ No errors are estimated in the calculated transformation coefficients.
And there is all the stuff that is not listed here...
"""
if input_mag_error is None:
input_mag_error = np.zeros_like(input_mag)
if catalog_mag_error is None:
catalog_mag_error = np.zeros_like(catalog_mag)
if gain is None:
gain = 1.0
# Independent variable is the color, dependent variable is the
# difference between the measured (input) magnitude and the catalog
# magnitude.
mag_diff = catalog_mag - (input_mag - 2.5 * np.log10(gain))
# The error is the errors of those combined in quadrature.
combined_error = np.sqrt(input_mag_error**2 + catalog_mag_error**2)
# If both errors are zero then the error is omitted.
if (combined_error == 0).all():
dy = None
else:
dy = combined_error
g_init = models.Polynomial1D(order)
fit = fitting.LinearLSQFitter()
or_fit = fitting.FittingWithOutlierRemoval(fit,
sigma_clip,
niter=2,
sigma=sigma)
if faintest_mag is not None:
bright = (catalog_mag < faintest_mag).filled(False)
else:
bright = np.ones_like(mag_diff, dtype='bool')
bright_index = np.nonzero(bright)
if verbose:
print(color[bright].max())
# get fitted model and filtered data
or_fitted_model, filtered_data_mask = or_fit(g_init, color[bright],
mag_diff[bright])
# Restore the filtered_data to the same size as the input
# magnitudes. Unmasked values were included in the fit,
# masked were not, either because they were too faint
# or because they were sigma clipped out.
restored_mask = np.zeros_like(mag_diff, dtype='bool')
restored_mask[bright_index] = filtered_data_mask
restored_mask[~bright] = True
restored_filtered = mag_diff.copy()
restored_filtered.mask = restored_mask
if extended_output:
return (restored_filtered, or_fitted_model, (color[bright],
mag_diff[bright]),
(color[bright][~filtered_data_mask],
mag_diff[bright][~filtered_data_mask]))
else:
return (restored_filtered, or_fitted_model)
def optimal_extraction(self,
data,
mask,
var,
aper_lower,
aper_upper,
cr_rej=5,
max_iters=None):
"""Optimal extraction following Horne (1986, PASP 98, 609)"""
BAD_BITS = DQ.bad_pixel | DQ.cosmic_ray | DQ.no_data | DQ.unilluminated
slitlength, npix = data.shape
pixels = np.arange(npix)
all_x1 = self._center_pixels + aper_lower
all_x2 = self._center_pixels + aper_upper
ix1 = max(int(min(all_x1) + 0.5), 0)
ix2 = min(int(max(all_x2) + 1.5), slitlength)
fit_it = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(),
outlier_func=sigma_clip,
sigma_upper=3,
sigma_lower=None)
# If we don't have a VAR plane, assume uniform variance based
# on the pixel-to-pixel variations in the data
if var is None:
var_model = models.Const1D(
sigma_clipped_stats(data, mask=mask)[2]**2)
var = np.full_like(data[ix1:ix2], var_model.amplitude)
var_mask = np.zeros_like(var, dtype=bool)
else:
mvar_init = models.Polynomial1D(degree=1)
var_model, var_mask = fit_it(
mvar_init, np.ma.masked_where(mask.ravel(),
abs(data).ravel()), var.ravel())
var_mask = var_mask.reshape(var.shape)[ix1:ix2]
var = np.where(var_mask, var[ix1:ix2], var_model(data[ix1:ix2]))
if mask is None:
mask = np.zeros((ix2 - ix1, npix), dtype=DQ.datatype)
else:
mask = mask[ix1:ix2]
data = data[ix1:ix2]
# Step 4; first calculation of spectrum. We don't do any masking
# here since we need all the flux
spectrum = data.sum(axis=0)
weights = np.where(var > 0, var, 0)
unmask = np.ones_like(data, dtype=bool)
iter = 0
while True:
# Step 5: construct spatial profile for each wavelength pixel
profile = np.divide(data,
spectrum,
out=np.zeros_like(data, dtype=np.float32),
where=spectrum > 0)
profile_models = []
for row, wt_row in zip(profile, weights):
m_init = models.Chebyshev1D(degree=3, domain=[0, npix - 1])
m_final, _ = fit_it(m_init, pixels, row, weights=wt_row)
profile_models.append(m_final(pixels))
profile_model_spectrum = np.array(
[np.where(pm < 0, 0, pm) for pm in profile_models])
sums = profile_model_spectrum.sum(axis=0)
model_profile = divide0(profile_model_spectrum,
sums,
like_num=True)
# Step 6: revise variance estimates
var = np.where(var_mask | mask & BAD_BITS, var,
var_model(abs(model_profile * spectrum)))
weights = divide0(1.0, var)
# Step 7: identify cosmic ray hits: we're (probably) OK
# to flag more than 1 per wavelength
sigma_deviations = divide0(data - model_profile * spectrum,
np.sqrt(var)) * unmask
mask[sigma_deviations > cr_rej] |= DQ.cosmic_ray
# most_deviant = np.argmax(sigma_deviations, axis=0)
# for i, most in enumerate(most_deviant):
# if sigma_deviations[most, i] > cr_rej:
# mask[most, i] |= DQ.cosmic_ray
last_unmask = unmask
unmask = (mask & BAD_BITS) == 0
spec_numerator = np.sum(unmask * model_profile * data * weights,
axis=0)
spec_denominator = np.sum(unmask * model_profile**2 * weights,
axis=0)
self.data = divide0(spec_numerator, spec_denominator)
self.var = divide0(np.sum(unmask * model_profile, axis=0),
spec_denominator)
self.mask = np.bitwise_and.reduce(mask, axis=0)
spectrum = self.data
iter += 1
# An unchanged mask means XORing always produces False
if ((np.sum(unmask ^ last_unmask) == 0)
or (max_iters is not None and iter > max_iters)):
break
def create_polynomial_transform(transform,
in_coords,
ref_coords,
order=3,
max_iters=5,
match_radius=0.1,
clip=True,
log=None):
"""
This function maps a set of 2D input coordinates to a set of 2D reference
coordiantes using a pair of Polynomial2D object (one for each ordinate),
given an initial transforming model.
Parameters
----------
transform : astropy.models.Model
the initial guess of the transform between coordinate frames
in_coords : 2-tuple of sequences
input coordinates being mapped to reference
ref_coords : 2-tuple of sequences
reference coordinates
order : int
order of polynomial fit in each ordinate
max_iters : int
maximum number of iterations to perform
match_radius : float
matching radius for sources (in units of the reference coords)
clip : bool
sigma-clip sources after matching?
log : logging object
Returns
-------
transform : Model
a model (and its inverse) to map in_coords to ref_coords
matched: ndarray
matched incoord for each refcoord
"""
affine = adwcs.calculate_affine_matrices(transform, shape=(100, 100))
num_params = [
len(models.Polynomial2D(degree=i).parameters) for i in range(order + 1)
]
orig_order = last_order = order
xref, yref = ref_coords
xin, yin = in_coords
if clip:
fit_it = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(),
sigma_clip,
sigma=3)
else:
fit_it = fitting.LinearLSQFitter()
matched = match_sources((xref, yref),
transform(xin, yin),
radius=match_radius)
num_matched = np.sum(matched >= 0)
niter = 0
while True:
# No point trying to compute a more complex model if it will
# be insufficiently constrained
order = min(np.searchsorted(num_params, num_matched, side="right"),
orig_order)
if order < last_order:
log.warning(f"Downgrading fit to order {order} due to "
"limited number of matches.")
elif order > last_order:
log.stdinfo(f"Upgrading fit to order {order} due to "
"increased number of matches.")
xmodel = models.Polynomial2D(degree=order,
c0_0=affine.offset[1],
c1_0=affine.matrix[1, 1],
c0_1=affine.matrix[1, 0])
ymodel = models.Polynomial2D(degree=order,
c0_0=affine.offset[0],
c1_0=affine.matrix[0, 1],
c0_1=affine.matrix[0, 0])
old_num_matched = num_matched
xobj_matched, yobj_matched = [], []
xref_matched, yref_matched = [], []
for i, m in enumerate(matched):
if m >= 0:
xref_matched.append(xref[i])
yref_matched.append(yref[i])
xobj_matched.append(xin[m])
yobj_matched.append(yin[m])
xmodel = fit_it(xmodel, np.array(xobj_matched), np.array(yobj_matched),
xref_matched)
ymodel = fit_it(ymodel, np.array(xobj_matched), np.array(yobj_matched),
yref_matched)
if clip:
xmodel, ymodel = xmodel[0], ymodel[0]
transform = models.Mapping((0, 1, 0, 1)) | (xmodel & ymodel)
matched = match_sources((xref, yref),
transform(xin, yin),
radius=match_radius)
num_matched = np.sum(matched >= 0)
last_order = order
niter += 1
log.debug(f"Iteration {niter}: Matched {num_matched} objects")
if num_matched == old_num_matched or niter > max_iters:
break
xmodel_inv = fit_it(xmodel, np.array(xref_matched), np.array(yref_matched),
xobj_matched)
ymodel_inv = fit_it(ymodel, np.array(xref_matched), np.array(yref_matched),
yobj_matched)
if clip:
xmodel_inv, ymodel_inv = xmodel_inv[0], ymodel_inv[0]
transform.inverse = models.Mapping(
(0, 1, 0, 1)) | (xmodel_inv & ymodel_inv)
return transform, matched
def fit_refraction_function(self,
steps=10,
plot=False,
sample=None,
debug=False):
"""
Fits a refraction function using a 3rd order Legendre
Polynomial to the x and y pixel offsets caused by
atmospheric refraction.
Parameters
----------
steps : int
Number of segments of the data cube to consider. The
larger this number, the finer the detail in fitting
offsets.
plot : bool
Plots the function and the corresponding data points.
sample : tuple, (w_0, w_1)
Wavelength interval to be considered in the fit.
debug : bool
Plots debugging graphs to confirm that the shifts
provided are actually matching the reference.
Returns
-------
None
"""
data = copy.deepcopy(self.science)
d = np.array(
[ma.median(_, axis=0) for _ in np.array_split(data, steps)])
planes = np.array([((_[-1] + _[0]) / 2)
for _ in np.array_split(self.wavelength, steps)])
for i, j in enumerate(d):
d[i] /= j.max()
md = d[int(len(d) / 2.0)]
mid_point = planes[int(len(d) / 2.0)]
if sample is None:
sample = self.wavelength[[0, -1]]
sample_mask = (planes >= sample[0]) & (planes <= sample[1])
d = d[sample_mask]
planes = planes[sample_mask]
x_off, y_off = self._check_registration(reference=md, images=d)
x_off *= self.sampling
y_off *= self.sampling
offsets, angle = self._offsets_rotation(x_off, y_off)
self.refraction_angle = angle
if debug:
print(self.file_name)
print('OFFSETS')
print(offsets)
model = DifferentialRefraction(temperature=self.temperature,
pressure=self.pressure,
air_mass=self.air_mass,
wl_0=mid_point)
model.wl_0.fixed = True
fitter = fitting.FittingWithOutlierRemoval(fitting.SLSQPLSQFitter(),
sigma_clip,
niter=3,
sigma=3.0)
rejected, shift = fitter(model, planes, offsets, acc=1e-12)
if plot:
fig, ax = plt.subplots(1, 1, sharex='col')
ax.scatter(planes, offsets, c=planes)
ax.set_ylabel('Differential refraction (arcsec)')
ax.set_xlabel('Wavelength')
ax.plot(self.wavelength, shift(self.wavelength))
pars = [getattr(shift, _).value for _ in ['air_mass', 'wl_0']]
pars.append(np.rad2deg(angle))
ax.set_title(
'secz = {:.2f}; wl_0 = {:.0f}; angle = {:.2f}'.format(*pars))
ax.grid()
plt.show()
self.atmospheric_shift = shift
if debug:
self._debug_plots(d, planes)
def cal_color(self,
matched,
m_inst,
filt,
color,
C=None,
mlim=[14, 18],
gmi_lim=[0.2, 3.0]):
"""Estimate calibration constant with color correction.
Parameters
----------
matched : array-like
Object IDs matched to sources. May be a
`~numpy.ma.MaskedArray`.
m_inst : array-like
Instrumental magnitudes for each source in matched.
filt : string
Filter to calibrate to, e.g., 'r'.
color : string
Color to consider, e.g., 'g-r'.
C : float, optional
Set to a value to hold the color correction fixed.
mlim : list, optional
Only fit stars with this magnitude range in filter ``filt``.
gmi_lim : list, optional
Only fit stars with this g-i color range, or `None` to disable.
Returns
-------
zp, C, unc : float
Zero-point magnitude, color slope, and uncertainty.
m - m_inst = C * color + zp
m, cindex : MaskedArray
Catalog magnitude and color index.
gmi : ndarray
g-i color for each source.
"""
if filt not in self.table.filter2col:
raise ValueError('Filter must be one of {}.'.format(
self.table.filter2col.keys()))
blue, red = color.split('-')
if gmi_lim is None:
limits = [-np.inf, np.inf, min(mlim), max(mlim)]
else:
limits = [min(gmi_lim), max(gmi_lim), min(mlim), max(mlim)]
columns = ("{filt[mag]},{filt[err]},{b[mag]}-{r[mag]},"
"{g[mag]}-{i[mag]}").format(
filt=self.table.filter2col[filt],
b=self.table.filter2col[blue],
r=self.table.filter2col[red],
g=self.table.filter2col['g'],
i=self.table.filter2col['i'])
cat = self.lookup(matched, columns)
m = np.ma.MaskedArray(np.zeros(len(matched)),
mask=np.ones(len(matched), bool))
gmi = np.zeros_like(m.data)
cindex = m.copy()
for i in range(len(cat)):
if len(cat[i]) > 0:
m[i], merr, cindex[i], gmi[i] = cat[i]
if all(
(gmi[i] >= limits[0], gmi[i] <= limits[1],
m[i] >= limits[2], m[i] <= limits[3], m[i] / merr > 2)):
m.mask[i] = False
cindex.mask[i] = False
else:
m.mask[i] = True
cindex.mask[i] = True
dm = m - m_inst
model = models.Linear1D(slope=0, intercept=28)
if C is not None:
model.slope.value = C
model.slope.fixed = True
fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(),
sigma_clip)
i = np.isfinite(dm) * ~dm.mask
if sum(i) == 0:
raise CalibrationError('All sources masked.')
if (astropy_version[0] > 3
or (astropy_version[0] == 3 and astropy_version[1] >= 1)):
# Return order changed in astropy 3.1
# (http://docs.astropy.org/en/stable/changelog.html#id10)
# Also now returns a boolean mask array rather than a
# MaskedArray of the data which could be applied back to
# reconstuct 'line' if needed (not currently used)
fit, mask = fitter(model, cindex[i], dm[i])
else:
line, fit = fitter(model, cindex[i], dm[i])
C = fit.slope.value
zp = fit.intercept.value
cal_unc = sigma_clipped_stats((dm - fit(cindex))[i])[2]
return zp, C, cal_unc, m, cindex, gmi
def _fit(self, image, weights=None, plot=False):
# Convert array-like input to a MaskedArray internally, to ensure it's
# an `ndarray` instance and that any mask gets passed through to the
# fitter:
origim = image # plotting works with the original image co-ords
image = np.ma.masked_array(image)
# Determine how many pixels we're fitting each vector over:
try:
npix = image.shape[self.axis]
except IndexError:
raise ValueError(
'axis={0} out of range for input shape {1}'.format(
self.axis, image.shape))
# Define pixel grid to fit on:
points = (np.arange(npix, dtype=np.int16)
if self.points is None else self.points)
if self.domain is None:
self.domain = (min(points), max(points))
# Classify the model to be fitted:
try:
astropy_model = issubclass(self.model_class, Model)
except TypeError:
astropy_model = False
# Convert user regions to a Boolean mask for slicing:
user_reg = ~at.create_mask_from_regions(points, self._slices)
# Record the actual fitted, inclusive ranges in 1-indexed pixels for
# this dataset (with no wildcards or adjacent/overlapping ranges etc.),
# mostly for plot annotation. Find the starting index of each selected
# or omitted range in user_reg (considering regions outside the array
# False and starting from the first True), then subtract 1 from every
# second index, pair them to get (start, end) ranges and add back the
# origin of 1:
reg_changes = np.where(
np.concatenate((user_reg[:1], user_reg[1:] != user_reg[:-1],
user_reg[-1:])))[0]
self.regions_pix = tuple(
(start, end)
for start, end in zip(reg_changes[::2] + 1, reg_changes[1::2]))
self.fitted_regions = tuple(
(points[start], points[end])
for start, end in zip(reg_changes[::2], reg_changes[1::2] - 1))
# To support fitting any axis of an N-dimensional array, we must
# flatten all the other dimensions into a single "model set axis"
# first; it's about 10-15% more efficient to stack astropy models along
# the second axis (for a 3k x 2k image), because that's what the linear
# fitter does internally, but for general callable functions we stack
# along the first axis in order to loop over the fits easily:
if astropy_model:
ax_before = 0
stack_shape = (npix, ) if image.size == npix else (npix, -1)
else:
ax_before = image.ndim
stack_shape = (-1, npix)
image = np.rollaxis(image, self.axis, ax_before)
self._tmpshape = image.shape
image = image.reshape(stack_shape)
if weights is not None:
weights = np.rollaxis(np.array(weights), self.axis,
ax_before).reshape(stack_shape)
# Record intermediate array properties for later evaluation of models
# onto the same grid, where only a subset of rows/cols may have been
# fitted due to some being entirely masked:
self._stack_shape = image.shape
self._dtype = image.dtype if image.dtype.kind == 'f' else np.float32
# Create a full-sized mask within which the fitter will populate the
# user-specified region(s) and the rest will remain masked out:
mask = np.ones(image.shape, dtype=bool)
# Branch pending on whether we're using an AstroPy model or some other
# supported fitting function (principally splines):
if astropy_model:
# Treat single model specially because FittingWithOutlierRemoval
# fails for "1-model sets" and it should be more efficient anyway:
image_to_fit = image
if image.ndim == 1:
n_models = 1
elif image.mask is np.ma.nomask:
n_models = image.shape[1]
else:
# remove fully masked columns otherwise this will lead to
# Runtime warnings from Numpy because of divisions by zero.
self._good_cols = (~image.mask).sum(axis=0) > self.order
n_models = np.sum(self._good_cols)
if n_models < image.shape[1]:
image_to_fit = image[:, self._good_cols]
if weights is not None:
weights = weights[:, self._good_cols]
model_set = self.model_class(
degree=self.order,
n_models=n_models,
domain=self.domain,
model_set_axis=(None if n_models == 1 else 1),
**self.model_args)
# Configure iterative linear fitter with rejection:
fitter = fitting.FittingWithOutlierRemoval(
fitting.LinearLSQFitter(),
sigma_clip,
niter=self.niter,
# additional args are passed to outlier_func, i.e. sigma_clip
sigma_lower=self.sigma_lower,
sigma_upper=self.sigma_upper,
maxiters=1,
cenfunc='mean',
stdfunc='std',
grow=self.grow # requires AstroPy 4.2 (#10613)
)
# Fit the pixel data with rejection of outlying points:
fitted_models, fitted_mask = fitter(
model_set,
points[user_reg],
image_to_fit[user_reg],
weights=None if weights is None else weights[user_reg])
# Incorporate mask for fitted columns into the full-sized mask:
if image.ndim > 1 and n_models < image.shape[1]:
# this is quite ugly, but seems the best way to assign to an
# array with a mask on both dimensions. This is equivalent to:
# mask[user_reg, masked_cols] = fitted_mask
mask[user_reg[:, None] & self._good_cols] = fitted_mask.flat
else:
mask[user_reg] = fitted_mask
else:
max_order = len(points) - self.model_args["k"]
if self.order is not None and self.order > max_order:
self.order = max_order
# If there are no weights, produce a None for every row:
weights = iter(lambda: None, True) if weights is None else weights
fitted_models = []
for n, (imrow, wrow) in enumerate(zip(image, weights)):
# Deal with weights being None or all undefined in regions
# without data (should we allow for NaNs as well?). The scipy
# spline-fitting routines state that weights should be inverse
# standard deviation, whereas fit_1D takes inverse-variance.
wrow = (wrow[user_reg]
if wrow is not None and np.any(wrow) else None)
# Could generalize this a bit further using another dict to map
# parameter names in function_map, eg. weights -> w?
single_model = self.model_class(points[user_reg],
imrow[user_reg],
order=self.order,
w=wrow,
niter=self.niter,
grow=int(self.grow),
sigma_lower=self.sigma_lower,
sigma_upper=self.sigma_upper,
maxiters=1,
**self.model_args)
fitted_models.append(single_model)
# Retrieve the mask from the spline object, but discard the
# fitted values because they only apply to a subsection
# (user_reg) of the original grid, rather than whatever points
# the user might request, and re-using them where possible only
# improves performance by ~1% for a single fit & evaluation:
mask[n][user_reg] = single_model.mask
single_model.data = None
# Save the set of fitted models in the flattened co-ordinate system,
# to allow later (re-)evaluation at arbitrary points:
self._models = fitted_models
# Convert the mask to the ordering & shape of the input array and
# save it. Calculate rms.
mask = mask.reshape(self._tmpshape)
if astropy_model:
start = (self.axis + 1) or mask.ndim
self.mask = np.rollaxis(mask, 0, start)
rms = (np.rollaxis(image.reshape(self._tmpshape), 0, start) -
self.evaluate())[~self.mask].std()
else:
self.mask = np.rollaxis(mask, -1, self.axis)
rms = (np.rollaxis(image.reshape(self._tmpshape), -1, self.axis) -
self.evaluate())[~self.mask].std()
self.rms = rms
# Plot the fit:
if plot:
self._plot(origim, index=None if plot is True else plot)