def MasterWave(self, fitsdict, det):
"""
Generate Master Wave frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
if self._mswave[det - 1] is not None:
msgs.info("An identical master arc frame already exists")
return False
if self._argflag['reduce']['usewave'] in ['wave']:
msgs.info("Preparing a master wave frame")
mswave = arutils.func_val(self._wvcalib[det - 1]['fitc'],
self._tilts[det - 1],
self._wvcalib[det - 1]['function'],
minv=self._wvcalib[det - 1]['fmin'],
maxv=self._wvcalib[det - 1]['fmax'])
else: # It must be the name of a file the user wishes to load
mswave_name = self._argflag['run'][
'masterdir'] + '/' + self._argflag['reduce']['usewave']
mswave = arload.load_master(mswave_name, frametype=None)
# Set and then delete the Master Arc frame
self.SetMasterFrame(mswave, "wave", det)
del mswave
return True
def extrapolate(outpar, ords, function='polynomial'):
nords = ords.size
x0ex = arutils.func_val(outpar['x0res'], ords, function,
minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
# Order centre
high_matr = np.zeros((nords, outpar['npc']))
for i in xrange(1, outpar['npc']+1):
if outpar['coeffstr'][i-1][0] == -9.99E9:
high_matr[:,i-1] = np.ones(nords)*outpar['high_fit'][0,i-1]
continue
high_matr[:,i-1] = arutils.func_val(outpar['coeffstr'][i-1], ords, function,
minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
extfit = np.dot(outpar['eigv'], high_matr.T) + np.outer(x0ex, np.ones(outpar['eigv'].shape[0])).T
outpar['high_matr'] = high_matr
return extfit, outpar
def refine_iter(outpar, orders, mask, irshft, relshift, fitord, function='polynomial'):
fail = False
x0ex = arutils.func_val(outpar['x0res'], orders, function, minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
# Make the refinement
x0ex[irshft] += relshift
# Refit the data to improve the refinement
good = np.where(mask != 0.0)[0]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
null, x0res = arutils.robust_polyfit(orders[good], x0ex[good], fitord, sigma=2.0, function=function,
minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
#x0res = arutils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1])
x0fit = arutils.func_val(x0res, orders, function, minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
chisq = (x0ex[good]-x0fit[good])**2.0
chisqnu = np.sum(chisq)/np.sum(mask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu > 0.5: # The refinement went wrong, ignore the refinement
fail = True
outpar['x0res'] = x0res
extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(x0fit, np.ones(outpar['eigv'].shape[0])).T
return extfit, outpar, fail
def refine_iter(outpar,
orders,
mask,
irshft,
relshift,
fitord,
function='polynomial'):
fail = False
x0ex = arutils.func_val(outpar['x0res'],
orders,
function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
# Make the refinement
x0ex[irshft] += relshift
# Refit the data to improve the refinement
good = np.where(mask != 0.0)[0]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
null, x0res = arutils.robust_polyfit(orders[good],
x0ex[good],
fitord,
sigma=2.0,
function=function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
#x0res = arutils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1])
x0fit = arutils.func_val(x0res,
orders,
function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
chisq = (x0ex[good] - x0fit[good])**2.0
chisqnu = np.sum(chisq) / np.sum(mask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu > 0.5: # The refinement went wrong, ignore the refinement
fail = True
outpar['x0res'] = x0res
extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(
x0fit, np.ones(outpar['eigv'].shape[0])).T
return extfit, outpar, fail
def extrapolate(outpar, ords, function='polynomial'):
nords = ords.size
x0ex = arutils.func_val(outpar['x0res'],
ords,
function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
# Order centre
high_matr = np.zeros((nords, outpar['npc']))
for i in xrange(1, outpar['npc'] + 1):
if outpar['coeffstr'][i - 1][0] == -9.99E9:
high_matr[:, i - 1] = np.ones(nords) * outpar['high_fit'][0, i - 1]
continue
high_matr[:, i - 1] = arutils.func_val(outpar['coeffstr'][i - 1],
ords,
function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
extfit = np.dot(outpar['eigv'], high_matr.T) + np.outer(
x0ex, np.ones(outpar['eigv'].shape[0])).T
outpar['high_matr'] = high_matr
return extfit, outpar
def apply_sensfunc(slf, scidx, fitsdict, MAX_EXTRAP=0.05):
"""
Apply the sensitivity function to the data
We also correct for extinction.
Parameters
----------
MAX_EXTRAP : float, optional [0.05]
Fractional amount to extrapolate sensitivity function
"""
# Load extinction data
extinct = load_extinction_data(slf)
airmass = fitsdict['airmass'][scidx]
# Loop on objects
for spobj in slf._specobjs:
# Loop on extraction modes
for extract_type in ['boxcar']:
try:
extract = getattr(spobj, extract_type)
except AttributeError:
continue
msgs.info("Fluxing {:s} extraction".format(extract_type))
wave = extract['wave'] # for convenience
scale = np.zeros(wave.size)
# Allow for some extrapolation
dwv = slf._sensfunc['wave_max'] - slf._sensfunc['wave_min']
inds = ((wave >= slf._sensfunc['wave_min'] - dwv * MAX_EXTRAP)
& (wave <= slf._sensfunc['wave_max'] + dwv * MAX_EXTRAP))
mag_func = arutils.func_val(slf._sensfunc['c'], wave[inds],
slf._sensfunc['func'])
sens = 10.0**(0.4 * mag_func)
# Extinction
ext_corr = extinction_correction(wave[inds], airmass, extinct)
scale[inds] = sens * ext_corr
# Fill
extract['flam'] = extract['counts'] * scale / slf._fitsdict[
'exptime'][scidx]
extract['flam_var'] = (
extract['var'] * (scale / slf._fitsdict['exptime'][scidx])**2)
def apply_sensfunc(slf, scidx, fitsdict, MAX_EXTRAP=0.05):
"""
Apply the sensitivity function to the data
We also correct for extinction.
Parameters
----------
MAX_EXTRAP : float, optional [0.05]
Fractional amount to extrapolate sensitivity function
"""
# Load extinction data
extinct = load_extinction_data(slf)
airmass = fitsdict['airmass'][scidx]
# Loop on objects
for spobj in slf._specobjs:
# Loop on extraction modes
for extract_type in ['boxcar']:
try:
extract = getattr(spobj,extract_type)
except AttributeError:
continue
msgs.info("Fluxing {:s} extraction".format(extract_type))
wave = extract['wave'] # for convenience
scale = np.zeros(wave.size)
# Allow for some extrapolation
dwv = slf._sensfunc['wave_max']-slf._sensfunc['wave_min']
inds = ((wave >= slf._sensfunc['wave_min']-dwv*MAX_EXTRAP)
& (wave <= slf._sensfunc['wave_max']+dwv*MAX_EXTRAP))
mag_func = arutils.func_val(slf._sensfunc['c'], wave[inds],
slf._sensfunc['func'])
sens = 10.0**(0.4*mag_func)
# Extinction
ext_corr = extinction_correction(wave[inds],airmass,extinct)
scale[inds] = sens*ext_corr
# Fill
extract['flam'] = extract['counts']*scale/slf._fitsdict['exptime'][scidx]
extract['flam_var'] = (extract['var']*
(scale/slf._fitsdict['exptime'][scidx])**2)
def arc_fit_qa(slf, fit, arc_spec, outfil=None):
"""
QA for Arc spectrum
Parameters
----------
fit : Wavelength fit
arc_spec : ndarray
Arc spectrum
outfil : str, optional
Name of output file
"""
if outfil is not None:
msgs.error("Not ready for this anymore")
# Begin
plt.figure(figsize=(8, 4.0))
plt.clf()
gs = gridspec.GridSpec(2, 2)
# Simple spectrum plot
ax_spec = plt.subplot(gs[:, 0])
ax_spec.plot(np.arange(len(arc_spec)), arc_spec)
ymin, ymax = 0., np.max(arc_spec)
ysep = ymax * 0.03
for kk, x in enumerate(fit['xfit'] * fit['xnorm']):
yline = np.max(arc_spec[int(x) - 2:int(x) + 2])
# Tick mark
ax_spec.plot([x, x], [yline + ysep * 0.25, yline + ysep], 'g-')
# label
ax_spec.text(x,
yline + ysep * 1.3,
'{:s} {:g}'.format(fit['ions'][kk], fit['yfit'][kk]),
ha='center',
va='bottom',
size='xx-small',
rotation=90.,
color='green')
ax_spec.set_xlim(0., len(arc_spec))
ax_spec.set_ylim(ymin, ymax * 1.2)
ax_spec.set_xlabel('Pixel')
ax_spec.set_ylabel('Flux')
# Arc Fit
ax_fit = plt.subplot(gs[0, 1])
# Points
ax_fit.scatter(fit['xfit'] * fit['xnorm'], fit['yfit'], marker='x')
if len(fit['xrej']) > 0:
ax_fit.scatter(fit['xrej'] * fit['xnorm'],
fit['yrej'],
marker='o',
edgecolor='gray',
facecolor='none')
# Solution
xval = np.arange(len(arc_spec))
wave = arutils.func_val(fit['fitc'],
xval / fit['xnorm'],
'legendre',
minv=fit['fmin'],
maxv=fit['fmax'])
ax_fit.plot(xval, wave, 'r-')
xmin, xmax = 0., len(arc_spec)
ax_fit.set_xlim(xmin, xmax)
ymin, ymax = np.min(wave) * .95, np.max(wave) * 1.05
ax_fit.set_ylim(np.min(wave) * .95, np.max(wave) * 1.05)
ax_fit.set_ylabel('Wavelength')
ax_fit.get_xaxis().set_ticks([]) # Suppress labeling
# Stats
wave_fit = arutils.func_val(fit['fitc'],
fit['xfit'],
'legendre',
minv=fit['fmin'],
maxv=fit['fmax'])
dwv_pix = np.median(np.abs(wave - np.roll(wave, 1)))
rms = np.sqrt(np.sum(
(fit['yfit'] - wave_fit)**2) / len(fit['xfit'])) # Ang
ax_fit.text(0.1 * len(arc_spec),
0.90 * ymin + (ymax - ymin),
r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(dwv_pix),
size='small')
ax_fit.text(0.1 * len(arc_spec),
0.80 * ymin + (ymax - ymin),
'RMS={:.3f} (pixels)'.format(rms / dwv_pix),
size='small')
# Arc Residuals
ax_res = plt.subplot(gs[1, 1])
res = fit['yfit'] - wave_fit
ax_res.scatter(fit['xfit'] * fit['xnorm'], res / dwv_pix, marker='x')
ax_res.plot([xmin, xmax], [0., 0], 'k--')
ax_res.set_xlim(xmin, xmax)
ax_res.set_xlabel('Pixel')
ax_res.set_ylabel('Residuals (Pix)')
# Finish
plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0)
slf._qa.savefig(bbox_inches='tight')
plt.close()
return
def basis(xfit,
yfit,
coeff,
npc,
pnpc,
weights=None,
skipx0=True,
x0in=None,
mask=None,
function='polynomial',
retmask=False):
nrow = xfit.shape[0]
ntrace = xfit.shape[1]
if x0in is None:
x0in = np.arange(float(ntrace))
# Mask out some orders if they are bad
if mask is None or mask.size == 0:
usetrace = np.arange(ntrace)
outmask = np.ones((nrow, ntrace))
else:
usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
outmask = np.ones((nrow, ntrace))
outmask[:, mask] = 0.0
# Do the PCA analysis
eigc, hidden = get_pc(coeff[1:npc + 1, usetrace], npc)
modl = arutils.func_vander(xfit[:, 0], function, npc)
eigv = np.dot(modl[:, 1:], eigc)
med_hidden = np.median(hidden, axis=1)
med_highorder = med_hidden.copy()
med_highorder[0] = 0
high_order_matrix = med_highorder.T[np.newaxis, :].repeat(ntrace, axis=0)
# y = hidden[0,:]
# coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)
# y = hidden[1,:]
# coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)
coeffstr = []
for i in xrange(1, npc + 1):
# if pnpc[i] == 0:
# coeffstr.append([-9.99E9])
# continue
# coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
if weights is not None:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
hidden[i - 1, :],
pnpc[i],
weights=weights[usetrace],
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1])
else:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
hidden[i - 1, :],
pnpc[i],
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1])
coeffstr.append(coeff0)
high_order_matrix[:, i - 1] = arutils.func_val(coeff0,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
# high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
high_fit = high_order_matrix.copy()
high_order_fit = np.dot(eigv, high_order_matrix.T)
sub = (yfit - high_order_fit) * outmask
numer = np.sum(sub, axis=0)
denom = np.sum(outmask, axis=0)
x0 = np.zeros(float(ntrace))
fitmask = np.zeros(float(ntrace))
#fitmask[mask] = 1
x0fit = np.zeros(float(ntrace))
chisqnu = 0.0
chisqold = 0.0
robust = True
#svx0 = numer/(denom+(denom == 0).astype(np.int))
if not skipx0:
fitmask = (np.abs(denom) > 10).astype(np.int)
if robust:
good = np.where(fitmask != 0)[0]
bad = np.where(fitmask == 0)[0]
x0[good] = numer[good] / denom[good]
imask = np.zeros(float(ntrace))
imask[bad] = 1.0
ttmask, x0res = arutils.robust_polyfit(x0in,
x0,
pnpc[0],
weights=weights,
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1],
initialmask=imask)
x0fit = arutils.func_val(x0res,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
good = np.where(ttmask == 0)[0]
xstd = 1.0 # This should represent the dispersion in the fit
chisq = ((x0[good] - x0fit[good]) / xstd)**2.0
chisqnu = np.sum(chisq) / np.sum(ttmask)
fitmask = 1.0 - ttmask
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
else:
for i in xrange(1, 5):
good = np.where(fitmask != 0)[0]
x0[good] = numer[good] / denom[good]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
x0res = arutils.func_fit(x0in[good],
x0[good],
function,
pnpc[0],
weights=weights,
minv=x0in[0],
maxv=x0in[-1])
x0fit = arutils.func_val(x0res,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
chisq = (x0[good] - x0fit[good])**2.0
fitmask[good] *= (chisq < np.sum(chisq) / 2.0).astype(np.int)
chisqnu = np.sum(chisq) / np.sum(fitmask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu == chisqold:
break
else:
chisqold = chisqnu
if chisqnu > 2.0:
msgs.warn("PCA has very large residuals")
elif chisqnu > 0.5:
msgs.warn("PCA has fairly large residuals")
#bad = np.where(fitmask==0)[0]
#x0[bad] = x0fit[bad]
else:
x0res = 0.0
x3fit = np.dot(eigv, high_order_matrix.T) + np.outer(x0fit,
np.ones(nrow)).T
outpar = dict({
'high_fit': high_fit,
'x0': x0,
'x0in': x0in,
'x0fit': x0fit,
'x0res': x0res,
'x0mask': fitmask,
'hidden': hidden,
'usetrc': usetrace,
'eigv': eigv,
'npc': npc,
'coeffstr': coeffstr
})
if retmask:
return x3fit, outpar, tmask
else:
return x3fit, outpar
def bspline_magfit(wave, flux, var, flux_std, nointerp=False, **kwargs):
"""
Perform a bspline fit to the flux ratio of standard to
observed counts. Used to generate a sensitivity function.
Parameters
----------
wave : ndarray
flux : ndarray
counts/s as observed
var : ndarray
variance
flux_std : Quantity array
standard star true flux (erg/s/cm^2/A)
nointer : bool, optional [False]
Skip interpolation over bad points (not recommended)?
**kwargs : keywords for robust_polyfit
Returns
-------
"""
invvar = (var > 0.)/(var + (var <= 0.))
nx = wave.size
pos_error = 1./np.sqrt(np.maximum(invvar,0.) + (invvar == 0))
pos_mask = (flux > pos_error/10.0) & (invvar > 0) & (flux_std > 0.0)
#pos = pos_mask==1 npos)
fluxlog = 2.5*np.log10(np.maximum(flux,pos_error/10))
logivar = invvar * flux**2 * pos_mask*1.08574
# cap the magfunc so that sensfunc < 1.0e10
magfunc = 2.5*np.log10(np.maximum(flux_std,1.0e-2)) - fluxlog
magfunc = np.minimum(magfunc,25.0)
sensfunc = 10.0**(0.4*magfunc)*pos_mask
# Interpolate over masked pixels
if not nointerp:
bad = logivar <= 0.
if np.sum(bad) > 0:
f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], magfunc[~bad], k=2)
magfunc[bad] = f(wave[bad])
fi = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], logivar[~bad], k=2)
logivar[bad] = fi(wave[bad])
# First iteration
mask, tck = arutils.robust_polyfit(wave, magfunc, 3, function='bspline', weights=np.sqrt(logivar), **kwargs)
logfit1 = arutils.func_val(tck,wave,'bspline')
modelfit1 = 10.0**(0.4*(logfit1))
residual = sensfunc/(modelfit1 + (modelfit1 == 0)) - 1.
new_mask = pos_mask & (sensfunc > 0)
residual_ivar = (modelfit1*flux/(sensfunc + (sensfunc == 0.0)))**2*invvar
residual_ivar = residual_ivar*new_mask
# Interpolate over masked pixels
if not nointerp:
if np.sum(bad) > 0:
f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], residual[~bad], k=2)
residual[bad] = f(wave[bad])
fi = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], residual_ivar[~bad], k=2)
residual_ivar[bad] = fi(wave[bad])
# Now do one more fit to the ratio of data/model - 1.
# Fuss with the knots first ()
inner_knots = arutils.bspline_inner_knots(tck[0])
#
mask, tck_residual = arutils.robust_polyfit(wave, residual, 3, function='bspline', weights=np.sqrt(residual_ivar), knots=inner_knots, **kwargs)
if tck_residual[1].size != tck[1].size:
msgs.error('Problem with bspline knots in bspline_magfit')
#bset_residual = bspline_iterfit(wave, residual, weights=np.sqrt(residual_ivar), knots = tck[0])
tck_log1 = list(tck)
tck_log1[1] = tck[1] + tck_residual[1]
sensfit = 10.0**(0.4*(arutils.func_val(tck_log1,wave, 'bspline')))
absdev = np.median(np.abs(sensfit/modelfit1-1))
msgs.info('Difference between fits is {:g}'.format(absdev))
# QA
msgs.work("Add QA for sensitivity function")
return tck_log1
def arc_fit_qa(slf, fit, arc_spec, outfil=None):
"""
QA for Arc spectrum
Parameters
----------
fit : Wavelength fit
arc_spec : ndarray
Arc spectrum
outfil : str, optional
Name of output file
"""
if outfil is not None:
msgs.error("Not ready for this anymore")
# Begin
plt.figure(figsize=(8, 4.0))
plt.clf()
gs = gridspec.GridSpec(2, 2)
# Simple spectrum plot
ax_spec = plt.subplot(gs[:,0])
ax_spec.plot(np.arange(len(arc_spec)), arc_spec)
ymin, ymax = 0., np.max(arc_spec)
ysep = ymax*0.03
for kk, x in enumerate(fit['xfit']*fit['xnorm']):
yline = np.max(arc_spec[int(x)-2:int(x)+2])
# Tick mark
ax_spec.plot([x,x], [yline+ysep*0.25, yline+ysep], 'g-')
# label
ax_spec.text(x, yline+ysep*1.3,
'{:s} {:g}'.format(fit['ions'][kk], fit['yfit'][kk]), ha='center', va='bottom',
size='xx-small', rotation=90., color='green')
ax_spec.set_xlim(0., len(arc_spec))
ax_spec.set_ylim(ymin, ymax*1.2)
ax_spec.set_xlabel('Pixel')
ax_spec.set_ylabel('Flux')
# Arc Fit
ax_fit = plt.subplot(gs[0, 1])
# Points
ax_fit.scatter(fit['xfit']*fit['xnorm'], fit['yfit'], marker='x')
if len(fit['xrej']) > 0:
ax_fit.scatter(fit['xrej']*fit['xnorm'], fit['yrej'], marker='o',
edgecolor='gray', facecolor='none')
# Solution
xval = np.arange(len(arc_spec))
wave = arutils.func_val(fit['fitc'], xval/fit['xnorm'], 'legendre',
minv=fit['fmin'], maxv=fit['fmax'])
ax_fit.plot(xval, wave, 'r-')
xmin, xmax = 0., len(arc_spec)
ax_fit.set_xlim(xmin, xmax)
ymin,ymax = np.min(wave)*.95, np.max(wave)*1.05
ax_fit.set_ylim(np.min(wave)*.95, np.max(wave)*1.05)
ax_fit.set_ylabel('Wavelength')
ax_fit.get_xaxis().set_ticks([]) # Suppress labeling
# Stats
wave_fit = arutils.func_val(fit['fitc'], fit['xfit'], 'legendre',
minv=fit['fmin'], maxv=fit['fmax'])
dwv_pix = np.median(np.abs(wave-np.roll(wave,1)))
rms = np.sqrt(np.sum((fit['yfit']-wave_fit)**2)/len(fit['xfit'])) # Ang
ax_fit.text(0.1*len(arc_spec), 0.90*ymin+(ymax-ymin),
r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(dwv_pix), size='small')
ax_fit.text(0.1*len(arc_spec), 0.80*ymin+(ymax-ymin),
'RMS={:.3f} (pixels)'.format(rms/dwv_pix), size='small')
# Arc Residuals
ax_res = plt.subplot(gs[1,1])
res = fit['yfit']-wave_fit
ax_res.scatter(fit['xfit']*fit['xnorm'], res/dwv_pix, marker='x')
ax_res.plot([xmin,xmax], [0.,0], 'k--')
ax_res.set_xlim(xmin, xmax)
ax_res.set_xlabel('Pixel')
ax_res.set_ylabel('Residuals (Pix)')
# Finish
plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0)
slf._qa.savefig(bbox_inches='tight')
plt.close()
return
def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial', retmask=False):
nrow = xfit.shape[0]
ntrace = xfit.shape[1]
if x0in is None:
x0in = np.arange(float(ntrace))
# Mask out some orders if they are bad
if mask is None or mask.size == 0:
usetrace = np.arange(ntrace)
outmask = np.ones((nrow, ntrace))
else:
usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
outmask = np.ones((nrow, ntrace))
outmask[:,mask] = 0.0
# Do the PCA analysis
eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc)
modl = arutils.func_vander(xfit[:,0], function, npc)
eigv = np.dot(modl[:,1:], eigc)
med_hidden = np.median(hidden, axis=1)
med_highorder = med_hidden.copy()
med_highorder[0] = 0
high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0)
# y = hidden[0,:]
# coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)
# y = hidden[1,:]
# coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)
coeffstr = []
for i in xrange(1, npc+1):
# if pnpc[i] == 0:
# coeffstr.append([-9.99E9])
# continue
# coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
if weights is not None:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
weights=weights[usetrace], sigma=2.0, function=function,
minv=x0in[0], maxv=x0in[-1])
else:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
sigma=2.0, function=function, minv=x0in[0], maxv=x0in[-1])
coeffstr.append(coeff0)
high_order_matrix[:,i-1] = arutils.func_val(coeff0, x0in, function, minv=x0in[0], maxv=x0in[-1])
# high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
high_fit = high_order_matrix.copy()
high_order_fit = np.dot(eigv, high_order_matrix.T)
sub = (yfit - high_order_fit) * outmask
numer = np.sum(sub, axis=0)
denom = np.sum(outmask, axis=0)
x0 = np.zeros(float(ntrace))
fitmask = np.zeros(float(ntrace))
#fitmask[mask] = 1
x0fit = np.zeros(float(ntrace))
chisqnu = 0.0
chisqold = 0.0
robust = True
#svx0 = numer/(denom+(denom == 0).astype(np.int))
if not skipx0:
fitmask = (np.abs(denom) > 10).astype(np.int)
if robust:
good = np.where(fitmask != 0)[0]
bad = np.where(fitmask == 0)[0]
x0[good] = numer[good]/denom[good]
imask = np.zeros(float(ntrace))
imask[bad] = 1.0
ttmask, x0res = arutils.robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0,
function=function, minv=x0in[0], maxv=x0in[-1], initialmask=imask)
x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
good = np.where(ttmask == 0)[0]
xstd = 1.0 # This should represent the dispersion in the fit
chisq = ((x0[good]-x0fit[good])/xstd)**2.0
chisqnu = np.sum(chisq)/np.sum(ttmask)
fitmask = 1.0-ttmask
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
else:
for i in xrange(1, 5):
good = np.where(fitmask != 0)[0]
x0[good] = numer[good]/denom[good]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
x0res = arutils.func_fit(x0in[good], x0[good], function, pnpc[0],
weights=weights, minv=x0in[0], maxv=x0in[-1])
x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
chisq = (x0[good]-x0fit[good])**2.0
fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int)
chisqnu = np.sum(chisq)/np.sum(fitmask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu == chisqold:
break
else:
chisqold = chisqnu
if chisqnu > 2.0:
msgs.warn("PCA has very large residuals")
elif chisqnu > 0.5:
msgs.warn("PCA has fairly large residuals")
#bad = np.where(fitmask==0)[0]
#x0[bad] = x0fit[bad]
else:
x0res = 0.0
x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T
outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask,
'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr})
if retmask:
return x3fit, outpar, tmask
else:
return x3fit, outpar
def bspline_magfit(wave, flux, var, flux_std, nointerp=False, **kwargs):
"""
Perform a bspline fit to the flux ratio of standard to
observed counts. Used to generate a sensitivity function.
Parameters
----------
wave : ndarray
flux : ndarray
counts/s as observed
var : ndarray
variance
flux_std : Quantity array
standard star true flux (erg/s/cm^2/A)
nointer : bool, optional [False]
Skip interpolation over bad points (not recommended)?
**kwargs : keywords for robust_polyfit
Returns
-------
"""
invvar = (var > 0.) / (var + (var <= 0.))
nx = wave.size
pos_error = 1. / np.sqrt(np.maximum(invvar, 0.) + (invvar == 0))
pos_mask = (flux > pos_error / 10.0) & (invvar > 0) & (flux_std > 0.0)
#pos = pos_mask==1 npos)
fluxlog = 2.5 * np.log10(np.maximum(flux, pos_error / 10))
logivar = invvar * flux**2 * pos_mask * 1.08574
# cap the magfunc so that sensfunc < 1.0e10
magfunc = 2.5 * np.log10(np.maximum(flux_std, 1.0e-2)) - fluxlog
magfunc = np.minimum(magfunc, 25.0)
sensfunc = 10.0**(0.4 * magfunc) * pos_mask
# Interpolate over masked pixels
if not nointerp:
bad = logivar <= 0.
if np.sum(bad) > 0:
f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad],
magfunc[~bad],
k=2)
magfunc[bad] = f(wave[bad])
fi = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad],
logivar[~bad],
k=2)
logivar[bad] = fi(wave[bad])
# First iteration
mask, tck = arutils.robust_polyfit(wave,
magfunc,
3,
function='bspline',
weights=np.sqrt(logivar),
**kwargs)
logfit1 = arutils.func_val(tck, wave, 'bspline')
modelfit1 = 10.0**(0.4 * (logfit1))
residual = sensfunc / (modelfit1 + (modelfit1 == 0)) - 1.
new_mask = pos_mask & (sensfunc > 0)
residual_ivar = (modelfit1 * flux / (sensfunc +
(sensfunc == 0.0)))**2 * invvar
residual_ivar = residual_ivar * new_mask
# Interpolate over masked pixels
if not nointerp:
if np.sum(bad) > 0:
f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad],
residual[~bad],
k=2)
residual[bad] = f(wave[bad])
fi = scipy.interpolate.InterpolatedUnivariateSpline(
wave[~bad], residual_ivar[~bad], k=2)
residual_ivar[bad] = fi(wave[bad])
# Now do one more fit to the ratio of data/model - 1.
# Fuss with the knots first ()
inner_knots = arutils.bspline_inner_knots(tck[0])
#
mask, tck_residual = arutils.robust_polyfit(wave,
residual,
3,
function='bspline',
weights=np.sqrt(residual_ivar),
knots=inner_knots,
**kwargs)
if tck_residual[1].size != tck[1].size:
msgs.error('Problem with bspline knots in bspline_magfit')
#bset_residual = bspline_iterfit(wave, residual, weights=np.sqrt(residual_ivar), knots = tck[0])
tck_log1 = list(tck)
tck_log1[1] = tck[1] + tck_residual[1]
sensfit = 10.0**(0.4 * (arutils.func_val(tck_log1, wave, 'bspline')))
absdev = np.median(np.abs(sensfit / modelfit1 - 1))
msgs.info('Difference between fits is {:g}'.format(absdev))
# QA
msgs.work("Add QA for sensitivity function")
return tck_log1