def test_func_fit():
""" Run the parameter setup script
"""
x = np.pi * np.linspace(0, 1., 100)
y = np.sin(x)
# Polynomial
pcoeff = arut.func_fit(x, y, 'polynomial', 3)
np.testing.assert_allclose(pcoeff,
np.array([
-4.74660344e-02, 1.30745471e+00,
-4.16175760e-01, 3.08557167e-18
]),
atol=1e-9)
# Legendre
lcoeff = arut.func_fit(x, y, 'legendre', 3)
np.testing.assert_allclose(lcoeff,
np.array([
6.37115652e-01, 6.83317251e-17,
-6.84581686e-01, -7.59352737e-17
]),
atol=1e-9)
# bspline
bcoeff = arut.func_fit(x, y, 'bspline', 2)
np.testing.assert_allclose(bcoeff[0], [
0., 0., 0., 0.31733259, 0.95199777, 1.58666296, 2.22132814, 3.14159265,
3.14159265, 3.14159265
],
atol=1e-5)
def fit_arcspec(xarray, yarray, pixt, fitp):
# Setup the arrays with fit parameters
sz_p = pixt.size
sz_a = yarray.size
ampl, cent, widt = -1.0*np.ones(sz_p, dtype=np.float),\
-1.0*np.ones(sz_p, dtype=np.float),\
-1.0*np.ones(sz_p, dtype=np.float)
for p in range(sz_p):
pmin = pixt[p] - (fitp - 1) // 2
pmax = pixt[p] - (fitp - 1) // 2 + fitp
if pmin < 0:
pmin = 0
if pmax > sz_a:
pmax = sz_a
if pmin == pmax:
continue
if pixt[p] - pmin <= 1 or pmax - pixt[p] <= 1:
continue # Probably won't be a good solution
# Fit the gaussian
try:
popt = arutils.func_fit(xarray[pmin:pmax], yarray[pmin:pmax],
"gaussian", 3)
ampl[p] = popt[0]
cent[p] = popt[1]
widt[p] = popt[2]
except RuntimeError:
pass
return ampl, cent, widt
def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial', retmask=False):
nrow = xfit.shape[0]
ntrace = xfit.shape[1]
if x0in is None:
x0in = np.arange(float(ntrace))
# Mask out some orders if they are bad
if mask is None or mask.size == 0:
usetrace = np.arange(ntrace)
outmask = np.ones((nrow, ntrace))
else:
usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
outmask = np.ones((nrow, ntrace))
outmask[:,mask] = 0.0
# Do the PCA analysis
eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc)
modl = arutils.func_vander(xfit[:,0], function, npc)
eigv = np.dot(modl[:,1:], eigc)
med_hidden = np.median(hidden, axis=1)
med_highorder = med_hidden.copy()
med_highorder[0] = 0
high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0)
# y = hidden[0,:]
# coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)
# y = hidden[1,:]
# coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)
coeffstr = []
for i in xrange(1, npc+1):
# if pnpc[i] == 0:
# coeffstr.append([-9.99E9])
# continue
# coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
if weights is not None:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
weights=weights[usetrace], sigma=2.0, function=function,
minv=x0in[0], maxv=x0in[-1])
else:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
sigma=2.0, function=function, minv=x0in[0], maxv=x0in[-1])
coeffstr.append(coeff0)
high_order_matrix[:,i-1] = arutils.func_val(coeff0, x0in, function, minv=x0in[0], maxv=x0in[-1])
# high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
high_fit = high_order_matrix.copy()
high_order_fit = np.dot(eigv, high_order_matrix.T)
sub = (yfit - high_order_fit) * outmask
numer = np.sum(sub, axis=0)
denom = np.sum(outmask, axis=0)
x0 = np.zeros(float(ntrace))
fitmask = np.zeros(float(ntrace))
#fitmask[mask] = 1
x0fit = np.zeros(float(ntrace))
chisqnu = 0.0
chisqold = 0.0
robust = True
#svx0 = numer/(denom+(denom == 0).astype(np.int))
if not skipx0:
fitmask = (np.abs(denom) > 10).astype(np.int)
if robust:
good = np.where(fitmask != 0)[0]
bad = np.where(fitmask == 0)[0]
x0[good] = numer[good]/denom[good]
imask = np.zeros(float(ntrace))
imask[bad] = 1.0
ttmask, x0res = arutils.robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0,
function=function, minv=x0in[0], maxv=x0in[-1], initialmask=imask)
x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
good = np.where(ttmask == 0)[0]
xstd = 1.0 # This should represent the dispersion in the fit
chisq = ((x0[good]-x0fit[good])/xstd)**2.0
chisqnu = np.sum(chisq)/np.sum(ttmask)
fitmask = 1.0-ttmask
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
else:
for i in xrange(1, 5):
good = np.where(fitmask != 0)[0]
x0[good] = numer[good]/denom[good]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
x0res = arutils.func_fit(x0in[good], x0[good], function, pnpc[0],
weights=weights, minv=x0in[0], maxv=x0in[-1])
x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
chisq = (x0[good]-x0fit[good])**2.0
fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int)
chisqnu = np.sum(chisq)/np.sum(fitmask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu == chisqold:
break
else:
chisqold = chisqnu
if chisqnu > 2.0:
msgs.warn("PCA has very large residuals")
elif chisqnu > 0.5:
msgs.warn("PCA has fairly large residuals")
#bad = np.where(fitmask==0)[0]
#x0[bad] = x0fit[bad]
else:
x0res = 0.0
x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T
outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask,
'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr})
if retmask:
return x3fit, outpar, tmask
else:
return x3fit, outpar
def simple_calib(slf, det, get_poly=False):
"""Simple calibration algorithm for longslit wavelengths
Uses slf._arcparam to guide the analysis
Parameters
----------
get_poly : bool, optional
Pause to record the polynomial pix = b0 + b1*lambda + b2*lambda**2
Returns
-------
final_fit : dict
Dict of fit info
"""
# Extract the arc
msgs.work("Detecting lines..")
tampl, tcent, twid, w, satsnd, yprep = detect_lines(
slf, det, slf._msarc[det - 1])
# Cut down to the good ones
tcent = tcent[w]
tampl = tampl[w]
msgs.info('Detected {:d} lines in the arc spectrum.'.format(len(w[0])))
# Parameters (just for convenience)
aparm = slf._arcparam[det - 1]
# Read Arc linelist
llist = aparm['llist']
# IDs were input by hand
if len(settings.argflag['arc']['calibrate']['IDpixels']) > 0:
# Check that there are at least 5 values
pixels = np.array(settings.argflag['arc']['calibrate']['IDpixels'])
if np.sum(pixels > 0.) < 5:
msgs.error("Need to give at least 5 pixel values!")
#
msgs.info("Using input lines to seed the wavelength solution")
# Calculate median offset
mdiff = [
np.min(np.abs(tcent - pix))
for pix in settings.argflag['arc']['calibrate']['IDpixels']
]
med_poff = np.median(np.array(mdiff))
msgs.info("Will apply a median offset of {:g} pixels".format(med_poff))
# Match input lines to observed spectrum
nid = len(settings.argflag['arc']['calibrate']['IDpixels'])
idx_str = np.ones(nid).astype(int)
ids = np.zeros(nid)
idsion = np.array([' '] * nid)
gd_str = np.arange(nid).astype(int)
for jj, pix in enumerate(
settings.argflag['arc']['calibrate']['IDpixels']):
diff = np.abs(tcent - pix - med_poff)
if np.min(diff) > 2.:
debugger.set_trace()
msgs.error("No match with input pixel {:g}!".format(pix))
else:
imn = np.argmin(diff)
# Set
idx_str[jj] = imn
# Take wavelength from linelist instead of input value
wdiff = np.abs(llist['wave'] -
settings.argflag['arc']['calibrate']['IDwaves'][jj])
imnw = np.argmin(wdiff)
if wdiff[imnw] > 0.015: # Arbitrary tolerance
msgs.error(
"Input IDwaves={:g} is not in the linelist. Fix".format(
settings.argflag['arc']['calibrate']['IDwaves'][jj]))
else:
ids[jj] = llist['wave'][imnw]
idsion[jj] = llist['Ion'][imnw]
msgs.info("Identifying arc line: {:s} {:g}".format(
idsion[jj], ids[jj]))
else:
# Generate dpix pairs
msgs.info("Using pair algorithm for wavelength solution")
nlist = len(llist)
dpix_list = np.zeros((nlist, nlist))
for kk, row in enumerate(llist):
#dpix_list[kk,:] = (np.array(row['wave'] - llist['wave']))/disp
dpix_list[kk, :] = slf._msarc[det - 1].shape[0] * (
aparm['b1'] * (np.array(row['wave'] - llist['wave'])) +
aparm['b2'] * np.array(row['wave']**2 - llist['wave']**2))
# Lambda pairs for the strongest N lines
srt = np.argsort(tampl)
idx_str = srt[-aparm['Nstrong']:]
idx_str.sort()
dpix_obs = np.zeros((aparm['Nstrong'], aparm['Nstrong']))
for kk, idx in enumerate(idx_str):
dpix_obs[kk, :] = np.array(tcent[idx] - tcent[idx_str])
# Match up (ugly loops)
ids = np.zeros(aparm['Nstrong'])
idsion = np.array([' '] * aparm['Nstrong'])
for kk in range(aparm['Nstrong']):
med_off = np.zeros(nlist)
for ss in range(nlist):
dpix = dpix_list[ss]
min_off = []
for jj in range(aparm['Nstrong']):
min_off.append(np.min(np.abs(dpix_obs[kk, jj] - dpix)))
med_off[ss] = np.median(min_off)
# Set by minimum
idm = np.argmin(med_off)
ids[kk] = llist['wave'][idm]
idsion[kk] = llist['Ion'][idm]
# Calculate disp of the strong lines
disp_str = np.zeros(aparm['Nstrong'])
for kk in range(aparm['Nstrong']):
disp_val = (ids[kk] - ids) / (tcent[idx_str[kk]] - tcent[idx_str])
isf = np.isfinite(disp_val)
disp_str[kk] = np.median(disp_val[isf])
# Consider calculating the RMS with clipping
gd_str = np.where(
np.abs(disp_str - aparm['disp']) /
aparm['disp'] < aparm['disp_toler'])[0]
msgs.info('Found {:d} lines within the dispersion threshold'.format(
len(gd_str)))
if len(gd_str) < 5:
if msgs._debug['arc']:
msgs.warn('You should probably try your best to ID lines now.')
debugger.set_trace()
debugger.plot1d(yprep)
else:
msgs.error('Insufficient lines to auto-fit.')
# Debug
#debug=True
if msgs._debug['arc']:
#tmp = list(gd_str)
#tmp.pop(1)
#gd_str = np.array(tmp)
#xdb.xpcol(tcent[idx_str[gd_str]],ids[gd_str])
#xdb.xplot(tcent[idx_str[gd_str]],ids[gd_str],scatter=True)
# debugger.xplot(yprep)
debugger.set_trace()
msgs.work('Cross correlate here?')
# Setup for fitting
ifit = idx_str[gd_str]
sv_ifit = list(ifit) # Keep the originals
all_ids = -999. * np.ones(len(tcent))
all_idsion = np.array(['12345'] * len(tcent))
all_ids[ifit] = ids[gd_str]
all_idsion[ifit] = idsion[gd_str]
# Fit
n_order = aparm['n_first']
flg_quit = False
fmin, fmax = -1., 1.
msgs.info('Iterative wavelength fitting..')
while (n_order <= aparm['n_final']) and (flg_quit is False):
#msgs.info('n_order={:d}'.format(n_order))
# Fit with rejection
xfit, yfit = tcent[ifit], all_ids[ifit]
mask, fit = arutils.robust_polyfit(xfit,
yfit,
n_order,
function=aparm['func'],
sigma=aparm['nsig_rej'],
minv=fmin,
maxv=fmax)
# Reject but keep originals (until final fit)
ifit = list(ifit[mask == 0]) + sv_ifit
# Find new points (should we allow removal of the originals?)
twave = arutils.func_val(fit,
tcent,
aparm['func'],
minv=fmin,
maxv=fmax)
for ss, iwave in enumerate(twave):
mn = np.min(np.abs(iwave - llist['wave']))
if mn / aparm['disp'] < aparm['match_toler']:
imn = np.argmin(np.abs(iwave - llist['wave']))
#if msgs._debug['arc']:
# print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
# Update and append
all_ids[ss] = llist['wave'][imn]
all_idsion[ss] = llist['Ion'][imn]
ifit.append(ss)
# Keep unique ones
ifit = np.unique(np.array(ifit, dtype=int))
#if msgs._debug['arc']:
# debugger.set_trace()
# Increment order
if n_order < aparm['n_final']:
n_order += 1
else:
# This does 2 iterations at the final order
flg_quit = True
# Final fit (originals can now be rejected)
fmin, fmax = 0., 1.
xfit, yfit = tcent[ifit] / (slf._msarc[det - 1].shape[0] -
1), all_ids[ifit]
mask, fit = arutils.robust_polyfit(xfit,
yfit,
n_order,
function=aparm['func'],
sigma=aparm['nsig_rej_final'],
minv=fmin,
maxv=fmax) #, debug=True)
irej = np.where(mask == 1)[0]
if len(irej) > 0:
xrej = xfit[irej]
yrej = yfit[irej]
for imask in irej:
msgs.info('Rejecting arc line {:g}'.format(yfit[imask]))
else:
xrej = []
yrej = []
xfit = xfit[mask == 0]
yfit = yfit[mask == 0]
ions = all_idsion[ifit][mask == 0]
#
if msgs._debug['arc']:
msarc = slf._msarc[det - 1]
wave = arutils.func_val(fit,
np.arange(msarc.shape[0]) /
float(msarc.shape[0]),
'legendre',
minv=fmin,
maxv=fmax)
debugger.set_trace()
#debugger.xplot(xfit, np.ones(len(xfit)), scatter=True,
# xtwo=np.arange(msarc.shape[0]),ytwo=yprep)
#debugger.xplot(xfit,yfit, scatter=True, xtwo=np.arange(msarc.shape[0]),
# ytwo=wave)
#debugger.set_trace()
#wave = arutils.func_val(fit, np.arange(msarc.shape[0])/float(msarc.shape[0]),
# 'legendre', min=fmin, max=fmax)
# 2nd order Poly fit for archival
#get_poly=True
if get_poly:
poly_fit = arutils.func_fit(yfit,
xfit,
'polynomial',
2,
minv=fmin,
maxv=fmax)
print(' Most likely you with to record these values:')
print(poly_fit)
debugger.set_trace()
# Pack up fit
final_fit = dict(fitc=fit,
function=aparm['func'],
xfit=xfit,
yfit=yfit,
ions=ions,
fmin=fmin,
fmax=fmax,
xnorm=float(slf._msarc[det - 1].shape[0]),
xrej=xrej,
yrej=yrej,
mask=mask,
spec=yprep,
nrej=aparm['nsig_rej_final'],
shift=0.,
tcent=tcent)
# QA
arqa.arc_fit_qa(slf, final_fit)
# RMS
rms_ang = arutils.calc_fit_rms(xfit,
yfit,
fit,
aparm['func'],
minv=fmin,
maxv=fmax)
wave = arutils.func_val(fit,
np.arange(slf._msarc[det - 1].shape[0]) /
float(slf._msarc[det - 1].shape[0]),
aparm['func'],
minv=fmin,
maxv=fmax)
rms_pix = rms_ang / np.median(np.abs(wave - np.roll(wave, 1)))
msgs.info("Fit RMS = {} pix".format(rms_pix))
# Return
return final_fit
def flex_shift(slf, det, obj_skyspec, arx_skyspec):
""" Calculate shift between object sky spectrum and archive sky spectrum
Parameters
----------
slf
det
obj_skyspec
arx_skyspec
Returns
-------
flex_dict
"""
#Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_cent, arx_wid, arx_w, arx_satsnd, arx_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=arx_skyspec.flux.value, MK_SATMASK=False)
obj_amp, obj_cent, obj_wid, obj_w, obj_satsnd, obj_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=obj_skyspec.flux.value, MK_SATMASK=False)
#Keep only 5 brightest amplitude lines (xxx_keep is array of indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
#Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(arx_skyspec.wavelength.value[1]-arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:]-arx_skyspec.wavelength.value[:-1])
#arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
obj_disp = np.append(obj_skyspec.wavelength.value[1]-obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:]-obj_skyspec.wavelength.value[:-1])
#obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size
#Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need this? can just use sigmas
arx_idx = (arx_cent+0.5).astype(np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent+0.5).astype(np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
#obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
# obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
msgs.info("Resolution of Archive={:g} and Observation={:g}".format(
np.median(arx_res), np.median(obj_res)))
#Determine sigma of gaussian for smoothing
arx_sig2 = (arx_disp[arx_idx]*arx_wid[arx_w][arx_keep])**2.
obj_sig2 = (obj_disp[obj_idx]*obj_wid[obj_w][obj_keep])**2.
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2-arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value), np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value), np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
if np.median(obj_res) >= np.median(arx_res):
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value>=min_wave) &
(obj_skyspec.wavelength.value<=max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.error("Not enough overlap between sky spectra")
else: #rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
#deal with bad pixels
msgs.work("Need to mask bad pixels")
#deal with underlying continuum
msgs.work("Need to deal with underlying continuum")
#Cross correlation of spectra
corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within max_shift of zero lag
lag0 = corr.size/2
mxshft = slf._argflag['reduce']['flexure']['max_shift']
max_corr = np.argmax(corr[lag0-mxshft:lag0+mxshft]) + lag0-mxshft
subpix_grid = np.linspace(max_corr-3., max_corr+3., 7.)
#Fit a 2-degree polynomial to peak of correlation function
fit = arutils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)], 'polynomial', 2)
max_fit = -0.5*fit[1]/fit[2]
#Calculate and apply shift in wavelength
shift = float(max_fit)-lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
if msgs._debug['flexure']:
debugger.xplot(arx_skyspec.wavelength, arx_skyspec.flux, xtwo=obj_skyspec.wavelength, ytwo=obj_skyspec.flux)
#debugger.xplot(arx_sky.wavelength, arx_sky.flux, xtwo=np.roll(obj_sky.wavelength.value,9), ytwo=obj_sky.flux*100)
debugger.set_trace()
flex_dict = dict(polyfit=fit, shift=shift, subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size/2, smooth=smooth_sig_pix)
# Return
return flex_dict
def flex_shift(slf, det, obj_skyspec, arx_skyspec):
""" Calculate shift between object sky spectrum and archive sky spectrum
Parameters
----------
slf
det
obj_skyspec
arx_skyspec
Returns
-------
flex_dict
"""
#Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_cent, arx_wid, arx_w, arx_satsnd, arx_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=arx_skyspec.flux.value, MK_SATMASK=False)
obj_amp, obj_cent, obj_wid, obj_w, obj_satsnd, obj_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=obj_skyspec.flux.value, MK_SATMASK=False)
#Keep only 5 brightest amplitude lines (xxx_keep is array of indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
#Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(arx_skyspec.wavelength.value[1]-arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:]-arx_skyspec.wavelength.value[:-1])
#arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
obj_disp = np.append(obj_skyspec.wavelength.value[1]-obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:]-obj_skyspec.wavelength.value[:-1])
#obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size
#Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need this? can just use sigmas
arx_idx = (arx_cent+0.5).astype(np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent+0.5).astype(np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
#obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
# obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
msgs.info("Resolution of Archive={:g} and Observation={:g}".format(
np.median(arx_res), np.median(obj_res)))
#Determine sigma of gaussian for smoothing
arx_sig2 = (arx_disp[arx_idx]*arx_wid[arx_w][arx_keep])**2.
obj_sig2 = (obj_disp[obj_idx]*obj_wid[obj_w][obj_keep])**2.
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2-arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value), np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value), np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
if np.median(obj_res) >= np.median(arx_res):
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value>=min_wave) &
(obj_skyspec.wavelength.value<=max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.error("Not enough overlap between sky spectra")
else: #rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
#deal with bad pixels
msgs.work("Need to mask bad pixels")
#deal with underlying continuum
msgs.work("Need to deal with underlying continuum")
#Cross correlation of spectra
corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within max_shift of zero lag
lag0 = corr.size/2
mxshft = slf._argflag['reduce']['flexure']['max_shift']
max_corr = np.argmax(corr[lag0-mxshft:lag0+mxshft]) + lag0-mxshft
subpix_grid = np.linspace(max_corr-3., max_corr+3., 7.)
#Fit a 2-degree polynomial to peak of correlation function
fit = arutils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)], 'polynomial', 2)
max_fit = -0.5*fit[1]/fit[2]
#Calculate and apply shift in wavelength
shift = float(max_fit)-lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
if msgs._debug['flexure']:
debugger.xplot(arx_skyspec.wavelength, arx_skyspec.flux, xtwo=obj_skyspec.wavelength, ytwo=obj_skyspec.flux)
#debugger.xplot(arx_sky.wavelength, arx_sky.flux, xtwo=np.roll(obj_sky.wavelength.value,9), ytwo=obj_sky.flux*100)
debugger.set_trace()
flex_dict = dict(polyfit=fit, shift=shift, subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size/2, smooth=smooth_sig_pix)
# Return
return flex_dict
def basis(xfit,
yfit,
coeff,
npc,
pnpc,
weights=None,
skipx0=True,
x0in=None,
mask=None,
function='polynomial',
retmask=False):
nrow = xfit.shape[0]
ntrace = xfit.shape[1]
if x0in is None:
x0in = np.arange(float(ntrace))
# Mask out some orders if they are bad
if mask is None or mask.size == 0:
usetrace = np.arange(ntrace)
outmask = np.ones((nrow, ntrace))
else:
usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
outmask = np.ones((nrow, ntrace))
outmask[:, mask] = 0.0
# Do the PCA analysis
eigc, hidden = get_pc(coeff[1:npc + 1, usetrace], npc)
modl = arutils.func_vander(xfit[:, 0], function, npc)
eigv = np.dot(modl[:, 1:], eigc)
med_hidden = np.median(hidden, axis=1)
med_highorder = med_hidden.copy()
med_highorder[0] = 0
high_order_matrix = med_highorder.T[np.newaxis, :].repeat(ntrace, axis=0)
# y = hidden[0,:]
# coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)
# y = hidden[1,:]
# coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)
coeffstr = []
for i in xrange(1, npc + 1):
# if pnpc[i] == 0:
# coeffstr.append([-9.99E9])
# continue
# coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
if weights is not None:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
hidden[i - 1, :],
pnpc[i],
weights=weights[usetrace],
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1])
else:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
hidden[i - 1, :],
pnpc[i],
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1])
coeffstr.append(coeff0)
high_order_matrix[:, i - 1] = arutils.func_val(coeff0,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
# high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
high_fit = high_order_matrix.copy()
high_order_fit = np.dot(eigv, high_order_matrix.T)
sub = (yfit - high_order_fit) * outmask
numer = np.sum(sub, axis=0)
denom = np.sum(outmask, axis=0)
x0 = np.zeros(float(ntrace))
fitmask = np.zeros(float(ntrace))
#fitmask[mask] = 1
x0fit = np.zeros(float(ntrace))
chisqnu = 0.0
chisqold = 0.0
robust = True
#svx0 = numer/(denom+(denom == 0).astype(np.int))
if not skipx0:
fitmask = (np.abs(denom) > 10).astype(np.int)
if robust:
good = np.where(fitmask != 0)[0]
bad = np.where(fitmask == 0)[0]
x0[good] = numer[good] / denom[good]
imask = np.zeros(float(ntrace))
imask[bad] = 1.0
ttmask, x0res = arutils.robust_polyfit(x0in,
x0,
pnpc[0],
weights=weights,
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1],
initialmask=imask)
x0fit = arutils.func_val(x0res,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
good = np.where(ttmask == 0)[0]
xstd = 1.0 # This should represent the dispersion in the fit
chisq = ((x0[good] - x0fit[good]) / xstd)**2.0
chisqnu = np.sum(chisq) / np.sum(ttmask)
fitmask = 1.0 - ttmask
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
else:
for i in xrange(1, 5):
good = np.where(fitmask != 0)[0]
x0[good] = numer[good] / denom[good]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
x0res = arutils.func_fit(x0in[good],
x0[good],
function,
pnpc[0],
weights=weights,
minv=x0in[0],
maxv=x0in[-1])
x0fit = arutils.func_val(x0res,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
chisq = (x0[good] - x0fit[good])**2.0
fitmask[good] *= (chisq < np.sum(chisq) / 2.0).astype(np.int)
chisqnu = np.sum(chisq) / np.sum(fitmask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu == chisqold:
break
else:
chisqold = chisqnu
if chisqnu > 2.0:
msgs.warn("PCA has very large residuals")
elif chisqnu > 0.5:
msgs.warn("PCA has fairly large residuals")
#bad = np.where(fitmask==0)[0]
#x0[bad] = x0fit[bad]
else:
x0res = 0.0
x3fit = np.dot(eigv, high_order_matrix.T) + np.outer(x0fit,
np.ones(nrow)).T
outpar = dict({
'high_fit': high_fit,
'x0': x0,
'x0in': x0in,
'x0fit': x0fit,
'x0res': x0res,
'x0mask': fitmask,
'hidden': hidden,
'usetrc': usetrace,
'eigv': eigv,
'npc': npc,
'coeffstr': coeffstr
})
if retmask:
return x3fit, outpar, tmask
else:
return x3fit, outpar
