def refine_iter(outpar, orders, mask, irshft, relshift, fitord, function='polynomial'):
fail = False
x0ex = utils.func_val(outpar['x0res'], orders, function, minx=outpar['x0in'][0], maxx=outpar['x0in'][-1])
# Make the refinement
x0ex[irshft] += relshift
# Refit the data to improve the refinement
good = np.where(mask != 0.0)[0]
# x0res = utils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
null, x0res = utils.robust_polyfit(orders[good], x0ex[good], fitord, sigma=2.0, function=function,
minx=outpar['x0in'][0], maxx=outpar['x0in'][-1])
#x0res = utils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1])
x0fit = utils.func_val(x0res, orders, function, minx=outpar['x0in'][0], maxx=outpar['x0in'][-1])
chisq = (x0ex[good]-x0fit[good])**2.0
chisqnu = np.sum(chisq)/np.sum(mask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu > 0.5: # The refinement went wrong, ignore the refinement
fail = True
outpar['x0res'] = x0res
extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(x0fit, np.ones(outpar['eigv'].shape[0])).T
return extfit, outpar, fail
def flex_shift(obj_skyspec, arx_skyspec, mxshft=20):
""" Calculate shift between object sky spectrum and archive sky spectrum
Parameters
----------
obj_skyspec
arx_skyspec
Returns
-------
flex_dict: dict
Contains flexure info
"""
# TODO None of these routines should have dependencies on XSpectrum1d!
# Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_amp_cont, arx_cent, arx_wid, _, arx_w, arx_yprep, nsig = arc.detect_lines(
arx_skyspec.flux.value)
obj_amp, obj_amp_cont, obj_cent, obj_wid, _, obj_w, obj_yprep, nsig_obj = arc.detect_lines(
obj_skyspec.flux.value)
# Keep only 5 brightest amplitude lines (xxx_keep is array of
# indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
# Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(
arx_skyspec.wavelength.value[1] - arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:] - arx_skyspec.wavelength.value[:-1])
#arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
obj_disp = np.append(
obj_skyspec.wavelength.value[1] - obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:] - obj_skyspec.wavelength.value[:-1])
#obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size
# Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need
# this? can just use sigmas
arx_idx = (arx_cent + 0.5).astype(
np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent + 0.5).astype(
np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
#obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
# obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
if not np.all(np.isfinite(obj_res)):
msgs.warn(
'Failed to measure the resolution of the object spectrum, likely due to error '
'in the wavelength image.')
return None
msgs.info("Resolution of Archive={0} and Observation={1}".format(
np.median(arx_res), np.median(obj_res)))
# Determine sigma of gaussian for smoothing
arx_sig2 = np.power(arx_disp[arx_idx] * arx_wid[arx_w][arx_keep], 2)
obj_sig2 = np.power(obj_disp[obj_idx] * obj_wid[obj_w][obj_keep], 2)
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2 - arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix * 2 *
np.sqrt(2 * np.log(2)))
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value),
np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value),
np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
# if np.median(obj_res) >= np.median(arx_res):
# msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
# else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value >= min_wave)
& (obj_skyspec.wavelength.value <= max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.warn("Not enough overlap between sky spectra")
return None
else: #rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
# Trim edges (rebinning is junk there)
arx_skyspec.data['flux'][0, :2] = 0.
arx_skyspec.data['flux'][0, -2:] = 0.
obj_skyspec.data['flux'][0, :2] = 0.
obj_skyspec.data['flux'][0, -2:] = 0.
# Normalize spectra to unit average sky count
norm = np.sum(obj_skyspec.flux.value) / obj_skyspec.npix
obj_skyspec.flux = obj_skyspec.flux / norm
norm2 = np.sum(arx_skyspec.flux.value) / arx_skyspec.npix
arx_skyspec.flux = arx_skyspec.flux / norm2
if (norm < 0.):
msgs.warn("Bad normalization of object in flexure algorithm")
msgs.warn("Will try the median")
norm = np.median(obj_skyspec.flux.value)
if (norm < 0.):
msgs.warn("Improper sky spectrum for flexure. Is it too faint??")
return None
if (norm2 < 0.):
msgs.warn(
'Bad normalization of archive in flexure. You are probably using wavelengths '
'well beyond the archive.')
return None
# Deal with bad pixels
msgs.work("Need to mask bad pixels")
# Deal with underlying continuum
msgs.work("Consider taking median first [5 pixel]")
everyn = obj_skyspec.npix // 20
bspline_par = dict(everyn=everyn)
mask, ct = utils.robust_polyfit(obj_skyspec.wavelength.value,
obj_skyspec.flux.value,
3,
function='bspline',
sigma=3.,
bspline_par=bspline_par)
obj_sky_cont = utils.func_val(ct, obj_skyspec.wavelength.value, 'bspline')
obj_sky_flux = obj_skyspec.flux.value - obj_sky_cont
mask, ct_arx = utils.robust_polyfit(arx_skyspec.wavelength.value,
arx_skyspec.flux.value,
3,
function='bspline',
sigma=3.,
bspline_par=bspline_par)
arx_sky_cont = utils.func_val(ct_arx, arx_skyspec.wavelength.value,
'bspline')
arx_sky_flux = arx_skyspec.flux.value - arx_sky_cont
# Consider sharpness filtering (e.g. LowRedux)
msgs.work("Consider taking median first [5 pixel]")
#Cross correlation of spectra
#corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
corr = np.correlate(arx_sky_flux, obj_sky_flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within maxshift of zero lag
lag0 = corr.size // 2
#mxshft = settings.argflag['reduce']['flexure']['maxshift']
max_corr = np.argmax(corr[lag0 - mxshft:lag0 + mxshft]) + lag0 - mxshft
subpix_grid = np.linspace(max_corr - 3., max_corr + 3., 7)
#Fit a 2-degree polynomial to peak of correlation function. JFH added this if/else to not crash for bad slits
if np.any(np.isfinite(corr[subpix_grid.astype(np.int)])):
fit = utils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)],
'polynomial', 2)
success = True
max_fit = -0.5 * fit[1] / fit[2]
else:
fit = utils.func_fit(subpix_grid, 0.0 * subpix_grid, 'polynomial', 2)
success = False
max_fit = 0.0
msgs.warn('Flexure compensation failed for one of your objects')
#Calculate and apply shift in wavelength
shift = float(max_fit) - lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
flex_dict = dict(polyfit=fit,
shift=shift,
subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size / 2,
smooth=smooth_sig_pix,
success=success)
# Return
return flex_dict
std = np.std(yreal-yfit)
nsigma = rand.uniform(1,3,nout)
sign = rand.choice([-1,1.],nout)
yfit[indrand] = yreal[indrand] + sign*nsigma*std
#yfit[0] = yfit[0]+3*np.std(yreal-yfit)
#yfit[10] = yfit[10]+4*np.std(yreal-yfit)
#yfit[19] = yfit[19]-3.5*np.std(yreal-yfit)
#yfit[49] = yfit[49]-2.8*np.std(yreal-yfit)
#yfit[69] = yfit[69]+5*np.std(yreal-yfit)
norder =3
xvec = np.linspace(xfit.min(), xfit.max(), num=200)
## Old robust_olyfit
msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma=1.5, function='polynomial')
msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit,yfit,norder, \
function = 'polynomial', minv = None, maxv = None, bspline_par = None,\
guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\
lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\
groupbadpix=False, grow=0,sticky=True,use_mad=True)
msk_nosticky, poly_coeff_nosticky = utils.robust_polyfit_djs(xfit,yfit,norder, \
function = 'polynomial', minv = None, maxv = None, bspline_par = None,\
guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\
lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\
groupbadpix=False, grow=0,sticky=False,use_mad=True)
robust_mask = msk == 0
robust_mask_new = msk_new == 1
robust_mask_nosticky = msk_nosticky == 1
def pca_trace(xcen, usepca=None, npca=2, npoly_cen=3, debug=True):
nspec = xcen.shape[0]
norders = xcen.shape[1]
if usepca is None:
usepca = np.zeros(norders, dtype=bool)
# use_order = True orders used to predict the usepca = True bad orders
use_order = np.invert(usepca)
ngood = np.sum(use_order)
if ngood < npca:
msgs.warn('Not enough good traces for a PCA fit: ngood = {:d}'.format(
ngood) + ' is < npca = {:d}'.format(npca))
msgs.warn('Using the input trace for now')
return xcen
pca = PCA(n_components=npca)
xcen_use = (xcen[:, use_order] - np.mean(xcen[:, use_order], 0)).T
pca_coeffs_use = pca.fit_transform(xcen_use)
pca_vectors = pca.components_
# Fit first pca dimension (with largest variance) with a higher order npoly depending on number of good orders.
# Fit all higher dimensions (with lower variance) with a line
npoly = int(np.fmin(np.fmax(np.floor(3.3 * ngood / norders), 1.0), 3.0))
npoly_vec = np.full(npca, npoly)
order_vec = np.arange(norders, dtype=float)
# pca_coeffs = np.zeros((norders, npca))
pca_coeffs_new = np.zeros((norders, npca))
# Now loop over the dimensionality of the compression and perform a polynomial fit to
for idim in range(npca):
# ToDO robust_polyfit is garbage remove it entirely from PypeIT!
xfit = order_vec[use_order]
yfit = pca_coeffs_use[:, idim]
norder = npoly_vec[idim]
# msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma = 3.0, function='polynomial')
# pca_coeffs[:,idim] = utils.func_val(poly_coeff, order_vec, 'polynomial')
# TESTING traceset fitting
xtemp = xfit.reshape(1, xfit.size)
ytemp = yfit.reshape(1, yfit.size)
tset = pydl.xy2traceset(xtemp, ytemp, ncoeff=norder, func='polynomial')
#tset_yfit = tset.yfit.reshape(tset.yfit.shape[1])
## Test new robust fitting with djs_reject
msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit, yfit, norder, \
function='polynomial', minv=None, maxv=None, bspline_par=None, \
guesses=None, maxiter=10, inmask=None, sigma=None, invvar=None, \
lower=5, upper=5, maxdev=None, maxrej=None, groupdim=None,
groupsize=None, \
groupbadpix=False, grow=0, sticky=False)
pca_coeffs_new[:, idim] = utils.func_val(poly_coeff_new, order_vec,
'polynomial')
if debug:
# Evaluate the fit
xvec = np.linspace(order_vec.min(), order_vec.max(), num=100)
(_, tset_fit) = tset.xy(xpos=xvec.reshape(1, xvec.size))
yfit_tset = tset_fit[0, :]
#robust_mask = msk == 0
robust_mask_new = msk_new == 1
tset_mask = tset.outmask[0, :]
plt.plot(xfit,
yfit,
'ko',
mfc='None',
markersize=8.0,
label='pca coeff')
#plt.plot(xfit[~robust_mask], yfit[~robust_mask], 'ms', mfc='None', markersize=10.0,label='robust_polyfit rejected')
#plt.plot(xfit[~robust_mask_new], yfit[~robust_mask_new], 'r+', markersize=20.0,label='robust_polyfit_djs rejected')
plt.plot(xfit[~tset_mask],
yfit[~tset_mask],
'bo',
markersize=10.0,
label='traceset rejected')
#plt.plot(xvec, utils.func_val(poly_coeff, xvec, 'polynomial'),ls='--', color='m', label='robust polyfit')
plt.plot(xvec,
utils.func_val(poly_coeff_new, xvec, 'polynomial'),
ls='-.',
color='r',
label='new robust polyfit')
plt.plot(xvec, yfit_tset, ls=':', color='b', label='traceset')
plt.legend()
plt.show()
#ToDo should we be masking the bad orders here and interpolating/extrapolating?
spat_mean = np.mean(xcen, 0)
msk_spat, poly_coeff_spat = utils.robust_polyfit(order_vec,
spat_mean,
npoly_cen,
sigma=3.0,
function='polynomial')
ibad = np.where(msk_spat == 1)
spat_mean[ibad] = utils.func_val(poly_coeff_spat, order_vec[ibad],
'polynomial')
#pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs, pca_vectors)).T
pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(
pca.mean_, np.ones(norders)) + (np.dot(pca_coeffs_new, pca_vectors)).T
return pca_fit
def findpeaks(fldpth,
fnamecorrs,
verbose=True,
printorig=True,
xmin=0,
xmax=None,
height=None,
pk_window=5,
apertures=None,
ypeaks00=None):
""" Find new traces (bspecs and slopes; YFULL = bspec + slope * XFULL)
XFULL and YFULL are combined from all fnamecorrs.
First, we fit a line YFULL = bspec + slope * XFULL, and find new slopes. For FUV we ignore this step,
and slopes are zero.
Next, we correct YFULL for the slopes, and find new bspec using cosredux.trace.refine_peak.
Using the code to change WCA traces might not be reliable in some cases (for default input parameters).
It is recomended using plottraces and plothist to check the results.
Parameters
----------
fldpth : str
path to the files
fnamecorrs : list of str
corrtag files. These files need to have the same settings (optical element, central wavelength, FUVA/FUVB).
printorig :bool
print original peaks and slopes
xmin : float
minimum XFULL that is taken into account when finding traces
xmax : float
maximum XFULL that is taken into account when finding traces
height : float
height (YFULL range) used when finding traces
apertures : list of str
apertures
Returns
-------
allnewypeaks : list of lists of floats
new yspecs
allnewslopes : list of lists of floats
new slopes
ypeaks : list of floats
previous yspecs
slopes : list of floats
previous slopes
"""
# read fnamecorrs header: cenwave, opt_elem, detector, LP2_1dx_file
hdu = fits.open(fnamecorrs[0])
hd = hdu[0].header
hdu.close()
cenwave = hd['CENWAVE']
opt_elem = hd['OPT_ELEM']
if hd['DETECTOR'] == 'NUV':
segments = ['NUVA', 'NUVB', 'NUVC']
LP2_1dx_file = fldpth + hd['XTRACTAB'][5:-5] + '_copy1.fits'
if apertures is None:
apertures = ['PSA', 'WCA']
elif hd['DETECTOR'] == 'FUV':
segments = [hd['SEGMENT']]
LP2_1dx_file = fldpth + hd['TWOZXTAB'][5:-5] + '_copy1.fits'
if apertures is None:
apertures = ['PSA']
else:
raise IOError("Detector not well defined in ", fnamecorrs[0])
# LP2_1dx_file: ypeaks, slopes, heights (should be LP3, etc. This is just notation.)
hdu = fits.open(LP2_1dx_file)
lp2 = Table(hdu[1].data)
#
ypeaks = []
slopes = []
heights = []
for aperture in apertures:
for segment in segments:
irow = np.where((lp2['CENWAVE'] == cenwave)
& (lp2['APERTURE'] == aperture)
& (lp2['SEGMENT'] == segment)
& (lp2['OPT_ELEM'] == opt_elem))[0]
if len(irow) != 1:
print('irow error')
pdb.set_trace()
else:
irow = irow[0]
if verbose:
if hd['DETECTOR'] == 'NUV':
print(lp2[irow]['B_SPEC'], lp2[irow]['SLOPE'])
if hd['DETECTOR'] == 'FUV':
print(lp2[irow]['B_SPEC'])
ypeaks.append(lp2[irow]['B_SPEC'])
if hd['DETECTOR'] == 'NUV':
slopes.append(lp2[irow]['SLOPE'])
else:
slopes.append(0)
if height is not None:
heights.append(height)
else:
heights.append(lp2[irow]['HEIGHT'])
if ypeaks00 is not None:
ypeaks = ypeaks00
if printorig:
print('Original values:')
print('slopes: ', slopes)
print('bspecs: ', ypeaks)
####################################################
# read and combine xfull, yfull for corrtag files
xfull = []
yfull = []
for fnamecorr in fnamecorrs:
hdu = fits.open(fnamecorr)
data1 = hdu[1].data
if len(data1['XFULL']) < 1:
print(' No data for ', fnamecorr)
else:
xfull = xfull + list(data1['XFULL'])
yfull = yfull + list(data1['YFULL'])
xfull = np.asarray(xfull)
yfull = np.asarray(yfull)
if len(yfull) < 1:
print(' No data available for', fldpth)
return
##################################################
# find new slopes
newslopes1 = []
newypeaks1 = []
for i in range(len(slopes)):
if xmax is None:
xmax = np.max(xfull)
k = np.where((abs(yfull - slopes[i] * xfull - ypeaks[i]) < heights[i] /
2) & (xfull >= xmin) & (xfull <= xmax))[0]
yfull1 = yfull[k]
xfull1 = xfull[k]
# fit
if hd['DETECTOR'] == 'NUV':
x0, x1 = pyutils.robust_polyfit(xfull1, yfull1, 1)[1]
newslopes1.append(x1)
# assuming that central point in the trace did not change, i.e.:
# newslopes1 * (np.max(xfull)+np.min(xfull))*0.5 + newypeaks1 = const
newypeaks1.append((slopes[i] - x1) *
(np.max(xfull) + np.min(xfull)) * 0.5 +
ypeaks[i])
else:
newypeaks1.append(ypeaks[i])
newslopes1.append(slopes[i])
newypeaks1 = np.asarray(newypeaks1)
newslopes1 = np.asarray(newslopes1)
# find new bspecs
dyfulls = []
for i in range(len(newslopes1)):
dyfulls.append(abs(yfull - newslopes1[i] * xfull - newypeaks1[i]))
jj = np.argmin(dyfulls, axis=0)
#
inewslopes = newslopes1[jj]
yfull2 = yfull - inewslopes * xfull # correct YFULL for the slope
newypeaks = crude_histogram_multi(yfull2, newypeaks1, pk_window=pk_window)
newslopes = newslopes1
#
if verbose:
print(newypeaks)
print(ypeaks)
print(newslopes)
print(slopes)
return newypeaks, newslopes, ypeaks, slopes
def pca_trace(xcen, usepca = None, npca = 2, npoly_cen = 3, debug=True):
nspec = xcen.shape[0]
norders = xcen.shape[1]
if usepca is None:
usepca = np.zeros(norders,dtype=bool)
# use_order = True orders used to predict the usepca = True bad orders
use_order = np.invert(usepca)
ngood = np.sum(use_order)
if ngood < npca:
msgs.warn('Not enough good traces for a PCA fit: ngood = {:d}'.format(ngood) + ' is < npca = {:d}'.format(npca))
msgs.warn('Using the input trace for now')
return xcen
pca = PCA(n_components=npca)
xcen_use = (xcen[:,use_order] - np.mean(xcen[:,use_order],0)).T
pca_coeffs_use = pca.fit_transform(xcen_use)
pca_vectors = pca.components_
# Fit first pca dimension (with largest variance) with a higher order npoly depending on number of good orders.
# Fit all higher dimensions (with lower variance) with a line
npoly = int(np.fmin(np.fmax(np.floor(3.3*ngood/norders),1.0),3.0))
npoly_vec = np.full(npca, npoly)
order_vec = np.arange(norders,dtype=float)
# pca_coeffs = np.zeros((norders, npca))
pca_coeffs_new = np.zeros((norders, npca))
# Now loop over the dimensionality of the compression and perform a polynomial fit to
for idim in range(npca):
# ToDO robust_polyfit is garbage remove it entirely from PypeIT!
xfit = order_vec[use_order]
yfit = pca_coeffs_use[:,idim]
norder = npoly_vec[idim]
# msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma = 3.0, function='polynomial')
# pca_coeffs[:,idim] = utils.func_val(poly_coeff, order_vec, 'polynomial')
# TESTING traceset fitting
xtemp = xfit.reshape(1, xfit.size)
ytemp = yfit.reshape(1, yfit.size)
tset = pydl.xy2traceset(xtemp, ytemp, ncoeff=norder,func='polynomial')
#tset_yfit = tset.yfit.reshape(tset.yfit.shape[1])
## Test new robust fitting with djs_reject
msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit, yfit, norder, \
function='polynomial', minv=None, maxv=None, bspline_par=None, \
guesses=None, maxiter=10, inmask=None, sigma=None, invvar=None, \
lower=5, upper=5, maxdev=None, maxrej=None, groupdim=None,
groupsize=None, \
groupbadpix=False, grow=0, sticky=False)
pca_coeffs_new[:,idim] = utils.func_val(poly_coeff_new, order_vec, 'polynomial')
if debug:
# Evaluate the fit
xvec = np.linspace(order_vec.min(),order_vec.max(),num=100)
(_,tset_fit) = tset.xy(xpos=xvec.reshape(1,xvec.size))
yfit_tset = tset_fit[0,:]
#robust_mask = msk == 0
robust_mask_new = msk_new == 1
tset_mask = tset.outmask[0,:]
plt.plot(xfit, yfit, 'ko', mfc='None', markersize=8.0, label='pca coeff')
#plt.plot(xfit[~robust_mask], yfit[~robust_mask], 'ms', mfc='None', markersize=10.0,label='robust_polyfit rejected')
#plt.plot(xfit[~robust_mask_new], yfit[~robust_mask_new], 'r+', markersize=20.0,label='robust_polyfit_djs rejected')
plt.plot(xfit[~tset_mask],yfit[~tset_mask], 'bo', markersize = 10.0, label = 'traceset rejected')
#plt.plot(xvec, utils.func_val(poly_coeff, xvec, 'polynomial'),ls='--', color='m', label='robust polyfit')
plt.plot(xvec, utils.func_val(poly_coeff_new, xvec, 'polynomial'),ls='-.', color='r', label='new robust polyfit')
plt.plot(xvec, yfit_tset,ls=':', color='b',label='traceset')
plt.legend()
plt.show()
#ToDo should we be masking the bad orders here and interpolating/extrapolating?
spat_mean = np.mean(xcen,0)
msk_spat, poly_coeff_spat = utils.robust_polyfit(order_vec, spat_mean, npoly_cen, sigma = 3.0, function = 'polynomial')
ibad = np.where(msk_spat == 1)
spat_mean[ibad] = utils.func_val(poly_coeff_spat,order_vec[ibad],'polynomial')
#pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs, pca_vectors)).T
pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs_new, pca_vectors)).T
return pca_fit
def iterative_fitting(spec, tcent, ifit, IDs, llist, disp,
match_toler = 2.0, func = 'legendre', n_first=2, sigrej_first=2.0,
n_final=4, sigrej_final=3.0,
weights=None, plot_fil=None, verbose=False):
""" Routine for iteratively fitting wavelength solutions.
Parameters
----------
spec : ndarray, shape = (nspec,)
arcline spectrum
tcent : ndarray
Centroids in pixels of lines identified in spec
ifit : ndarray
Indices of the lines that will be fit
IDs: ndarray
wavelength IDs of the lines that will be fit (I think?)
llist: dict
Linelist dictionary
disp: float
dispersion
Optional Parameters
-------------------
match_toler: float, default = 3.0
Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to
an arc line by an iteration of the wavelength solution to the wavelength in the line list.
func: str, default = 'legendre'
Name of function used for the wavelength solution
n_first: int, default = 2
Order of first guess to the wavelength solution.
sigrej_first: float, default = 2.0
Number of sigma for rejection for the first guess to the wavelength solution.
n_final: int, default = 4
Order of the final wavelength solution fit
sigrej_final: float, default = 3.0
Number of sigma for rejection for the final fit to the wavelength solution.
weights: ndarray
Weights to be used?
verbose : bool
If True, print out more information.
plot_fil:
Filename for plotting some QA?
Returns
-------
final_fit: dict
Dictionary containing the full fitting results and the final best guess of the line IDs
"""
#TODO JFH add error checking here to ensure that IDs and ifit have the same size!
if weights is None:
weights = np.ones(tcent.size)
nspec = spec.size
xnspecmin1 = float(nspec-1)
# Setup for fitting
sv_ifit = list(ifit) # Keep the originals
all_ids = -999.*np.ones(len(tcent))
all_idsion = np.array(['UNKNWN']*len(tcent))
all_ids[ifit] = IDs
# Fit
n_order = n_first
flg_continue = True
flg_penultimate = False
fmin, fmax = 0.0, 1.0
# Note the number of parameters is actually n_order and not n_order+1
while flg_continue:
if flg_penultimate:
flg_continue = False
# Fit with rejection
xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit]
mask, fit = utils.robust_polyfit(xfit/xnspecmin1, yfit, n_order, function=func, sigma=sigrej_first,
minx=fmin, maxx=fmax, verbose=verbose, weights=wfit)
rms_ang = utils.calc_fit_rms(xfit[mask == 0]/xnspecmin1, yfit[mask == 0], fit, func, minx=fmin, maxx=fmax,
weights=wfit[mask == 0])
rms_pix = rms_ang/disp
if verbose:
msgs.info('n_order = {:d}'.format(n_order) + ': RMS = {:g}'.format(rms_pix))
# Reject but keep originals (until final fit)
ifit = list(ifit[mask == 0]) + sv_ifit
# Find new points (should we allow removal of the originals?)
twave = utils.func_val(fit, tcent/xnspecmin1, func, minx=fmin, maxx=fmax)
for ss, iwave in enumerate(twave):
mn = np.min(np.abs(iwave-llist['wave']))
if mn/disp < match_toler:
imn = np.argmin(np.abs(iwave-llist['wave']))
#if verbose:
# print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
# Update and append
all_ids[ss] = llist['wave'][imn]
all_idsion[ss] = llist['ion'][imn]
ifit.append(ss)
# Keep unique ones
ifit = np.unique(np.array(ifit, dtype=int))
# Increment order?
if n_order < n_final:
n_order += 1
else:
flg_penultimate = True
# Final fit (originals can now be rejected)
#fmin, fmax = 0., 1.
#xfit, yfit, wfit = tcent[ifit]/(nspec-1), all_ids[ifit], weights[ifit]
xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit]
mask, fit = utils.robust_polyfit(xfit/xnspecmin1, yfit, n_order, function=func, sigma=sigrej_final,
minx=fmin, maxx=fmax, verbose=verbose, weights=wfit)#, debug=True)
irej = np.where(mask == 1)[0]
if len(irej) > 0:
xrej = xfit[irej]
yrej = yfit[irej]
if verbose:
for kk, imask in enumerate(irej):
wave = utils.func_val(fit, xrej[kk]/xnspecmin1, func, minx=fmin, maxx=fmax)
msgs.info('Rejecting arc line {:g}; {:g}'.format(yfit[imask], wave))
else:
xrej = []
yrej = []
#xfit = xfit[mask == 0]
#yfit = yfit[mask == 0]
#wfit = wfit[mask == 0]
ions = all_idsion[ifit]
# ions = all_idsion[ifit][mask == 0]
# Final RMS
rms_ang = utils.calc_fit_rms(xfit[mask==0]/xnspecmin1, yfit[mask==0], fit, func,
minx=fmin, maxx=fmax, weights=wfit[mask==0])
# rms_ang = utils.calc_fit_rms(xfit, yfit, fit, func,
# minx=fmin, maxx=fmax, weights=wfit)
rms_pix = rms_ang/disp
# Pack up fit
spec_vec = np.arange(nspec)
wave_soln = utils.func_val(fit,spec_vec/xnspecmin1, func, minx=fmin, maxx=fmax)
cen_wave = utils.func_val(fit, float(nspec)/2/xnspecmin1, func, minx=fmin, maxx=fmax)
cen_wave_min1 = utils.func_val(fit, (float(nspec)/2 - 1.0)/xnspecmin1, func, minx=fmin, maxx=fmax)
cen_disp = cen_wave - cen_wave_min1
final_fit = dict(fitc=fit, function=func, pixel_fit=xfit, wave_fit=yfit, weights=wfit, ions=ions,
fmin=fmin, fmax=fmax, xnorm = xnspecmin1, nspec=nspec, cen_wave = cen_wave, cen_disp = cen_disp,
xrej=xrej, yrej=yrej, mask=(mask == 0), spec=spec, wave_soln = wave_soln, nrej=sigrej_final,
shift=0., tcent=tcent, rms=rms_pix)
# If set to True, this will output a file that can then be included in the tests
saveit = False
if saveit:
from linetools import utils as ltu
jdict = ltu.jsonify(final_fit)
if plot_fil is None:
outname = "temp"
print("You should have set the plot_fil directory to save wavelength fits... using 'temp' as a filename")
else:
outname = plot_fil
ltu.savejson(outname + '.json', jdict, easy_to_read=True, overwrite=True)
print(" Wrote: {:s}".format(outname + '.json'))
# QA
if plot_fil is not None:
autoid.arc_fit_qa(final_fit, plot_fil)
# Return
return final_fit
def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None,
function='polynomial'):
nrow = xfit.shape[0]
ntrace = xfit.shape[1]
if x0in is None:
x0in = np.arange(float(ntrace))
# Mask out some orders if they are bad
if mask is None or mask.size == 0:
usetrace = np.arange(ntrace)
outmask = np.ones((nrow, ntrace))
else:
usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
outmask = np.ones((nrow, ntrace))
outmask[:,mask] = 0.0
# Do the PCA analysis
eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc)
modl = func_vander(xfit[:,0], function, npc)
eigv = np.dot(modl[:,1:], eigc)
med_hidden = np.median(hidden, axis=1)
med_highorder = med_hidden.copy()
med_highorder[0] = 0
high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0)
# y = hidden[0,:]
# coeff0 = utils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)
# y = hidden[1,:]
# coeff1 = utils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)
coeffstr = []
for i in range(1, npc+1):
# if pnpc[i] == 0:
# coeffstr.append([-9.99E9])
# continue
# coeff0 = utils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
if weights is not None:
tmask, coeff0 = utils.robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i],
weights=weights[usetrace], sigma=2.0, function=function,
minx=x0in[0], maxx=x0in[-1])
else:
tmask, coeff0 = utils.robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i],
sigma=2.0, function=function,
minx=x0in[0], maxx=x0in[-1])
coeffstr.append(coeff0)
high_order_matrix[:, i-1] = utils.func_val(coeff0, x0in, function, minx=x0in[0], maxx=x0in[-1])
# high_order_matrix[:,1] = utils.func_val(coeff1, x0in, function)
high_fit = high_order_matrix.copy()
high_order_fit = np.dot(eigv, high_order_matrix.T)
sub = (yfit - high_order_fit) * outmask
numer = np.sum(sub, axis=0)
denom = np.sum(outmask, axis=0)
x0 = np.zeros(ntrace, dtype=np.float)
fitmask = np.zeros(ntrace, dtype=np.float)
#fitmask[mask] = 1
x0fit = np.zeros(ntrace, dtype=np.float)
chisqnu = 0.0
chisqold = 0.0
robust = True
#svx0 = numer/(denom+(denom == 0).astype(np.int))
if not skipx0:
fitmask = (np.abs(denom) > 10).astype(np.int)
if robust:
good = np.where(fitmask != 0)[0]
bad = np.where(fitmask == 0)[0]
x0[good] = numer[good]/denom[good]
imask = np.zeros(ntrace, dtype=np.float)
imask[bad] = 1.0
ttmask, x0res = utils.robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0,
function=function, minx=x0in[0], maxx=x0in[-1], initialmask=imask)
x0fit = utils.func_val(x0res, x0in, function, minx=x0in[0], maxx=x0in[-1])
good = np.where(ttmask == 0)[0]
xstd = 1.0 # This should represent the dispersion in the fit
chisq = ((x0[good]-x0fit[good])/xstd)**2.0
chisqnu = np.sum(chisq)/np.sum(ttmask)
fitmask = 1.0-ttmask
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
else:
for i in range(1, 5):
good = np.where(fitmask != 0)[0]
x0[good] = numer[good]/denom[good]
# x0res = utils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
x0res = utils.func_fit(x0in[good], x0[good], function, pnpc[0],
weights=weights, minx=x0in[0], maxx=x0in[-1])
x0fit = utils.func_val(x0res, x0in, function, minx=x0in[0], maxx=x0in[-1])
chisq = (x0[good]-x0fit[good])**2.0
fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int)
chisqnu = np.sum(chisq)/np.sum(fitmask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu == chisqold:
break
else:
chisqold = chisqnu
if chisqnu > 2.0:
msgs.warn("PCA has very large residuals")
elif chisqnu > 0.5:
msgs.warn("PCA has fairly large residuals")
#bad = np.where(fitmask==0)[0]
#x0[bad] = x0fit[bad]
else:
x0res = 0.0
x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T
outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask,
'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr})
return x3fit, outpar
def do_it_all(slit, instr, plot_fil=None, IDtol=1., subpix=None):
if instr == '600':
wv_dict, template = load_600()
fwhm = 4.
n_order = 3
n_first = 2
elif instr == '300':
wv_dict, template = load_300()
fwhm = 8.
n_order = 3
n_first = 3
else:
pdb.set_trace()
nwwv = template['wave'].data
nwspec = template['flux'].data
# Snippet
tspec, mspec, mwv, targ_wv, targ_x, targ_y = target_lines(wv_dict,
slit,
nwwv,
nwspec,
fwhm=fwhm,
subpix=subpix)
# Find new lines
all_tcent, all_ecent, cut_tcent, icut, arc_cont_sub = wvutils.arc_lines_from_spec(
tspec, fwhm=fwhm, debug=True)
# Grab initial guess
func = 'legendre'
sigrej_first = 3.
fmin, fmax = 0., 1.
xfit = np.arange(mwv.size)
yfit = mwv
initial_mask = (mwv > targ_wv.min()) & (mwv < targ_wv.max())
i1 = np.where(initial_mask)[0][0]
disp = mwv[i1 + 1] - mwv[i1]
#
mask, fit = utils.robust_polyfit(xfit,
yfit,
n_order,
function=func,
sigma=sigrej_first,
initialmask=~initial_mask,
minx=fmin,
maxx=fmax,
verbose=True) # , weights=wfit)
rms_ang = utils.calc_fit_rms(xfit[mask == 0],
yfit[mask == 0],
fit,
func,
minx=fmin,
maxx=fmax)
# weights=wfit[mask == 0])
rms_pix = rms_ang / disp
print("Initial fit: {}; RMS = {}".format(fit, rms_pix))
# Target grid
targ_grid = np.outer(targ_wv, np.ones(all_tcent.size))
# Check the loss
loss = loss_func(fit[0:2], all_tcent, targ_grid, targ_y, fit, 2)
# Bounds
bounds = [[fit[0] - 50., fit[0] + 50], [fit[1] * 0.9, fit[1] * 1.1]]
#for ifit in fit[2:]:
# bounds.append([np.abs(ifit)*-2, np.abs(ifit)*2])
# Differential evolution
#diff_result = fitme(bounds, all_tcent, targ_grid, targ_y)
diff_result = fitme(bounds, all_tcent, targ_grid, targ_y, fit.copy())
new_fit = fit.copy()
new_fit[:len(bounds)] = diff_result.x
loss = loss_func(new_fit[0:len(bounds)],
all_tcent,
targ_grid,
targ_y,
fit,
len(bounds),
verbose=True)
print("Refined fit: {}".format(diff_result.x))
# Now cut on saturation
all_tcent, all_ecent, cut_tcent, icut, arc_cont_sub = wvutils.arc_lines_from_spec(
tspec, fwhm=fwhm, nonlinear_counts=55000)
#debug=True)
final_guess = utils.func_val(new_fit,
all_tcent,
func,
minx=fmin,
maxx=fmax)
# Match to line list
IDs = []
dwv = mwv[i1 + 1] - mwv[i1]
lmask = np.zeros(final_guess.size, dtype=bool)
for kk, iwv in enumerate(final_guess):
imin = np.argmin(np.abs(iwv - llist['wave']))
if np.abs(iwv - llist['wave'][imin]) < dwv * IDtol:
lmask[kk] = True
IDs.append(llist['wave'][imin])
print("IDs: {}".format(IDs))
# Final fit
final_fit = fitting.iterative_fitting(tspec,
all_tcent,
np.where(lmask)[0],
np.array(IDs),
llist,
dwv,
verbose=True,
plot_fil=plot_fil,
n_first=n_first)
# Return
return final_fit
sign = rand.choice([-1, 1.], nout)
yfit[indrand] = yreal[indrand] + sign * nsigma * std
#yfit[0] = yfit[0]+3*np.std(yreal-yfit)
#yfit[10] = yfit[10]+4*np.std(yreal-yfit)
#yfit[19] = yfit[19]-3.5*np.std(yreal-yfit)
#yfit[49] = yfit[49]-2.8*np.std(yreal-yfit)
#yfit[69] = yfit[69]+5*np.std(yreal-yfit)
norder = 3
xvec = np.linspace(xfit.min(), xfit.max(), num=200)
## Old robust_olyfit
msk, poly_coeff = utils.robust_polyfit(xfit,
yfit,
norder,
sigma=1.5,
function='polynomial')
msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit,yfit,norder, \
function = 'polynomial', minv = None, maxv = None, bspline_par = None,\
guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\
lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\
groupbadpix=False, grow=0,sticky=True,use_mad=True)
msk_nosticky, poly_coeff_nosticky = utils.robust_polyfit_djs(xfit,yfit,norder, \
function = 'polynomial', minv = None, maxv = None, bspline_par = None,\
guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\
lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\
groupbadpix=False, grow=0,sticky=False,use_mad=True)
robust_mask = msk == 0
