def refine_iter(outpar, orders, mask, irshft, relshift, fitord, function='polynomial'): fail = False x0ex = utils.func_val(outpar['x0res'], orders, function, minx=outpar['x0in'][0], maxx=outpar['x0in'][-1]) # Make the refinement x0ex[irshft] += relshift # Refit the data to improve the refinement good = np.where(mask != 0.0)[0] # x0res = utils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function) null, x0res = utils.robust_polyfit(orders[good], x0ex[good], fitord, sigma=2.0, function=function, minx=outpar['x0in'][0], maxx=outpar['x0in'][-1]) #x0res = utils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1]) x0fit = utils.func_val(x0res, orders, function, minx=outpar['x0in'][0], maxx=outpar['x0in'][-1]) chisq = (x0ex[good]-x0fit[good])**2.0 chisqnu = np.sum(chisq)/np.sum(mask) msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu)) if chisqnu > 0.5: # The refinement went wrong, ignore the refinement fail = True outpar['x0res'] = x0res extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(x0fit, np.ones(outpar['eigv'].shape[0])).T return extfit, outpar, fail
def flex_shift(obj_skyspec, arx_skyspec, mxshft=20): """ Calculate shift between object sky spectrum and archive sky spectrum Parameters ---------- obj_skyspec arx_skyspec Returns ------- flex_dict: dict Contains flexure info """ # TODO None of these routines should have dependencies on XSpectrum1d! # Determine the brightest emission lines msgs.warn("If we use Paranal, cut down on wavelength early on") arx_amp, arx_amp_cont, arx_cent, arx_wid, _, arx_w, arx_yprep, nsig = arc.detect_lines( arx_skyspec.flux.value) obj_amp, obj_amp_cont, obj_cent, obj_wid, _, obj_w, obj_yprep, nsig_obj = arc.detect_lines( obj_skyspec.flux.value) # Keep only 5 brightest amplitude lines (xxx_keep is array of # indices within arx_w of the 5 brightest) arx_keep = np.argsort(arx_amp[arx_w])[-5:] obj_keep = np.argsort(obj_amp[obj_w])[-5:] # Calculate wavelength (Angstrom per pixel) arx_disp = np.append( arx_skyspec.wavelength.value[1] - arx_skyspec.wavelength.value[0], arx_skyspec.wavelength.value[1:] - arx_skyspec.wavelength.value[:-1]) #arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size obj_disp = np.append( obj_skyspec.wavelength.value[1] - obj_skyspec.wavelength.value[0], obj_skyspec.wavelength.value[1:] - obj_skyspec.wavelength.value[:-1]) #obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size # Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need # this? can just use sigmas arx_idx = (arx_cent + 0.5).astype( np.int)[arx_w][arx_keep] # The +0.5 is for rounding arx_res = arx_skyspec.wavelength.value[arx_idx]/\ (arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep]) obj_idx = (obj_cent + 0.5).astype( np.int)[obj_w][obj_keep] # The +0.5 is for rounding obj_res = obj_skyspec.wavelength.value[obj_idx]/ \ (obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep]) #obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/( # obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep]) if not np.all(np.isfinite(obj_res)): msgs.warn( 'Failed to measure the resolution of the object spectrum, likely due to error ' 'in the wavelength image.') return None msgs.info("Resolution of Archive={0} and Observation={1}".format( np.median(arx_res), np.median(obj_res))) # Determine sigma of gaussian for smoothing arx_sig2 = np.power(arx_disp[arx_idx] * arx_wid[arx_w][arx_keep], 2) obj_sig2 = np.power(obj_disp[obj_idx] * obj_wid[obj_w][obj_keep], 2) arx_med_sig2 = np.median(arx_sig2) obj_med_sig2 = np.median(obj_sig2) if obj_med_sig2 >= arx_med_sig2: smooth_sig = np.sqrt(obj_med_sig2 - arx_med_sig2) # Ang smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx]) arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix * 2 * np.sqrt(2 * np.log(2))) else: msgs.warn("Prefer archival sky spectrum to have higher resolution") smooth_sig_pix = 0. msgs.warn("New Sky has higher resolution than Archive. Not smoothing") #smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2) #Determine region of wavelength overlap min_wave = max(np.amin(arx_skyspec.wavelength.value), np.amin(obj_skyspec.wavelength.value)) max_wave = min(np.amax(arx_skyspec.wavelength.value), np.amax(obj_skyspec.wavelength.value)) #Smooth higher resolution spectrum by smooth_sig (flux is conserved!) # if np.median(obj_res) >= np.median(arx_res): # msgs.warn("New Sky has higher resolution than Archive. Not smoothing") #obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig) # else: #tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig) # arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2))) #arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig) # Define wavelengths of overlapping spectra keep_idx = np.where((obj_skyspec.wavelength.value >= min_wave) & (obj_skyspec.wavelength.value <= max_wave))[0] #keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave] #Rebin both spectra onto overlapped wavelength range if len(keep_idx) <= 50: msgs.warn("Not enough overlap between sky spectra") return None else: #rebin onto object ALWAYS keep_wave = obj_skyspec.wavelength[keep_idx] arx_skyspec = arx_skyspec.rebin(keep_wave) obj_skyspec = obj_skyspec.rebin(keep_wave) # Trim edges (rebinning is junk there) arx_skyspec.data['flux'][0, :2] = 0. arx_skyspec.data['flux'][0, -2:] = 0. obj_skyspec.data['flux'][0, :2] = 0. obj_skyspec.data['flux'][0, -2:] = 0. # Normalize spectra to unit average sky count norm = np.sum(obj_skyspec.flux.value) / obj_skyspec.npix obj_skyspec.flux = obj_skyspec.flux / norm norm2 = np.sum(arx_skyspec.flux.value) / arx_skyspec.npix arx_skyspec.flux = arx_skyspec.flux / norm2 if (norm < 0.): msgs.warn("Bad normalization of object in flexure algorithm") msgs.warn("Will try the median") norm = np.median(obj_skyspec.flux.value) if (norm < 0.): msgs.warn("Improper sky spectrum for flexure. Is it too faint??") return None if (norm2 < 0.): msgs.warn( 'Bad normalization of archive in flexure. You are probably using wavelengths ' 'well beyond the archive.') return None # Deal with bad pixels msgs.work("Need to mask bad pixels") # Deal with underlying continuum msgs.work("Consider taking median first [5 pixel]") everyn = obj_skyspec.npix // 20 bspline_par = dict(everyn=everyn) mask, ct = utils.robust_polyfit(obj_skyspec.wavelength.value, obj_skyspec.flux.value, 3, function='bspline', sigma=3., bspline_par=bspline_par) obj_sky_cont = utils.func_val(ct, obj_skyspec.wavelength.value, 'bspline') obj_sky_flux = obj_skyspec.flux.value - obj_sky_cont mask, ct_arx = utils.robust_polyfit(arx_skyspec.wavelength.value, arx_skyspec.flux.value, 3, function='bspline', sigma=3., bspline_par=bspline_par) arx_sky_cont = utils.func_val(ct_arx, arx_skyspec.wavelength.value, 'bspline') arx_sky_flux = arx_skyspec.flux.value - arx_sky_cont # Consider sharpness filtering (e.g. LowRedux) msgs.work("Consider taking median first [5 pixel]") #Cross correlation of spectra #corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same") corr = np.correlate(arx_sky_flux, obj_sky_flux, "same") #Create array around the max of the correlation function for fitting for subpixel max # Restrict to pixels within maxshift of zero lag lag0 = corr.size // 2 #mxshft = settings.argflag['reduce']['flexure']['maxshift'] max_corr = np.argmax(corr[lag0 - mxshft:lag0 + mxshft]) + lag0 - mxshft subpix_grid = np.linspace(max_corr - 3., max_corr + 3., 7) #Fit a 2-degree polynomial to peak of correlation function. JFH added this if/else to not crash for bad slits if np.any(np.isfinite(corr[subpix_grid.astype(np.int)])): fit = utils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)], 'polynomial', 2) success = True max_fit = -0.5 * fit[1] / fit[2] else: fit = utils.func_fit(subpix_grid, 0.0 * subpix_grid, 'polynomial', 2) success = False max_fit = 0.0 msgs.warn('Flexure compensation failed for one of your objects') #Calculate and apply shift in wavelength shift = float(max_fit) - lag0 msgs.info("Flexure correction of {:g} pixels".format(shift)) #model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0] flex_dict = dict(polyfit=fit, shift=shift, subpix=subpix_grid, corr=corr[subpix_grid.astype(np.int)], sky_spec=obj_skyspec, arx_spec=arx_skyspec, corr_cen=corr.size / 2, smooth=smooth_sig_pix, success=success) # Return return flex_dict
std = np.std(yreal-yfit) nsigma = rand.uniform(1,3,nout) sign = rand.choice([-1,1.],nout) yfit[indrand] = yreal[indrand] + sign*nsigma*std #yfit[0] = yfit[0]+3*np.std(yreal-yfit) #yfit[10] = yfit[10]+4*np.std(yreal-yfit) #yfit[19] = yfit[19]-3.5*np.std(yreal-yfit) #yfit[49] = yfit[49]-2.8*np.std(yreal-yfit) #yfit[69] = yfit[69]+5*np.std(yreal-yfit) norder =3 xvec = np.linspace(xfit.min(), xfit.max(), num=200) ## Old robust_olyfit msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma=1.5, function='polynomial') msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit,yfit,norder, \ function = 'polynomial', minv = None, maxv = None, bspline_par = None,\ guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\ lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\ groupbadpix=False, grow=0,sticky=True,use_mad=True) msk_nosticky, poly_coeff_nosticky = utils.robust_polyfit_djs(xfit,yfit,norder, \ function = 'polynomial', minv = None, maxv = None, bspline_par = None,\ guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\ lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\ groupbadpix=False, grow=0,sticky=False,use_mad=True) robust_mask = msk == 0 robust_mask_new = msk_new == 1 robust_mask_nosticky = msk_nosticky == 1
def pca_trace(xcen, usepca=None, npca=2, npoly_cen=3, debug=True): nspec = xcen.shape[0] norders = xcen.shape[1] if usepca is None: usepca = np.zeros(norders, dtype=bool) # use_order = True orders used to predict the usepca = True bad orders use_order = np.invert(usepca) ngood = np.sum(use_order) if ngood < npca: msgs.warn('Not enough good traces for a PCA fit: ngood = {:d}'.format( ngood) + ' is < npca = {:d}'.format(npca)) msgs.warn('Using the input trace for now') return xcen pca = PCA(n_components=npca) xcen_use = (xcen[:, use_order] - np.mean(xcen[:, use_order], 0)).T pca_coeffs_use = pca.fit_transform(xcen_use) pca_vectors = pca.components_ # Fit first pca dimension (with largest variance) with a higher order npoly depending on number of good orders. # Fit all higher dimensions (with lower variance) with a line npoly = int(np.fmin(np.fmax(np.floor(3.3 * ngood / norders), 1.0), 3.0)) npoly_vec = np.full(npca, npoly) order_vec = np.arange(norders, dtype=float) # pca_coeffs = np.zeros((norders, npca)) pca_coeffs_new = np.zeros((norders, npca)) # Now loop over the dimensionality of the compression and perform a polynomial fit to for idim in range(npca): # ToDO robust_polyfit is garbage remove it entirely from PypeIT! xfit = order_vec[use_order] yfit = pca_coeffs_use[:, idim] norder = npoly_vec[idim] # msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma = 3.0, function='polynomial') # pca_coeffs[:,idim] = utils.func_val(poly_coeff, order_vec, 'polynomial') # TESTING traceset fitting xtemp = xfit.reshape(1, xfit.size) ytemp = yfit.reshape(1, yfit.size) tset = pydl.xy2traceset(xtemp, ytemp, ncoeff=norder, func='polynomial') #tset_yfit = tset.yfit.reshape(tset.yfit.shape[1]) ## Test new robust fitting with djs_reject msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit, yfit, norder, \ function='polynomial', minv=None, maxv=None, bspline_par=None, \ guesses=None, maxiter=10, inmask=None, sigma=None, invvar=None, \ lower=5, upper=5, maxdev=None, maxrej=None, groupdim=None, groupsize=None, \ groupbadpix=False, grow=0, sticky=False) pca_coeffs_new[:, idim] = utils.func_val(poly_coeff_new, order_vec, 'polynomial') if debug: # Evaluate the fit xvec = np.linspace(order_vec.min(), order_vec.max(), num=100) (_, tset_fit) = tset.xy(xpos=xvec.reshape(1, xvec.size)) yfit_tset = tset_fit[0, :] #robust_mask = msk == 0 robust_mask_new = msk_new == 1 tset_mask = tset.outmask[0, :] plt.plot(xfit, yfit, 'ko', mfc='None', markersize=8.0, label='pca coeff') #plt.plot(xfit[~robust_mask], yfit[~robust_mask], 'ms', mfc='None', markersize=10.0,label='robust_polyfit rejected') #plt.plot(xfit[~robust_mask_new], yfit[~robust_mask_new], 'r+', markersize=20.0,label='robust_polyfit_djs rejected') plt.plot(xfit[~tset_mask], yfit[~tset_mask], 'bo', markersize=10.0, label='traceset rejected') #plt.plot(xvec, utils.func_val(poly_coeff, xvec, 'polynomial'),ls='--', color='m', label='robust polyfit') plt.plot(xvec, utils.func_val(poly_coeff_new, xvec, 'polynomial'), ls='-.', color='r', label='new robust polyfit') plt.plot(xvec, yfit_tset, ls=':', color='b', label='traceset') plt.legend() plt.show() #ToDo should we be masking the bad orders here and interpolating/extrapolating? spat_mean = np.mean(xcen, 0) msk_spat, poly_coeff_spat = utils.robust_polyfit(order_vec, spat_mean, npoly_cen, sigma=3.0, function='polynomial') ibad = np.where(msk_spat == 1) spat_mean[ibad] = utils.func_val(poly_coeff_spat, order_vec[ibad], 'polynomial') #pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs, pca_vectors)).T pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer( pca.mean_, np.ones(norders)) + (np.dot(pca_coeffs_new, pca_vectors)).T return pca_fit
def findpeaks(fldpth, fnamecorrs, verbose=True, printorig=True, xmin=0, xmax=None, height=None, pk_window=5, apertures=None, ypeaks00=None): """ Find new traces (bspecs and slopes; YFULL = bspec + slope * XFULL) XFULL and YFULL are combined from all fnamecorrs. First, we fit a line YFULL = bspec + slope * XFULL, and find new slopes. For FUV we ignore this step, and slopes are zero. Next, we correct YFULL for the slopes, and find new bspec using cosredux.trace.refine_peak. Using the code to change WCA traces might not be reliable in some cases (for default input parameters). It is recomended using plottraces and plothist to check the results. Parameters ---------- fldpth : str path to the files fnamecorrs : list of str corrtag files. These files need to have the same settings (optical element, central wavelength, FUVA/FUVB). printorig :bool print original peaks and slopes xmin : float minimum XFULL that is taken into account when finding traces xmax : float maximum XFULL that is taken into account when finding traces height : float height (YFULL range) used when finding traces apertures : list of str apertures Returns ------- allnewypeaks : list of lists of floats new yspecs allnewslopes : list of lists of floats new slopes ypeaks : list of floats previous yspecs slopes : list of floats previous slopes """ # read fnamecorrs header: cenwave, opt_elem, detector, LP2_1dx_file hdu = fits.open(fnamecorrs[0]) hd = hdu[0].header hdu.close() cenwave = hd['CENWAVE'] opt_elem = hd['OPT_ELEM'] if hd['DETECTOR'] == 'NUV': segments = ['NUVA', 'NUVB', 'NUVC'] LP2_1dx_file = fldpth + hd['XTRACTAB'][5:-5] + '_copy1.fits' if apertures is None: apertures = ['PSA', 'WCA'] elif hd['DETECTOR'] == 'FUV': segments = [hd['SEGMENT']] LP2_1dx_file = fldpth + hd['TWOZXTAB'][5:-5] + '_copy1.fits' if apertures is None: apertures = ['PSA'] else: raise IOError("Detector not well defined in ", fnamecorrs[0]) # LP2_1dx_file: ypeaks, slopes, heights (should be LP3, etc. This is just notation.) hdu = fits.open(LP2_1dx_file) lp2 = Table(hdu[1].data) # ypeaks = [] slopes = [] heights = [] for aperture in apertures: for segment in segments: irow = np.where((lp2['CENWAVE'] == cenwave) & (lp2['APERTURE'] == aperture) & (lp2['SEGMENT'] == segment) & (lp2['OPT_ELEM'] == opt_elem))[0] if len(irow) != 1: print('irow error') pdb.set_trace() else: irow = irow[0] if verbose: if hd['DETECTOR'] == 'NUV': print(lp2[irow]['B_SPEC'], lp2[irow]['SLOPE']) if hd['DETECTOR'] == 'FUV': print(lp2[irow]['B_SPEC']) ypeaks.append(lp2[irow]['B_SPEC']) if hd['DETECTOR'] == 'NUV': slopes.append(lp2[irow]['SLOPE']) else: slopes.append(0) if height is not None: heights.append(height) else: heights.append(lp2[irow]['HEIGHT']) if ypeaks00 is not None: ypeaks = ypeaks00 if printorig: print('Original values:') print('slopes: ', slopes) print('bspecs: ', ypeaks) #################################################### # read and combine xfull, yfull for corrtag files xfull = [] yfull = [] for fnamecorr in fnamecorrs: hdu = fits.open(fnamecorr) data1 = hdu[1].data if len(data1['XFULL']) < 1: print(' No data for ', fnamecorr) else: xfull = xfull + list(data1['XFULL']) yfull = yfull + list(data1['YFULL']) xfull = np.asarray(xfull) yfull = np.asarray(yfull) if len(yfull) < 1: print(' No data available for', fldpth) return ################################################## # find new slopes newslopes1 = [] newypeaks1 = [] for i in range(len(slopes)): if xmax is None: xmax = np.max(xfull) k = np.where((abs(yfull - slopes[i] * xfull - ypeaks[i]) < heights[i] / 2) & (xfull >= xmin) & (xfull <= xmax))[0] yfull1 = yfull[k] xfull1 = xfull[k] # fit if hd['DETECTOR'] == 'NUV': x0, x1 = pyutils.robust_polyfit(xfull1, yfull1, 1)[1] newslopes1.append(x1) # assuming that central point in the trace did not change, i.e.: # newslopes1 * (np.max(xfull)+np.min(xfull))*0.5 + newypeaks1 = const newypeaks1.append((slopes[i] - x1) * (np.max(xfull) + np.min(xfull)) * 0.5 + ypeaks[i]) else: newypeaks1.append(ypeaks[i]) newslopes1.append(slopes[i]) newypeaks1 = np.asarray(newypeaks1) newslopes1 = np.asarray(newslopes1) # find new bspecs dyfulls = [] for i in range(len(newslopes1)): dyfulls.append(abs(yfull - newslopes1[i] * xfull - newypeaks1[i])) jj = np.argmin(dyfulls, axis=0) # inewslopes = newslopes1[jj] yfull2 = yfull - inewslopes * xfull # correct YFULL for the slope newypeaks = crude_histogram_multi(yfull2, newypeaks1, pk_window=pk_window) newslopes = newslopes1 # if verbose: print(newypeaks) print(ypeaks) print(newslopes) print(slopes) return newypeaks, newslopes, ypeaks, slopes
def pca_trace(xcen, usepca = None, npca = 2, npoly_cen = 3, debug=True): nspec = xcen.shape[0] norders = xcen.shape[1] if usepca is None: usepca = np.zeros(norders,dtype=bool) # use_order = True orders used to predict the usepca = True bad orders use_order = np.invert(usepca) ngood = np.sum(use_order) if ngood < npca: msgs.warn('Not enough good traces for a PCA fit: ngood = {:d}'.format(ngood) + ' is < npca = {:d}'.format(npca)) msgs.warn('Using the input trace for now') return xcen pca = PCA(n_components=npca) xcen_use = (xcen[:,use_order] - np.mean(xcen[:,use_order],0)).T pca_coeffs_use = pca.fit_transform(xcen_use) pca_vectors = pca.components_ # Fit first pca dimension (with largest variance) with a higher order npoly depending on number of good orders. # Fit all higher dimensions (with lower variance) with a line npoly = int(np.fmin(np.fmax(np.floor(3.3*ngood/norders),1.0),3.0)) npoly_vec = np.full(npca, npoly) order_vec = np.arange(norders,dtype=float) # pca_coeffs = np.zeros((norders, npca)) pca_coeffs_new = np.zeros((norders, npca)) # Now loop over the dimensionality of the compression and perform a polynomial fit to for idim in range(npca): # ToDO robust_polyfit is garbage remove it entirely from PypeIT! xfit = order_vec[use_order] yfit = pca_coeffs_use[:,idim] norder = npoly_vec[idim] # msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma = 3.0, function='polynomial') # pca_coeffs[:,idim] = utils.func_val(poly_coeff, order_vec, 'polynomial') # TESTING traceset fitting xtemp = xfit.reshape(1, xfit.size) ytemp = yfit.reshape(1, yfit.size) tset = pydl.xy2traceset(xtemp, ytemp, ncoeff=norder,func='polynomial') #tset_yfit = tset.yfit.reshape(tset.yfit.shape[1]) ## Test new robust fitting with djs_reject msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit, yfit, norder, \ function='polynomial', minv=None, maxv=None, bspline_par=None, \ guesses=None, maxiter=10, inmask=None, sigma=None, invvar=None, \ lower=5, upper=5, maxdev=None, maxrej=None, groupdim=None, groupsize=None, \ groupbadpix=False, grow=0, sticky=False) pca_coeffs_new[:,idim] = utils.func_val(poly_coeff_new, order_vec, 'polynomial') if debug: # Evaluate the fit xvec = np.linspace(order_vec.min(),order_vec.max(),num=100) (_,tset_fit) = tset.xy(xpos=xvec.reshape(1,xvec.size)) yfit_tset = tset_fit[0,:] #robust_mask = msk == 0 robust_mask_new = msk_new == 1 tset_mask = tset.outmask[0,:] plt.plot(xfit, yfit, 'ko', mfc='None', markersize=8.0, label='pca coeff') #plt.plot(xfit[~robust_mask], yfit[~robust_mask], 'ms', mfc='None', markersize=10.0,label='robust_polyfit rejected') #plt.plot(xfit[~robust_mask_new], yfit[~robust_mask_new], 'r+', markersize=20.0,label='robust_polyfit_djs rejected') plt.plot(xfit[~tset_mask],yfit[~tset_mask], 'bo', markersize = 10.0, label = 'traceset rejected') #plt.plot(xvec, utils.func_val(poly_coeff, xvec, 'polynomial'),ls='--', color='m', label='robust polyfit') plt.plot(xvec, utils.func_val(poly_coeff_new, xvec, 'polynomial'),ls='-.', color='r', label='new robust polyfit') plt.plot(xvec, yfit_tset,ls=':', color='b',label='traceset') plt.legend() plt.show() #ToDo should we be masking the bad orders here and interpolating/extrapolating? spat_mean = np.mean(xcen,0) msk_spat, poly_coeff_spat = utils.robust_polyfit(order_vec, spat_mean, npoly_cen, sigma = 3.0, function = 'polynomial') ibad = np.where(msk_spat == 1) spat_mean[ibad] = utils.func_val(poly_coeff_spat,order_vec[ibad],'polynomial') #pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs, pca_vectors)).T pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs_new, pca_vectors)).T return pca_fit
def iterative_fitting(spec, tcent, ifit, IDs, llist, disp, match_toler = 2.0, func = 'legendre', n_first=2, sigrej_first=2.0, n_final=4, sigrej_final=3.0, weights=None, plot_fil=None, verbose=False): """ Routine for iteratively fitting wavelength solutions. Parameters ---------- spec : ndarray, shape = (nspec,) arcline spectrum tcent : ndarray Centroids in pixels of lines identified in spec ifit : ndarray Indices of the lines that will be fit IDs: ndarray wavelength IDs of the lines that will be fit (I think?) llist: dict Linelist dictionary disp: float dispersion Optional Parameters ------------------- match_toler: float, default = 3.0 Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to an arc line by an iteration of the wavelength solution to the wavelength in the line list. func: str, default = 'legendre' Name of function used for the wavelength solution n_first: int, default = 2 Order of first guess to the wavelength solution. sigrej_first: float, default = 2.0 Number of sigma for rejection for the first guess to the wavelength solution. n_final: int, default = 4 Order of the final wavelength solution fit sigrej_final: float, default = 3.0 Number of sigma for rejection for the final fit to the wavelength solution. weights: ndarray Weights to be used? verbose : bool If True, print out more information. plot_fil: Filename for plotting some QA? Returns ------- final_fit: dict Dictionary containing the full fitting results and the final best guess of the line IDs """ #TODO JFH add error checking here to ensure that IDs and ifit have the same size! if weights is None: weights = np.ones(tcent.size) nspec = spec.size xnspecmin1 = float(nspec-1) # Setup for fitting sv_ifit = list(ifit) # Keep the originals all_ids = -999.*np.ones(len(tcent)) all_idsion = np.array(['UNKNWN']*len(tcent)) all_ids[ifit] = IDs # Fit n_order = n_first flg_continue = True flg_penultimate = False fmin, fmax = 0.0, 1.0 # Note the number of parameters is actually n_order and not n_order+1 while flg_continue: if flg_penultimate: flg_continue = False # Fit with rejection xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit] mask, fit = utils.robust_polyfit(xfit/xnspecmin1, yfit, n_order, function=func, sigma=sigrej_first, minx=fmin, maxx=fmax, verbose=verbose, weights=wfit) rms_ang = utils.calc_fit_rms(xfit[mask == 0]/xnspecmin1, yfit[mask == 0], fit, func, minx=fmin, maxx=fmax, weights=wfit[mask == 0]) rms_pix = rms_ang/disp if verbose: msgs.info('n_order = {:d}'.format(n_order) + ': RMS = {:g}'.format(rms_pix)) # Reject but keep originals (until final fit) ifit = list(ifit[mask == 0]) + sv_ifit # Find new points (should we allow removal of the originals?) twave = utils.func_val(fit, tcent/xnspecmin1, func, minx=fmin, maxx=fmax) for ss, iwave in enumerate(twave): mn = np.min(np.abs(iwave-llist['wave'])) if mn/disp < match_toler: imn = np.argmin(np.abs(iwave-llist['wave'])) #if verbose: # print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss])) # Update and append all_ids[ss] = llist['wave'][imn] all_idsion[ss] = llist['ion'][imn] ifit.append(ss) # Keep unique ones ifit = np.unique(np.array(ifit, dtype=int)) # Increment order? if n_order < n_final: n_order += 1 else: flg_penultimate = True # Final fit (originals can now be rejected) #fmin, fmax = 0., 1. #xfit, yfit, wfit = tcent[ifit]/(nspec-1), all_ids[ifit], weights[ifit] xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit] mask, fit = utils.robust_polyfit(xfit/xnspecmin1, yfit, n_order, function=func, sigma=sigrej_final, minx=fmin, maxx=fmax, verbose=verbose, weights=wfit)#, debug=True) irej = np.where(mask == 1)[0] if len(irej) > 0: xrej = xfit[irej] yrej = yfit[irej] if verbose: for kk, imask in enumerate(irej): wave = utils.func_val(fit, xrej[kk]/xnspecmin1, func, minx=fmin, maxx=fmax) msgs.info('Rejecting arc line {:g}; {:g}'.format(yfit[imask], wave)) else: xrej = [] yrej = [] #xfit = xfit[mask == 0] #yfit = yfit[mask == 0] #wfit = wfit[mask == 0] ions = all_idsion[ifit] # ions = all_idsion[ifit][mask == 0] # Final RMS rms_ang = utils.calc_fit_rms(xfit[mask==0]/xnspecmin1, yfit[mask==0], fit, func, minx=fmin, maxx=fmax, weights=wfit[mask==0]) # rms_ang = utils.calc_fit_rms(xfit, yfit, fit, func, # minx=fmin, maxx=fmax, weights=wfit) rms_pix = rms_ang/disp # Pack up fit spec_vec = np.arange(nspec) wave_soln = utils.func_val(fit,spec_vec/xnspecmin1, func, minx=fmin, maxx=fmax) cen_wave = utils.func_val(fit, float(nspec)/2/xnspecmin1, func, minx=fmin, maxx=fmax) cen_wave_min1 = utils.func_val(fit, (float(nspec)/2 - 1.0)/xnspecmin1, func, minx=fmin, maxx=fmax) cen_disp = cen_wave - cen_wave_min1 final_fit = dict(fitc=fit, function=func, pixel_fit=xfit, wave_fit=yfit, weights=wfit, ions=ions, fmin=fmin, fmax=fmax, xnorm = xnspecmin1, nspec=nspec, cen_wave = cen_wave, cen_disp = cen_disp, xrej=xrej, yrej=yrej, mask=(mask == 0), spec=spec, wave_soln = wave_soln, nrej=sigrej_final, shift=0., tcent=tcent, rms=rms_pix) # If set to True, this will output a file that can then be included in the tests saveit = False if saveit: from linetools import utils as ltu jdict = ltu.jsonify(final_fit) if plot_fil is None: outname = "temp" print("You should have set the plot_fil directory to save wavelength fits... using 'temp' as a filename") else: outname = plot_fil ltu.savejson(outname + '.json', jdict, easy_to_read=True, overwrite=True) print(" Wrote: {:s}".format(outname + '.json')) # QA if plot_fil is not None: autoid.arc_fit_qa(final_fit, plot_fil) # Return return final_fit
def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial'): nrow = xfit.shape[0] ntrace = xfit.shape[1] if x0in is None: x0in = np.arange(float(ntrace)) # Mask out some orders if they are bad if mask is None or mask.size == 0: usetrace = np.arange(ntrace) outmask = np.ones((nrow, ntrace)) else: usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0] outmask = np.ones((nrow, ntrace)) outmask[:,mask] = 0.0 # Do the PCA analysis eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc) modl = func_vander(xfit[:,0], function, npc) eigv = np.dot(modl[:,1:], eigc) med_hidden = np.median(hidden, axis=1) med_highorder = med_hidden.copy() med_highorder[0] = 0 high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0) # y = hidden[0,:] # coeff0 = utils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function) # y = hidden[1,:] # coeff1 = utils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function) coeffstr = [] for i in range(1, npc+1): # if pnpc[i] == 0: # coeffstr.append([-9.99E9]) # continue # coeff0 = utils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1]) if weights is not None: tmask, coeff0 = utils.robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i], weights=weights[usetrace], sigma=2.0, function=function, minx=x0in[0], maxx=x0in[-1]) else: tmask, coeff0 = utils.robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i], sigma=2.0, function=function, minx=x0in[0], maxx=x0in[-1]) coeffstr.append(coeff0) high_order_matrix[:, i-1] = utils.func_val(coeff0, x0in, function, minx=x0in[0], maxx=x0in[-1]) # high_order_matrix[:,1] = utils.func_val(coeff1, x0in, function) high_fit = high_order_matrix.copy() high_order_fit = np.dot(eigv, high_order_matrix.T) sub = (yfit - high_order_fit) * outmask numer = np.sum(sub, axis=0) denom = np.sum(outmask, axis=0) x0 = np.zeros(ntrace, dtype=np.float) fitmask = np.zeros(ntrace, dtype=np.float) #fitmask[mask] = 1 x0fit = np.zeros(ntrace, dtype=np.float) chisqnu = 0.0 chisqold = 0.0 robust = True #svx0 = numer/(denom+(denom == 0).astype(np.int)) if not skipx0: fitmask = (np.abs(denom) > 10).astype(np.int) if robust: good = np.where(fitmask != 0)[0] bad = np.where(fitmask == 0)[0] x0[good] = numer[good]/denom[good] imask = np.zeros(ntrace, dtype=np.float) imask[bad] = 1.0 ttmask, x0res = utils.robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0, function=function, minx=x0in[0], maxx=x0in[-1], initialmask=imask) x0fit = utils.func_val(x0res, x0in, function, minx=x0in[0], maxx=x0in[-1]) good = np.where(ttmask == 0)[0] xstd = 1.0 # This should represent the dispersion in the fit chisq = ((x0[good]-x0fit[good])/xstd)**2.0 chisqnu = np.sum(chisq)/np.sum(ttmask) fitmask = 1.0-ttmask msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu)) else: for i in range(1, 5): good = np.where(fitmask != 0)[0] x0[good] = numer[good]/denom[good] # x0res = utils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function) x0res = utils.func_fit(x0in[good], x0[good], function, pnpc[0], weights=weights, minx=x0in[0], maxx=x0in[-1]) x0fit = utils.func_val(x0res, x0in, function, minx=x0in[0], maxx=x0in[-1]) chisq = (x0[good]-x0fit[good])**2.0 fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int) chisqnu = np.sum(chisq)/np.sum(fitmask) msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu)) if chisqnu == chisqold: break else: chisqold = chisqnu if chisqnu > 2.0: msgs.warn("PCA has very large residuals") elif chisqnu > 0.5: msgs.warn("PCA has fairly large residuals") #bad = np.where(fitmask==0)[0] #x0[bad] = x0fit[bad] else: x0res = 0.0 x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask, 'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr}) return x3fit, outpar
def do_it_all(slit, instr, plot_fil=None, IDtol=1., subpix=None): if instr == '600': wv_dict, template = load_600() fwhm = 4. n_order = 3 n_first = 2 elif instr == '300': wv_dict, template = load_300() fwhm = 8. n_order = 3 n_first = 3 else: pdb.set_trace() nwwv = template['wave'].data nwspec = template['flux'].data # Snippet tspec, mspec, mwv, targ_wv, targ_x, targ_y = target_lines(wv_dict, slit, nwwv, nwspec, fwhm=fwhm, subpix=subpix) # Find new lines all_tcent, all_ecent, cut_tcent, icut, arc_cont_sub = wvutils.arc_lines_from_spec( tspec, fwhm=fwhm, debug=True) # Grab initial guess func = 'legendre' sigrej_first = 3. fmin, fmax = 0., 1. xfit = np.arange(mwv.size) yfit = mwv initial_mask = (mwv > targ_wv.min()) & (mwv < targ_wv.max()) i1 = np.where(initial_mask)[0][0] disp = mwv[i1 + 1] - mwv[i1] # mask, fit = utils.robust_polyfit(xfit, yfit, n_order, function=func, sigma=sigrej_first, initialmask=~initial_mask, minx=fmin, maxx=fmax, verbose=True) # , weights=wfit) rms_ang = utils.calc_fit_rms(xfit[mask == 0], yfit[mask == 0], fit, func, minx=fmin, maxx=fmax) # weights=wfit[mask == 0]) rms_pix = rms_ang / disp print("Initial fit: {}; RMS = {}".format(fit, rms_pix)) # Target grid targ_grid = np.outer(targ_wv, np.ones(all_tcent.size)) # Check the loss loss = loss_func(fit[0:2], all_tcent, targ_grid, targ_y, fit, 2) # Bounds bounds = [[fit[0] - 50., fit[0] + 50], [fit[1] * 0.9, fit[1] * 1.1]] #for ifit in fit[2:]: # bounds.append([np.abs(ifit)*-2, np.abs(ifit)*2]) # Differential evolution #diff_result = fitme(bounds, all_tcent, targ_grid, targ_y) diff_result = fitme(bounds, all_tcent, targ_grid, targ_y, fit.copy()) new_fit = fit.copy() new_fit[:len(bounds)] = diff_result.x loss = loss_func(new_fit[0:len(bounds)], all_tcent, targ_grid, targ_y, fit, len(bounds), verbose=True) print("Refined fit: {}".format(diff_result.x)) # Now cut on saturation all_tcent, all_ecent, cut_tcent, icut, arc_cont_sub = wvutils.arc_lines_from_spec( tspec, fwhm=fwhm, nonlinear_counts=55000) #debug=True) final_guess = utils.func_val(new_fit, all_tcent, func, minx=fmin, maxx=fmax) # Match to line list IDs = [] dwv = mwv[i1 + 1] - mwv[i1] lmask = np.zeros(final_guess.size, dtype=bool) for kk, iwv in enumerate(final_guess): imin = np.argmin(np.abs(iwv - llist['wave'])) if np.abs(iwv - llist['wave'][imin]) < dwv * IDtol: lmask[kk] = True IDs.append(llist['wave'][imin]) print("IDs: {}".format(IDs)) # Final fit final_fit = fitting.iterative_fitting(tspec, all_tcent, np.where(lmask)[0], np.array(IDs), llist, dwv, verbose=True, plot_fil=plot_fil, n_first=n_first) # Return return final_fit
sign = rand.choice([-1, 1.], nout) yfit[indrand] = yreal[indrand] + sign * nsigma * std #yfit[0] = yfit[0]+3*np.std(yreal-yfit) #yfit[10] = yfit[10]+4*np.std(yreal-yfit) #yfit[19] = yfit[19]-3.5*np.std(yreal-yfit) #yfit[49] = yfit[49]-2.8*np.std(yreal-yfit) #yfit[69] = yfit[69]+5*np.std(yreal-yfit) norder = 3 xvec = np.linspace(xfit.min(), xfit.max(), num=200) ## Old robust_olyfit msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma=1.5, function='polynomial') msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit,yfit,norder, \ function = 'polynomial', minv = None, maxv = None, bspline_par = None,\ guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\ lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\ groupbadpix=False, grow=0,sticky=True,use_mad=True) msk_nosticky, poly_coeff_nosticky = utils.robust_polyfit_djs(xfit,yfit,norder, \ function = 'polynomial', minv = None, maxv = None, bspline_par = None,\ guesses = None, maxiter = 20, inmask = None, sigma = None,invvar = None,\ lower = 1.5, upper = 1.5,maxdev=None,maxrej=3,groupdim=None,groupsize=None,\ groupbadpix=False, grow=0,sticky=False,use_mad=True) robust_mask = msk == 0