def test_func_fit(): """ Run the parameter setup script """ x = np.pi * np.linspace(0, 1., 100) y = np.sin(x) # Polynomial pcoeff = utils.func_fit(x, y, 'polynomial', 3) np.testing.assert_allclose(pcoeff, np.array([ -4.74660344e-02, 1.30745471e+00, -4.16175760e-01, 3.08557167e-18 ]), atol=1e-9) # Legendre lcoeff = utils.func_fit(x, y, 'legendre', 3) np.testing.assert_allclose(lcoeff, np.array([ 6.37115652e-01, 6.83317251e-17, -6.84581686e-01, -7.59352737e-17 ]), atol=1e-9) # bspline bcoeff = utils.func_fit(x, y, 'bspline', 2) np.testing.assert_allclose(bcoeff[0], [ 0., 0., 0., 0.31733259, 0.95199777, 1.58666296, 2.22132814, 3.14159265, 3.14159265, 3.14159265 ], atol=1e-5)
def flex_shift(obj_skyspec, arx_skyspec, mxshft=20): """ Calculate shift between object sky spectrum and archive sky spectrum Parameters ---------- obj_skyspec arx_skyspec Returns ------- flex_dict: dict Contains flexure info """ # TODO None of these routines should have dependencies on XSpectrum1d! # Determine the brightest emission lines msgs.warn("If we use Paranal, cut down on wavelength early on") arx_amp, arx_amp_cont, arx_cent, arx_wid, _, arx_w, arx_yprep, nsig = arc.detect_lines( arx_skyspec.flux.value) obj_amp, obj_amp_cont, obj_cent, obj_wid, _, obj_w, obj_yprep, nsig_obj = arc.detect_lines( obj_skyspec.flux.value) # Keep only 5 brightest amplitude lines (xxx_keep is array of # indices within arx_w of the 5 brightest) arx_keep = np.argsort(arx_amp[arx_w])[-5:] obj_keep = np.argsort(obj_amp[obj_w])[-5:] # Calculate wavelength (Angstrom per pixel) arx_disp = np.append( arx_skyspec.wavelength.value[1] - arx_skyspec.wavelength.value[0], arx_skyspec.wavelength.value[1:] - arx_skyspec.wavelength.value[:-1]) #arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size obj_disp = np.append( obj_skyspec.wavelength.value[1] - obj_skyspec.wavelength.value[0], obj_skyspec.wavelength.value[1:] - obj_skyspec.wavelength.value[:-1]) #obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size # Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need # this? can just use sigmas arx_idx = (arx_cent + 0.5).astype( np.int)[arx_w][arx_keep] # The +0.5 is for rounding arx_res = arx_skyspec.wavelength.value[arx_idx]/\ (arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep]) obj_idx = (obj_cent + 0.5).astype( np.int)[obj_w][obj_keep] # The +0.5 is for rounding obj_res = obj_skyspec.wavelength.value[obj_idx]/ \ (obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep]) #obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/( # obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep]) if not np.all(np.isfinite(obj_res)): msgs.warn( 'Failed to measure the resolution of the object spectrum, likely due to error ' 'in the wavelength image.') return None msgs.info("Resolution of Archive={0} and Observation={1}".format( np.median(arx_res), np.median(obj_res))) # Determine sigma of gaussian for smoothing arx_sig2 = np.power(arx_disp[arx_idx] * arx_wid[arx_w][arx_keep], 2) obj_sig2 = np.power(obj_disp[obj_idx] * obj_wid[obj_w][obj_keep], 2) arx_med_sig2 = np.median(arx_sig2) obj_med_sig2 = np.median(obj_sig2) if obj_med_sig2 >= arx_med_sig2: smooth_sig = np.sqrt(obj_med_sig2 - arx_med_sig2) # Ang smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx]) arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix * 2 * np.sqrt(2 * np.log(2))) else: msgs.warn("Prefer archival sky spectrum to have higher resolution") smooth_sig_pix = 0. msgs.warn("New Sky has higher resolution than Archive. Not smoothing") #smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2) #Determine region of wavelength overlap min_wave = max(np.amin(arx_skyspec.wavelength.value), np.amin(obj_skyspec.wavelength.value)) max_wave = min(np.amax(arx_skyspec.wavelength.value), np.amax(obj_skyspec.wavelength.value)) #Smooth higher resolution spectrum by smooth_sig (flux is conserved!) # if np.median(obj_res) >= np.median(arx_res): # msgs.warn("New Sky has higher resolution than Archive. Not smoothing") #obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig) # else: #tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig) # arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2))) #arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig) # Define wavelengths of overlapping spectra keep_idx = np.where((obj_skyspec.wavelength.value >= min_wave) & (obj_skyspec.wavelength.value <= max_wave))[0] #keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave] #Rebin both spectra onto overlapped wavelength range if len(keep_idx) <= 50: msgs.warn("Not enough overlap between sky spectra") return None else: #rebin onto object ALWAYS keep_wave = obj_skyspec.wavelength[keep_idx] arx_skyspec = arx_skyspec.rebin(keep_wave) obj_skyspec = obj_skyspec.rebin(keep_wave) # Trim edges (rebinning is junk there) arx_skyspec.data['flux'][0, :2] = 0. arx_skyspec.data['flux'][0, -2:] = 0. obj_skyspec.data['flux'][0, :2] = 0. obj_skyspec.data['flux'][0, -2:] = 0. # Normalize spectra to unit average sky count norm = np.sum(obj_skyspec.flux.value) / obj_skyspec.npix obj_skyspec.flux = obj_skyspec.flux / norm norm2 = np.sum(arx_skyspec.flux.value) / arx_skyspec.npix arx_skyspec.flux = arx_skyspec.flux / norm2 if (norm < 0.): msgs.warn("Bad normalization of object in flexure algorithm") msgs.warn("Will try the median") norm = np.median(obj_skyspec.flux.value) if (norm < 0.): msgs.warn("Improper sky spectrum for flexure. Is it too faint??") return None if (norm2 < 0.): msgs.warn( 'Bad normalization of archive in flexure. You are probably using wavelengths ' 'well beyond the archive.') return None # Deal with bad pixels msgs.work("Need to mask bad pixels") # Deal with underlying continuum msgs.work("Consider taking median first [5 pixel]") everyn = obj_skyspec.npix // 20 bspline_par = dict(everyn=everyn) mask, ct = utils.robust_polyfit(obj_skyspec.wavelength.value, obj_skyspec.flux.value, 3, function='bspline', sigma=3., bspline_par=bspline_par) obj_sky_cont = utils.func_val(ct, obj_skyspec.wavelength.value, 'bspline') obj_sky_flux = obj_skyspec.flux.value - obj_sky_cont mask, ct_arx = utils.robust_polyfit(arx_skyspec.wavelength.value, arx_skyspec.flux.value, 3, function='bspline', sigma=3., bspline_par=bspline_par) arx_sky_cont = utils.func_val(ct_arx, arx_skyspec.wavelength.value, 'bspline') arx_sky_flux = arx_skyspec.flux.value - arx_sky_cont # Consider sharpness filtering (e.g. LowRedux) msgs.work("Consider taking median first [5 pixel]") #Cross correlation of spectra #corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same") corr = np.correlate(arx_sky_flux, obj_sky_flux, "same") #Create array around the max of the correlation function for fitting for subpixel max # Restrict to pixels within maxshift of zero lag lag0 = corr.size // 2 #mxshft = settings.argflag['reduce']['flexure']['maxshift'] max_corr = np.argmax(corr[lag0 - mxshft:lag0 + mxshft]) + lag0 - mxshft subpix_grid = np.linspace(max_corr - 3., max_corr + 3., 7) #Fit a 2-degree polynomial to peak of correlation function. JFH added this if/else to not crash for bad slits if np.any(np.isfinite(corr[subpix_grid.astype(np.int)])): fit = utils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)], 'polynomial', 2) success = True max_fit = -0.5 * fit[1] / fit[2] else: fit = utils.func_fit(subpix_grid, 0.0 * subpix_grid, 'polynomial', 2) success = False max_fit = 0.0 msgs.warn('Flexure compensation failed for one of your objects') #Calculate and apply shift in wavelength shift = float(max_fit) - lag0 msgs.info("Flexure correction of {:g} pixels".format(shift)) #model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0] flex_dict = dict(polyfit=fit, shift=shift, subpix=subpix_grid, corr=corr[subpix_grid.astype(np.int)], sky_spec=obj_skyspec, arx_spec=arx_skyspec, corr_cen=corr.size / 2, smooth=smooth_sig_pix, success=success) # Return return flex_dict
def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial'): nrow = xfit.shape[0] ntrace = xfit.shape[1] if x0in is None: x0in = np.arange(float(ntrace)) # Mask out some orders if they are bad if mask is None or mask.size == 0: usetrace = np.arange(ntrace) outmask = np.ones((nrow, ntrace)) else: usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0] outmask = np.ones((nrow, ntrace)) outmask[:,mask] = 0.0 # Do the PCA analysis eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc) modl = func_vander(xfit[:,0], function, npc) eigv = np.dot(modl[:,1:], eigc) med_hidden = np.median(hidden, axis=1) med_highorder = med_hidden.copy() med_highorder[0] = 0 high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0) # y = hidden[0,:] # coeff0 = utils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function) # y = hidden[1,:] # coeff1 = utils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function) coeffstr = [] for i in range(1, npc+1): # if pnpc[i] == 0: # coeffstr.append([-9.99E9]) # continue # coeff0 = utils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1]) if weights is not None: tmask, coeff0 = utils.robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i], weights=weights[usetrace], sigma=2.0, function=function, minx=x0in[0], maxx=x0in[-1]) else: tmask, coeff0 = utils.robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i], sigma=2.0, function=function, minx=x0in[0], maxx=x0in[-1]) coeffstr.append(coeff0) high_order_matrix[:, i-1] = utils.func_val(coeff0, x0in, function, minx=x0in[0], maxx=x0in[-1]) # high_order_matrix[:,1] = utils.func_val(coeff1, x0in, function) high_fit = high_order_matrix.copy() high_order_fit = np.dot(eigv, high_order_matrix.T) sub = (yfit - high_order_fit) * outmask numer = np.sum(sub, axis=0) denom = np.sum(outmask, axis=0) x0 = np.zeros(ntrace, dtype=np.float) fitmask = np.zeros(ntrace, dtype=np.float) #fitmask[mask] = 1 x0fit = np.zeros(ntrace, dtype=np.float) chisqnu = 0.0 chisqold = 0.0 robust = True #svx0 = numer/(denom+(denom == 0).astype(np.int)) if not skipx0: fitmask = (np.abs(denom) > 10).astype(np.int) if robust: good = np.where(fitmask != 0)[0] bad = np.where(fitmask == 0)[0] x0[good] = numer[good]/denom[good] imask = np.zeros(ntrace, dtype=np.float) imask[bad] = 1.0 ttmask, x0res = utils.robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0, function=function, minx=x0in[0], maxx=x0in[-1], initialmask=imask) x0fit = utils.func_val(x0res, x0in, function, minx=x0in[0], maxx=x0in[-1]) good = np.where(ttmask == 0)[0] xstd = 1.0 # This should represent the dispersion in the fit chisq = ((x0[good]-x0fit[good])/xstd)**2.0 chisqnu = np.sum(chisq)/np.sum(ttmask) fitmask = 1.0-ttmask msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu)) else: for i in range(1, 5): good = np.where(fitmask != 0)[0] x0[good] = numer[good]/denom[good] # x0res = utils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function) x0res = utils.func_fit(x0in[good], x0[good], function, pnpc[0], weights=weights, minx=x0in[0], maxx=x0in[-1]) x0fit = utils.func_val(x0res, x0in, function, minx=x0in[0], maxx=x0in[-1]) chisq = (x0[good]-x0fit[good])**2.0 fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int) chisqnu = np.sum(chisq)/np.sum(fitmask) msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu)) if chisqnu == chisqold: break else: chisqold = chisqnu if chisqnu > 2.0: msgs.warn("PCA has very large residuals") elif chisqnu > 0.5: msgs.warn("PCA has fairly large residuals") #bad = np.where(fitmask==0)[0] #x0[bad] = x0fit[bad] else: x0res = 0.0 x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask, 'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr}) return x3fit, outpar