def flag_tell(w, f, w_tpl, f_tpl, bp, o): # another way of flagging tellurics # more testing needed x = np.arange(len(f_tpl)) ba = (np.load("lib/CRIRES/blaze_K2166.npy"))[o - 1] bandp = (np.poly1d(ba)(x)) # print(len(x),len(bandp)) # exit() good = np.where( f_tpl / np.nanmean(f_tpl) > 0.8 * bandp / np.nanmean(bandp)) bp[f_tpl / np.nanmean(f_tpl) < 0.8 * bandp / np.nanmean(bandp)] |= 64 # good = np.where((f_tpl/np.nanmean(f_tpl) > 0.9*bandp/np.nanmean(bandp)) & (f/np.nanmean(f) > 0.9*bandp/np.nanmean(bandp))) # bp[(f_tpl/np.nanmean(f_tpl) < 0.9*bandp/np.nanmean(bandp)) & (f/np.nanmean(f) < 0.9*bandp/np.nanmean(bandp))] |= 64 w1 = w * 1. gplot(w, f / np.nanmean(f), 'w l t "data flag",', w_tpl, f_tpl / np.nanmean(f_tpl), 'w l t "tpl flag",', w1, bandp / np.mean(bandp) * 0.85, 'w l t "blaze"') pause() w = w[good] f = f[good] w_tpl = w_tpl[good] f_tpl = f_tpl[good] # return w,f,w_tpl,f_tpl return bp
def qfacmask(lnw, f, sz=26, thres=3000., plot=False): '''mask the parts of the spectrum which have a a Q-factor higher than thres calculate moving Q factor''' lnf = np.log(f) qi = (np.gradient(lnf) / np.gradient(lnw))**2 # ((lnf[1:]-lnf[:-1])/(lnw[1:]-lnw[:-1]))**2 Q = np.zeros(len(qi) + sz) # includes padded zeros Q[sz / 2:-sz / 2] = np.cumsum(qi) Q[-sz / 2:] = Q[-sz / 2 - 1] # pad the last; the first have zero num = np.empty_like(qi) num[sz / 2:-sz / 2] = sz num[:sz / 2] = np.arange(sz / 2, sz) num[-sz / 2:] = np.arange(sz / 2, sz)[::-1] Qb = np.sqrt((Q[sz:] - Q[:-sz]) / num) # normalise idx = Qb > thres # core index reg = True * idx for i, msk in enumerate(idx): if msk: reg[max(i - sz / 2, 1):i + sz / 2] = True if plot: from gplot import * gplot( lnw, f, Qb, " ,'' us 1:3 axis x1y2, " + str(thres) + " axis x1y2 t 'threshold'") ogplot(lnw[reg], f[reg], ",", lnw[idx], f[idx]) pause() return reg
def spaghetti(self): o = np.tile(self.orders, (self.rvc.shape[0], 1)) bjd = np.tile(self.bjd, (self.rvc.shape[1], 1)).T # color bjd with lc variable => same color for same floor(bjd) #gplot(ovis, vis.rvc.ravel(), bjdvis.ravel(), ' lc variable pt 7 ps 0.5,', onir+60, nir.rvc.ravel(), bjdnir.ravel(), ' lc variable pt 7 ps 0.5') gplot.key('tit "%s"' % self.keytitle) gplot.palette('defined') gplot.xlabel('"Order"').ylabel('"RV [m/s]"').cblabel( '"BJD - 2 450 000"') gplot(o.ravel(), self.rvc.ravel(), bjd.ravel(), 'us 1:2:($3-2450000) palette') pause('spaghetti %s' % self.keytitle)
def __init__(self, chi2map, vrange, RV, e_RV, rv, e_rv, orders, keytitle, rchi=None, No=None, name=''): self.args = RV, e_RV, rv, e_rv, orders, keytitle, rchi self.No = No # Number of orders self.chi2map = chi2map self.name = name self.info = name.split('/')[-1] self.vrange = v_lo, v_step = vrange self.vgrid = v_lo + v_step * np.arange(chi2map.shape[1]) # normalised master map self.mCCF = chi2map[orders].sum(axis=0) # master CCF from rescaled order-wise CCF if rchi is not None: #self.rCCF self.mCCF = (chi2map[orders] / rchi[orders][:, np.newaxis]**2).sum(axis=0) self.nmCCF = self.mCCF / self.mCCF.max() self.mlRV, self.e_mlRV, _, _ = SSRstat(self.vgrid, self.mCCF, dk=1, plot=0) self.mlRV, self.e_mlRV = self.mlRV * 1000, self.e_mlRV * 1000 # Derive again the RV from the CCF minima v, e_v, _, self.SSRv = zip(*[ SSRstat(self.vgrid, cmo, dk=1, plot=0) for cmo in chi2map[orders] ]) V, e_V = wsem(np.array(v), e=np.array(e_v), rescale=0) V, e_V = wsem(np.array(v), e=np.array(e_v)) if 0: print V, e_V print RV, e_RV gplot(rv, e_rv, 'us 0:1:2 w e,', v, e_v, 'us 0:1:2 w e lc 3, %s,%s' % (RV, V)) gplot(rv, e_rv, 'us 0:1:2 w e,', v, e_v * rchi[1:][orders], 'us 0:1:2 w e lc 3, %s,%s' % (RV, V)) pause()
def disp(self): orddisp = self.rv - self.RV[:, np.newaxis] #ok &= np.abs(orddisp-d_ordmean) <= 3*ordstd # clip and update mask ordstd, d_ordmean = wstd(orddisp, self.e_rv, axis=0) #ordmean += d_ordmean # update ordmean #orddisp gplot.reset() gplot.tit('"Order dispersion %s";' % self.keytitle) gplot.xlabel('"Order"').ylabel('"RV_{n,o} - RV_n [m/s]"') gplot( orddisp.T, ' matrix us (%s-1):3 t ""' % "".join(['$1==%s?%s:' % io for io in enumerate(self.orders)])) ogplot(self.orders, ordstd, ' w lp lt 3 t "", "" us 1:(-$2) w lp t ""') pause()
def plot_dlwno(self): '''Show RVs over order for each observation.''' (bjd, dLW, e_dLW), dlw, e_rv = self.dlw.T[[0, 1, 2]], self.dlw[:, 3:], self.allerr[:, 5:] n = 0 while 0 <= n < self.N: gplot.key('tit "%s %s %s"' % (n + 1, bjd[n], self.info[n])) if 1: gplot.multiplot('layout 2,1') # bottom panel gplot.xlabel('"BJD - 2 450 000"').ylabel('"dLW [1000(m/s)^2]"') hypertext = ' "" us 1:2:(sprintf("No: %d\\nID: %s\\nBJD: %f\\ndLW: %f+/-%f",$0+1, stringcolumn(4), $1, $2, $3)) w labels hypertext point pt 0 lt 1 t "",' gplot(bjd - 2450000, dLW, e_dLW, self.info, 'us 1:2:3 w e pt 6,' + hypertext, [bjd[n] - 2450000], [dLW[n]], [e_dLW[n]], 'us 1:2:3 w e pt 7 t""', flush=' \n') gplot.xlabel('"order"').ylabel('"dLW [m/s]"') gplot(self.orders, dlw[n, self.orders], self.e_rv[n] * 0, 'us 1:2:3 w e pt 7') ''' , %s lt 3 t "%s +/- %sm/s", "+" us 1:(%s):(%s) w filledcurves lt 3 fs transparent solid 0.50 t ""' %(RVc[n], RVc[n], e_RVc[n], RVc[n]-e_RVc[n], RVc[n]+e_RVc[n])) gplot.xlabel('"wavelength"').ylabel('"RV [m/s]"') gplot.autoscale('noextend') gplot.put('i=2; bind "$" "i = i%2+1; xlab=i==1?\\"order\\":\\"wavelength\\"; set xlabel xlab; set xra [*:*]; print i; if (i==1) {unset log} else {set log x} ; repl"') #gplot(self.orders, lam_o[n,self.orders], self.rvc[n], self.e_rv[n], 'us i:3:4 w e pt 7, %s lt 3 t "%s +/- %sm/s", "+" us 1:(%s):(%s) w filledcurves lt 3 fs transparent solid 0.20 t "", %s*(log(x)-%s)+%s lt 2, "+" us 1:(%s*(log(x)-%s)+%s):(%s*(log(x)-%s)+%s) w filledcurves lt 2 fs transparent solid 0.20 t ""' %(RVc[n], RVc[n], e_RVc[n], RVc[n]-e_RVc[n], RVc[n]+e_RVc[n], crx[n], lnlv[n], RVc[n], crx[n]-e_crx[n], lnlv[n], RVc[n], crx[n]+e_crx[n], lnlv[n], RVc[n])) RVmod = crx[n]*(np.log(lam_o[n,self.orders])-lnlv[n])+RVc[n] RVlow = (crx[n]-e_crx[n])*(np.log(lam_o[n,self.orders])-lnlv[n])+RVc[n] RVupp = (crx[n]+e_crx[n])*(np.log(lam_o[n,self.orders])-lnlv[n])+RVc[n] gplot(self.orders, lam_o[n,self.orders], self.rvc[n], self.e_rv[n], RVmod, RVlow, RVupp,'us i:3:4 w e pt 7, "" us i:5 w l lt 2 t "CRX = %g +/- %g m/s/Np", "" us i:6:7 w filledcurves lt 2 fs transparent solid 0.20 t "", %s lt 3 t "%g +/- %g m/s", "+" us 1:(%s):(%s) w filledcurves lt 3 fs transparent solid 0.20 t ""' %(crx[n], e_crx[n], RVc[n], RVc[n], e_RVc[n], RVc[n]-e_RVc[n], RVc[n]+e_RVc[n])) ''' gplot.unset('multiplot') nn = pause('%i/%i %s %s' % (n + 1, self.N, bjd[n], self.info[n])) try: n += int(nn) except: if nn in ('-', "D", "B"): n -= 1 elif nn in ('^'): n = 0 elif nn in ('$'): n = self.N - 1 else: n += 1
def plotrv(self): '''Show RVs over order for each observation.''' bjd, RVc, e_RVc = self.bjd, self.RVc, self.e_RVc arg = '' if not self.has_d.all(): arg += 'us 1:2:3 w e pt 6 lt 7 t "RV no drift"' if self.has_d.any(): if arg: arg += ', "" ' arg += 'us 1:($2/$4):3 w e pt 7 lt 7 t "RVc"' hypertext = ', "" us 1:2:(sprintf("No: %d\\nID: %s\\nBJD: %f\\nRV: %f +/- %f",$0+1, stringcolumn(5),$1, $2, $3)) w labels hypertext point pt 0 lt 1 t ""' arg += hypertext gplot.key('tit "%s (rms = %.3g m/s)"' % (self.keytitle, self.mlrms[0])) gplot.xlabel('"BJD - 2 450 000"').ylabel('"RV [m/s]"') gplot(bjd - 2450000, RVc, e_RVc, self.has_d, self.info, arg) pause('rv ', self.tag)
def mlc(self): '''Show RVs over order for each observation.''' _, mlRVc, e_mlRVc = self.tmlc.T[0:3] bjd, RVc, e_RVc = self.bjd, self.RVc, self.e_RVc if 1: gplot.key('tit "%s"' % self.keytitle) gplot.xlabel('"BJD - 2 450 000"').ylabel('"RV [m/s]"') gplot( bjd - 2450000, RVc, e_RVc, ' us 1:2:3 w e pt 5 lc 1 t "RVc (rms = %1.3g m/s)' % self.mlrms[0]) ogplot( bjd - 2450000, self.tmlc.T[1:], 'us 1:2:3 w e pt 7 lc 3 t "\\nmlRVc (rms = %1.3g m/s)\\nrms(diff)=%.2f m/s"' % (mlrms(mlRVc, e=e_mlRVc)[0], mlrms(mlRVc - RVc, e_RVc)[0])) pause('rv ', self.tag) # , "" us 1:2:($3/$4) w e pt 7 lt 1
def deconv_RL(f_tpl, w_tpl, bb=4): # Deconvolution of f_tpl using Richardson Lucy algorithm # needed for CRIRES data psf = np.ones((1, bb)) / bb f2 = f_tpl + 0. f_tpl1 = np.reshape(f_tpl, (1, -1)) deconv_RL = restoration.richardson_lucy(f_tpl1 / np.max(f_tpl1) / 2., psf, iterations=30)[0] f_tpl = deconv_RL * np.max(f_tpl1) * 2 rv, ccc = pyasl.crosscorrRV(w_tpl[200:-200], f_tpl[200:-200], w_tpl[200:-200], f2[200:-200], -10., 10., 0.01, skipedge=20) pol = (np.poly1d(np.polyfit(rv, ccc, 15)))(rv) shift = rv[np.argmax(pol)] print('shift RL: ', shift, rv[np.argmax(ccc)]) # there seems to appear a shift in the spectra after the deconvolution # why ??? # correction ??? # more testing needed, or another way of deconvolution plot_RL = 0 if plot_RL: gplot(rv, ccc, 'w l,', rv, pol, 'w l') # gplot(w_tpl, f2, 'w l,', w_tpl /(1+shift/3e5),f_tpl,'w l') pause() # w_tpl /= (1+shift/3e5) ''' bo = 10 f_tpl = f_tpl[bo:-bo] w_tpl = w_tpl[bo:-bo] f_ok = f_ok[bo:-bo] w_ok = w_ok[bo:-bo] x_ok = x_ok[bo:-bo] ''' return f_tpl, w_tpl
def plot_halpha(self): '''Show RVs over order for each observation.''' bjd, halpha, e_halpha = self.halpha[0:3] arg = '' if not self.has_d.all(): arg += 'us 1:2:3 w e pt 6 lt 7 t "dLW no drift"' if self.has_d.any(): if arg: arg += ', "" ' arg += 'us 1:2:($3/$4) w e pt 7 lt 7 t "dLW"' hypertext = ', "" us 1:2:(sprintf("No: %d\\nID: %s\\nBJD: %f\\ndLW: %f +/- %f",$0+1, stringcolumn(5),$1, $2, $3)) w labels hypertext point pt 0 lt 1 t "",' arg += hypertext gplot.key('tit "%s"' % (self.keytitle)) gplot.xlabel('"BJD - 2 450 000"').ylabel('"Halpha index"') gplot(bjd - 2450000, halpha, e_halpha, e_halpha, self.info, arg) pause('halpha ', self.tag)
def example(): # a noisy time serie x = np.arange(10490) / 40.0 + 0.1 yorg = np.sin(x * 1.65) + 0.1 * np.sin(x * 1.65 * 6.32) y = yorg + 0.021 * np.random.randn(x.size) mod, yfit = ucbspl_fit(x, y, w=1 + 0 * y, K=1800, lam=0.11, retfit=True) gplot(x, y, yorg, ' lt 2,"" us 1:3 w l lt 1 t "org"') ogplot(x, yfit, yfit - yorg, 'w l,"" us 1:3') A = np.array([[20, -5, 0, 0], [-5, 15, -5, 0], [0, -5, 15, -5], [0, 0, -5, 10]]) D = np.array([1100, 100, 100, 100]) print "SciPy - Solve Banded" print "A:", A print "D:", D print "Result:", SolveBanded(A, D) pause()
def prerv(self): '''Show RVs over order for each observation.''' prebjd, preRV, pree_RVc = self.tpre.T[0:3] preRVc = preRV - np.nan_to_num(self.trvc[8]) bjd, RVc, e_RVc = self.bjd, self.RVc, self.e_RVc if 1: gplot.key('tit "%s"' % self.keytitle) gplot.xlabel('"BJD - 2 450 000"').ylabel('"RV [m/s]"') #gplot(';set ytics nomirr; set y2tics;', flush='') gplot( prebjd - 2450000, preRVc, pree_RVc, 'us 1:2:3 w e pt 5 lt 3 t "preRVc (rms = %1.3g m/s)"' % mlrms(preRVc, e=pree_RVc)[0]) # ogplot(bjd-2450000, RVc-np.median(RVc)+np.median(drsRVc), e_RVc, ' us 1:2:3 w e pt 7 lt 1 t "RVc-med(RVc)+med(DRS)"') ogplot( bjd - 2450000, RVc - np.median(RVc) + np.median(preRVc), e_RVc, ' us 1:2:3 w e pt 7 lt 1 t "RVc (rms = %1.3g m/s)"' % self.mlrms[0]) pause('rv ', self.tag) # , "" us 1:2:($3/$4) w e pt 7 lt 1
def drsrv(self): '''Show RVs over order for each observation.''' drsbjd, drsRVc, drse_RVc = self.drs.T[0:3] drsRVc = drsRVc - np.nan_to_num(self.trvc[8]) bjd, RVc, e_RVc = self.bjd, self.RVc, self.e_RVc if 1: gplot.key('tit "%s"' % self.keytitle) gplot.xlabel('"BJD - 2 450 000"').ylabel('"RV [m/s]"') #gplot(';set ytics nomirr; set y2tics;', flush='') gplot( drsbjd - 2450000, drsRVc, drse_RVc, 'us 1:2:3 w e pt 7 lt 1 t "DRS (rms = %1.3g m/s)"' % mlrms(drsRVc, e=drse_RVc)[0]) # ogplot(bjd-2450000, RVc-np.median(RVc)+np.median(drsRVc), e_RVc, ' us 1:2:3 w e pt 7 lt 1 t "RVc-med(RVc)+med(DRS)"') ogplot( bjd - 2450000, RVc - np.median(RVc) + np.nanmedian(drsRVc), e_RVc, ' us 1:2:3 w e pt 5 lt 3 t "\\nRVc (rms = %1.3g m/s)\\nrms(diff) = %.2f m/s"' % (self.mlrms[0], mlrms(drsRVc - RVc, e_RVc)[0])) pause('rv ', self.tag) # , "" us 1:2:($3/$4) w e pt 7 lt 1
def plot_dlwo(self): bjd, RVc, e_RVc, RVd, e_RVd, RV, e_RV, BRV, RVsa = self.trvc allrv = self.allrv bjd, RV, e_RV, rv, e_rv = allrv[:, 0], allrv[:, 1], allrv[:, 2], allrv[:, 5:], self.allerr[:, 5:] dlw = self (bjd, dLW, e_dLW), dlw, e_rv = self.dlw.T[[0, 1, 2]], self.dlw[:, 3:], self.allerr[:, 5:] bjdmap = np.tile(bjd[:, np.newaxis], rv.shape[1]) omap = np.tile(np.arange(len(self.dlw.T)), dlw.shape[0]).T # Drift and sa yet no applied to rvo # gplot(bjd, RV, e_RV, 'us 1:2:3 w e pt 7',) # gplot(bjdmap.ravel(), rv.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 palette, "'+obj+'/'+obj+'.rvo.dat'+'" us 1:2:3 w e pt 7 lt 7') rvc = rv - (np.nan_to_num(RVd) + np.nan_to_num(RVsa))[:, np.newaxis] gplot.palette('define (0 "blue", 1 "green", 2 "red")').key( 'tit "%s"' % self.keytitle) gplot.xlabel('"BJD - 2 450 000"').ylabel('"dLW [1000 (m/s)^2]') # not drift corrected #gplot(bjdmap.ravel(), rv.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 palette,', bjd, RV, e_RV, 'us 1:2:3 w e pt 7 lt 7') # drift corrected arg = '' if not self.has_d.all(): arg += 'us 1:2:3 w e pt 6 lt 7 t "dLWo (no drift)"' if self.has_d.any(): if arg: arg += ', "" ' arg += 'us 1:2:($3/$4) w e pt 7 lt 7 t "dLWo"' gplot( bjdmap.ravel() - 2450000, dlw.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 ps 0.5 palette t "RV_o",', bjd - 2450000, dLW, e_dLW, self.has_d, arg) # 'us 1:2:3 w e pt 6 lt 7, "" us 1:2:($3/$4) w e pt 7 lt 7') pause('dLWo ', self.tag)
def plotrvo(self): bjd, RVc, e_RVc, RVd, e_RVd, RV, e_RV, BRV, RVsa = self.trvc allrv = self.allrv bjd, RV, e_RV, rv, e_rv = allrv[:, 0], allrv[:, 1], allrv[:, 2], allrv[:, 5:], self.allerr[:, 5:] bjdmap = np.tile(bjd[:, np.newaxis], rv.shape[1]) omap = np.tile(np.arange(len(rv.T)), rv.shape[0]).T # Drift and sa yet no applied to rvo # gplot(bjd, RV, e_RV, 'us 1:2:3 w e pt 7',) # gplot(bjdmap.ravel(), rv.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 palette, "'+obj+'/'+obj+'.rvo.dat'+'" us 1:2:3 w e pt 7 lt 7') rvc = rv - (np.nan_to_num(RVd) + np.nan_to_num(RVsa))[:, np.newaxis] gplot.palette('define (0 "blue", 1 "green", 2 "red")').key( 'tit "%s"' % self.keytitle) gplot.xlabel('"BJD - 2 450 000"').ylabel('"RV [m/s]') # not drift corrected #gplot(bjdmap.ravel(), rv.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 palette,', bjd, RV, e_RV, 'us 1:2:3 w e pt 7 lt 7') # drift corrected arg = '' hypertext = '' if not self.has_d.all(): arg += 'us 1:2:3 w e pt 6 lt 7 t "RV no drift"' if self.has_d.any(): if arg: arg += ', "" ' arg += 'us 1:2:($3/$4) w e pt 7 lt 7 t "RVc"' if 1: hypertext = ' "" us 1:2:(sprintf("o: %d\\nBJD: %f\\nRV: %f", $4, $1, $2)) w labels hypertext point pt 0 lt 1 t "",' arg += ', "" us 1:2:(sprintf("No: %d\\nID: %s\\nBJD: %f\\nRV: %f+/-%f",$0+1, stringcolumn(5),$1, $2, $3)) w labels hypertext point pt 0 lt 1 t ""' gplot( bjdmap.ravel() - 2450000, rvc.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 ps 0.5 palette t "RV_o",' + hypertext, bjd - 2450000, RVc, e_RVc, self.has_d, self.info, arg) # 'us 1:2:3 w e pt 6 lt 7, "" us 1:2:($3/$4) w e pt 7 lt 7') pause('rvo ', self.tag)
def plot_input(*args): RV, e_RV, rv, e_rv, orders, keytitle, rchi = args gplot.palette('defined (0 "blue", 1 "green", 2 "red")') mm = np.nanmean(chi2map, axis=0) gplot.multiplot("layout 1,2") # Plot order,RV,chi2map and overplot RV and master map # easy to plot as image gplot.xlabel('"order"').ylabel('"RV [km/s]"').cblabel( '"{/Symbol c^2}"') gplot(chi2map, 'matrix us 1:($2*%s+%s):3 w image, ' % (v_step, v_lo), rv, e_rv, 'us 0:1:2 w e pt 6 lc 7, ', mm, mm, 'us (-$1-5):($2*%s+%s):3 matrix w image' % (v_step, v_lo), ',', [-5.5], [RV], [e_RV], 'us 1:2:3 w e pt 7 lc 7') gplot.xlabel('"RV [km/s]"').ylabel('""').cblabel('"order"') gplot(chi2map.T, 'matrix us ($1*%s+%s):3:2 w l palette , ' % (v_step, v_lo), chi2map.min(axis=1), rv, e_rv, 'us 2:1:3 w xe pt 7, ', mm, 'us ($0*%s+%s):1 w l' % (v_step, v_lo), ' lc 7 lw 2,', [mm.min()], [RV], [e_RV], 'us 2:1:3 w xe pt 7 lc 7 ') gplot.unset("multiplot")
def read_spec(self, s, inst, plot=False, **kwargs): #print s, inst sp = inst.read(self, s, **kwargs) return sp if '.tar' in s: s = file_from_tar(s, inst=inst, fib=self.fib, **kwargs) if 'HARP' in inst: sp = read_harps(self, s, inst=inst, **kwargs) elif inst == 'CARM_VIS': sp = read_carm_vis(self, s, **kwargs) elif inst == 'CARM_NIR': sp = read_carm_nir(self, s, **kwargs) elif inst == 'FEROS': sp = read_feros(self, s, **kwargs) elif inst == 'FTS': sp = read_fts(self, s, **kwargs) elif inst == 'HPF': sp = read_hpf(self, s, **kwargs) else: return None #if hasattr(s, 'close'): #s.close() if plot: gplot.xlabel("'wavelength'").ylabel("'intensity'") for o in range(len(sp.w)): gplot(sp.w[o], sp.f[o], " w lp t 'order %i'" % o) pause(o) return sp
def SSRstat(vgrid, SSR, dk=1, plot='maybe'): '''taken from serval.py and modified with SSRv (chi2 minimum)''' v_step = vgrid[1] - vgrid[0] # analyse peak k = SSR[dk:-dk].argmin() + dk # best point (exclude borders) vpeak = vgrid[k - dk:k + dk + 1] SSRpeak = SSR[k - dk:k + dk + 1] - SSR[k] # interpolating parabola (direct solution) through the three pixels in the minimum a = np.array([ 0, (SSR[k + dk] - SSR[k - dk]) / (2 * v_step), (SSR[k + dk] - 2 * SSR[k] + SSR[k - dk]) / (2 * v_step**2) ]) # interpolating parabola for even grid v = (SSR[k + dk] - SSR[k - dk]) / (SSR[k + dk] - 2 * SSR[k] + SSR[k - dk]) * 0.5 * v_step SSR_v = SSR[k] - a[1]**2 / a[ 2] / 4. # a1*v+a2*v**2 # position of parabola minimum = a1*v+a2*v**2 = - a[1]^2/2./a[2]+a[1]^2/4./a[2] = -a[1]^2/a[2]/4. v = vgrid[k] - a[1] / 2. / a[2] # parabola minimum e_v = np.nan if -1 in SSR: print 'opti warning: bad ccf.' elif a[2] <= 0: print 'opti warning: a[2]=%f<=0.' % a[2] elif not vgrid[0] <= v <= vgrid[-1]: print 'opti warning: v not in [va,vb].' else: e_v = 1. / a[2]**0.5 if (plot == 1 and np.isnan(e_v)) or plot == 2: gplot.yrange('[*:%f]' % SSR.max()) gplot( vgrid, SSR - SSR[k], " w lp, v1=" + str(vgrid[k]) + ", %f+(x-v1)*%f+(x-v1)**2*%f," % tuple(a), [v, v], [0, SSR[1]], 'w l t "%f km/s"' % v) ogplot(vpeak, SSRpeak, ' lt 1 pt 6; set yrange [*:*]') pause(v) return v, e_v, a, SSR_v
def plot(self): RV, e_RV, rv, e_rv, orders, keytitle, rchi = self.args chi2map = self.chi2map[orders] # Plot all maps normalised gplot.key('Left center rev top title "%s"' % keytitle) gplot.palette('defined (0 "blue", 1 "green", 2 "red")') gplot.xlabel( '"v [km/s]"; set ylabel "chi^2 / max(chi^2)"; set cblabel "order"') #gplot(chi2map, ' matrix us ($1*%s+%s):3:2 w l palette'%self.vrange) gplot((chi2map / chi2map.max(axis=1)[:, np.newaxis]).T, ' matrix us (%s+$1*%s):3:2 w l palette t "%s"' % (self.vrange + (self.info, )), flush='') #ogplot(self.rCCF/self.rCCF.max(), ' us (%s+$0*%s):1 w l lt 4 lw 2 t "total"'%self.vrange, flush='') # the master CCF ogplot(self.nmCCF, ' us (%s+$0*%s):1 w l lt 7 lw 2 t "lnL"' % self.vrange, flush='') # the parabola for the master ogplot( '((x-%s)**2/%s**2+%s)/%s lc 7 dt 2 t "lnL (parabola %.5g +/- %.5g m/s)"' % (self.mlRV / 1000, self.e_mlRV / 1000., self.mCCF.min(), self.mCCF.max(), self.mlRV, self.e_mlRV), flush='') ogplot("((x-%s)**2/%s**2+%s)/%s lc 9 dt 3 t 'RV,e\_RV (parabola)'" % (RV, e_RV, self.mCCF.min(), self.mCCF.max()), flush='') ogplot([RV] * 2, [0, 1], [e_RV] * 2, 'us 1:2:3 w xerrorlines pt 2 lt 9 t "%.5g +/- %.5g m/s"' % (RV * 1000, e_RV * 1000), flush='') ogplot(rv, (chi2map / chi2map.max(axis=1)[:, np.newaxis]).min(axis=1), 'us 1:2 lt 7 pt 1 t "chi^2_{min}"')
#! /usr/bin/ipython import numpy as np import ke from gplot import * from pause import * ke._Ecs(0.0001, 1., n=15) ke.E(0.0001, 1., n=15, typ='atr') x=np.arange(0,10,0.001); y=1*x; ke._ke_newton.v_cos(x,y, x.size); gplot(x,y-np.cos(x)) x=np.arange(0,10,0.001); y=1*x; ke._ke_newton.v_sin(x,y, x.size); gplot(x,y-np.sin(x)) e = 1.0 E = np.arange(-10,6,0.001) E = np.arange(0,3,0.001); M = ke.M(E, e) if 0: gplot(M, E, ke._E_newton(M, e), ke.E(M, e, n=55, typ='atr'), ' us 1:3, "" us 1:4') gplot('"" us 1:($3-$2), "" us 1:($4-$2)') gplot(M, np.log10(np.abs(ke._E(M, e, n=55)-E))) pause() M = np.arange(0, np.pi, np.pi/1000); timeit ke._E(M, e, n=29) #1000 loops, best of 3: 1.51 ms per loop timeit ke.E(M, e, n=29) #1000 loops, best of 3: 1.51 ms per loop
if not case: import doctest doctest.testmod() elif case == 1: # chi2map for a linear fit and comparison with curve_fit from scipy.optimize import curve_fit # model trend = lambda x, a, b: a + b * x # = np.polynomial.polyval(x, a) a0, a1 = 2, 2 # data x = np.arange(20) e = 2 + np.random.rand(20) y = trend(x, a0, a1) + 2 * np.random.normal(0, e) # scale e by 2 gplot(x, y, e, "w e pt 7") # curve_fit a_cf, cov_cf = curve_fit(trend, x, y, [0.0, 0.0], e) print(a_cf) cov_cf # paraboloid # parameter and chi2 samples P = [[1, 0], [0, 0], [-1, 0], [0, 1], [0, -1], [1, -1]] # chi2map z = [np.dot((y - trend(x, *p))**2, 1 / e**2) for p in P] pause() cov = covmat_fit(P, z, N=len(x)) #cov = covmat_fit(zip(*P), z)
def plotrvno(self): '''Show RVs over order for each observation.''' bjd, RVc, e_RVc = self.bjd, self.RVc, self.e_RVc crx, e_crx = self.tcrx[1:3] lam_o = np.exp(self.tcrx[6:].T) lnlv = np.log(self.tcrx[5]) print "use '$' to toggle between orders and wavelength" n = 0 while 0 <= n < self.N: gplot.key('tit "%s %s %s"' % (n + 1, bjd[n], self.info[n])) if 1: gplot.multiplot('layout 2,1') # top panel gplot.xlabel('"BJD - 2 450 000"').ylabel('"RV [m/s]"') gplot.unset('log') arg = '' if not self.has_d.all(): arg += 'us 1:2:3 w e pt 6 lt 7 t "RV no drift",' if self.has_d.any(): if arg: arg += ' "" ' arg += 'us 1:($2/$4):3 w e pt 7 lt 7 t "RVc",' hypertext = '"" us 1:2:(sprintf("No: %d\\nID: %s\\nBJD: %f\\nRV: %f +/- %f",$0+1, stringcolumn(5),$1, $2, $3)) w labels hypertext point pt 0 lt 1 t "",' gplot(bjd - 2450000, RVc, e_RVc, self.has_d, self.info, arg, hypertext, bjd[n] - 2450000, RVc[n], e_RVc[n], 'us 1:2:3 w e pt 7 t ""', flush=' \n') # bottom panel gplot.xlabel('"order"').ylabel('"RV [m/s]"') gplot.xlabel('"wavelength"').ylabel('"RV [m/s]"') gplot.autoscale('noextend') gplot.put( 'i=2; bind "$" "i = i%2+1; xlab=i==1?\\"order\\":\\"wavelength\\"; set xlabel xlab; set xra [*:*]; print i; if (i==1) {unset log} else {set log x} ; repl"' ) RVmod = crx[n] * (np.log(lam_o[n, self.orders]) - lnlv[n]) + RVc[n] RVlow = (crx[n] - e_crx[n]) * (np.log(lam_o[n, self.orders]) - lnlv[n]) + RVc[n] RVupp = (crx[n] + e_crx[n]) * (np.log(lam_o[n, self.orders]) - lnlv[n]) + RVc[n] hypertext = ', "" us i:3:(sprintf("No: %d\\nID: %s\\nBJD: %f\\nRV: %f +/- %f\\n' % ( n + 1, self.info[n], bjd[n], RVc[n], e_RVc[n] ) + 'o: %d\\nrv[o]: %f +/- %f", $1, $3, $4)) w labels hypertext point pt 0 lt 1 t ""' gplot( self.orders, lam_o[n, self.orders], self.rvc[n], self.e_rv[n], RVmod, RVlow, RVupp, 'us i:3:4 w e pt 7' + hypertext + ', "" us i:5 w l lt 2 t "CRX = %g +/- %g m/s/Np", "" us i:6:7 w filledcurves lt 2 fs transparent solid 0.20 t "", %s lt 3 t "%g +/- %g m/s", "+" us 1:(%s):(%s) w filledcurves lt 3 fs transparent solid 0.20 t ""' % (crx[n], e_crx[n], RVc[n], RVc[n], e_RVc[n], RVc[n] - e_RVc[n], RVc[n] + e_RVc[n])) gplot.unset('multiplot') nn = pause('%i/%i %s %s' % (n + 1, self.N, bjd[n], self.info[n])) try: n += int(nn) except: if nn in ('-', "D", "B"): n -= 1 elif nn in ('^'): n = 0 elif nn in ('$'): n = self.N - 1 else: n += 1
def plot_fit(self): RV, e_RV, rv, e_rv, orders, keytitle, rchi = self.args x, xc, ind = self.xarg vv = self.vv lnL0 = np.array([cs_o(vv) for cs_o in self.cs]) gplot.palette('defined (0 "blue", 1 "green", 2 "red")') gplot.multiplot("layout 1,2") gplot.xlabel( '"v [m/s]"; set ylabel "chi^2 / max(chi^2)"; set cblabel "order"') nlnL0 = lnL0.T / lnL0.max(axis=1) mm = nlnL0.sum(axis=1) # left panel: the lnL parabolas in RV[km/s] gplot( nlnL0, ' matrix us ($1*%s+%s):3:2 w l palette,' % (vv[1] - vv[0], vv[0]), nlnL0.min(axis=0), rv, e_rv, 'us 2:1:3 w xe pt 6 lc 7, ', mm / mm.max(), 'us ($0*%s+%s):1 w l' % (vv[1] - vv[0], vv[0]), ' lc 7 lw 2,', #[(mm/mm.max()).min()], [RV[i]/1000], [e_RVc[i]/1000], 'us 2:1:3 w xe pt 7 lc 7 ' ) # right panel: the map ln(lam) vs RV [m/s] gplot.xlabel( '"ln(wavelength)"; set ylabel "velocity v [m/s]"; set cblabel "chi^2 / max(chi^2)"' ) gplot.yrange('[%s:%s]' % ((rv * 1000 - e_rv * 1000).min(), 1000 * (rv + e_rv).max())) # lnL as maps ,width shoul be addjust # often some maps are not drawn gplot( "a='0 " + " ".join(map(str, self.ll)) + "'" + ', for [i=2:%s+1]' % len(lnL0), vv * 1000, lnL0 / lnL0.max(axis=1)[:, np.newaxis], ' us (word(a,i)+0):1:(0.002):(%s):i with boxxy palette fs solid t "",' % ((vv[1] - vv[0]) * 1000 / 2), self.ll, rv * 1000, e_rv * 1000, " w e pt 7 lc 7 t 'RV_o',", #x,(x-xc)* self.crx+pval[0], 'w l lc 1,', x, ((x - xc) * self.crx / 1000.) * 1000 + self.zz, 'w l lc 3 t "%s"' % self.crx) gplot.unset("multiplot") gplot.yrange('[*:*]') #pause() if 0: gplot.xlabel( '"v [m/s]"; set ylabel "chi^2/max(chi^2)"; set cblabel "order"' ) #gplot(chi2map, ' matrix us ($1*%s+%s):3:2 w l palette'%(v_step, v_lo)) gplot(chi2map.T / chi2map.max(axis=1), ' matrix us ($1*%s+%s):3:2 w l palette' % (v_step, v_lo)) gplot(lnLvbint, 'matrix us 1:3:2 w l palette,', uu.sum(axis=0), "us ($0-30):1") #gplot(chi2map, 'matrix us 1:2:3 w l palette') ij = 19 gplot(lnLvb[ij], 'matrix us 1:2:3 w image,', np.argmin(lnLvb[ij], axis=1)) gplot.splot(np.log(np.array(lnLvbint).T - lnLvbint.min()), 'matrix us 1:2:3 w l palette')
def mlcrx(self, x, xc, ind): '''Maximum likelihood''' self.xarg = x, xc, ind self.crx, self.e_crx = np.nan, np.nan chi2map = self.chi2map #- self.chi2map.min(axis=1)[:,np.newaxis] v_lo, v_step = self.vrange vgrid = v_lo + v_step * np.arange(chi2map.shape[1]) if 0: # test data oo = ~np.isnan(chi2map[:, 0]) rv[i] = (x - 8.6) * 0.6 * 1000 + np.sin(x * 100) * 100 + 50 e_rv[i] = rv[i] * 0 + 10 rv[i][~oo] = np.nan gplot(x, rv[i], e_rv[i], " w e") #pval, cov = curve_fit(func, x[ind]-xc, rv[i][ind], np.array([0.0, 0.0]), e_rv[i][ind]) #crx[i] = pval[1] #chi2map = (1./2*(vgrid[:,np.newaxis]-rv[i]/1000)**2/(e_rv[i]/1000)**2).T def plot_input(*args): RV, e_RV, rv, e_rv, orders, keytitle, rchi = args gplot.palette('defined (0 "blue", 1 "green", 2 "red")') mm = np.nanmean(chi2map, axis=0) gplot.multiplot("layout 1,2") # Plot order,RV,chi2map and overplot RV and master map # easy to plot as image gplot.xlabel('"order"').ylabel('"RV [km/s]"').cblabel( '"{/Symbol c^2}"') gplot(chi2map, 'matrix us 1:($2*%s+%s):3 w image, ' % (v_step, v_lo), rv, e_rv, 'us 0:1:2 w e pt 6 lc 7, ', mm, mm, 'us (-$1-5):($2*%s+%s):3 matrix w image' % (v_step, v_lo), ',', [-5.5], [RV], [e_RV], 'us 1:2:3 w e pt 7 lc 7') gplot.xlabel('"RV [km/s]"').ylabel('""').cblabel('"order"') gplot(chi2map.T, 'matrix us ($1*%s+%s):3:2 w l palette , ' % (v_step, v_lo), chi2map.min(axis=1), rv, e_rv, 'us 2:1:3 w xe pt 7, ', mm, 'us ($0*%s+%s):1 w l' % (v_step, v_lo), ' lc 7 lw 2,', [mm.min()], [RV], [e_RV], 'us 2:1:3 w xe pt 7 lc 7 ') gplot.unset("multiplot") if 0: plot_input(*self.args) pause() # Spline interpolate each chi2map self.ll = ll = x[ind] rchi = self.args[-1] self.cs = cs = [ spl.ucbspl_fit(vgrid, chi_o, K=len(chi_o)) for chi_o in (chi2map[ind] - np.array(self.SSRv)[:, np.newaxis]) / rchi[ind][:, np.newaxis]**2 ] self.vv = vv = np.arange(vgrid[30], vgrid[-30], 0.01) if 0: # discrete brute force minisation lnLvb = [] bb = np.arange(-0.8, 0.8, 0.005) for bbi in bb: lnLv = [] for x_o, cs_o in zip(ll, cs): lnLv += [cs_o(vv + (x_o - xc) * bbi)] lnLvb += [lnLv] lnLvb = np.array(lnLvb) lnLvbint = lnLvb.sum(axis=1).T ii = np.unravel_index(lnLvbint.argmin(), lnLvbint.shape) crx_i = bb[ii[1]] * 1000 # km to m/s zz = vv[ii[0]] * 1000 if 1: # paraboloid #imin,jmin = np.unravel_index(np.argmin(lnLvbint), lnLvbint.shape) #sl = np.s_[imin-10:imin+11, jmin-10:jmin+11] #X = np.tile(vv, bb.shape+(1,)).T[sl].ravel() #Y = np.tile(bb, vv.shape+(1,))[sl].ravel() #z = lnLvbint[sl].ravel() # ln L = chi^2/2 #cov = paraboloid.covmat_fit([X,Y], z, N=self.No[ind].sum()) #cov =bbi paraboloid.covmat_fit([X,Y], z) try: # setup a grid around the start guess mlRv and mlCRX=0 par = map( np.ravel, np.meshgrid( np.arange(self.mlRV / 1000. - 1, self.mlRV / 1000. + 1, 0.2), np.arange(-500, 500, 50) / 1000.)) # par = map(np.ravel, np.meshgrid(np.arange(self.mlRV/1000.-0.1, self.mlRV/1000.+.1,0.02), np.arange(-500,500,50)/1000.)) z = [ np.sum([ cs_o(vvi + (x_o - xc) * bbi) for x_o, cs_o in zip(ll, cs) ]) for vvi, bbi in zip(*par) ] # gplot.splot(par, z, ' palette') #pause() cov = paraboloid.covmat_fit(zip(*par), z, N=len(cs)) zmod = cov.p(zip(*par)) #pause("\n", cov.Va*1000**2, cov.Xc*1000) #cov = paraboloid.covmat_fit(par, z) #cov = paraboloid.covmat_fit(par, z, N=self.No[ind].sum()) v, b = cov.Xc e_v, e_b = cov.e_a #gplot.splot(par, z, ' palette,', v,b, cov.min); pause("v", v, "b", b) #gplot.splot(par, z, zmod, ' palette, "" us 1:2:4 palette pt 6 , "" us 1:2:(($3-$4)*10),', v,b, cov.min); pause("v", v, "b", b) if -1 < v < 1 and -0.5 < b < 0.5 and np.isfinite( cov.e_a).all(): # refit closer to minimum par = map( np.ravel, np.meshgrid( np.arange(v - 3 * e_v, v + 3 * e_v, e_v / 2), np.arange(b - 3 * e_b, b + 3 * e_b, e_b / 2))) z = [ np.sum([ cs_o(vvi + (x_o - xc) * bbi) for x_o, cs_o in zip(ll, cs) ]) for vvi, bbi in zip(*par) ] zmod = cov.p(zip(*par)) cov = paraboloid.covmat_fit(zip(*par), z, N=len(cs)) self.crx = cov.Xc[1] * 1000 self.e_crx = cov.e_a[1] * 1000 except: print "warning: mlCRX failed." if 0: # scipy optimisation def lnL(a): return np.sum([ cs_o(a[0] + a[1] * (x_o - xc)) for x_o, cs_o in zip(ll, cs) ]) from scipy.optimize import fmin pp = fmin(lnL, [0, 0], disp=False) #print pp * 1000 self.zz = pp[0] * 1000 self.crx = pp[1] * 1000 self.e_crx = None # e_crx[i] #SSRstat(self.vgrid, self.mCCF, dk=1, plot=0)[0:2] #ii = np.argmin(self.mCCF); ss = np.s_[ii-3:ii+3] #uu = paraboloid.covmat_fit([self.vgrid[ss]], self.mCCF[ss]) #uu.Xc, uu.e_a return self.crx, self.e_crx
def ls(self, suf='_A_mod.fits'): '''Show last square fit form fits file.''' #prebjd, preRV, pree_RVc = self.tpre.T[0:3] #for n in range(self.N): gplot.key("Left left rev bottom title '%s'" % self.keytitle) ''' if def_wlog: w2 = np.exp(w2) res = np.nan * f2 res[pmin:pmax] = (f2[pmin:pmax]-f2mod[pmin:pmax]) / e2[pmin:pmax] # normalised residuals b = str(stat['std']) #gplot_set('set key left Left rev samplen 2 tit "%s (o=%s, v=%.2fm/s)"'%(obj,o,rvo)) #gplot_set('set ytics nomirr; set y2tics; set y2range [-5*%f:35*%f]; set bar 0.5'%(rchi,rchi)) #gplot_set('i=1; bind "$" "i = i%2+1; xlab=i==1?\\"pixel\\":\\"wavelength\\"; set xlabel xlab; set xra [*:*]; print i; repl"') #gplot('[][][][-5:35]',x2, w2, f2, e2.clip(0.,f2.max()), 'us (column(i)):3:4 w errorli t "'+sp.timeid+' all"', flush='') ogplot(x2,w2, f2, ((b2==0)|(b2==flag.clip))*0.5, 1+4*(b2==flag.clip), 'us (column(i)):3:4:5 w p pt 7 lc var ps var t "'+sp.timeid+' telluric free"', flush='') ogplot(x2,w2, f2mod,(b2==0)*0.5, 'us (column(i)):3:4 w lp lt 3 pt 7 ps var t "Fmod"', flush='') ogplot(x2,w2, res, b2, "us (column(i)):3:4 w lp pt 7 ps 0.5 lc var axis x1y2 t 'residuals'", flush='') # legend with translation of bpmap, plot dummy using NaN function ogplot(", ".join(["NaN w p pt 7 ps 0.5 lc "+str(f) +" t '"+str(f)+" "+",".join(flag.translate(f))+"'" for f in np.unique(b2)]), flush='') ogplot("0 axis x1y2 lt 3 t'',"+b+" axis x1y2 lt 1,-"+b+" axis x1y2 lt 1 t ''", flush='') ogplot(x2,w2, ((b2&flag.atm)!=flag.atm)*40-5, 'us (column(i)):3 w filledcurve x2 fs transparent solid 0.5 noborder lc 9 axis x1y2 t "tellurics"', flush='') ogplot(x2,w2, ((b2&flag.sky)!=flag.sky)*40-5, 'us (column(i)):3 w filledcurve x2 fs transparent solid 0.5 noborder lc 6 axis x1y2 t "sky"') pause('large RV ' if abs(rvo/1000-rvguess+vref)>rvwarn else 'look ', o, ' rv = %.3f +/- %.3f m/s rchi = %.2f' %(rvo, e_rv[i,o],rchi2[i,o])) # end loop over orders ''' n = 0 o = 40 while True: name = self.dir + '/res/' + self.info[n][:-5] + suf hdu = pyfits.open(name) resmap = hdu['res'].data fmod = hdu['fmod'].data e_f = hdu['err'].data wave = hdu['wave'].data flux = fmod + resmap bpmap = 0 * wave rvo = 1 if 1: gplot_set('set xlabel "wavelength [A]"; set ylabel "flux"') gplot_set( 'set key default left Left rev samplen 2 tit "%s (o=%s, v=%.2fm/s)"' % (self.tag, o, rvo)) gplot_set( 'set ytics nomirr; set y2tics; set y2range [-5*%f:35*%f]; set bar 0.5' % (1, 1)) gplot_set( 'i=1; bind "$" "i = i%2+1; xlab=i==1?\\"pixel\\":\\"wavelength\\"; set xlabel xlab; set xra [*:*]; print i; repl"' ) x2 = wave[o] w2 = wave[o] f2 = flux[o] f2mod = fmod[o] e2 = e_f[o] b2 = bpmap[o] res = resmap[o] / f2 gplot('[][][][-5:35]', x2, w2, f2, e2.clip(0., f2.max()), 'us (column(i)):3:4 w errorli t "' + self.info[n] + ' all"', flush='') #ogplot(x2,w2, f2, ((b2==0)|(b2==flag.clip))*0.5, 1+4*(b2==flag.clip), 'us (column(i)):3:4:5 w p pt 7 lc var ps var t "'+sp.timeid+' telluric free"', flush='') ogplot(x2, w2, f2mod, (b2 == 0) * 0.5, 'us (column(i)):3:4 w lp lt 3 pt 7 ps var t "Fmod"', flush='') ogplot( x2, w2, res / e2, b2, "us (column(i)):3:4 w lp pt 7 ps 0.5 lc var axis x1y2 t 'residuals'" ) #pause() # legend with translation of bpmap, plot dummy using NaN function #ogplot(", ".join(["NaN w p pt 7 ps 0.5 lc "+str(f) +" t '"+str(f)+" "+",".join(flag.translate(f))+"'" for f in np.unique(b2)]), flush='') no = pause('%i/%i %s %s %s' % (n + 1, self.N, o, self.bjd[n], self.info[n])) try: o = int(no) except: if no in ('-', "D", "B"): n -= 1 elif no in ('e'): break elif no in ('^'): n = 0 elif no in ('$'): n = self.N - 1 elif no in (')'): o += 1 elif no in ('('): o -= 1 else: n += 1 n = np.clip(n, 0, self.N - 1) o = np.clip(o, 0, resmap.shape[0] - 1)
def postrv(self, postiter=1, fibsuf='', oidx=None, safemode=False, pdf=False, plotrvo=True): """ """ #pause() rv, e_rv = self.allrv[:, 5 + self.orders], self.allerr[:, 5 + self.orders] RV, e_RV = nanwsem(rv, e=e_rv, axis=1) allrv = self.allrv allerr = self.allerr bjd, RVc, e_RVc, RVd, e_RVd, RV, e_RV, BRV, RVsa = self.trvc bjd, RV, e_RV, rv, e_rv = allrv[:, 0], allrv[:, 1], allrv[:, 2], allrv[:, 5:], allerr[:, 5:] bjdmap = np.tile(bjd[:, np.newaxis], rv.shape[1]) omap = np.tile(np.arange(len(rv.T)), rv.shape[0]).T #gplot(bjd, RV, e_RV, 'us 1:2:3 w e pt 7',) #gplot(bjdmap.ravel(), rv.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 palette, "'+obj+'/'+obj+'.rvo.dat'+'" us 1:2:3 w e pt 7 lt 7') rvc = rv - (np.nan_to_num(RVd) + np.nan_to_num(RVsa))[:, np.newaxis] gplot.palette('define (0 "blue", 1 "green", 2 "red")').key( 'tit "%s"' % self.keytitle) gplot.xlabel('"BJD - 2 450 000"').ylabel('"RV [m/s]') # not drift corrected gplot(bjdmap.ravel() - 2450000, rv.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 palette,', bjd - 2450000, RV, e_RV, 'us 1:2:3 w e pt 7 lt 7') # drift corrected #gplot(bjdmap.ravel(), rvc.ravel(), e_rv.ravel(), omap.ravel(), 'us 1:2:4 w p pt 7 ps 0.5 palette,', bjd, RVc, e_RVc, 'us 1:2:3 w e pt 7 lt 7') pause(self.tag) # minimise (rv_io - rv_i - m_o)^2 / (e_rv_io^2 + e_rv_i^2 + e_rv_o^2) e_n = 0 e_o = 0 m_o = 0 for e_n in [0, 0.1, 1, 2, 5, 10, 20, 30, 50, 100, 200]: e_r = np.sqrt(e_rv**2 + e_n**2 + e_o**2) RV, e_RV = nanwsem(rv - m_o, e=e_r, axis=1) RV = RV[:, np.newaxis] #stop() m_o = (wmean(rv - RV, 1 / e_r**2, axis=0) * 0)[np.newaxis, :] # order offset r = rv - RV - m_o # residuals stop() e_n = np.sqrt((np.nansum( (r**2 - e_rv**2 - e_o**2) / e_r**2, axis=1) / np.nansum(1 / e_r**2, axis=1)).clip( min=0))[:, np.newaxis] # keepdims not in v1.6.1 e_o = np.sqrt( (np.nansum((r**2 - e_rv**2 - e_n**2) / e_r**2, axis=0) / np.nansum(1 / e_r**2, axis=0)).clip(min=0))[np.newaxis, :] L = -0.5 * np.nansum(r**2 / e_r**2) - 0.5 * np.nansum( np.log(2. * np.pi * e_r**2)) print L, e_n.ravel() gplot.unset(' multiplot') gplot.datafile('missing "nan"') gplot.multiplot('layout 1,2;') gplot.xlabel('"order"').ylabel('"rvo - RV [m/s]"') gplot(rv - RV, ' matrix,', m_o.ravel(), 'us 0:1') gplot.xlabel('"obs No."').ylabel('"rvo - RV [m/s]"') gplot((rv - RV).T, ' matrix') gplot.unset('multiplot') pause() # ind, = where(np.isfinite(e_rv[i])) # do not use the failed and last order RV, e_RV = nanwsem(rv, e=e_rv, axis=1) RVc = RV - np.nan_to_num(RVd) - np.nan_to_num(RVsa) e_RVc = np.sqrt(e_RV**2 + np.nan_to_num(e_RVd)**2) pause()
def ucbspl_fit(x, y=None, w=None, K=10, xmin=None, xmax=None, lam=0., pord=2, mu=None, e_mu=None, nat=True, retfit=False, var=False, e_yk=False, cov=False, plot=False, c=True): ''' Fit a uniform cubic spline to data. Parameters ---------- x : array_like Data position. y : array_like Data values. w : array_like Data weights (1/e_y^2). K : integer Number of uniform knots. xmin : float Position of the first knot (default minimum of x). xmax : float Position of the last knot (default maximum of x). lam : float Penality, smoothing value (default 0, i.e. spline regression). pord : int Penality order (default 2, i.e. second derivative/curvature). mu : array_like Analog to mean value of a Gaussian process. e_mu : array_like Deviation of mu. nat : boolean Natural spline. (Unclear how to treat this for pord!=2.) e_yk : boolean Error estimates for the knot values. In unweighted case, it estimates the np.sqrt((N-1)/(K-1)). retfit: boolean If true the returned tuple contains the predicted values y(x) at the data position x. var : boolean Variance of the model prediction. cov : boolean If false band matrices are used for efficient solution of equation system with band solver. If true covariances are estimated using matrix inversion. Returns ------- ucbspl The spline model. ucbspl, yfit, varmod, v_cspl The content of tuple depends on keywords retfit, var, cov. Examples -------- >>> x = np.r_[0:100., 400:500, 600:700] >>> xx = np.r_[0:x[-1]:0.1] >>> y = (np.sin(0.1*x)+1)*1000 Comparison of simple and proper variance estimation >>> spl, v_spl, c = ucbspl_fit(x, y, w=1./1000**2, K=50, lam=0.00000001, pord=1, nat=0, var=True, cov=True) >>> gplot(xx, spl(xx), np.sqrt(v_spl(xx)), np.sqrt(c(xx)), ' us 1:($2-$3):($2+$3) with filledcurves fill transparent solid 0.5, "" us 1:($2-$4):($2+$4) with filledcurves fill transparent solid 0.5 lt 3, "" w l lt 7,', spl.xk, spl(), ' lt 7 ps 0.5 pt 7,', x, y, ' pt 7 lt 1') >>> spl, v_spl = ucbspl_fit(x, y, w=0.00001, K=50, mu=800, e_mu=800, nat=0, var=True) >>> gplot(x, y, ' pt 7') >>> ogplot(xx, spl(xx), np.sqrt(v_spl(xx)), ' w l, "" us 1:($2-$3):($2+$3) with filledcurves fill transparent solid 0.5') Comparison of penality order >>> gplot(x, y, ' pt 7') >>> for e_mu in [5, 1, 0.1]: ... spl = ucbspl_fit(x, y, K=50, mu=500, e_mu=e_mu) ... ogplot(xx, spl(xx), 'w l t "mu=0.5 +/- %s"'%e_mu) >>> for lam in [0.1, 1, 10, 100, 10000]: ... spl = ucbspl_fit(x, y, K=50, lam=lam, pord=1, nat=0) ... ogplot(xx,spl(xx), 'w l t "lam=%s",' % lam, spl.xk, spl(), ' ps 0.3 pt 6 lt 3 t ""') >>> for lam in [0.0000001, 0.01, 0.1, 1, 10, 100000]: ... spl = ucbspl_fit(x, y, K=50, lam=lam, pord=2, nat=0) ... ogplot(xx,spl(xx), 'w l t "lam=%s",' % lam, spl.xk, spl(), ' ps 0.3 pt 6 lt 3 t ""') >>> for lam in [1/0.1**2, 1, 1/5.**2]: ... spl = ucbspl_fit(x, y, K=50, lam=lam, pord=0, nat=0) ... ogplot(xx, spl(xx), 'w l t "lam=%s"' % lam) ''' if y is None: # uniform data y = x x = np.linspace(xmin or 0, y.size - 1 if xmax is None else xmax, num=y.size) if w is None: w = 1 wy = y else: wy = w * y wy = np.ascontiguousarray(wy) if xmin is None: xmin = x.min() if xmax is None: xmax = x.max() G, kk = [cbspline_Bk, _cbspline_Bk][c](x, K, xmin, xmax) if nat: Gorg = 1 * G bk2bknat(G, kk, K) nk = K + 2 BTy = np.zeros(nk) BTBbnd = np.zeros((4, nk)) if 0: # the slow way without band storage print 'bspline2' B = bspline2((x - x.min()) / (x.max() - x.min()) * K, K, D=3) n = B.shape[1] D = np.diff(np.eye(n), n=pord).T print 'BTy' BTy = np.dot(B.T, y) print 'BTB' BTB = np.dot(B.T, B) + lam * np.dot(D.T, D) #BTB = B.T.dot(B)+lam*D.T.dot(D) #a = np.linalg.solve(BTB, BTy) # too slow a = solve_banded((3, 3), bandmat(BTB, bw=7), BTy) print a yfit = np.dot(B, a) if c: # C version for band matrix _cbspline.rhs_fill(BTy, G, wy, kk, kk.size) # w.size=1 not broadcasted in lhsbnd_fill _cbspline.lhsbnd_fill(BTBbnd, G, w if np.size(w) > 1 else np.full_like(y, w), kk, kk.size, nk) if nat: BTy = 1 * BTy[1:K + 1] BTBbnd = 1 * BTBbnd[:, 1:K + 1] if lam: _cbspline.add_DTD(BTBbnd, BTy.size, lam, pord) else: # Python version for band matrix # compute sum_jk B_j(x_i) B_k(x_i) w_i for k in range(4): BTy += np.bincount(kk + k, wy * G[k], nk) #for j in range(k+1): #BTBbnd[3-k+j] += np.bincount(kk+k, w*G[k]*G[j], nk) # fill upper #for j in range(k+1): #BTBbnd[k-j] += np.bincount(kk+j, w*G[k]*G[j], nk) # fill lower for j in range(k, 4): BTBbnd[j - k] += np.bincount(kk + k, w * G[k] * G[j], nk) # fill lower if nat: BTy = BTy[1:K + 1] * 1 BTBbnd = BTBbnd[:, 1:K + 1] * 1 if lam: # Add penalty lam*DTD print lam if pord == 0: # diagonal BTBbnd[0] += lam elif pord == 1: # 1 2 ... 2 2 1 # -1 -1 ... -1 -1 # diagonal BTBbnd[0, [0, -1]] += lam BTBbnd[0, 1:-1] += 2 * lam # subdiagonals BTBbnd[1, :-1] -= lam elif pord == 2: # 1 5 6 ... 6 5 1 # -2 -4 -4 ... -4 -2 # 1 1 1 ... 1 # diagonal BTBbnd[0, [0, -1]] += lam BTBbnd[0, [1, -2]] += 5 * lam BTBbnd[0, 2:-2] += 6 * lam # first subdiagonal BTBbnd[1, [0, -2]] -= 2 * lam BTBbnd[1, 1:-2] -= 4 * lam # second subdiagonal BTBbnd[2, :-2] += lam else: # generic version D = np.diff(np.eye(nk), n=pord) DTD = lam * np.dot(D, D.T) for k in range(4): BTBbnd[k, :-k] += np.diag(DTD, k) #ds9(BTBbnd) #pause() if mu is not None and e_mu: # GP like penality with mu and variance BTy += mu / e_mu**2 BTBbnd[0] += 1. / e_mu**2 if cov: # invert matrix BTB = np.diag(BTBbnd[0]) for i in range(1, 4): BTB += np.diag(BTBbnd[i, :-i], i) + np.diag(BTBbnd[i, :-i], -i) covmat = np.linalg.inv(BTB) BTy = covmat.dot(BTy) else: if c: # python version #BTB = np.diag(BTBbnd[0]) #tt = BTy*1 #for i in range(1,4): BTB += np.diag(BTBbnd[i,:-i], i) + np.diag(BTBbnd[i,:-i], -i) #ds9(BTB) #print _cbspline.cholsol(BTB, tt, BTy.size) _cbspline.cholbnd(BTBbnd, BTy, BTy.size, 3) #_cbspline.cholbnd_upper(gg, uu, BTy.size, 3) elif 1: # python version solveh_banded(BTBbnd, BTy, lower=True, overwrite_ab=True, overwrite_b=True) else: # old c version with Gauss elimination # symmetric part BTBbnd[4, :-1] = BTBbnd[2, 1:] BTBbnd[5, :-2] = BTBbnd[1, 2:] BTBbnd[6, :-3] = BTBbnd[0, 3:] #pause() _cbspline.bandsol(BTBbnd, BTy, BTy.size, 7) # only works with "1*" (copy needed: np.ctypeslib.as_ctypes(BTBbnd): TypeError: strided arrays not supported) a = BTy if nat: G = Gorg a = np.r_[2 * a[0] - a[1], a, 2 * a[K - 1] - a[K - 2]] mod = ucbspl(a, xmin, xmax) out = mod if retfit or var or cov: out = out, if retfit: # same as yfit = mod(x), but re-using G[k] yfit = 0. for k in [0, 1, 2, 3]: yfit += a[kk + k] * G[k] out += yfit, # error estimation for knot coefficients ak # sig(ak)^-2 = sum_i Bk(x_i) * w_i if var or e_yk: wa = 0. # = sum_i Bk(x_i) * w_i = 1/var(ak) for k, Gk in enumerate( G): # short loop, but bincount could also have overhead wa += np.bincount(kk + k, Gk * w, nk) if mu is not None and e_mu: wa += 1. / e_mu**2 with np.errstate(divide='ignore'): # yet lam is not handled vara = 1. / wa varmod = ucbspl(vara, xmin, xmax) if var: out += varmod, if cov: # compute sum_i,j B_j(x) * cov_jk B_k(x) out += v_cspl(covmat, xmin, xmax), #gplot(x, v_f(x), varmod(x), ', "" us 1:3,', varmod.xk, varmod(), v_f(varmod.xk),', "" us 1:3') # error estimation for knot values # var(yk) = 1/6 var(ak-1) + 4/6 var(ak) + 1/6 var(ak+1) if e_yk: mod.e_yk = np.sqrt(varmod()) if plot: gplot( mod.osamp(20), ' w l lt 1,', # oversample the knots mod.xk, mod(), ' lt 1 pt 7,', x, y, mod(x), ' lt 3, "" us 1:3 w l lt 2') return out
def read_feros(self, s, orders=None, pfits=True, verb=True): """ SYNTAX: read_feros(filename) OUTPUT: namedtuple('spectrum', 'w f berv bjd blaze drift timeid sn55 ') w - wavelength f - flux berv - Barycentric Earth Radial Velocity bjd - Barycentric Julian Day blaze - Blaze filename drift - Used RV Drift sn55 - S_N order center55 """ hdulist = pyfits.open(s) # slow 30 ms if orders is None or self.header is None: HIERARCH = 'HIERARCH ' hdr = hdulist[0].header self.inst = hdr['INSTRUME'] #data,hdr = fitsio.read(s,header=True) indberv = [ i for i, s in enumerate(hdr.values()) if 'BARY_CORR' in str(s) ][0] + 1 self.drsberv = float(hdr[indberv]) # FEROS drsberv are wrong and already applied to the spectra # remove it from the spectrum self.drsbjd = hdr.get('MJD-OBS', np.nan) #hdr.get(HIERARCH+'ESO DRS BJD',0) if self.drsbjd: self.drsbjd += 2400000.5 if self.fib == 'B': self.drsberv = np.nan self.tmmean = 0.5 # hdr['HIERARCH ESO INS DET1 TMMEAN'] not available self.exptime = hdr['EXPTIME'] self.dateobs = hdr['ARCFILE'][6:-5] # hdr['DATE-OBS'] # FEROS has no S/N estimates; judge with airmass and seeing #print hdr['DATASUM'], self.exptime, hdr['HIERARCH ESO TEL AIRM START'], hdr['HIERARCH ESO TEL AMBI FWHM START'] seefhwm = hdr['HIERARCH ESO TEL AMBI FWHM START'] # if -1 assume 3.0 self.sn55 = 100 / hdr['HIERARCH ESO TEL AIRM START'] / ( seefhwm if seefhwm > 0 else 3.0) self.blaze = '' #hdr[HIERARCH+'ESO DRS BLAZE FILE'] self.drift = hdr.get(HIERARCH + 'ESO DRS DRIFT RV USED', np.nan) #fileid = str(hdr.ascardlist()['MJD-OBS']) # read the comment fileid = hdr.get('ARCFILE', 0) #fileid[fileid.index('(')+1:fileid.index(')')] self.calmode = hdr.get( 'ORIGFILE', 0).split("_")[3] #fileid[fileid.index('(')+1:fileid.index(')')] calmodedict = {'objcal': 'OBJ,ThAr', 'objsky': 'OBJ,SKY'} if self.calmode in calmodedict: self.calmode = calmodedict[self.calmode] self.timeid = self.dateobs #fileid if self.calmode != 'OBJ,ThAr' and hdr.get( 'FILENAME') != 'rebinned1.bdf': self.flag = 1 # hdr['OBJECT'] = hdr.get('OBJECT','FOX') self.header = hdr hdr = self.header if verb: print "read_feros:", self.timeid, self.header[ 'OBJECT'], self.drsbjd, self.sn55, self.drsberv, self.drift, self.flag, self.calmode if orders is not None: # read order data if self.filename.endswith('.mt'): realname = os.path.realpath(self.filename) data = [ pyfits.getdata(realname.replace('01.mt', '%02i.mt' % (o + 1))) for o in range(39) ] fsize = max([len(fo) for fo in data]) f = np.zeros((39, fsize)).astype(float) for o, fo in enumerate(data): f[o, 0:len(fo)] = fo * 100000. else: f = hdulist[0].data * 100000. bpmap = np.isnan(f).astype(int) # flag 1 for nan bpmap[f < 0.0001] |= flag.neg # flag 2 for zero and negative flux # print " applying wavelength solution ", file CRPIX1 = -49. CDELT1 = 0.03 w = 0 * f.astype(float) his = list(hdr.get_history()) wind = [i for i, x in enumerate(his) if 'WSTART' in x][0] wstart = ' '.join(his[wind + 1:wind + 14]) wstart = np.fromstring(wstart, dtype=float, sep=' ') for i in range(39): #wstart = 3.527250000000000E+03 w[i] = wstart[i] + CDELT1 * np.arange(w[0].size) c = 299792.4580 # [km/s] w *= (1 - self.drsberv / c) w = airtovac(w) #from gplot import *; gplot(lam[i],data[i]) #pause() #lam = hdulist['WAVE'].data e = np.ones_like(f) * 100. e[bpmap == 0] = 1.0 * 10. #np.sqrt(data[bpmap==0]) # slow 50 ms if self.fib != 'B': # filter for cosmics look = (None, ) kap = 4.5 for o in range(39): idx, = np.where(bpmap[o] == 0) if 1: # using a robust 4.5 68 percentile clipping #hh = np.argsort(f[o][idx]) #ii = hh[len(hh)*0.98:] # reject 2% p1sig = (100 - 68.2) / 2 p16, p50, p84 = np.percentile(f[o][idx], (p1sig, 50, 100 - p1sig)) rbsig = (p84 - p16) / 2 # robust five sigma ii = idx[f[o][idx] > p50 + kap * rbsig] if o in look: gplot( w[o], f[o], ',', w[o][idx], f[o][idx], ',', w[o][ii], f[o][ii], ",%f, %f, %f, %f" % (p16, p50, p84, p50 + kap * rbsig)) if 0: # moving robust 4.5sigma clipping csum = np.cumsum(f[o][idx]) mvmean = (csum[400:] - csum[:-400]) / 400 #mvmean = np.lib.pad(mvmean, (400,400), 'edge') mvmean = np.concatenate((np.zeros(200) + mvmean[0], mvmean, np.zeros(200) + mvmean[-1])) csum = np.cumsum(abs(f[o][idx] - mvmean)) mvrbsig = (csum[400:] - csum[:-400]) / 400 mvrbsig = np.concatenate( (np.zeros(200) + mvrbsig[0], mvrbsig, np.zeros(200) + mvrbsig[-1])) ii = idx[f[o][idx] > mvmean + kap * mvrbsig] if o in look: gplot(w[o], f[o], ',', w[o][idx], f[o][idx], mvmean, mvmean + kap * mvrbsig, ',"" us 1:3, "" us 1:4,', w[o][ii], sf[o][ii]) bpmap[o][ii] |= flag_cosm if o in look: pause(o) hdulist.close() return w, f, e, bpmap hdulist.close()
def atm_model(w, f, berv, o): # divides spectra by atmosphere model # much more testing needed ! modelfile = 'inst/CRIRES_atm/stdAtmos_crires_airmass1.fits' hdu = fits.open(modelfile, ignore_blank=True) atm_model = hdu[1].data w_atm2 = atm_model.field(0).astype(np.float64) f_atm2 = atm_model.field(1).astype(np.float64) # file_obs2 = "data/CRIRES/210220_PiOri/cr2res_obs_nodding_extracted_combined.fits" file_obs2 = "/data/jana/VIPER/210220_tetVir/K2166/cr2res_obs_nodding_extracted_combined.fits" x2, w_atm, f_atm, bp2, bjd2, berv2 = Spectrum(file_obs2, o=o) bw = (np.load('wave_solution_tetvir.npy'))[o - 1] w_atm = np.poly1d(bw[::-1])(x2) # bb = [-0.457193, 314.784913, 5675608.219445] ba = (np.load("lib/CRIRES/blaze_K2166.npy"))[o - 1] bandp = np.poly1d(ba)(x2) lmin = w[0] lmax = w[-1] lmin = max(w[0], w_atm[0]) lmax = min(w[-1], w_atm[-1]) smo = slice(*np.searchsorted(w_atm, [lmin, lmax])) w_atm1 = w_atm[smo] f_atm1 = f_atm[smo] smo = slice(*np.searchsorted(w, [lmin, lmax])) w = w[smo] f = f[smo] bandp = bandp[smo] gplot(w_atm, f_atm / np.nanmean(f_atm), 'w l t "tell",', w, f / np.nanmean(f), 'w l t "data",', w, bandp / np.mean(bandp) * 0.9, 'w l t "bandp"') #, w_atm1,f_atm1/np.nanmean(f_atm1),'w l') pause() # correction of shift between model and observed data rv, ccc = pyasl.crosscorrRV(w, f, w_atm1, f_atm1, -5., 5., 0.005, skipedge=20) pol = (np.poly1d(np.polyfit(rv, ccc, 15)))(rv) shift = rv[np.argmax(pol)] print('shift atmmod: ', shift) print(w, w_atm1) w_atm /= (1 + shift / 3e5) smo = slice(*np.searchsorted(w_atm, [lmin, lmax])) w_atm1 = w_atm[smo] f_atm1 = f_atm[smo] print(w, w_atm1) f_atm1 = np.interp(w, w_atm1, f_atm1) # f_div = f/np.nanmean(f)/(f_atm1/np.mean(f_atm1)) f_div = f / f_atm1 print('mean', np.nanmean(abs(f / np.mean(f) - f_div / np.mean(f_div)))) i_flag = np.where( abs(f_div) <= (np.median(f_div) + 2 * np.std(abs(f_div))))[0] # i_flag = np.where(abs(f/np.nanmean(f)-f_div/np.nanmean(f_div)) <= 0.2)[0] # f_div[f_div<=0] = 1000 # i_flag = np.where(abs(f_div) <= (np.median(f_div)*1.5))[0] w1 = w * 1. w = w[i_flag] f = f[i_flag] f_div = f_div[i_flag] #*np.median(f) plot_tell = 1 if plot_tell: print('mean', np.nanmean(abs(f / np.mean(f) - f_div / np.mean(f_div)))) # gplot(w,f/np.mean(f),'w l,', w,f_atm1/np.mean(f_atm1),'w l') gplot(w, f / np.mean(f), 'w l t "data",', w, f_div / np.mean(f_div), 'w l t "div data",', w1, f_atm1 / np.mean(f_atm1), 'w l t "tell",', w, abs(f / np.nanmean(f) - f_div / np.nanmean(f_div)), 'w p pt 7 ps 0.4 t "res"') pause() return w, f_div, i_flag #f_atm