def kepimages(infile,outfix,imtype,ranges,clobber,verbose,logfile,status): # startup parameters status = 0 # log the call hashline = '----------------------------------------------------------------------------' kepmsg.log(logfile,hashline,verbose) call = 'KEPIMAGES -- ' call += 'infile='+infile+' ' call += 'outfix='+outfix+' ' call += 'imtype='+imtype+' ' call += 'ranges='+str(ranges)+' ' overwrite = 'n' if (clobber): overwrite = 'y' call += 'clobber='+overwrite+ ' ' chatter = 'n' if (verbose): chatter = 'y' call += 'verbose='+chatter+' ' call += 'logfile='+logfile kepmsg.log(logfile,call+'\n',verbose) # start time kepmsg.clock('KEPIMAGES started at',logfile,verbose) # test log file logfile = kepmsg.test(logfile) # open input file status = 0 print ' ' instr = pyfits.open(infile,mode='readonly',memmap=True) cards0 = instr[0].header.cards cards1 = instr[1].header.cards cards2 = instr[2].header.cards # fudge non-compliant FITS keywords with no values if status == 0: instr = kepkey.emptykeys(instr,file,logfile,verbose) # ingest time series data if status == 0: time = instr[1].data.field('TIME')[:] + 2454833.0 timecorr = instr[1].data.field('TIMECORR')[:] cadenceno = instr[1].data.field('CADENCENO')[:] raw_cnts = instr[1].data.field('RAW_CNTS')[:] flux = instr[1].data.field('FLUX')[:] flux_err = instr[1].data.field('FLUX_ERR')[:] flux_bkg = instr[1].data.field('FLUX_BKG')[:] flux_bkg_err = instr[1].data.field('FLUX_BKG_ERR')[:] cosmic_rays = instr[1].data.field('COSMIC_RAYS')[:] quality = instr[1].data.field('QUALITY')[:] pos_corr1 = instr[1].data.field('POS_CORR1')[:] pos_corr2 = instr[1].data.field('POS_CORR2')[:] # choose output image if status == 0: if imtype.lower() == 'raw_cnts': outim = raw_cnts elif imtype.lower() == 'flux_err': outim = flux_err elif imtype.lower() == 'flux_bkg': outim = flux_bkg elif imtype.lower() == 'flux_bkg_err': outim = flux_bkg_err elif imtype.lower() == 'cosmic_rays': outim = cosmic_rays else: outim = flux # identify images to be exported if status == 0: tim = array([]); dat = array([]); err = array([]) tstart, tstop, status = kepio.timeranges(ranges,logfile,verbose) if status == 0: cadencelis, status = kepstat.filterOnRange(time,tstart,tstop) # provide name for each output file and clobber if file exists if status == 0: for cadence in cadencelis: outfile = outfix + '_BJD%.4f' % time[cadence] + '.fits' if clobber and status == 0: status = kepio.clobber(outfile,logfile,verbose) if kepio.fileexists(outfile) and status == 0: message = 'ERROR -- KEPIMAGES: ' + outfile + ' exists. Use --clobber' status = kepmsg.err(logfile,message,True) # construct output primary extension if status == 0: ncad = 0 for cadence in cadencelis: outfile = outfix + '_BJD%.4f' % time[cadence] + '.fits' hdu0 = pyfits.PrimaryHDU() for i in range(len(cards0)): try: if cards0[i].key not in hdu0.header.keys(): hdu0.header.update(cards0[i].key, cards0[i].value, cards0[i].comment) else: hdu0.header.cards[cards0[i].key].comment = cards0[i].comment except: pass status = kepkey.history(call,hdu0,outfile,logfile,verbose) outstr = HDUList(hdu0) # construct output image extension hdu1 = ImageHDU(flux[cadence]) for i in range(len(cards2)): try: if cards2[i].key not in hdu1.header.keys(): hdu1.header.update(cards2[i].key, cards2[i].value, cards2[i].comment) except: pass for i in range(len(cards1)): if (cards1[i].key not in hdu1.header.keys() and cards1[i].key[:4] not in ['TTYP','TFOR','TUNI','TDIS','TDIM','WCAX','1CTY', '2CTY','1CRP','2CRP','1CRV','2CRV','1CUN','2CUN', '1CDE','2CDE','1CTY','2CTY','1CDL','2CDL','11PC', '12PC','21PC','22PC','WCSN','TFIE']): hdu1.header.update(cards1[i].key, cards1[i].value, cards1[i].comment) try: int_time = cards1['INT_TIME'].value except: kepmsg.warn(logfile,'WARNING -- KEPIMAGES: cannot find INT_TIME keyword') try: frametim = cards1['FRAMETIM'].value except: kepmsg.warn(logfile,'WARNING -- KEPIMAGES: cannot find FRAMETIM keyword') try: num_frm = cards1['NUM_FRM'].value except: kepmsg.warn(logfile,'WARNING -- KEPIMAGES: cannot find NUM_FRM keyword') hdu1.header.update('EXTNAME','IMAGE','name of extension') try: hdu1.header.update('TELAPSE',frametim * num_frm,'[s] elapsed time for exposure') except: hdu1.header.update('TELAPSE',-999,'[s] elapsed time for exposure') try: hdu1.header.update('LIVETIME',int_time * num_frm,'[s] TELASPE multiplied by DEADC') except: hdu1.header.update('LIVETIME',-999,'[s] TELASPE multiplied by DEADC') try: hdu1.header.update('EXPOSURE',int_time * num_frm,'[s] time on source') except: hdu1.header.update('EXPOSURE',-999,'[s] time on source') try: hdu1.header.update('MIDTIME',time[cadence],'[BJD] mid-time of exposure') except: hdu1.header.update('MIDTIME',-999,'[BJD] mid-time of exposure') try: hdu1.header.update('TIMECORR',timecorr[cadence],'[d] barycenter - timeslice correction') except: hdu1.header.update('TIMECORR',-999,'[d] barycenter - timeslice correction') try: hdu1.header.update('CADENCEN',cadenceno[cadence],'unique cadence number') except: hdu1.header.update('CADENCEN',-999,'unique cadence number') try: hdu1.header.update('QUALITY',quality[cadence],'pixel quality flag') except: hdu1.header.update('QUALITY',-999,'pixel quality flag') try: if True in numpy.isfinite(cosmic_rays[cadence]): hdu1.header.update('COSM_RAY',True,'cosmic ray detected?') else: hdu1.header.update('COSM_RAY',False,'cosmic ray detected?') except: hdu1.header.update('COSM_RAY',-999,'cosmic ray detected?') try: pc1 = str(pos_corr1[cadence]) pc2 = str(pos_corr2[cadence]) hdu1.header.update('POSCORR1',pc1,'[pix] column position correction') hdu1.header.update('POSCORR2',pc2,'[pix] row position correction') except: hdu1.header.update('POSCORR1',-999,'[pix] column position correction') hdu1.header.update('POSCORR2',-999,'[pix] row position correction') outstr.append(hdu1) # write output file if status == 0: outstr.writeto(outfile,checksum=True) ncad += 1 txt = '\r%3d%% ' % (float(ncad) / float(len(cadencelis)) * 100.0) txt += '%s ' % outfile sys.stdout.write(txt) sys.stdout.flush() # close input structure if status == 0: status = kepio.closefits(instr,logfile,verbose) print '\n' # end time kepmsg.clock('KEPIMAGES finished at',logfile,verbose)
def kepoutlier(infile,outfile,datacol,nsig,stepsize,npoly,niter, operation,ranges,plot,plotfit,clobber,verbose,logfile,status, cmdLine=False): # startup parameters status = 0 labelsize = 24 ticksize = 16 xsize = 16 ysize = 6 lcolor = '#0000ff' lwidth = 1.0 fcolor = '#ffff00' falpha = 0.2 # log the call hashline = '----------------------------------------------------------------------------' kepmsg.log(logfile,hashline,verbose) call = 'KEPOUTLIER -- ' call += 'infile='+infile+' ' call += 'outfile='+outfile+' ' call += 'datacol='+str(datacol)+' ' call += 'nsig='+str(nsig)+' ' call += 'stepsize='+str(stepsize)+' ' call += 'npoly='+str(npoly)+' ' call += 'niter='+str(niter)+' ' call += 'operation='+str(operation)+' ' call += 'ranges='+str(ranges)+' ' plotit = 'n' if (plot): plotit = 'y' call += 'plot='+plotit+ ' ' plotf = 'n' if (plotfit): plotf = 'y' call += 'plotfit='+plotf+ ' ' overwrite = 'n' if (clobber): overwrite = 'y' call += 'clobber='+overwrite+ ' ' chatter = 'n' if (verbose): chatter = 'y' call += 'verbose='+chatter+' ' call += 'logfile='+logfile kepmsg.log(logfile,call+'\n',verbose) # start time kepmsg.clock('KEPOUTLIER started at',logfile,verbose) # test log file logfile = kepmsg.test(logfile) # clobber output file if clobber: status = kepio.clobber(outfile,logfile,verbose) if kepio.fileexists(outfile): message = 'ERROR -- KEPOUTLIER: ' + outfile + ' exists. Use clobber=yes' status = kepmsg.err(logfile,message,verbose) # open input file if status == 0: instr, status = kepio.openfits(infile,'readonly',logfile,verbose) if status == 0: tstart, tstop, bjdref, cadence, status = kepio.timekeys(instr,infile,logfile,verbose,status) if status == 0: try: work = instr[0].header['FILEVER'] cadenom = 1.0 except: cadenom = cadence # fudge non-compliant FITS keywords with no values if status == 0: instr = kepkey.emptykeys(instr,file,logfile,verbose) # read table structure if status == 0: table, status = kepio.readfitstab(infile,instr[1],logfile,verbose) # filter input data table if status == 0: try: nanclean = instr[1].header['NANCLEAN'] except: naxis2 = 0 try: for i in range(len(table.field(0))): if numpy.isfinite(table.field('barytime')[i]) and \ numpy.isfinite(table.field(datacol)[i]): table[naxis2] = table[i] naxis2 += 1 instr[1].data = table[:naxis2] except: for i in range(len(table.field(0))): if numpy.isfinite(table.field('time')[i]) and \ numpy.isfinite(table.field(datacol)[i]): table[naxis2] = table[i] naxis2 += 1 instr[1].data = table[:naxis2] comment = 'NaN cadences removed from data' status = kepkey.new('NANCLEAN',True,comment,instr[1],outfile,logfile,verbose) # read table columns if status == 0: try: intime = instr[1].data.field('barytime') + 2.4e6 except: intime, status = kepio.readfitscol(infile,instr[1].data,'time',logfile,verbose) indata, status = kepio.readfitscol(infile,instr[1].data,datacol,logfile,verbose) if status == 0: intime = intime + bjdref indata = indata / cadenom # time ranges for region to be corrected if status == 0: t1, t2, status = kepio.timeranges(ranges,logfile,verbose) cadencelis, status = kepstat.filterOnRange(intime,t1,t2) # find limits of each time step if status == 0: tstep1 = []; tstep2 = [] work = intime[0] while work < intime[-1]: tstep1.append(work) tstep2.append(array([work+stepsize,intime[-1]],dtype='float64').min()) work += stepsize # find cadence limits of each time step if status == 0: cstep1 = []; cstep2 = [] work1 = 0; work2 = 0 for i in range(len(intime)): if intime[i] >= intime[work1] and intime[i] < intime[work1] + stepsize: work2 = i else: cstep1.append(work1) cstep2.append(work2) work1 = i; work2 = i cstep1.append(work1) cstep2.append(work2) outdata = indata * 1.0 # comment keyword in output file if status == 0: status = kepkey.history(call,instr[0],outfile,logfile,verbose) # clean up x-axis unit if status == 0: intime0 = float(int(tstart / 100) * 100.0) ptime = intime - intime0 xlab = 'BJD $-$ %d' % intime0 # clean up y-axis units if status == 0: pout = indata * 1.0 nrm = len(str(int(pout.max())))-1 pout = pout / 10**nrm ylab = '10$^%d$ e$^-$ s$^{-1}$' % nrm # data limits xmin = ptime.min() xmax = ptime.max() ymin = pout.min() ymax = pout.max() xr = xmax - xmin yr = ymax - ymin ptime = insert(ptime,[0],[ptime[0]]) ptime = append(ptime,[ptime[-1]]) pout = insert(pout,[0],[0.0]) pout = append(pout,0.0) # plot light curve if status == 0 and plot: plotLatex = True try: params = {'backend': 'png', 'axes.linewidth': 2.5, 'axes.labelsize': labelsize, 'axes.font': 'sans-serif', 'axes.fontweight' : 'bold', 'text.fontsize': 12, 'legend.fontsize': 12, 'xtick.labelsize': ticksize, 'ytick.labelsize': ticksize} rcParams.update(params) except: plotLatex = False if status == 0 and plot: pylab.figure(figsize=[xsize,ysize]) pylab.clf() # plot data ax = pylab.axes([0.06,0.1,0.93,0.87]) # force tick labels to be absolute rather than relative pylab.gca().xaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() setp(labels, 'rotation', 90, fontsize=12) pylab.plot(ptime,pout,color=lcolor,linestyle='-',linewidth=lwidth) fill(ptime,pout,color=fcolor,linewidth=0.0,alpha=falpha) xlabel(xlab, {'color' : 'k'}) if not plotLatex: ylab = '10**%d electrons/sec' % nrm ylabel(ylab, {'color' : 'k'}) grid() # loop over each time step, fit data, determine rms if status == 0: masterfit = indata * 0.0 mastersigma = zeros(len(masterfit)) functype = 'poly' + str(npoly) for i in range(len(cstep1)): pinit = [indata[cstep1[i]:cstep2[i]+1].mean()] if npoly > 0: for j in range(npoly): pinit.append(0.0) pinit = array(pinit,dtype='float32') try: coeffs, errors, covar, iiter, sigma, chi2, dof, fit, plotx, ploty, status = \ kepfit.lsqclip(functype,pinit,intime[cstep1[i]:cstep2[i]+1]-intime[cstep1[i]], indata[cstep1[i]:cstep2[i]+1],None,nsig,nsig,niter,logfile, verbose) for j in range(len(coeffs)): masterfit[cstep1[i]:cstep2[i]+1] += coeffs[j] * \ (intime[cstep1[i]:cstep2[i]+1] - intime[cstep1[i]])**j for j in range(cstep1[i],cstep2[i]+1): mastersigma[j] = sigma if plotfit: pylab.plot(plotx+intime[cstep1[i]]-intime0,ploty / 10**nrm, 'g',lw='3') except: for j in range(cstep1[i],cstep2[i]+1): masterfit[j] = indata[j] mastersigma[j] = 1.0e10 message = 'WARNING -- KEPOUTLIER: could not fit range ' message += str(intime[cstep1[i]]) + '-' + str(intime[cstep2[i]]) kepmsg.warn(None,message) # reject outliers if status == 0: rejtime = []; rejdata = []; naxis2 = 0 for i in range(len(masterfit)): if abs(indata[i] - masterfit[i]) > nsig * mastersigma[i] and i in cadencelis: rejtime.append(intime[i]) rejdata.append(indata[i]) if operation == 'replace': [rnd] = kepstat.randarray([masterfit[i]],[mastersigma[i]]) table[naxis2] = table[i] table.field(datacol)[naxis2] = rnd naxis2 += 1 else: table[naxis2] = table[i] naxis2 += 1 instr[1].data = table[:naxis2] rejtime = array(rejtime,dtype='float64') rejdata = array(rejdata,dtype='float32') pylab.plot(rejtime-intime0,rejdata / 10**nrm,'ro') # plot ranges xlim(xmin-xr*0.01,xmax+xr*0.01) if ymin >= 0.0: ylim(ymin-yr*0.01,ymax+yr*0.01) else: ylim(1.0e-10,ymax+yr*0.01) # render plot if cmdLine: pylab.show() else: pylab.ion() pylab.plot([]) pylab.ioff() # write output file if status == 0: instr.writeto(outfile) # close input file if status == 0: status = kepio.closefits(instr,logfile,verbose) ## end time if (status == 0): message = 'KEPOUTLIER completed at' else: message = '\nKEPOUTLIER aborted at' kepmsg.clock(message,logfile,verbose)
def kepdetrend(infile, outfile, datacol, errcol, ranges1, npoly1, nsig1, niter1, ranges2, npoly2, nsig2, niter2, popnans, plot, clobber, verbose, logfile, status, cmdLine=False): # startup parameters status = 0 labelsize = 24 ticksize = 16 xsize = 16 ysize = 9 lcolor = '#0000ff' lwidth = 1.0 fcolor = '#ffff00' falpha = 0.2 # log the call hashline = '----------------------------------------------------------------------------' kepmsg.log(logfile, hashline, verbose) call = 'KEPDETREND -- ' call += 'infile=' + infile + ' ' call += 'outfile=' + outfile + ' ' call += 'datacol=' + str(datacol) + ' ' call += 'errcol=' + str(errcol) + ' ' call += 'ranges1=' + str(ranges1) + ' ' call += 'npoly1=' + str(npoly1) + ' ' call += 'nsig1=' + str(nsig1) + ' ' call += 'niter1=' + str(niter1) + ' ' call += 'ranges2=' + str(ranges2) + ' ' call += 'npoly2=' + str(npoly2) + ' ' call += 'nsig2=' + str(nsig2) + ' ' call += 'niter2=' + str(niter2) + ' ' popn = 'n' if (popnans): popn = 'y' call += 'popnans=' + popn + ' ' plotit = 'n' if (plot): plotit = 'y' call += 'plot=' + plotit + ' ' overwrite = 'n' if (clobber): overwrite = 'y' call += 'clobber=' + overwrite + ' ' chatter = 'n' if (verbose): chatter = 'y' call += 'verbose=' + chatter + ' ' call += 'logfile=' + logfile kepmsg.log(logfile, call + '\n', verbose) # start time kepmsg.clock('KEPDETREND started at', logfile, verbose) # test log file logfile = kepmsg.test(logfile) # clobber output file if clobber: status = kepio.clobber(outfile, logfile, verbose) if kepio.fileexists(outfile): message = 'ERROR -- KEPDETREND: ' + outfile + ' exists. Use clobber=yes' status = kepmsg.err(logfile, message, verbose) # open input file if status == 0: instr, status = kepio.openfits(infile, 'readonly', logfile, verbose) tstart, tstop, bjdref, cadence, status = kepio.timekeys( instr, infile, logfile, verbose, status) # fudge non-compliant FITS keywords with no values if status == 0: instr = kepkey.emptykeys(instr, file, logfile, verbose) # read table structure if status == 0: table, status = kepio.readfitstab(infile, instr[1], logfile, verbose) # filter input data table if status == 0: work1 = numpy.array( [table.field('time'), table.field(datacol), table.field(errcol)]) work1 = numpy.rot90(work1, 3) work1 = work1[~numpy.isnan(work1).any(1)] # read table columns if status == 0: intime = work1[:, 2] + bjdref indata = work1[:, 1] inerr = work1[:, 0] print intime # time ranges for region 1 (region to be corrected) if status == 0: time1 = [] data1 = [] err1 = [] t1start, t1stop, status = kepio.timeranges(ranges1, logfile, verbose) if status == 0: cadencelis1, status = kepstat.filterOnRange(intime, t1start, t1stop) if status == 0: for i in range(len(cadencelis1)): time1.append(intime[cadencelis1[i]]) data1.append(indata[cadencelis1[i]]) if errcol.lower() != 'none': err1.append(inerr[cadencelis1[i]]) t0 = time1[0] time1 = array(time1, dtype='float64') - t0 data1 = array(data1, dtype='float32') if errcol.lower() != 'none': err1 = array(err1, dtype='float32') else: err1 = None # fit function to range 1 if status == 0: functype = 'poly' + str(npoly1) pinit = [data1.mean()] if npoly1 > 0: for i in range(npoly1): pinit.append(0) pinit = array(pinit, dtype='float32') coeffs, errors, covar, iiter, sigma, chi2, dof, fit, plotx1, ploty1, status = \ kepfit.lsqclip(functype,pinit,time1,data1,err1,nsig1,nsig1,niter1, logfile,verbose) fit1 = indata * 0.0 for i in range(len(coeffs)): fit1 += coeffs[i] * (intime - t0)**i for i in range(len(intime)): if i not in cadencelis1: fit1[i] = 0.0 plotx1 += t0 print coeffs # time ranges for region 2 (region that is correct) if status == 0: time2 = [] data2 = [] err2 = [] t2start, t2stop, status = kepio.timeranges(ranges2, logfile, verbose) cadencelis2, status = kepstat.filterOnRange(intime, t2start, t2stop) for i in range(len(cadencelis2)): time2.append(intime[cadencelis2[i]]) data2.append(indata[cadencelis2[i]]) if errcol.lower() != 'none': err2.append(inerr[cadencelis2[i]]) t0 = time2[0] time2 = array(time2, dtype='float64') - t0 data2 = array(data2, dtype='float32') if errcol.lower() != 'none': err2 = array(err2, dtype='float32') else: err2 = None # fit function to range 2 if status == 0: functype = 'poly' + str(npoly2) pinit = [data2.mean()] if npoly2 > 0: for i in range(npoly2): pinit.append(0) pinit = array(pinit, dtype='float32') coeffs, errors, covar, iiter, sigma, chi2, dof, fit, plotx2, ploty2, status = \ kepfit.lsqclip(functype,pinit,time2,data2,err2,nsig2,nsig2,niter2, logfile,verbose) fit2 = indata * 0.0 for i in range(len(coeffs)): fit2 += coeffs[i] * (intime - t0)**i for i in range(len(intime)): if i not in cadencelis1: fit2[i] = 0.0 plotx2 += t0 # normalize data if status == 0: outdata = indata - fit1 + fit2 if errcol.lower() != 'none': outerr = inerr * 1.0 # comment keyword in output file if status == 0: status = kepkey.history(call, instr[0], outfile, logfile, verbose) # clean up x-axis unit if status == 0: intime0 = float(int(tstart / 100) * 100.0) if intime0 < 2.4e6: intime0 += 2.4e6 ptime = intime - intime0 plotx1 = plotx1 - intime0 plotx2 = plotx2 - intime0 xlab = 'BJD $-$ %d' % intime0 # clean up y-axis units if status == 0: pout = outdata ploty1 ploty2 nrm = len(str(int(numpy.nanmax(indata)))) - 1 indata = indata / 10**nrm pout = pout / 10**nrm ploty1 = ploty1 / 10**nrm ploty2 = ploty2 / 10**nrm ylab = '10$^%d$ e$^-$ s$^{-1}$' % nrm # data limits xmin = ptime.min() xmax = ptime.max() ymin = indata.min() ymax = indata.max() omin = pout.min() omax = pout.max() xr = xmax - xmin yr = ymax - ymin oo = omax - omin ptime = insert(ptime, [0], [ptime[0]]) ptime = append(ptime, [ptime[-1]]) indata = insert(indata, [0], [0.0]) indata = append(indata, [0.0]) pout = insert(pout, [0], [0.0]) pout = append(pout, 0.0) # plot light curve if status == 0 and plot: try: params = { 'backend': 'png', 'axes.linewidth': 2.5, 'axes.labelsize': labelsize, 'axes.font': 'sans-serif', 'axes.fontweight': 'bold', 'text.fontsize': 12, 'legend.fontsize': 12, 'xtick.labelsize': ticksize, 'ytick.labelsize': ticksize } rcParams.update(params) except: pass pylab.figure(figsize=[xsize, ysize]) pylab.clf() # plot original data ax = pylab.axes([0.06, 0.523, 0.93, 0.45]) # force tick labels to be absolute rather than relative pylab.gca().xaxis.set_major_formatter( pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter( pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() pylab.setp(labels, 'rotation', 90, fontsize=12) pylab.plot(ptime, indata, color=lcolor, linestyle='-', linewidth=lwidth) pylab.fill(ptime, indata, color=fcolor, linewidth=0.0, alpha=falpha) pylab.plot(plotx1, ploty1, color='r', linestyle='-', linewidth=2.0) pylab.plot(plotx2, ploty2, color='g', linestyle='-', linewidth=2.0) pylab.xlim(xmin - xr * 0.01, xmax + xr * 0.01) if ymin > 0.0: pylab.ylim(ymin - yr * 0.01, ymax + yr * 0.01) else: pylab.ylim(1.0e-10, ymax + yr * 0.01) pylab.ylabel(ylab, {'color': 'k'}) pylab.grid() # plot detrended data ax = pylab.axes([0.06, 0.073, 0.93, 0.45]) # force tick labels to be absolute rather than relative pylab.gca().xaxis.set_major_formatter( pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter( pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() pylab.setp(labels, 'rotation', 90, fontsize=12) pylab.plot(ptime, pout, color=lcolor, linestyle='-', linewidth=lwidth) pylab.fill(ptime, pout, color=fcolor, linewidth=0.0, alpha=falpha) pylab.xlim(xmin - xr * 0.01, xmax + xr * 0.01) if ymin > 0.0: pylab.ylim(omin - oo * 0.01, omax + oo * 0.01) else: pylab.ylim(1.0e-10, omax + oo * 0.01) pylab.xlabel(xlab, {'color': 'k'}) try: pylab.ylabel(ylab, {'color': 'k'}) except: ylab = '10**%d e-/s' % nrm pylab.ylabel(ylab, {'color': 'k'}) # render plot if status == 0: if cmdLine: pylab.show() else: pylab.ion() pylab.plot([]) pylab.ioff() # write output file if status == 0 and popnans: instr[1].data.field(datacol)[good_data] = outdata instr[1].data.field(errcol)[good_data] = outerr instr[1].data.field(datacol)[bad_data] = None instr[1].data.field(errcol)[bad_data] = None instr.writeto(outfile) elif status == 0 and not popnans: for i in range(len(outdata)): instr[1].data.field(datacol)[i] = outdata[i] if errcol.lower() != 'none': instr[1].data.field(errcol)[i] = outerr[i] instr.writeto(outfile) # close input file if status == 0: status = kepio.closefits(instr, logfile, verbose) ## end time if (status == 0): message = 'KEPDETREND completed at' else: message = '\nKEPDETREND aborted at' kepmsg.clock(message, logfile, verbose)
def kepimages(infile,outfix,imtype,ranges,clobber,verbose,logfile,status): # startup parameters status = 0 # log the call hashline = '----------------------------------------------------------------------------' kepmsg.log(logfile,hashline,verbose) call = 'KEPIMAGES -- ' call += 'infile='+infile+' ' call += 'outfix='+outfix+' ' call += 'imtype='+imtype+' ' call += 'ranges='+str(ranges)+' ' overwrite = 'n' if (clobber): overwrite = 'y' call += 'clobber='+overwrite+ ' ' chatter = 'n' if (verbose): chatter = 'y' call += 'verbose='+chatter+' ' call += 'logfile='+logfile kepmsg.log(logfile,call+'\n',verbose) # start time kepmsg.clock('KEPIMAGES started at',logfile,verbose) # test log file logfile = kepmsg.test(logfile) # open input file status = 0 print(' ') instr = pyfits.open(infile,mode='readonly',memmap=True) cards0 = instr[0].header.cards cards1 = instr[1].header.cards cards2 = instr[2].header.cards # fudge non-compliant FITS keywords with no values if status == 0: instr = kepkey.emptykeys(instr,file,logfile,verbose) # ingest time series data if status == 0: time = instr[1].data.field('TIME')[:] + 2454833.0 timecorr = instr[1].data.field('TIMECORR')[:] cadenceno = instr[1].data.field('CADENCENO')[:] raw_cnts = instr[1].data.field('RAW_CNTS')[:] flux = instr[1].data.field('FLUX')[:] flux_err = instr[1].data.field('FLUX_ERR')[:] flux_bkg = instr[1].data.field('FLUX_BKG')[:] flux_bkg_err = instr[1].data.field('FLUX_BKG_ERR')[:] cosmic_rays = instr[1].data.field('COSMIC_RAYS')[:] quality = instr[1].data.field('QUALITY')[:] pos_corr1 = instr[1].data.field('POS_CORR1')[:] pos_corr2 = instr[1].data.field('POS_CORR2')[:] # choose output image if status == 0: if imtype.lower() == 'raw_cnts': outim = raw_cnts elif imtype.lower() == 'flux_err': outim = flux_err elif imtype.lower() == 'flux_bkg': outim = flux_bkg elif imtype.lower() == 'flux_bkg_err': outim = flux_bkg_err elif imtype.lower() == 'cosmic_rays': outim = cosmic_rays else: outim = flux # identify images to be exported if status == 0: tim = array([]); dat = array([]); err = array([]) tstart, tstop, status = kepio.timeranges(ranges,logfile,verbose) if status == 0: cadencelis, status = kepstat.filterOnRange(time,tstart,tstop) # provide name for each output file and clobber if file exists if status == 0: for cadence in cadencelis: outfile = outfix + '_BJD%.4f' % time[cadence] + '.fits' if clobber and status == 0: status = kepio.clobber(outfile,logfile,verbose) if kepio.fileexists(outfile) and status == 0: message = 'ERROR -- KEPIMAGES: ' + outfile + ' exists. Use --clobber' status = kepmsg.err(logfile,message,True) # construct output primary extension if status == 0: ncad = 0 for cadence in cadencelis: outfile = outfix + '_BJD%.4f' % time[cadence] + '.fits' hdu0 = pyfits.PrimaryHDU() for i in range(len(cards0)): try: if cards0[i].key not in list(hdu0.header.keys()): hdu0.header.update(cards0[i].key, cards0[i].value, cards0[i].comment) else: hdu0.header.cards[cards0[i].key].comment = cards0[i].comment except: pass status = kepkey.history(call,hdu0,outfile,logfile,verbose) outstr = HDUList(hdu0) # construct output image extension hdu1 = ImageHDU(flux[cadence]) for i in range(len(cards2)): try: if cards2[i].key not in list(hdu1.header.keys()): hdu1.header.update(cards2[i].key, cards2[i].value, cards2[i].comment) except: pass for i in range(len(cards1)): if (cards1[i].key not in list(hdu1.header.keys()) and cards1[i].key[:4] not in ['TTYP','TFOR','TUNI','TDIS','TDIM','WCAX','1CTY', '2CTY','1CRP','2CRP','1CRV','2CRV','1CUN','2CUN', '1CDE','2CDE','1CTY','2CTY','1CDL','2CDL','11PC', '12PC','21PC','22PC','WCSN','TFIE']): hdu1.header.update(cards1[i].key, cards1[i].value, cards1[i].comment) try: int_time = cards1['INT_TIME'].value except: kepmsg.warn(logfile,'WARNING -- KEPIMAGES: cannot find INT_TIME keyword') try: frametim = cards1['FRAMETIM'].value except: kepmsg.warn(logfile,'WARNING -- KEPIMAGES: cannot find FRAMETIM keyword') try: num_frm = cards1['NUM_FRM'].value except: kepmsg.warn(logfile,'WARNING -- KEPIMAGES: cannot find NUM_FRM keyword') hdu1.header.update('EXTNAME','IMAGE','name of extension') try: hdu1.header.update('TELAPSE',frametim * num_frm,'[s] elapsed time for exposure') except: hdu1.header.update('TELAPSE',-999,'[s] elapsed time for exposure') try: hdu1.header.update('LIVETIME',int_time * num_frm,'[s] TELASPE multiplied by DEADC') except: hdu1.header.update('LIVETIME',-999,'[s] TELASPE multiplied by DEADC') try: hdu1.header.update('EXPOSURE',int_time * num_frm,'[s] time on source') except: hdu1.header.update('EXPOSURE',-999,'[s] time on source') try: hdu1.header.update('MIDTIME',time[cadence],'[BJD] mid-time of exposure') except: hdu1.header.update('MIDTIME',-999,'[BJD] mid-time of exposure') try: hdu1.header.update('TIMECORR',timecorr[cadence],'[d] barycenter - timeslice correction') except: hdu1.header.update('TIMECORR',-999,'[d] barycenter - timeslice correction') try: hdu1.header.update('CADENCEN',cadenceno[cadence],'unique cadence number') except: hdu1.header.update('CADENCEN',-999,'unique cadence number') try: hdu1.header.update('QUALITY',quality[cadence],'pixel quality flag') except: hdu1.header.update('QUALITY',-999,'pixel quality flag') try: if True in numpy.isfinite(cosmic_rays[cadence]): hdu1.header.update('COSM_RAY',True,'cosmic ray detected?') else: hdu1.header.update('COSM_RAY',False,'cosmic ray detected?') except: hdu1.header.update('COSM_RAY',-999,'cosmic ray detected?') try: pc1 = str(pos_corr1[cadence]) pc2 = str(pos_corr2[cadence]) hdu1.header.update('POSCORR1',pc1,'[pix] column position correction') hdu1.header.update('POSCORR2',pc2,'[pix] row position correction') except: hdu1.header.update('POSCORR1',-999,'[pix] column position correction') hdu1.header.update('POSCORR2',-999,'[pix] row position correction') outstr.append(hdu1) # write output file if status == 0: outstr.writeto(outfile,checksum=True) ncad += 1 txt = '\r%3d%% ' % (float(ncad) / float(len(cadencelis)) * 100.0) txt += '%s ' % outfile sys.stdout.write(txt) sys.stdout.flush() # close input structure if status == 0: status = kepio.closefits(instr,logfile,verbose) print('\n') # end time kepmsg.clock('KEPIMAGES finished at',logfile,verbose)
def kepoutlier(infile,outfile,datacol,nsig,stepsize,npoly,niter, operation,ranges,plot,plotfit,clobber,verbose,logfile,status, cmdLine=False): # startup parameters status = 0 labelsize = 24 ticksize = 16 xsize = 16 ysize = 6 lcolor = '#0000ff' lwidth = 1.0 fcolor = '#ffff00' falpha = 0.2 # log the call hashline = '----------------------------------------------------------------------------' kepmsg.log(logfile,hashline,verbose) call = 'KEPOUTLIER -- ' call += 'infile='+infile+' ' call += 'outfile='+outfile+' ' call += 'datacol='+str(datacol)+' ' call += 'nsig='+str(nsig)+' ' call += 'stepsize='+str(stepsize)+' ' call += 'npoly='+str(npoly)+' ' call += 'niter='+str(niter)+' ' call += 'operation='+str(operation)+' ' call += 'ranges='+str(ranges)+' ' plotit = 'n' if (plot): plotit = 'y' call += 'plot='+plotit+ ' ' plotf = 'n' if (plotfit): plotf = 'y' call += 'plotfit='+plotf+ ' ' overwrite = 'n' if (clobber): overwrite = 'y' call += 'clobber='+overwrite+ ' ' chatter = 'n' if (verbose): chatter = 'y' call += 'verbose='+chatter+' ' call += 'logfile='+logfile kepmsg.log(logfile,call+'\n',verbose) # start time kepmsg.clock('KEPOUTLIER started at',logfile,verbose) # test log file logfile = kepmsg.test(logfile) # clobber output file if clobber: status = kepio.clobber(outfile,logfile,verbose) if kepio.fileexists(outfile): message = 'ERROR -- KEPOUTLIER: ' + outfile + ' exists. Use clobber=yes' status = kepmsg.err(logfile,message,verbose) # open input file if status == 0: instr, status = kepio.openfits(infile,'readonly',logfile,verbose) if status == 0: tstart, tstop, bjdref, cadence, status = kepio.timekeys(instr,infile,logfile,verbose,status) if status == 0: try: work = instr[0].header['FILEVER'] cadenom = 1.0 except: cadenom = cadence # fudge non-compliant FITS keywords with no values if status == 0: instr = kepkey.emptykeys(instr,file,logfile,verbose) # read table structure if status == 0: table, status = kepio.readfitstab(infile,instr[1],logfile,verbose) # filter input data table if status == 0: try: nanclean = instr[1].header['NANCLEAN'] except: naxis2 = 0 try: for i in range(len(table.field(0))): if numpy.isfinite(table.field('barytime')[i]) and \ numpy.isfinite(table.field(datacol)[i]): table[naxis2] = table[i] naxis2 += 1 instr[1].data = table[:naxis2] except: for i in range(len(table.field(0))): if numpy.isfinite(table.field('time')[i]) and \ numpy.isfinite(table.field(datacol)[i]): table[naxis2] = table[i] naxis2 += 1 instr[1].data = table[:naxis2] comment = 'NaN cadences removed from data' status = kepkey.new('NANCLEAN',True,comment,instr[1],outfile,logfile,verbose) # read table columns if status == 0: try: intime = instr[1].data.field('barytime') + 2.4e6 except: intime, status = kepio.readfitscol(infile,instr[1].data,'time',logfile,verbose) indata, status = kepio.readfitscol(infile,instr[1].data,datacol,logfile,verbose) if status == 0: intime = intime + bjdref indata = indata / cadenom # time ranges for region to be corrected if status == 0: t1, t2, status = kepio.timeranges(ranges,logfile,verbose) cadencelis, status = kepstat.filterOnRange(intime,t1,t2) # find limits of each time step if status == 0: tstep1 = []; tstep2 = [] work = intime[0] while work < intime[-1]: tstep1.append(work) tstep2.append(array([work+stepsize,intime[-1]],dtype='float64').min()) work += stepsize # find cadence limits of each time step if status == 0: cstep1 = []; cstep2 = [] work1 = 0; work2 = 0 for i in range(len(intime)): if intime[i] >= intime[work1] and intime[i] < intime[work1] + stepsize: work2 = i else: cstep1.append(work1) cstep2.append(work2) work1 = i; work2 = i cstep1.append(work1) cstep2.append(work2) outdata = indata * 1.0 # comment keyword in output file if status == 0: status = kepkey.history(call,instr[0],outfile,logfile,verbose) # clean up x-axis unit if status == 0: intime0 = float(int(tstart / 100) * 100.0) ptime = intime - intime0 xlab = 'BJD $-$ %d' % intime0 # clean up y-axis units if status == 0: pout = indata * 1.0 nrm = len(str(int(pout.max())))-1 pout = pout / 10**nrm ylab = '10$^%d$ e$^-$ s$^{-1}$' % nrm # data limits xmin = ptime.min() xmax = ptime.max() ymin = pout.min() ymax = pout.max() xr = xmax - xmin yr = ymax - ymin ptime = insert(ptime,[0],[ptime[0]]) ptime = append(ptime,[ptime[-1]]) pout = insert(pout,[0],[0.0]) pout = append(pout,0.0) # plot light curve if status == 0 and plot: plotLatex = True try: params = {'backend': 'png', 'axes.linewidth': 2.5, 'axes.labelsize': labelsize, 'axes.font': 'sans-serif', 'axes.fontweight' : 'bold', 'text.fontsize': 12, 'legend.fontsize': 12, 'xtick.labelsize': ticksize, 'ytick.labelsize': ticksize} rcParams.update(params) except: plotLatex = False if status == 0 and plot: pylab.figure(figsize=[xsize,ysize]) pylab.clf() # plot data ax = pylab.axes([0.06,0.1,0.93,0.87]) # force tick labels to be absolute rather than relative pylab.gca().xaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() setp(labels, 'rotation', 90, fontsize=12) pylab.plot(ptime,pout,color=lcolor,linestyle='-',linewidth=lwidth) fill(ptime,pout,color=fcolor,linewidth=0.0,alpha=falpha) xlabel(xlab, {'color' : 'k'}) if not plotLatex: ylab = '10**%d electrons/sec' % nrm ylabel(ylab, {'color' : 'k'}) grid() # loop over each time step, fit data, determine rms if status == 0: masterfit = indata * 0.0 mastersigma = zeros(len(masterfit)) functype = 'poly' + str(npoly) for i in range(len(cstep1)): pinit = [indata[cstep1[i]:cstep2[i]+1].mean()] if npoly > 0: for j in range(npoly): pinit.append(0.0) pinit = array(pinit,dtype='float32') try: coeffs, errors, covar, iiter, sigma, chi2, dof, fit, plotx, ploty, status = \ kepfit.lsqclip(functype,pinit,intime[cstep1[i]:cstep2[i]+1]-intime[cstep1[i]], indata[cstep1[i]:cstep2[i]+1],None,nsig,nsig,niter,logfile, verbose) for j in range(len(coeffs)): masterfit[cstep1[i]:cstep2[i]+1] += coeffs[j] * \ (intime[cstep1[i]:cstep2[i]+1] - intime[cstep1[i]])**j for j in range(cstep1[i],cstep2[i]+1): mastersigma[j] = sigma if plotfit: pylab.plot(plotx+intime[cstep1[i]]-intime0,ploty / 10**nrm, 'g',lw='3') except: for j in range(cstep1[i],cstep2[i]+1): masterfit[j] = indata[j] mastersigma[j] = 1.0e10 message = 'WARNING -- KEPOUTLIER: could not fit range ' message += str(intime[cstep1[i]]) + '-' + str(intime[cstep2[i]]) kepmsg.warn(None,message) # reject outliers if status == 0: rejtime = []; rejdata = []; naxis2 = 0 for i in range(len(masterfit)): if abs(indata[i] - masterfit[i]) > nsig * mastersigma[i] and i in cadencelis: rejtime.append(intime[i]) rejdata.append(indata[i]) if operation == 'replace': [rnd] = kepstat.randarray([masterfit[i]],[mastersigma[i]]) table[naxis2] = table[i] table.field(datacol)[naxis2] = rnd naxis2 += 1 else: table[naxis2] = table[i] naxis2 += 1 instr[1].data = table[:naxis2] rejtime = array(rejtime,dtype='float64') rejdata = array(rejdata,dtype='float32') pylab.plot(rejtime-intime0,rejdata / 10**nrm,'ro') # plot ranges xlim(xmin-xr*0.01,xmax+xr*0.01) if ymin >= 0.0: ylim(ymin-yr*0.01,ymax+yr*0.01) else: ylim(1.0e-10,ymax+yr*0.01) # render plot if cmdLine: pylab.show() else: pylab.ion() pylab.plot([]) pylab.ioff() # write output file if status == 0: instr.writeto(outfile) # close input file if status == 0: status = kepio.closefits(instr,logfile,verbose) # end time if (status == 0): message = 'KEPOUTLIER completed at' else: message = '\nKEPOUTLIER aborted at' kepmsg.clock(message,logfile,verbose)
def kepdetrend(infile,outfile,datacol,errcol,ranges1,npoly1,nsig1,niter1, ranges2,npoly2,nsig2,niter2,popnans,plot,clobber,verbose,logfile, status,cmdLine=False): # startup parameters status = 0 labelsize = 24 ticksize = 16 xsize = 16 ysize = 9 lcolor = '#0000ff' lwidth = 1.0 fcolor = '#ffff00' falpha = 0.2 # log the call hashline = '----------------------------------------------------------------------------' kepmsg.log(logfile,hashline,verbose) call = 'KEPDETREND -- ' call += 'infile='+infile+' ' call += 'outfile='+outfile+' ' call += 'datacol='+str(datacol)+' ' call += 'errcol='+str(errcol)+' ' call += 'ranges1='+str(ranges1)+' ' call += 'npoly1='+str(npoly1)+' ' call += 'nsig1='+str(nsig1)+' ' call += 'niter1='+str(niter1)+' ' call += 'ranges2='+str(ranges2)+' ' call += 'npoly2='+str(npoly2)+' ' call += 'nsig2='+str(nsig2)+' ' call += 'niter2='+str(niter2)+' ' popn = 'n' if (popnans): popn = 'y' call += 'popnans='+popn+ ' ' plotit = 'n' if (plot): plotit = 'y' call += 'plot='+plotit+ ' ' overwrite = 'n' if (clobber): overwrite = 'y' call += 'clobber='+overwrite+ ' ' chatter = 'n' if (verbose): chatter = 'y' call += 'verbose='+chatter+' ' call += 'logfile='+logfile kepmsg.log(logfile,call+'\n',verbose) # start time kepmsg.clock('KEPDETREND started at',logfile,verbose) # test log file logfile = kepmsg.test(logfile) # clobber output file if clobber: status = kepio.clobber(outfile,logfile,verbose) if kepio.fileexists(outfile): message = 'ERROR -- KEPDETREND: ' + outfile + ' exists. Use clobber=yes' status = kepmsg.err(logfile,message,verbose) # open input file if status == 0: instr, status = kepio.openfits(infile,'readonly',logfile,verbose) tstart, tstop, bjdref, cadence, status = kepio.timekeys(instr,infile,logfile,verbose,status) # fudge non-compliant FITS keywords with no values if status == 0: instr = kepkey.emptykeys(instr,file,logfile,verbose) # read table structure if status == 0: table, status = kepio.readfitstab(infile,instr[1],logfile,verbose) # filter input data table if status == 0: work1 = numpy.array([table.field('time'), table.field(datacol), table.field(errcol)]) work1 = numpy.rot90(work1,3) work1 = work1[~numpy.isnan(work1).any(1)] # read table columns if status == 0: intime = work1[:,2] + bjdref indata = work1[:,1] inerr = work1[:,0] print(intime) # time ranges for region 1 (region to be corrected) if status == 0: time1 = []; data1 = []; err1 = [] t1start, t1stop, status = kepio.timeranges(ranges1,logfile,verbose) if status == 0: cadencelis1, status = kepstat.filterOnRange(intime,t1start,t1stop) if status == 0: for i in range(len(cadencelis1)): time1.append(intime[cadencelis1[i]]) data1.append(indata[cadencelis1[i]]) if errcol.lower() != 'none': err1.append(inerr[cadencelis1[i]]) t0 = time1[0] time1 = array(time1,dtype='float64') - t0 data1 = array(data1,dtype='float32') if errcol.lower() != 'none': err1 = array(err1,dtype='float32') else: err1 = None # fit function to range 1 if status == 0: functype = 'poly' + str(npoly1) pinit = [data1.mean()] if npoly1 > 0: for i in range(npoly1): pinit.append(0) pinit = array(pinit,dtype='float32') coeffs, errors, covar, iiter, sigma, chi2, dof, fit, plotx1, ploty1, status = \ kepfit.lsqclip(functype,pinit,time1,data1,err1,nsig1,nsig1,niter1, logfile,verbose) fit1 = indata * 0.0 for i in range(len(coeffs)): fit1 += coeffs[i] * (intime - t0)**i for i in range(len(intime)): if i not in cadencelis1: fit1[i] = 0.0 plotx1 += t0 print(coeffs) # time ranges for region 2 (region that is correct) if status == 0: time2 = []; data2 = []; err2 = [] t2start, t2stop, status = kepio.timeranges(ranges2,logfile,verbose) cadencelis2, status = kepstat.filterOnRange(intime,t2start,t2stop) for i in range(len(cadencelis2)): time2.append(intime[cadencelis2[i]]) data2.append(indata[cadencelis2[i]]) if errcol.lower() != 'none': err2.append(inerr[cadencelis2[i]]) t0 = time2[0] time2 = array(time2,dtype='float64') - t0 data2 = array(data2,dtype='float32') if errcol.lower() != 'none': err2 = array(err2,dtype='float32') else: err2 = None # fit function to range 2 if status == 0: functype = 'poly' + str(npoly2) pinit = [data2.mean()] if npoly2 > 0: for i in range(npoly2): pinit.append(0) pinit = array(pinit,dtype='float32') coeffs, errors, covar, iiter, sigma, chi2, dof, fit, plotx2, ploty2, status = \ kepfit.lsqclip(functype,pinit,time2,data2,err2,nsig2,nsig2,niter2, logfile,verbose) fit2 = indata * 0.0 for i in range(len(coeffs)): fit2 += coeffs[i] * (intime - t0)**i for i in range(len(intime)): if i not in cadencelis1: fit2[i] = 0.0 plotx2 += t0 # normalize data if status == 0: outdata = indata - fit1 + fit2 if errcol.lower() != 'none': outerr = inerr * 1.0 # comment keyword in output file if status == 0: status = kepkey.history(call,instr[0],outfile,logfile,verbose) # clean up x-axis unit if status == 0: intime0 = float(int(tstart / 100) * 100.0) if intime0 < 2.4e6: intime0 += 2.4e6 ptime = intime - intime0 plotx1 = plotx1 - intime0 plotx2 = plotx2 - intime0 xlab = 'BJD $-$ %d' % intime0 # clean up y-axis units if status == 0: pout = outdata ploty1 ploty2 nrm = len(str(int(numpy.nanmax(indata))))-1 indata = indata / 10**nrm pout = pout / 10**nrm ploty1 = ploty1 / 10**nrm ploty2 = ploty2 / 10**nrm ylab = '10$^%d$ e$^-$ s$^{-1}$' % nrm # data limits xmin = ptime.min() xmax = ptime.max() ymin = indata.min() ymax = indata.max() omin = pout.min() omax = pout.max() xr = xmax - xmin yr = ymax - ymin oo = omax - omin ptime = insert(ptime,[0],[ptime[0]]) ptime = append(ptime,[ptime[-1]]) indata = insert(indata,[0],[0.0]) indata = append(indata,[0.0]) pout = insert(pout,[0],[0.0]) pout = append(pout,0.0) # plot light curve if status == 0 and plot: try: params = {'backend': 'png', 'axes.linewidth': 2.5, 'axes.labelsize': labelsize, 'axes.font': 'sans-serif', 'axes.fontweight' : 'bold', 'text.fontsize': 12, 'legend.fontsize': 12, 'xtick.labelsize': ticksize, 'ytick.labelsize': ticksize} rcParams.update(params) except: pass pylab.figure(figsize=[xsize,ysize]) pylab.clf() # plot original data ax = pylab.axes([0.06,0.523,0.93,0.45]) # force tick labels to be absolute rather than relative pylab.gca().xaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() pylab.setp(labels, 'rotation', 90, fontsize=12) pylab.plot(ptime,indata,color=lcolor,linestyle='-',linewidth=lwidth) pylab.fill(ptime,indata,color=fcolor,linewidth=0.0,alpha=falpha) pylab.plot(plotx1,ploty1,color='r',linestyle='-',linewidth=2.0) pylab.plot(plotx2,ploty2,color='g',linestyle='-',linewidth=2.0) pylab.xlim(xmin-xr*0.01,xmax+xr*0.01) if ymin > 0.0: pylab.ylim(ymin-yr*0.01,ymax+yr*0.01) else: pylab.ylim(1.0e-10,ymax+yr*0.01) pylab.ylabel(ylab, {'color' : 'k'}) pylab.grid() # plot detrended data ax = pylab.axes([0.06,0.073,0.93,0.45]) # force tick labels to be absolute rather than relative pylab.gca().xaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() pylab.setp(labels, 'rotation', 90, fontsize=12) pylab.plot(ptime,pout,color=lcolor,linestyle='-',linewidth=lwidth) pylab.fill(ptime,pout,color=fcolor,linewidth=0.0,alpha=falpha) pylab.xlim(xmin-xr*0.01,xmax+xr*0.01) if ymin > 0.0: pylab.ylim(omin-oo*0.01,omax+oo*0.01) else: pylab.ylim(1.0e-10,omax+oo*0.01) pylab.xlabel(xlab, {'color' : 'k'}) try: pylab.ylabel(ylab, {'color' : 'k'}) except: ylab = '10**%d e-/s' % nrm pylab.ylabel(ylab, {'color' : 'k'}) # render plot if status == 0: if cmdLine: pylab.show() else: pylab.ion() pylab.plot([]) pylab.ioff() # write output file if status == 0 and popnans: instr[1].data.field(datacol)[good_data] = outdata instr[1].data.field(errcol)[good_data] = outerr instr[1].data.field(datacol)[bad_data] = None instr[1].data.field(errcol)[bad_data] = None instr.writeto(outfile) elif status == 0 and not popnans: for i in range(len(outdata)): instr[1].data.field(datacol)[i] = outdata[i] if errcol.lower() != 'none': instr[1].data.field(errcol)[i] = outerr[i] instr.writeto(outfile) # close input file if status == 0: status = kepio.closefits(instr,logfile,verbose) ## end time if (status == 0): message = 'KEPDETREND completed at' else: message = '\nKEPDETREND aborted at' kepmsg.clock(message,logfile,verbose)
def kepflatten(infile,outfile,datacol,errcol,nsig,stepsize,winsize,npoly,niter,ranges, plot,clobber,verbose,logfile,status,cmdLine=False): # startup parameters status = 0 labelsize = 32 ticksize = 18 xsize = 16 ysize = 10 lcolor = '#0000ff' lwidth = 1.0 fcolor = '#ffff00' falpha = 0.2 # log the call hashline = '----------------------------------------------------------------------------' kepmsg.log(logfile,hashline,verbose) call = 'KEPFLATTEN -- ' call += 'infile='+infile+' ' call += 'outfile='+outfile+' ' call += 'datacol='+str(datacol)+' ' call += 'errcol='+str(errcol)+' ' call += 'nsig='+str(nsig)+' ' call += 'stepsize='+str(stepsize)+' ' call += 'winsize='+str(winsize)+' ' call += 'npoly='+str(npoly)+' ' call += 'niter='+str(niter)+' ' call += 'ranges='+str(ranges)+' ' plotit = 'n' if (plot): plotit = 'y' call += 'plot='+plotit+ ' ' overwrite = 'n' if (clobber): overwrite = 'y' call += 'clobber='+overwrite+ ' ' chatter = 'n' if (verbose): chatter = 'y' call += 'verbose='+chatter+' ' call += 'logfile='+logfile kepmsg.log(logfile,call+'\n',verbose) # start time kepmsg.clock('KEPFLATTEN started at',logfile,verbose) # test log file logfile = kepmsg.test(logfile) # test winsize > stepsize if winsize < stepsize: message = 'ERROR -- KEPFLATTEN: winsize must be greater than stepsize' status = kepmsg.err(logfile,message,verbose) # clobber output file if clobber: status = kepio.clobber(outfile,logfile,verbose) if kepio.fileexists(outfile): message = 'ERROR -- KEPFLATTEN: ' + outfile + ' exists. Use clobber=yes' status = kepmsg.err(logfile,message,verbose) # open input file if status == 0: instr, status = kepio.openfits(infile,'readonly',logfile,verbose) if status == 0: tstart, tstop, bjdref, cadence, status = kepio.timekeys(instr,infile,logfile,verbose,status) if status == 0: try: work = instr[0].header['FILEVER'] cadenom = 1.0 except: cadenom = cadence # fudge non-compliant FITS keywords with no values if status == 0: instr = kepkey.emptykeys(instr,file,logfile,verbose) # read table structure if status == 0: table, status = kepio.readfitstab(infile,instr[1],logfile,verbose) # filter input data table if status == 0: try: datac = table.field(datacol) except: message = 'ERROR -- KEPFLATTEN: cannot find or read data column ' + datacol status = kepmsg.err(logfile,message,verbose) if status == 0: try: err = table.field(errcol) except: message = 'WARNING -- KEPFLATTEN: cannot find or read error column ' + errcol errcol = 'None' if status == 0: if errcol.lower() == 'none' or errcol == 'PSF_FLUX_ERR': err = datac * cadence err = numpy.sqrt(numpy.abs(err)) / cadence work1 = numpy.array([table.field('time'), datac, err]) else: work1 = numpy.array([table.field('time'), datac, err]) work1 = numpy.rot90(work1,3) work1 = work1[~numpy.isnan(work1).any(1)] # read table columns if status == 0: intime = work1[:,2] + bjdref indata = work1[:,1] inerr = work1[:,0] if len(intime) == 0: message = 'ERROR -- KEPFLATTEN: one of the input arrays is all NaN' status = kepmsg.err(logfile,message,verbose) # time ranges for region to be corrected if status == 0: t1, t2, status = kepio.timeranges(ranges,logfile,verbose) cadencelis, status = kepstat.filterOnRange(intime,t1,t2) # find limits of each time step if status == 0: tstep1 = []; tstep2 = [] work = intime[0] while work <= intime[-1]: tstep1.append(work) tstep2.append(array([work+winsize,intime[-1]],dtype='float64').min()) work += stepsize # find cadence limits of each time step if status == 0: cstep1 = []; cstep2 = [] for n in range(len(tstep1)): for i in range(len(intime)-1): if intime[i] <= tstep1[n] and intime[i+1] > tstep1[n]: for j in range(i,len(intime)-1): if intime[j] < tstep2[n] and intime[j+1] >= tstep2[n]: cstep1.append(i) cstep2.append(j+1) # comment keyword in output file if status == 0: status = kepkey.history(call,instr[0],outfile,logfile,verbose) # clean up x-axis unit if status == 0: intime0 = float(int(tstart / 100) * 100.0) ptime = intime - intime0 xlab = 'BJD $-$ %d' % intime0 # clean up y-axis units if status == 0: pout = copy(indata) nrm = len(str(int(pout.max())))-1 pout = pout / 10**nrm ylab = '10$^%d$ e$^-$ s$^{-1}$' % nrm # data limits xmin = ptime.min() xmax = ptime.max() ymin = pout.min() ymax = pout.max() xr = xmax - xmin yr = ymax - ymin ptime = insert(ptime,[0],[ptime[0]]) ptime = append(ptime,[ptime[-1]]) pout = insert(pout,[0],[0.0]) pout = append(pout,0.0) # plot light curve if status == 0 and plot: plotLatex = True try: params = {'backend': 'png', 'axes.linewidth': 2.5, 'axes.labelsize': labelsize, 'axes.font': 'sans-serif', 'axes.fontweight' : 'bold', 'text.fontsize': 12, 'legend.fontsize': 12, 'xtick.labelsize': ticksize, 'ytick.labelsize': ticksize} rcParams.update(params) except: plotLatex = False if status == 0 and plot: pylab.figure(figsize=[xsize,ysize]) pylab.clf() # plot data ax = pylab.axes([0.06,0.54,0.93,0.43]) # force tick labels to be absolute rather than relative pylab.gca().xaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() pylab.setp(labels, 'rotation', 90) pylab.setp(pylab.gca(),xticklabels=[]) pylab.plot(ptime[1:-1],pout[1:-1],color=lcolor,linestyle='-',linewidth=lwidth) pylab.fill(ptime,pout,color=fcolor,linewidth=0.0,alpha=falpha) if not plotLatex: ylab = '10**%d electrons/sec' % nrm ylabel(ylab, {'color' : 'k'}) grid() # loop over each time step, fit data, determine rms if status == 0: fitarray = numpy.zeros((len(indata),len(cstep1)),dtype='float32') sigarray = numpy.zeros((len(indata),len(cstep1)),dtype='float32') fitarray[:,:] = numpy.nan sigarray[:,:] = numpy.nan masterfit = indata * 0.0 mastersigma = numpy.zeros(len(masterfit)) functype = 'poly' + str(npoly) for i in range(len(cstep1)): timeSeries = intime[cstep1[i]:cstep2[i]+1]-intime[cstep1[i]] dataSeries = indata[cstep1[i]:cstep2[i]+1] fitTimeSeries = numpy.array([],dtype='float32') fitDataSeries = numpy.array([],dtype='float32') pinit = [dataSeries.mean()] if npoly > 0: for j in range(npoly): pinit.append(0.0) pinit = array(pinit,dtype='float32') try: if len(fitarray[cstep1[i]:cstep2[i]+1,i]) > len(pinit): coeffs, errors, covar, iiter, sigma, chi2, dof, fit, plotx, ploty, status = \ kepfit.lsqclip(functype,pinit,timeSeries,dataSeries,None,nsig,nsig,niter, logfile,verbose) fitarray[cstep1[i]:cstep2[i]+1,i] = 0.0 sigarray[cstep1[i]:cstep2[i]+1,i] = sigma for j in range(len(coeffs)): fitarray[cstep1[i]:cstep2[i]+1,i] += coeffs[j] * timeSeries**j except: for j in range(cstep1[i],cstep2[i]+1): fitarray[cstep1[i]:cstep2[i]+1,i] = 0.0 sigarray[cstep1[i]:cstep2[i]+1,i] = 1.0e-10 message = 'WARNING -- KEPFLATTEN: could not fit range ' message += str(intime[cstep1[i]]) + '-' + str(intime[cstep2[i]]) kepmsg.warn(None,message) # find mean fit for each timestamp if status == 0: for i in range(len(indata)): masterfit[i] = scipy.stats.nanmean(fitarray[i,:]) mastersigma[i] = scipy.stats.nanmean(sigarray[i,:]) masterfit[-1] = masterfit[-4] #fudge masterfit[-2] = masterfit[-4] #fudge masterfit[-3] = masterfit[-4] #fudge pylab.plot(intime-intime0, masterfit / 10**nrm,'g',lw='3') # reject outliers if status == 0: rejtime = []; rejdata = []; naxis2 = 0 for i in range(len(masterfit)): if abs(indata[i] - masterfit[i]) > nsig * mastersigma[i] and i in cadencelis: rejtime.append(intime[i]) rejdata.append(indata[i]) rejtime = array(rejtime,dtype='float64') rejdata = array(rejdata,dtype='float32') if plot: pylab.plot(rejtime-intime0,rejdata / 10**nrm,'ro') # new data for output file if status == 0: outdata = indata / masterfit outerr = inerr / masterfit # plot ranges if status == 0 and plot: pylab.xlim(xmin-xr*0.01,xmax+xr*0.01) if ymin >= 0.0: pylab.ylim(ymin-yr*0.01,ymax+yr*0.01) else: pylab.ylim(1.0e-10,ymax+yr*0.01) # plot residual data if status == 0 and plot: ax = pylab.axes([0.06,0.09,0.93,0.43]) # force tick labels to be absolute rather than relative if status == 0 and plot: pylab.gca().xaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) pylab.gca().yaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False)) # rotate y labels by 90 deg labels = ax.get_yticklabels() setp(labels, 'rotation', 90) # clean up y-axis units if status == 0: pout = copy(outdata) ylab = 'Normalized Flux' # data limits if status == 0 and plot: ymin = pout.min() ymax = pout.max() yr = ymax - ymin pout = insert(pout,[0],[0.0]) pout = append(pout,0.0) pylab.plot(ptime[1:-1],pout[1:-1],color=lcolor,linestyle='-',linewidth=lwidth) pylab.fill(ptime,pout,color=fcolor,linewidth=0.0,alpha=falpha) pylab.xlabel(xlab, {'color' : 'k'}) pylab.ylabel(ylab, {'color' : 'k'}) pylab.grid() # plot ranges if status == 0 and plot: pylab.xlim(xmin-xr*0.01,xmax+xr*0.01) if ymin >= 0.0: pylab.ylim(ymin-yr*0.01,ymax+yr*0.01) else: pylab.ylim(1.0e-10,ymax+yr*0.01) # render plot if status == 0 and plot: pylab.savefig(re.sub('.fits','.png',outfile)) if cmdLine: pylab.show(block=True) else: pylab.ion() pylab.plot([]) pylab.ioff() # add NaNs back into data if status == 0: n = 0 work1 = array([],dtype='float32') work2 = array([],dtype='float32') instr, status = kepio.openfits(infile,'readonly',logfile,verbose) table, status = kepio.readfitstab(infile,instr[1],logfile,verbose) tn = table.field('time') dn = table.field(datacol) for i in range(len(table.field(0))): if numpy.isfinite(tn[i]) and numpy.isfinite(dn[i]) and numpy.isfinite(err[i]): try: work1 = numpy.append(work1,outdata[n]) work2 = numpy.append(work2,outerr[n]) n += 1 except: pass else: work1 = numpy.append(work1,numpy.nan) work2 = numpy.append(work2,numpy.nan) # history keyword in output file if status == 0: status = kepkey.history(call,instr[0],outfile,logfile,verbose) # write output file try: col1 = pyfits.Column(name='DETSAP_FLUX',format='E13.7',array=work1) col2 = pyfits.Column(name='DETSAP_FLUX_ERR',format='E13.7',array=work2) cols = instr[1].data.columns + col1 + col2 instr[1] = pyfits.new_table(cols,header=instr[1].header) instr.writeto(outfile) except ValueError: try: instr[1].data.field('DETSAP_FLUX')[:] = work1 instr[1].data.field('DETSAP_FLUX_ERR')[:] = work2 instr.writeto(outfile) except: message = 'ERROR -- KEPFLATTEN: cannot add DETSAP_FLUX data to FITS file' status = kepmsg.err(logfile,message,verbose) # close input file if status == 0: status = kepio.closefits(instr,logfile,verbose) ## end time if (status == 0): message = 'KEPFLATTEN completed at' else: message = '\nKEPFLATTEN aborted at' kepmsg.clock(message,logfile,verbose)