def poetSave(event, filedir='..', topdir=None): #Append system path if topdir == None: r = os.getcwd().split("/") topdir = "/".join(r[:r.index("run")]) sys.path.append(topdir + '/lib/') me.saveevent(event, filedir + "/" + event.eventname + "_p5c")
def reduceWFC3(eventname, eventdir, madVariable=False, madVarSet=False, isplots=False): ''' Reduces data images and calculated optimal spectra. Parameters ---------- isplots : Set True to produce plots Returns ------- None Remarks ------- Requires eventname_params file to intialize event object Steps ----- 1. Read in all data frames and header info 2. Record JD, scan direction, etc 3. Group images by frame, batch, and orbit number 4. Calculate centroid of direct image(s) 5. Calculate trace and 1D+2D wavelength solutions 6. Make flats, apply flat field correction 7. Manually mask regions 8. Apply light-time correction 9. Compute difference frames 10. Compute scan length 11. Perform outlier rejection of BG region 12. Background subtraction 13. Compute 2D drift, apply rough (integer-pixel) correction 14. Full-frame outlier rejection for time-series stack of NDRs 15. Apply sub-pixel 2D drift correction 16. Extract spectrum through summation 17. Compute median frame 18. Optimal spectral extraction 19. Save results, plot figures History ------- Written by Kevin Stevenson January 2017 ''' evpname = eventname + '_params' #exec 'import ' + evpname + ' as evp' in locals() #exec('import ' + evpname + ' as evp', locals()) exec('import ' + evpname + ' as evp', globals()) reload(evp) t0 = time.time() # Initialize event object # All parameters are specified in this file ev = evp.event_init() try: aux = evp.aux_init() except: print("Need to update event file to include auxiliary object.") return ev.eventdir = eventdir # Create directories if not os.path.exists(ev.eventdir): os.makedirs(ev.eventdir) if not os.path.exists(ev.eventdir + "/figs"): os.makedirs(ev.eventdir + "/figs") # Copy ev_params file shutil.copyfile(evpname + '.py', ev.eventdir + '/' + evpname + '.py') # Reset attribute for MAD variable (added by K. Showalter) if madVariable: setattr(ev, madVariable, madVarSet) ev.madVarStr = madVariable ev.madVariable = madVarSet # Object ev.obj_list = [] #Do not rename ev.obj_list! if ev.objfile == None: #Retrieve files within specified range for i in range(ev.objstart, ev.objend): ev.obj_list.append(ev.loc_sci + ev.filebase + str(i).zfill(4) + ".fits") elif ev.objfile == 'all': #Retrieve all files from science directory for fname in os.listdir(ev.loc_sci): ev.obj_list.append(ev.loc_sci + '/' + fname) ev.obj_list = sn.sort_nicely(ev.obj_list) else: #Retrieve filenames from list files = np.genfromtxt(ev.objfile, dtype=str, comments='#') for fname in files: ev.obj_list.append(ev.loc_sci + '/' + fname) # handle = open(ev.objfile) # for line in handle: # print(line) # ev.obj_list.append(ev.loc_sci + line) # handle.close() ev.n_files = len(ev.obj_list) #Determine image size and filter/grism hdulist = pf.open(ev.obj_list[0].rstrip()) nx = hdulist['SCI', 1].header['NAXIS1'] ny = hdulist['SCI', 1].header['NAXIS2'] ev.grism = hdulist[0].header['FILTER'] ev.detector = hdulist[0].header['DETECTOR'] ev.flatoffset = [[ -1 * hdulist['SCI', 1].header['LTV2'], -1 * hdulist['SCI', 1].header['LTV1'] ]] n_reads = hdulist['SCI', 1].header['SAMPNUM'] hdulist.close() # Record JD and exposure times print('Reading data & headers, recording JD and exposure times...') ywindow = ev.ywindow[0] xwindow = ev.xwindow[0] subny = ywindow[1] - ywindow[0] subnx = xwindow[1] - xwindow[0] subdata = np.zeros((ev.n_files, n_reads, subny, subnx)) suberr = np.zeros((ev.n_files, n_reads, subny, subnx)) data_mhdr = [] data_hdr = [] ev.jd = np.zeros(ev.n_files) ev.exptime = np.zeros(ev.n_files) for m in range(ev.n_files): data, err, hdr, mhdr = hst.read(ev.obj_list[m].rstrip()) subdata[m] = data[0, :, ywindow[0]:ywindow[1], xwindow[0]:xwindow[1]] suberr[m] = err[0, :, ywindow[0]:ywindow[1], xwindow[0]:xwindow[1]] data_mhdr.append(mhdr[0]) data_hdr.append(hdr[0]) ev.jd[m] = 2400000.5 + 0.5 * (data_mhdr[m]['EXPSTART'] + data_mhdr[m]['EXPEND']) ev.exptime[m] = data_mhdr[m]['EXPTIME'] # Assign scan direction ev.scandir = np.zeros(ev.n_files, dtype=int) ev.n_scan0 = 0 ev.n_scan1 = 0 try: scan0 = data_mhdr[0]['POSTARG2'] scan1 = data_mhdr[1]['POSTARG2'] for m in range(ev.n_files): if data_mhdr[m]['POSTARG2'] == scan0: ev.n_scan0 += 1 elif data_mhdr[m]['POSTARG2'] == scan1: ev.scandir[m] = 1 ev.n_scan1 += 1 else: print('WARNING: Unknown scan direction for file ' + str(m) + '.') print("# of files in scan direction 0: " + str(ev.n_scan0)) print("# of files in scan direction 1: " + str(ev.n_scan1)) except: ev.n_scan0 = ev.n_files print("Unable to determine scan direction, assuming unidirectional.") # Group frames into frame, batch, and orbit number ev.framenum, ev.batchnum, ev.orbitnum = hst.groupFrames(ev.jd) # Determine read noise and gain ev.readNoise = np.mean( (data_mhdr[0]['READNSEA'], data_mhdr[0]['READNSEB'], data_mhdr[0]['READNSEC'], data_mhdr[0]['READNSED'])) print('Read noise: ' + str(ev.readNoise)) print('Gain: ' + str(ev.gain)) #ev.v0 = (ev.readNoise/ev.gain)**2 #Units of ADU ev.v0 = ev.readNoise**2 #Units of electrons # Calculate centroid of direct image(s) ev.img_list = [] if isinstance(ev.directfile, str) and ev.directfile.endswith('.fits'): ev.img_list.append(ev.loc_cal + ev.directfile) else: #Retrieve filenames from list handle = open(ev.directfile) for line in handle: ev.img_list.append(ev.loc_cal + line) handle.close() ev.n_img = len(ev.img_list) ev.centroid, ev.directim = hst.imageCentroid(ev.img_list, ev.centroidguess, ev.centroidtrim, ny, ev.obj_list[0]) """ # Calculate theoretical centroids along spatial scan direction ev.centroids = [] for j in range(ev.n_img): ev.centroids.append([]) for i in range(ev.n_spec): # Can assume that scan direction is only in y direction (no x component) # because we will apply drift correction to make it so ev.centroids[j].append([np.zeros(subny)+ev.centroid[j][0],ev.centroid[j][1]]) # Calculate trace print("Calculating 2D trace and wavelength assuming " + ev.grism + " filter/grism...") ev.xrange = [] for i in range(ev.n_spec): ev.xrange.append(np.arange(ev.xwindow[i][0],ev.xwindow[i][1])) ev.trace2d = [] ev.wave2d = [] for j in range(ev.n_img): ev.trace2d.append([]) ev.wave2d.append([]) for i in range(ev.n_spec): ev.trace2d[j].append(hst.calcTrace(ev.xrange[i], ev.centroids[j][i], ev.grism)) ev.wave2d[j].append(hst.calibrateLambda(ev.xrange[i], ev.centroids[j][i], ev.grism)/1e4) #wavelength in microns if ev.detector == 'IR': print("Calculating slit shift values using last frame...") i = 0 #Use first spectrum j = -1 #Use last image #spectrum = subdata[j] spectrum = pf.getdata(ev.obj_list[j]) ev.slitshift, ev.shift_values, ev.yfit = hst.calc_slitshift2(spectrum, ev.xrange[i], ev.ywindow[i], ev.xwindow[i]) ev.wavegrid = ev.wave2d ev.wave = [] for j in range(ev.n_img): ev.wave.append([]) for i in range(ev.n_spec): ev.wave[j].append(np.mean(ev.wavegrid[j][i],axis=0)) else: # Assume no slitshift for UVIS ev.yfit = range(ev.ywindow[0][1] - ev.ywindow[0][0]) ev.slitshift = np.zeros(ev.ywindow[0][1] - ev.ywindow[0][0]) ev.shift_values = np.zeros(len(ev.yfit)) # Make list of master flat field frames subflat = np.ones((ev.n_img,ev.n_spec,subny,subnx)) flatmask = np.ones((ev.n_img,ev.n_spec,subny,subnx)) if ev.flatfile == None: print('No flat frames found.') flat_hdr = None flat_mhdr = None else: print('Loading flat frames...') for j in range(ev.n_img): tempflat, tempmask = hst.makeflats(ev.flatfile, ev.wavegrid[j], ev.xwindow, ev.ywindow, ev.flatoffset, ev.n_spec, ny, nx, sigma=ev.flatsigma) for i in range(ev.n_spec): subflat[j][i] = tempflat[i][ywindow[0]:ywindow[1],xwindow[0]:xwindow[1]] flatmask[j][i] = tempmask[i][ywindow[0]:ywindow[1],xwindow[0]:xwindow[1]] # Manually mask regions [specnum, colstart, colend] if hasattr(ev, 'manmask'): print("\rMasking manually identified bad pixels.") for j in range(ev.n_img): for i in range(len(ev.manmask)): ind, colstart, colend, rowstart, rowend = ev.manmask[i] n = ind % ev.n_spec flatmask[j][n][rowstart:rowend,colstart:colend] = 0 #ev.window[:,ind][0]:ev.window[:,ind][1] # Calculate reduced image for m in range(ev.n_files): #Select appropriate flat, mask, and slitshift if ev.n_img == (np.max(ev.orbitnum)+1): j = int(ev.orbitnum[m]) else: j = 0 for n in range(n_reads): subdata[m][n] /= subflat[j][0] """ # Read in drift2D from previous iteration # np.save("drift2D.npy",ev.drift2D) #try: # drift2D = np.load("drift2D.npy") #except: # print("drift2D.npy not found.") drift2D = np.zeros((ev.n_files, n_reads - 1, 2)) # Calculate centroids for each grism frame ev.centroids = np.zeros((ev.n_files, n_reads - 1, 2)) for m in range(ev.n_files): for n in range(n_reads - 1): ev.centroids[m, n] = np.array([ ev.centroid[0][0] + drift2D[m, n, 0], ev.centroid[0][1] + drift2D[m, n, 1] ]) #ev.centroids[m,n] = np.array([np.zeros(subny)+ev.centroid[0][0]+drift2D[m,n,0], # np.zeros(subnx)+ev.centroid[0][1]+drift2D[m,n,1]]) # Calculate trace print("Calculating 2D trace and wavelength assuming " + ev.grism + " filter/grism...") ev.xrange = np.arange(ev.xwindow[0][0], ev.xwindow[0][1]) trace2d = np.zeros((ev.n_files, n_reads - 1, subny, subnx)) wave2d = np.zeros((ev.n_files, n_reads - 1, subny, subnx)) for m in range(ev.n_files): for n in range(n_reads - 1): trace2d[m, n] = hst.calcTrace(ev.xrange, ev.centroids[m, n], ev.grism) wave2d[m, n] = hst.calibrateLambda( ev.xrange, ev.centroids[m, n], ev.grism) / 1e4 #wavelength in microns # Assume no slitshift ev.yfit = range(ev.ywindow[0][1] - ev.ywindow[0][0]) ev.slitshift = np.zeros(ev.ywindow[0][1] - ev.ywindow[0][0]) ev.shift_values = np.zeros(len(ev.yfit)) ev.wave = np.mean(wave2d, axis=2) print("Wavelength Range: %.3f - %.3f" % (np.min(ev.wave), np.max(ev.wave))) #iwmax = np.where(ev.wave[0][0]>1.65)[0][0] #print(ev.wave[0,0]) #print(ev.wave[0,1]) #print(ev.centroids) # Make list of master flat field frames subflat = np.ones((ev.n_files, subny, subnx)) flatmask = np.ones((ev.n_files, subny, subnx)) if ev.flatfile == None: print('No flat frames found.') flat_hdr = None flat_mhdr = None else: print('Loading flat frames...') print(ev.flatfile) for m in range(ev.n_files): tempflat, tempmask = hst.makeflats(ev.flatfile, [np.mean(wave2d[m], axis=0)], ev.xwindow, ev.ywindow, ev.flatoffset, ev.n_spec, ny, nx, sigma=ev.flatsigma) #tempflat = [pf.getdata(ev.flatfile)] #tempmask = [np.ones(tempflat[0].shape)] subflat[m] = tempflat[0][ywindow[0]:ywindow[1], xwindow[0]:xwindow[1]] flatmask[m] = tempmask[0][ywindow[0]:ywindow[1], xwindow[0]:xwindow[1]] # Manually mask regions [specnum, colstart, colend] if hasattr(ev, 'manmask'): print("\rMasking manually identified bad pixels.") for m in range(ev.n_files): for i in range(len(ev.manmask)): ind, colstart, colend, rowstart, rowend = ev.manmask[i] flatmask[m][rowstart:rowend, colstart:colend] = 0 #FINDME: Change flat field #subflat[:,:,28] /= 1.015 #subflat[:,:,50] /= 1.015 #subflat[:,:,70] *= 1.01 """ plt.figure(2) plt.clf() plt.imshow(np.copy(subdata[10,-1]),origin='lower',aspect='auto', vmin=0,vmax=25000,cmap=plt.cm.RdYlBu_r) plt.ylim(65,95) plt.show() """ # Calculate reduced image subdata /= subflat[:, np.newaxis] #subdata /= np.mean(subflat,axis=0)[np.newaxis,np.newaxis] """ # FINDME # Perform self flat field calibration # drift2D_int = np.round(edrift2D,0) # Identify frames outside SAA iNoSAA = np.where(np.round(drift2D[:,0,0],0)==0)[0] # Select subregion with lots of photons normdata = np.copy(subdata[iNoSAA,-1,69:91,15:147]) normmask = flatmask[iNoSAA,69:91,15:147] normdata[np.where(normmask==0)] = 0 # Normalize flux in each row to remove ramp/transit/variable scan rate normdata /= np.sum(normdata,axis=2)[:,:,np.newaxis] # Divide by mean spectrum to remove wavelength dependence normdata /= np.mean(normdata,axis=(0,1))[np.newaxis,np.newaxis,:] # Average frames to get flat-field correction flat_norm = np.mean(normdata,axis=0) flat_norm[np.where(np.mean(normmask,axis=0)<1)] = 1 ''' normdata /= np.mean(normdata,axis=(1,2))[:,np.newaxis,np.newaxis] flat_window = np.median(normdata,axis=0) medflat = np.median(flat_window, axis=0) flat_window /= medflat flat_window /= np.median(flat_window,axis=1)[:,np.newaxis] flat_norm = flat_window/np.mean(flat_window) ''' plt.figure(3) plt.clf() plt.imshow(np.copy(subdata[10,-1]),origin='lower',aspect='auto', vmin=0,vmax=25000,cmap=plt.cm.RdYlBu_r) plt.ylim(65,95) ff = np.load('ff.npy') subff = ff[ywindow[0]:ywindow[1],xwindow[0]:xwindow[1]] #subdata[:,:,69:91,15:147] /= flat_norm subdata /= subff plt.figure(4) plt.clf() plt.imshow(subdata[10,-1],origin='lower',aspect='auto',vmin=0,vmax=25000,cmap=plt.cm.RdYlBu_r) plt.ylim(65,95) plt.figure(1) plt.clf() plt.imshow(flat_norm,origin='lower',aspect='auto') plt.colorbar() plt.tight_layout() plt.pause(0.1) ev.flat_norm = flat_norm return ev """ """ if isplots: # Plot normalized flat fields plt.figure(1000, figsize=(12,8)) plt.clf() plt.suptitle('Master Flat Frames') for i in range(ev.n_spec): for j in range(ev.n_img): #plt.subplot(ev.n_spec,ev.n_img,i*ev.n_img+j+1) plt.subplot(2,np.ceil(ev.n_img/2.),i*ev.n_img+j+1) plt.title(str(j) +','+ str(i)) plt.imshow(subflat[j][i], origin='lower') plt.tight_layout() plt.savefig(ev.eventdir + '/figs/fig1000-Flats.png') # Plot masks plt.figure(1001, figsize=(12,8)) plt.clf() plt.suptitle('Mask Frames') for i in range(ev.n_spec): for j in range(ev.n_img): #plt.subplot(ev.n_spec,ev.n_img,i*ev.n_img+j+1) plt.subplot(2,np.ceil(ev.n_img/2.),i*ev.n_img+j+1) plt.title(str(j) +','+ str(i)) plt.imshow(flatmask[j][i], origin='lower') plt.tight_layout() plt.savefig(ev.eventdir + '/figs/fig1001-Masks.png') if ev.detector == 'IR': # Plot Slit shift plt.figure(1004, figsize=(12,8)) plt.clf() plt.suptitle('Model Slit Tilts/Shifts') plt.plot(ev.shift_values, ev.yfit, '.') plt.plot(ev.slitshift, range(ev.ywindow[0][0],ev.ywindow[0][1]), 'r-', lw=2) plt.xlim(-1,1) plt.savefig(ev.eventdir + '/figs/fig1004-SlitTilt.png') plt.pause(0.1) """ ev.ra = data_mhdr[0]['RA_TARG'] * np.pi / 180 ev.dec = data_mhdr[0]['DEC_TARG'] * np.pi / 180 if ev.horizonsfile != None: # Apply light-time correction, convert to BJD_TDB # Horizons file created for HST around time of observations print("Converting times to BJD_TDB...") ev.bjd_corr = suntimecorr.suntimecorr(ev.ra, ev.dec, ev.jd, ev.horizonsfile) bjdutc = ev.jd + ev.bjd_corr / 86400. ev.bjdtdb = utc_tt.utc_tt(bjdutc, ev.leapdir) print('BJD_corr range: ' + str(ev.bjd_corr[0]) + ', ' + str(ev.bjd_corr[-1])) else: print("No Horizons file found.") ev.bjdtdb = ev.jd if n_reads > 1: ev.n_reads = n_reads # Subtract pairs of subframes diffdata = np.zeros((ev.n_files, ev.n_reads - 1, subny, subnx)) differr = np.zeros((ev.n_files, ev.n_reads - 1, subny, subnx)) for m in range(ev.n_files): for n in range(n_reads - 1): #diffmask[m,n] = np.copy(flatmask[j][0]) #diffmask[m,n][np.where(suberr[m,n ] > diffthresh*np.std(suberr[m,n ]))] = 0 #diffmask[m,n][np.where(suberr[m,n+1] > diffthresh*np.std(suberr[m,n+1]))] = 0 diffdata[m, n] = subdata[m, n + 1] - subdata[m, n] differr[m, n] = np.sqrt(suberr[m, n + 1]**2 + suberr[m, n]**2) else: # FLT data has already been differenced # FLT files subtract first from last, 2 reads ev.n_reads = 2 diffdata = subdata differr = suberr diffmask = np.zeros((ev.n_files, ev.n_reads - 1, subny, subnx)) guess = np.zeros((ev.n_files, ev.n_reads - 1), dtype=int) for m in range(ev.n_files): #Select appropriate mask #if ev.n_img == (np.max(ev.orbitnum)+1): # j = int(ev.orbitnum[m]) #else: # j = 0 for n in range(n_reads - 1): diffmask[m, n] = np.copy(flatmask[m][0]) try: diffmask[m, n][np.where( differr[m, n] > ev.diffthresh * np.median(differr[m, n], axis=1)[:, np.newaxis])] = 0 #diffdata[m,n] *= diffmask[m,n] except: # May fail for FLT files print("Diffthresh failed.") foo = diffdata[m, n] * diffmask[m, n] guess[m, n] = np.median(np.where(foo > np.mean(foo))[0]).astype(int) # Guess may be skewed if first file is zeros if guess[m, 0] < 0 or guess[m, 0] > subny: guess[m, 0] = guess[m, 1] # Compute full scan length ev.scanHeight = np.zeros(ev.n_files) for m in range(ev.n_files): scannedData = np.sum(subdata[m, -1], axis=1) xmin = np.min(guess[m]) xmax = np.max(guess[m]) scannedData /= np.median(scannedData[xmin:xmax + 1]) scannedData -= 0.5 #leftEdge = np.where(scannedData > 0)/2)[0][0] #rightEdge = np.where(scannedData > 0)/2)[0][-1] #yrng = range(leftEdge-5, leftEdge+5, 1) yrng = range(subny) spline = spi.UnivariateSpline(yrng, scannedData[yrng], k=3, s=0) roots = spline.roots() try: ev.scanHeight[m] = roots[1] - roots[0] except: pass #Outlier rejection of sky background along time axis print("Performing background outlier rejection...") import sigrej, optspex for p in range(2): iscan = np.where(ev.scandir == p)[0] if len(iscan) > 0: for n in range(ev.n_reads - 1): # Set limits on the sky background x1 = (guess[iscan, n].min() - ev.fitbghw).astype(int) x2 = (guess[iscan, n].max() + ev.fitbghw).astype(int) bgdata1 = diffdata[iscan, n, :x1] bgmask1 = diffmask[iscan, n, :x1] bgdata2 = diffdata[iscan, n, x2:] bgmask2 = diffmask[iscan, n, x2:] bgerr1 = np.median(suberr[iscan, n, :x1]) bgerr2 = np.median(suberr[iscan, n, x2:]) estsig1 = [bgerr1 for j in range(len(ev.sigthresh))] estsig2 = [bgerr2 for j in range(len(ev.sigthresh))] diffmask[iscan, n, :x1] = sigrej.sigrej(bgdata1, ev.sigthresh, bgmask1, estsig1) diffmask[iscan, n, x2:] = sigrej.sigrej(bgdata2, ev.sigthresh, bgmask2, estsig2) # Write background #global bg, diffmask def writeBG(arg): background, mask, m, n = arg bg[m, n] = background diffmask[m, n] = mask return # STEP 3: Fit sky background with out-of-spectra data # FINDME: parallelrize bg subtraction print("Performing background subtraction...") x1 = np.zeros((ev.n_files, ev.n_reads - 1), dtype=int) x2 = np.zeros((ev.n_files, ev.n_reads - 1), dtype=int) bg = np.zeros((diffdata.shape)) if ev.ncpu == 1: # Only 1 CPU for m in range(ev.n_files): for n in range(ev.n_reads - 1): x1[m, n] = (guess[m, n] - ev.fitbghw).astype(int) x2[m, n] = (guess[m, n] + ev.fitbghw).astype(int) writeBG( hst.fitbg(diffdata[m, n], diffmask[m, n], x1[m, n], x2[m, n], ev.bgdeg, ev.p3thresh, isplots, m, n, ev.n_files)) else: # Multiple CPUs pool = mp.Pool(ev.ncpu) for m in range(ev.n_files): for n in range(ev.n_reads - 1): x1[m, n] = (guess[m, n] - ev.fitbghw).astype(int) x2[m, n] = (guess[m, n] + ev.fitbghw).astype(int) res = pool.apply_async(hst.fitbg, args=(diffdata[m, n], diffmask[m, n], x1[m, n], x2[m, n], ev.bgdeg, ev.p3thresh, isplots, m, n, ev.n_files), callback=writeBG) pool.close() pool.join() res.wait() print(" Done.") # STEP 2: Calculate variance bgerr = np.std(bg, axis=2) / np.sqrt(np.sum(diffmask, axis=2)) bgerr[np.where(np.isnan(bgerr))] = 0. ev.v0 += np.mean(bgerr**2) variance = abs(diffdata) / ev.gain + ev.v0 #variance = abs(subdata*submask) / gain + v0 # Perform background subtraction diffdata -= bg # ''' foo = np.sum(diffdata*diffmask, axis=2) guess = [] for i in range(nreads-1): guess.append(np.median(np.where(foo[i] > np.mean(foo[i]))[0]).astype(int)) guess = np.array(guess) # Guess may be skewed if first file is zeros if guess[0] < 0 or guess[0] > subnx: guess[0] = guess[1] ''' # Write drift2D def writeDrift2D(arg): drift2D, m, n = arg # Assign to array of spectra and uncertainties ev.drift2D[m, n] = drift2D return ''' # Calulate drift2D def calcDrift2D():#im1, im2, m, n): print("test") drift2D = imr.chi2_shift(im1, im2, boundary='constant', nthreads=4, zeromean=False, return_error=False) return (drift2D, m, n) ''' print("Calculating 2D drift...") #FINDME: instead of calculating scanHeight, consider fitting stretch factor ev.drift2D = np.zeros((ev.n_files, ev.n_reads - 1, 2)) if ev.ncpu == 1: # Only 1 CPU for m in range(ev.n_files): p = int(ev.scandir[m]) for n in range(ev.n_reads - 1): writeDrift2D( hst.calcDrift2D( diffdata[ev.iref[p], n] * diffmask[ev.iref[p], n], diffdata[m, n] * diffmask[m, n], m, n, ev.n_files)) else: # Multiple CPUs pool = mp.Pool(ev.ncpu) for m in range(ev.n_files): p = int(ev.scandir[m]) for n in range(ev.n_reads - 1): #res = pool.apply_async(hst.calcDrift2D) res = pool.apply_async( hst.calcDrift2D, args=(diffdata[ev.iref[p], n] * diffmask[ev.iref[p], n], diffdata[m, n] * diffmask[m, n], m, n, ev.n_files), callback=writeDrift2D) pool.close() pool.join() res.wait() print(" Done.") #np.save("drift2D.npy",ev.drift2D) #global shiftdata, shiftmask print("Performing rough, pixel-scale drift correction...") import scipy.ndimage.interpolation as spni ev.drift2D_int = np.round(ev.drift2D, 0) shiftdata = np.zeros(diffdata.shape) shiftmask = np.zeros(diffmask.shape) shiftvar = np.zeros(diffdata.shape) shiftbg = np.zeros(diffdata.shape) # Correct for drift by integer pixel numbers, no interpolation for m in range(ev.n_files): for n in range(ev.n_reads - 1): shiftdata[m, n] = spni.shift(diffdata[m, n], -1 * ev.drift2D_int[m, n, ::-1], order=0, mode='constant', cval=0) shiftmask[m, n] = spni.shift(diffmask[m, n], -1 * ev.drift2D_int[m, n, ::-1], order=0, mode='constant', cval=0) shiftvar[m, n] = spni.shift(variance[m, n], -1 * ev.drift2D_int[m, n, ::-1], order=0, mode='constant', cval=0) shiftbg[m, n] = spni.shift(bg[m, n], -1 * ev.drift2D_int[m, n, ::-1], order=0, mode='constant', cval=0) """ # spni.shift does not handle constant boundaries correctly if ev.drift2D_int[m,n,0] > 0: shiftdata[m,n,:,-1*ev.drift2D_int[m,n,0]:] = 0 shiftmask[m,n,:,-1*ev.drift2D_int[m,n,0]:] = 0 shiftvar [m,n,:,-1*ev.drift2D_int[m,n,0]:] = 0 shiftbg [m,n,:,-1*ev.drift2D_int[m,n,0]:] = 0 elif ev.drift2D_int[m,n,0] < 0: #print(m,n,-1*ev.drift2D_int[m,n,0]) shiftdata[m,n,:,:-1*ev.drift2D_int[m,n,0]] = 0 shiftmask[m,n,:,:-1*ev.drift2D_int[m,n,0]] = 0 shiftvar [m,n,:,:-1*ev.drift2D_int[m,n,0]] = 0 shiftbg [m,n,:,:-1*ev.drift2D_int[m,n,0]] = 0 """ # Outlier rejection of full frame along time axis print("Performing full-frame outlier rejection...") for p in range(2): iscan = np.where(ev.scandir == p)[0] if len(iscan) > 0: for n in range(ev.n_reads - 1): #y1 = guess[ev.iref,n] - ev.spec_width #y2 = guess[ev.iref,n] + ev.spec_width #estsig = [differr[ev.iref,n,y1:y2] for j in range(len(ev.sigthresh))] shiftmask[iscan, n] = sigrej.sigrej(shiftdata[iscan, n], ev.sigthresh, shiftmask[iscan, n]) #, estsig) """ # Replace bad pixels using 2D Gaussian kernal along x and time axes def writeReplacePixels(arg): shift, m, n, i, j = arg shiftdata[m,n,i,j] = shift return #import smoothing #reload(smoothing) ny, nx, sy, sx = (2,2,1,1) wherebad = np.array(np.where(shiftmask==0)).T #smdata = np.copy(shiftdata) print("Replacing " + str(len(wherebad)) + " bad pixels...") k = 0 ktot = len(wherebad) #FINDME: multiple CPUs is inefficient if ev.ncpu >= 1: # Only 1 CPU for m,n,i,j in wherebad: #sys.stdout.write('\r'+str(k+1)+'/'+str(len(wherebad))) #sys.stdout.flush() writeReplacePixels(hst.replacePixels(shiftdata[:,n,:,j], shiftmask[:,n,:,j], m, n, i, j, k, ktot, ny, nx, sy, sx)) #Pad image initially with zeros #newim = np.zeros(np.array(shiftdata[:,n,:,j].shape) + 2*np.array((ny, nx))) #newim[ny:-ny, nx:-nx] = shiftdata[:,n,:,j] #Calculate kernel #gk = smoothing.gauss_kernel_mask2((ny,nx), (sy,sx), (m,i), shiftmask[:,n,:,j]) #shiftdata[m,n,i,j] = np.sum(gk * newim[m:m+2*ny+1, i:i+2*nx+1]) k += 1 else: # Multiple CPUs pool = mp.Pool(ev.ncpu) for m,n,i,j in wherebad: res = pool.apply_async(hst.replacePixels, args=(shiftdata[:,n,:,j], shiftmask[:,n,:,j], m, n, i, j, k, ktot, ny, nx, sy, sx), callback=writeReplacePixels) k += 1 pool.close() pool.join() res.wait() print(" Done.") """ if isplots >= 3: for m in range(ev.n_files): for n in range(ev.n_reads - 1): plt.figure(1010) plt.clf() plt.suptitle(str(m) + "," + str(n)) plt.subplot(211) plt.imshow(shiftdata[m, n] * shiftmask[m, n], origin='lower', aspect='auto', vmin=0, vmax=500) plt.subplot(212) #plt.imshow(submask[i], origin='lower', aspect='auto', vmax=1) mean = np.median(shiftbg[m, n]) std = np.std(shiftbg[m, n]) plt.imshow(shiftbg[m, n], origin='lower', aspect='auto', vmin=mean - 3 * std, vmax=mean + 3 * std) plt.savefig(ev.eventdir + '/figs/fig1010-' + str(m) + '-' + str(n) + '-Image+Background.png') #plt.pause(0.1) """ apdata = np.zeros((ev.n_files,ev.n_reads-1,ev.spec_width*2,subnx)) apmask = np.zeros((ev.n_files,ev.n_reads-1,ev.spec_width*2,subnx)) apvar = np.zeros((ev.n_files,ev.n_reads-1,ev.spec_width*2,subnx)) apbg = np.zeros((ev.n_files,ev.n_reads-1,ev.spec_width*2,subnx)) for n in range(ev.n_reads-1): y1 = guess[ev.iref,n] - ev.spec_width y2 = guess[ev.iref,n] + ev.spec_width apdata[:,n] = shiftdata[:,n,y1:y2] apmask[:,n] = shiftmask[:,n,y1:y2] apvar [:,n] = shiftvar [:,n,y1:y2] apbg [:,n] = shiftbg [:,n,y1:y2] """ print("Performing sub-pixel drift correction...") istart = 0 #corrdata = np.zeros(diffdata.shape) #corrmask = np.zeros(diffdata.shape) # Select aperture data apdata = np.zeros((ev.n_files, ev.n_reads - 1, ev.spec_width * 2, subnx)) apmask = np.zeros((ev.n_files, ev.n_reads - 1, ev.spec_width * 2, subnx)) apvar = np.zeros((ev.n_files, ev.n_reads - 1, ev.spec_width * 2, subnx)) apbg = np.zeros((ev.n_files, ev.n_reads - 1, ev.spec_width * 2, subnx)) iy, ix = np.indices((ev.spec_width * 2, subnx)) iyy, ixx = np.indices((subny, subnx)) #FINDME: should be using (3,3) kx, ky = (1, 1) # Correct for drift for n in range(ev.n_reads - 1): #FINDME: change below to handle case of single scan direction y1 = [ guess[ev.iref[0], n] - ev.spec_width, guess[ev.iref[1], n] - ev.spec_width ] #y2 = guess[ev.iref,n] + ev.spec_width for m in range(ev.n_files): p = int(ev.scandir[m]) # Data spline = spi.RectBivariateSpline(range(subny), range(subnx), shiftdata[m, n], kx=kx, ky=ky, s=0) apdata[m,n] = (spline.ev((iy+y1[p]+ev.drift2D[m,n,1]-ev.drift2D_int[m,n,1]).flatten(), (ix+ev.drift2D[m,n,0]-ev.drift2D_int[m,n,0]).flatten())).reshape \ (ev.spec_width*2,subnx) #spni.shift works identically to spi.RectBivariateSpline #set prefilter=False and order=3 to enable spline filtering (smoothing) #apdata[m,n] = spni.shift(shiftdata[m,n], ev.drift2D_int[m,n,::-1]-ev.drift2D[m,n,::-1], order=1, # mode='constant', cval=0, prefilter=True)[y1:y2] # Mask spline = spi.RectBivariateSpline(range(subny), range(subnx), shiftmask[m, n], kx=kx, ky=ky, s=0) apmask[m,n] = np.round((spline.ev((iy+y1[p]+ev.drift2D[m,n,1]-ev.drift2D_int[m,n,1]).flatten(), (ix+ev.drift2D[m,n,0]-ev.drift2D_int[m,n,0]).flatten())).reshape \ (ev.spec_width*2,subnx),0).astype(int) # Variance spline = spi.RectBivariateSpline(range(subny), range(subnx), shiftvar[m, n], kx=kx, ky=ky, s=0) apvar[m,n] = (spline.ev((iy+y1[p]+ev.drift2D[m,n,1]-ev.drift2D_int[m,n,1]).flatten(), (ix+ev.drift2D[m,n,0]-ev.drift2D_int[m,n,0]).flatten())).reshape \ (ev.spec_width*2,subnx) # Background spline = spi.RectBivariateSpline(range(subny), range(subnx), shiftbg[m, n], kx=kx, ky=ky, s=0) apbg[m,n] = (spline.ev((iy+y1[p]+ev.drift2D[m,n,1]-ev.drift2D_int[m,n,1]).flatten(), (ix+ev.drift2D[m,n,0]-ev.drift2D_int[m,n,0]).flatten())).reshape \ (ev.spec_width*2,subnx) """ #Outlier rejection of aperture along time axis print("Performing aperture outlier rejection...") for n in range(ev.n_reads-1): #y1 = guess[ev.iref,n] - ev.spec_width #y2 = guess[ev.iref,n] + ev.spec_width #estsig = [differr[ev.iref,n,y1:y2] for j in range(len(ev.sigthresh))] apmask[:,n] = sigrej.sigrej(apdata[:,n], ev.sigthresh, apmask[:,n])#, estsig) """ # STEP 4: Extract standard spectrum and its variance #stdspec = np.zeros((ev.n_files,subnx)) #stdvar = np.zeros((ev.n_files,subnx)) #stdbg = np.zeros((ev.n_files,subnx)) #fracMaskReg = np.zeros(nreads-1) #for n in range(nreads-1): #stdspec = np.sum((apdata*apmask), axis=2) #stdvar = np.sum((apvar *apmask), axis=2) stdspec = np.sum(apdata, axis=2) stdvar = np.sum(apvar, axis=2) #stdbg = np.sum((bg *apmask), axis=2) # Compute fraction of masked pixels within regular spectral extraction window numpixels = 2. * ev.spec_width * subnx fracMaskReg = (numpixels - np.sum(apmask, axis=(2, 3))) / numpixels # Compute median frame ev.meddata = np.median(apdata, axis=0) # Extract optimal spectrum with uncertainties print("Performing optimal spectral extraction...") spectra = np.zeros((stdspec.shape)) specerr = np.zeros((stdspec.shape)) fracMaskOpt = np.zeros((ev.n_files, ev.n_reads - 1)) #tempmask = np.ones((ev.spec_width*2,subnx)) for m in range(ev.n_files): sys.stdout.write('\r' + str(m + 1) + '/' + str(ev.n_files)) sys.stdout.flush() for n in range(ev.n_reads - 1): #smoothspec = smooth.medfilt(stdspec[i], window_len) spectra[m, n], specerr[m, n], mask = optspex.optimize( apdata[m, n], apmask[m, n], apbg[m, n], stdspec[m, n], ev.gain, ev.v0, p5thresh=ev.p5thresh, p7thresh=ev.p7thresh, fittype=ev.fittype, window_len=ev.window_len, deg=ev.deg, n=m, iread=n, isplots=isplots, eventdir=ev.eventdir, meddata=ev.meddata[n]) # Compute fraction of masked pixels within optimal spectral extraction window numpixels = 1. * mask.size fracMaskOpt[m, n] = (np.sum(apmask[m, n]) - np.sum(mask)) / numpixels print(" Done.") if isplots >= 3: for m in range(ev.n_files): for n in range(ev.n_reads - 1): plt.figure(1011) plt.clf() plt.suptitle(str(m) + "," + str(n)) #plt.errorbar(ev.wave[m], stdspec, np.sqrt(stdvar), fmt='-') plt.errorbar(range(subnx), stdspec[m, n], np.sqrt(stdvar[m, n]), fmt='b-') plt.errorbar(range(subnx), spectra[m, n], specerr[m, n], fmt='g-') plt.savefig(ev.eventdir + '/figs/fig1011-' + str(m) + '-' + str(n) + '-Spectrum.png') #plt.pause(0.1) # Calculate total time total = (time.time() - t0) / 60. print('\nTotal time (min): ' + str(np.round(total, 2))) ev.guess = guess # Save results print('Saving results...') aux.spectra = spectra aux.specerr = specerr #aux.specbg = specbg aux.fracMaskReg = fracMaskReg aux.fracMaskOpt = fracMaskOpt aux.data_hdr = data_hdr aux.data_mhdr = data_mhdr aux.mask = mask #aux.trace2d = trace2d #aux.wave2d = wave2d #aux.bias_mhdr = bias_mhdr aux.subflat = subflat me.saveevent(aux, ev.eventdir + '/d-' + ev.eventname + '-data') me.saveevent(ev, ev.eventdir + '/d-' + ev.eventname + '-w2') #, delete=['flat_mhdr']) if isplots: # 2D light curve without drift correction plt.figure(1012, figsize=(8, ev.n_files / 20. + 0.8)) plt.clf() if ev.grism == 'G102': wmin = 0.82 wmax = 1.22 else: #G141 wmin = 1.125 wmax = 1.65 iwmin = np.where(ev.wave[0][0] > wmin)[0][0] iwmax = np.where(ev.wave[0][0] > wmax)[0][0] vmin = 0.97 vmax = 1.03 #FINDME normspec = np.zeros((ev.n_files, subnx)) for p in range(2): iscan = np.where(ev.scandir == p)[0] if len(iscan) > 0: normspec[iscan] = np.mean(spectra[iscan],axis=1)/ \ np.mean(spectra[iscan[ev.inormspec[0]:ev.inormspec[1]]],axis=(0,1)) #normspec = np.mean(spectra,axis=1)/np.mean(spectra[ev.inormspec[0]:ev.inormspec[1]],axis=(0,1)) #normspec = np.mean(spectra,axis=1)/np.mean(spectra[-6:],axis=(0,1)) #normspec = np.mean(ev.stdspec,axis=1)/np.mean(ev.stdspec[-6:],axis=(0,1)) ediff = np.zeros(ev.n_files) for m in range(ev.n_files): ediff[m] = 1e6 * np.median( np.abs(np.ediff1d(normspec[m, iwmin:iwmax]))) plt.scatter(ev.wave[0][0], np.zeros(subnx) + m, c=normspec[m], s=14, linewidths=0, vmin=vmin, vmax=vmax, marker='s', cmap=plt.cm.RdYlBu_r) plt.title("MAD = " + str(np.round(np.mean(ediff), 0).astype(int)) + " ppm") plt.xlim(wmin, wmax) plt.ylim(0, ev.n_files) plt.ylabel('Frame Number') plt.xlabel('Wavelength ($\mu m$)') plt.colorbar() plt.tight_layout() plt.savefig(ev.eventdir + '/figs/fig1012-2D_LC.png') ''' # Plot individual non-destructive reads vmin = 0.97 vmax = 1.03 iwmin = np.where(ev.wave[0][0]>wmin)[0][0] iwmax = np.where(ev.wave[0][0]>wmax)[0][0] normspec = spectra[:,istart:]/np.mean(spectra[-6:,istart:],axis=0) for n in range(ev.n_reads-1): plt.figure(1100+n, figsize=(8,6.5)) plt.clf() ediff = np.zeros(ev.n_files) for m in range(ev.n_files): ediff[m] = 1e6*np.median(np.abs(np.ediff1d(normspec[m,n,iwmin:iwmax]))) plt.scatter(ev.wave[0][0], np.zeros(normspec.shape[-1])+m, c=normspec[m,n], s=14,linewidths=0,vmin=vmin,vmax=vmax,marker='s',cmap=plt.cm.RdYlBu_r) plt.title("MAD = "+str(np.round(np.mean(ediff),0)) + " ppm") plt.xlim(wmin,wmax) plt.ylim(0,ev.n_files) plt.ylabel('Frame Number') plt.xlabel('Wavelength ($\mu m$)') plt.colorbar() plt.tight_layout() plt.savefig(ev.eventdir+'/figs/fig'+str(1100+n)+'-2D_LC.png') ''' #FINDME ev.spectra = spectra ev.subflat = subflat ev.subdata = subdata ev.suberr = suberr ev.diffdata = diffdata ev.differr = differr ev.diffmask = diffmask ev.shiftdata = shiftdata ev.shiftmask = shiftmask ev.bg = bg ev.apdata = apdata ev.apmask = apmask ev.stdspec = stdspec ''' #ev.mad2 = np.round(np.mean(ediff),0).astype(int) f = open('W2_MAD_'+ madVariable +'.txt','a+') f.write(str(madVarSet) + ',' + str(np.round(np.mean(ediff),0).astype(int)) + '\n') f.close() print('W2_MAD_'+ madVariable +'.txt saved\n') ''' return ev
def photometry(event, pcf, photdir, mute): tini = time.time() # Create photometry log logname = event.logname log = le.Logedit(photdir + "/" + logname, logname) log.writelog("\nStart " + photdir + " photometry: " + time.ctime()) parentdir = os.getcwd() + "/" os.chdir(photdir) # copy photom.pcf in photdir pcf.make_file("photom.pcf") # Parse the attributes from the control file to the event: attrib = vars(pcf) keys = attrib.keys() for key in keys: setattr(event, key, attrib.get(key).get()) maxnimpos, npos = event.maxnimpos, event.npos # allocating frame parameters: event.fp.aplev = np.zeros((npos, maxnimpos)) # aperture flux event.fp.aperr = np.zeros((npos, maxnimpos)) # aperture error event.fp.nappix = np.zeros((npos, maxnimpos)) # number of aperture pixels event.fp.skylev = np.zeros((npos, maxnimpos)) # background sky flux level event.fp.skyerr = np.zeros((npos, maxnimpos)) # sky error event.fp.nskypix = np.zeros((npos, maxnimpos)) # number of sky pixels event.fp.nskyideal = np.zeros( (npos, maxnimpos)) # ideal number of sky pixels event.fp.status = np.zeros((npos, maxnimpos)) # apphot return status event.fp.good = np.zeros((npos, maxnimpos)) # good flag # Aperture photometry: if not event.dooptimal or event.from_aper == None: # Multy Process set up: # Shared memory arrays allow only 1D Arrays :( aplev = Array("d", np.zeros(npos * maxnimpos)) aperr = Array("d", np.zeros(npos * maxnimpos)) nappix = Array("d", np.zeros(npos * maxnimpos)) skylev = Array("d", np.zeros(npos * maxnimpos)) skyerr = Array("d", np.zeros(npos * maxnimpos)) nskypix = Array("d", np.zeros(npos * maxnimpos)) nskyideal = Array("d", np.zeros(npos * maxnimpos)) status = Array("d", np.zeros(npos * maxnimpos)) good = Array("d", np.zeros(npos * maxnimpos)) # Size of chunk of data each core will process: chunksize = maxnimpos / event.ncores + 1 print("Number of cores: " + str(event.ncores)) # Start Muti Procecess: processes = [] for nc in np.arange(event.ncores): start = nc * chunksize # Starting index to process end = (nc + 1) * chunksize # Ending index to process proc = Process(target=do_aphot, args=(start, end, event, log, mute, aplev, aperr, nappix, skylev, skyerr, nskypix, nskyideal, status, good)) processes.append(proc) proc.start() # Make sure all processes finish their work: for nc in np.arange(event.ncores): processes[nc].join() # Put the results in the event. I need to reshape them: event.fp.aplev = np.asarray(aplev).reshape(npos, maxnimpos) event.fp.aperr = np.asarray(aperr).reshape(npos, maxnimpos) event.fp.nappix = np.asarray(nappix).reshape(npos, maxnimpos) event.fp.skylev = np.asarray(skylev).reshape(npos, maxnimpos) event.fp.skyerr = np.asarray(skyerr).reshape(npos, maxnimpos) event.fp.nskypix = np.asarray(nskypix).reshape(npos, maxnimpos) event.fp.nskyideal = np.asarray(nskyideal).reshape(npos, maxnimpos) event.fp.status = np.asarray(status).reshape(npos, maxnimpos) event.fp.good = np.asarray(good).reshape(npos, maxnimpos) # raw photometry (no sky subtraction): event.fp.apraw = (event.fp.aplev + (event.fp.skylev * event.fp.nappix)) # Print results into the log if it wans't done before: for pos in np.arange(npos): for i in np.arange(event.nimpos[pos]): log.writelog( '\nframe =%7d ' % i + 'pos =%5d ' % pos + 'y =%7.3f ' % event.fp.y[pos, i] + 'x =%7.3f' % event.fp.x[pos, i] + '\n' + 'aplev =%11.3f ' % event.fp.aplev[pos, i] + 'aperr =%9.3f ' % event.fp.aperr[pos, i] + 'nappix =%6.2f' % event.fp.nappix[pos, i] + '\n' + 'skylev=%11.3f ' % event.fp.skylev[pos, i] + 'skyerr=%9.3f ' % event.fp.skyerr[pos, i] + 'nskypix=%6.2f ' % event.fp.nskypix[pos, i] + 'nskyideal=%6.2f' % event.fp.nskyideal[pos, i] + '\n' + 'status=%7d ' % event.fp.status[pos, i] + 'good =%5d' % event.fp.good[pos, i], mute=True) elif event.from_aper != None: # Load previous aperture photometry if required for optimal: evt = me.loadevent(parentdir + event.from_aper + "/" + event.eventname + "_pht") event.fp.aplev = evt.fp.aplev event.fp.aperr = evt.fp.aperr event.fp.nappix = evt.fp.nappix event.fp.skylev = evt.fp.skylev event.fp.skyerr = evt.fp.skyerr event.fp.nskypix = evt.fp.nskypix event.fp.nskyideal = evt.fp.nskyideal event.fp.status = evt.fp.status event.fp.good = evt.fp.good event.fp.apraw = evt.fp.apraw if event.dooptimal: ofp, psf = do.dooptphot(event.data, event.uncd, event.mask, event.fp, event.srcest, event.nimpos, rejlim=[10.45, 1000, 1.5], order=1, resize=event.oresize, norm=1, trim=event.otrim, log=log) event.fp = ofp event.psf = psf elif event.ispsf: # PSF aperture correction: log.writelog('Calculating PSF aperture:') event.aperfrac, event.psfnappix, event.psfskylev, \ event.psfnskypix, event.psfnskyideal, event.psfstatus \ = ap.apphot(event.psfim, event.psfctr, event.photap * event.psfexpand, event.skyin * event.psfexpand, event.skyout * event.psfexpand, med = event.skymed, expand = event.apscale, nappix = True, skylev = True, nskypix = True, nskyideal = True, status = True) event.aperfrac += event.psfskylev * event.psfnappix event.fp.aplev /= event.aperfrac event.fp.aperr /= event.aperfrac log.writelog('Aperture contains %f of PSF.' % event.aperfrac) # For running pixel-level decorrelation (pld) if event.ispld and event.npos == 1: event.apdata = pld.pld_box(event.data, event.targpos, event.pldhw, event.fp.skylev) log.writelog( "Created " + str(event.pldhw * 2 + 1) + "x" + str(event.pldhw * 2 + 1) + " box around centroid for pixel-level decorrelation and normalized it in time." ) elif event.ispld and event.npos != 1: log.writelog( "Could not perform pixel-level decorrelation because there is more than 1 nod position." ) # save print("\nSaving ...") me.saveevent(event, event.eventname + "_pht", delete=['data', 'uncd', 'mask']) # Print time elapsed and close log: cwd = os.getcwd() + "/" log.writelog("Output files (" + event.photdir + "):") log.writelog("Data:") log.writelog(" " + cwd + event.eventname + "_pht.dat") log.writelog("Log:") log.writelog(" " + cwd + logname) dt = t.hms_time(time.time() - tini) log.writeclose("\nEnd Photometry. Time (h:m:s): %s " % dt + " (" + photdir + ")") print("-------------- ------------\n")
def centering(event, pcf, centerdir, owd): os.chdir(centerdir) tini = time.time() # Create centering log log = le.Logedit(event.logname, event.logname) log.writelog("\nStart " + centerdir + " centering: " + time.ctime()) # Parse the attributes from the control file to the event: attrib = vars(pcf) keys = attrib.keys() for key in keys: setattr(event, key, attrib.get(key)) # Check least asym parameters work: if event.method in ['lac', 'lag']: if event.ctrim < (event.cradius + event.csize) and event.ctrim != 0: event.ctrim = event.cradius + event.csize + 1 log.writelog('Trim radius is too small, changed to: %i' % event.ctrim) if event.psfctrim < (event.psfcrad + event.psfcsize) and event.psfctrim != 0: event.psfctrim = event.psfcrad + event.psfcsize + 1 log.writelog('PSF Trim radius is too small, changed to: %i' % event.psfctrim) # Centering bad pixel mask: centermask = np.ones((event.ny, event.nx)) if event.ymask is not None: ymask = np.asarray(event.ymask, int) xmask = np.asarray(event.xmask, int) for i in range(len(ymask)): centermask[ymask[i], xmask[i]] = 0 # PSF: # Re-evaluate if a PSF has been redefined: if event.newpsf is not None: event.ispsf = os.path.isfile(event.newpsf) if event.ispsf: event.psffile = event.newpsf log.writelog('The PSF file has been redefined!') log.writelog("PSF: " + event.psffile) # PSF Centering: if event.ispsf: event.psfim = fits.getdata(event.psffile) # Guess of the center of the PSF (center of psfim) psfctrguess = np.asarray(np.shape(event.psfim)) // 2 # Do not find center of PSF: if event.nopsfctr: event.psfctr = psfctrguess # Find center of PSF: else: if event.method == "bpf" or event.method == "ipf": method = "fgc" else: method = event.method event.psfctr, extra = cd.centerdriver(method, event.psfim, psfctrguess, event.psfctrim, event.psfcrad, event.psfcsize, npskyrad=(event.npskyin, event.npskyout)) log.writelog('PSF center found.') else: event.psfim = None event.psfctr = None log.writelog('No PSF supplied.') # Find center of the mean Image: event.targpos = np.zeros((2, event.npos)) # Override target position estimate if specified if type(pcf.srcesty) != type(None) and type(pcf.srcestx) != type(None): srcesty = str(pcf.srcesty).split(',') srcestx = str(pcf.srcestx).split(',') if len(srcestx) != len(srcesty): print("WARNING: Length of srcest inputs do not match!") if len(srcestx) != event.npos or len(srcesty) != event.npos: print("WARNING: Length of srcest inputs do not match npos!") if len(srcestx) > 1 or len(srcesty) > 1: print( "Verify that srcest override order matches telescope pos order." ) for pos in range(event.npos): event.srcest[0, pos] = srcesty[pos] event.srcest[1, pos] = srcestx[pos] for pos in range(event.npos): print("Fitting mean image at pos: " + str(pos)) meanim = event.meanim[:, :, pos] guess = event.srcest[:, pos] targpos, extra = cd.centerdriver(event.method, meanim, guess, event.ctrim, event.cradius, event.csize, fitbg=event.fitbg, psf=event.psfim, psfctr=event.psfctr, expand=event.expand, npskyrad=(event.npskyin, event.npskyout)) event.targpos[:, pos] = targpos log.writelog("Center position(s) of the mean Image(s):\n" + str(np.transpose(event.targpos))) # Inclusion :::::::: # Multy Process set up: # Shared memory arrays allow only 1D Arrays :( x = Array("d", np.zeros(event.npos * event.maxnimpos)) y = Array("d", np.zeros(event.npos * event.maxnimpos)) xerr = Array("d", np.zeros(event.npos * event.maxnimpos)) yerr = Array("d", np.zeros(event.npos * event.maxnimpos)) xsig = Array("d", np.zeros(event.npos * event.maxnimpos)) ysig = Array("d", np.zeros(event.npos * event.maxnimpos)) rot = Array("d", np.zeros(event.npos * event.maxnimpos)) noisepix = Array("d", np.zeros(event.npos * event.maxnimpos)) flux = Array("d", np.zeros(event.npos * event.maxnimpos)) sky = Array("d", np.zeros(event.npos * event.maxnimpos)) goodfit = Array("d", np.zeros(event.npos * event.maxnimpos)) # Size of chunk of data each core will process: chunksize = event.maxnimpos // event.ccores + 1 print("Number of cores: " + str(event.ccores)) # Start Muti Procecess: :::::::::::::::::::::::::::::::::::::: processes = [] for nc in range(event.ccores): start = nc * chunksize # Starting index to process end = (nc + 1) * chunksize # Ending index to process proc = Process(target=do_center, args=(start, end, event, centermask, log, x, y, flux, sky, goodfit, xerr, yerr, xsig, ysig, noisepix, rot)) processes.append(proc) proc.start() # Make sure all processes finish their work: for nc in range(event.ccores): processes[nc].join() # Put the results in the event. I need to reshape them: event.fp.x = np.asarray(x).reshape(event.npos, event.maxnimpos) event.fp.y = np.asarray(y).reshape(event.npos, event.maxnimpos) event.fp.xerr = np.asarray(xerr).reshape(event.npos, event.maxnimpos) event.fp.yerr = np.asarray(yerr).reshape(event.npos, event.maxnimpos) event.fp.noisepix = np.asarray(noisepix).reshape(event.npos, event.maxnimpos) # If Gaussian fit: if event.method == 'fgc' or event.method == 'rfgc': event.fp.xsig = np.asarray(xsig).reshape(event.npos, event.maxnimpos) event.fp.ysig = np.asarray(ysig).reshape(event.npos, event.maxnimpos) event.fp.rot = np.asarray(rot).reshape(event.npos, event.maxnimpos) # If PSF fit: if event.method in ["ipf", "bpf"]: event.fp.flux = np.asarray(flux).reshape(event.npos, event.maxnimpos) event.fp.psfsky = np.asarray(sky).reshape(event.npos, event.maxnimpos) event.fp.goodfit = np.asarray(goodfit).reshape(event.npos, event.maxnimpos) # :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: # Pixel R position: event.fp.r = np.sqrt((event.fp.x % 1.0 - 0.5)**2.0 + (event.fp.y % 1.0 - 0.5)**2.0) log.writelog("End frames centering.") # Save print("\nSaving") if event.denoised: me.saveevent(event, event.eventname + "_ctr", save=['dendata', 'data', 'uncd', 'mask']) else: me.saveevent(event, event.eventname + "_ctr", save=['data', 'uncd', 'mask']) # Print time elapsed and close log: cwd = os.getcwd() log.writelog("Output files (" + event.centerdir + "):") log.writelog("Data:") log.writelog(" " + cwd + '/' + event.eventname + "_ctr.dat") log.writelog(" " + cwd + '/' + event.eventname + "_ctr.h5") log.writelog("Log:") log.writelog(" " + cwd + '/' + event.logname) dt = t.hms_time(time.time() - tini) log.writeclose("\nEnd Centering. Time (h:m:s): %s" % dt + " (" + event.centerdir + ")") print("------------- ------------\n") os.chdir(owd) if event.runp4: os.system("python3 poet.py p4 %s" % event.centerdir)
def checks(eventname, period=None, ephtime=None, cwd=None): if cwd == None: cwd = os.getcwd() os.chdir(cwd) # Load the Event event = me.loadevent(eventname) # Create a log oldlogname = event.logname logname = event.eventname + "_p5.log" log = le.Logedit(logname, oldlogname) log.writelog('\nStart Checks: ' + time.ctime()) # If p5 run after p3: we are using results from PSFfit: if not hasattr(event, "phottype"): event.phottype = "psffit" try: os.mkdir("psffit/") except: pass os.chdir("psffit/") # Move frame parameters to fit Kevin's syntax: # event.fp.param --> event.param event.filenames = event.fp.filename event.x = event.fp.x event.y = event.fp.y event.sx = event.fp.sx event.sy = event.fp.sy event.time = event.fp.time event.pos = event.fp.pos event.frmvis = event.fp.frmvis event.filename = event.eventname if event.phottype == "aper": event.good = event.fp.good event.aplev = event.fp.aplev event.aperr = event.fp.aperr event.background = event.fp.skylev log.writelog('Photometry method is APERTURE') elif event.phottype == "psffit": event.aplev = event.fp.psfflux event.background = event.fp.psfsky # FINDME: do something with aperr and good event.aperr = 0.0025 * np.mean( event.fp.psfflux) * (event.aplev * 0 + 1) event.good = np.ones(np.shape(event.aplev)) log.writelog('Photometry method is PSF FITTING') elif event.phottype == "optimal": event.good = event.fp.ogood event.aplev = event.fp.ophotlev event.aperr = event.fp.ophoterr # FINDME: Background from optimal? event.background = event.fp.psfsky log.writelog('Photometry method is OPTIMAL') # UPDATE period AND ephtime if period != None: event.period = period[0] event.perioderr = period[1] if ephtime != None: event.ephtime = ephtime[0] event.ephtimeerr = ephtime[1] log.writelog("\nCurrent event = " + event.eventname) log.writelog("Kurucz file = " + event.kuruczfile) log.writelog("Filter file = " + event.filtfile) # Light-time correction to BJD: # Julian observation date #event.juldat = event.jdjf80 + event.fp.time / 86400.0 event.juldat = event.fp.juldat = event.j2kjd + event.fp.time / 86400.0 if not event.ishorvec: log.writeclose('\nHorizon file not found!') return print("Calculating BJD correction...") event.fp.bjdcor = stc.suntimecorr(event.ra, event.dec, event.fp.juldat, event.horvecfile) # Get bjd times: event.bjdcor = event.fp.bjdcor #event.bjddat = event.fp.juldat + event.fp.bjdcor / 86400.0 event.bjdutc = event.fp.juldat + event.fp.bjdcor / 86400.0 # utc bjd date event.bjdtdb = np.empty(event.bjdutc.shape) for i in range(event.bjdtdb.shape[0]): event.bjdtdb[i] = utc_tt.utc_tdb( event.bjdutc[i]) # terrestial bjd date # ccampo 3/18/2011: check which units phase should be in try: if event.tep.ttrans.unit == "BJDTDB": event.timestd = "tdb" event.fp.phase = tp.time2phase(event.bjdtdb, event.ephtime, event.period, event.ecltype) else: event.timestd = "utc" event.fp.phase = tp.time2phase(event.bjdutc, event.ephtime, event.period, event.ecltype) except: event.timestd = "utc" event.fp.phase = tp.time2phase(event.bjdutc, event.ephtime, event.period, event.ecltype) # assign phase variable event.phase = event.fp.phase # ccampo 3/18/2011: moved this above # Eclipse phase, BJD #event.fp.phase = tp.time2phase(event.fp.juldat + event.fp.bjdcor / 86400.0, # event.ephtime, event.period, event.ecltype) # verify leapsecond correction hfile = event.filenames[0, 0] try: image, event.header = pf.getdata(hfile.decode('utf-8'), header=True) dt = ((event.bjdtdb - event.bjdutc) * 86400.0)[0, 0] dt2 = event.header['ET_OBS'] - event.header['UTCS_OBS'] log.writelog('Leap second correction : ' + str(dt) + ' = ' + str(dt2)) except: log.writelog('Could not verify leap-second correction.') log.writelog('Min and Max light-time correction: ' + np.str(np.amin(event.fp.bjdcor)) + ', ' + np.str(np.amax(event.fp.bjdcor)) + ' seconds') # Verify light-time correction try: image, event.header = pf.getdata(hfile.decode('utf-8'), header=True) try: log.writelog('BJD Light-time correction: ' + str(event.bjdcor[0, 0]) + ' = ' + str((event.header['BMJD_OBS'] - event.header['MJD_OBS']) * 86400)) except: log.writelog('HJD Light-time correction: ' + str(event.bjdcor[0, 0]) + ' = ' + str((event.header['HMJD_OBS'] - event.header['MJD_OBS']) * 86400)) except: log.writelog('Could not verify light-time correction.') # Number of good frames should be > 95% log.writelog("Good Frames = %7.3f" % (np.mean(event.good) * 100) + " %") log.writelog('\nCentering: X mean X stddev Y mean Y stddev') for pos in np.arange(event.npos): log.writelog( 'position %2d:' % pos + ' %10.5f' % np.mean(event.x[pos, np.where(event.good[pos])]) + ' %9.5f' % np.std(event.x[pos, np.where(event.good[pos])]) + ' %10.5f' % np.mean(event.y[pos, np.where(event.good[pos])]) + ' %9.5f' % np.std(event.y[pos, np.where(event.good[pos])])) # COMPUTE RMS POSITION CONSISTENCY event.xprecision = np.sqrt(np.median(np.ediff1d(event.x)**2)) event.yprecision = np.sqrt(np.median(np.ediff1d(event.y)**2)) log.writelog('RMS of x precision = ' + str(np.round(event.xprecision, 4)) + ' pixels.') log.writelog('RMS of y precision = ' + str(np.round(event.yprecision, 4)) + ' pixels.') if event.phottype == "aper": log.writelog('\nCenter & photometry half-width/aperture sizes = ' + str(event.ctrim) + ', ' + str(event.photap) + ' pixels.') log.writelog('Period = ' + str(event.period) + ' +/- ' + str(event.perioderr) + ' days') log.writelog('Ephemeris = ' + str(event.ephtime) + ' +/- ' + str(event.ephtimeerr) + ' JD') fmt1 = [ 'C0o', 'C1o', 'C2o', 'ro', 'ko', 'co', 'mo', 'bs', 'gs', 'ys', 'rs', 'ks', 'cs', 'ms' ] fmt2 = ['b,', 'g,', 'y,', 'r,'] plt.figure(501) plt.clf() plt.figure(502, figsize=(8, 12)) plt.clf() plt.figure(503) plt.clf() plt.figure(504) plt.clf() plt.figure(505) plt.clf() plt.figure(506) plt.clf() for pos in np.arange(event.npos): wheregood = np.where(event.good[pos, :]) # CHOOSE ONLY GOOD FRAMES FOR PLOTTING phase = event.phase[pos, :][wheregood] aplev = event.aplev[pos, :][wheregood] jdtime = event.bjdutc[pos, :][wheregood] background = event.background[pos, :][wheregood] # COMPUTE X AND Y PIXEL LOCATION RELATIVE TO ... if event.npos > 1: # CENTER OF EACH PIXEL y = (event.y[pos, :] - np.round(event.y[pos, :]))[wheregood] x = (event.x[pos, :] - np.round(event.x[pos, :]))[wheregood] else: # CENTER OF MEDIAN PIXEL y = (event.y[pos, :] - np.round(np.median(event.y)))[wheregood] x = (event.x[pos, :] - np.round(np.median(event.x)))[wheregood] # SORT aplev BY x, y AND radial POSITIONS rad = np.sqrt(x**2 + y**2) xx = np.sort(x) yy = np.sort(y) sxx = np.sort(event.sx[0]) syy = np.sort(event.sy[0]) rr = np.sort(rad) xaplev = aplev[np.argsort(x)] yaplev = aplev[np.argsort(y)] raplev = aplev[np.argsort(rad)] # BIN RESULTS FOR PLOTTING POSITION SENSITIVITY EFFECT nobj = aplev.size nbins = int(120 / event.npos) binxx = np.zeros(nbins) binyy = np.zeros(nbins) binsxx = np.zeros(nbins) binsyy = np.zeros(nbins) binrr = np.zeros(nbins) binxaplev = np.zeros(nbins) binyaplev = np.zeros(nbins) binraplev = np.zeros(nbins) binxapstd = np.zeros(nbins) binyapstd = np.zeros(nbins) binrapstd = np.zeros(nbins) binphase = np.zeros(nbins) binaplev = np.zeros(nbins) binapstd = np.zeros(nbins) for i in range(nbins): start = int(1. * i * nobj / nbins) end = int(1. * (i + 1) * nobj / nbins) binxx[i] = np.mean(xx[start:end]) binyy[i] = np.mean(yy[start:end]) binsxx[i] = np.mean(sxx[start:end]) binsyy[i] = np.mean(syy[start:end]) binrr[i] = np.mean(rr[start:end]) binxaplev[i] = np.median(xaplev[start:end]) binyaplev[i] = np.median(yaplev[start:end]) binraplev[i] = np.median(raplev[start:end]) binxapstd[i] = np.std(xaplev[start:end]) / np.sqrt(end - start) binyapstd[i] = np.std(yaplev[start:end]) / np.sqrt(end - start) binrapstd[i] = np.std(raplev[start:end]) / np.sqrt(end - start) binphase[i] = np.mean(phase[start:end]) binaplev[i] = np.median(aplev[start:end]) binapstd[i] = np.std(aplev[start:end]) / np.sqrt(end - start) # PLOT 1: flux plt.figure(501) plt.errorbar(binphase, binaplev, binapstd, fmt=fmt1[pos], linewidth=1, label=('pos %i' % (pos))) plt.title(event.planetname + ' Phase vs. Binned Flux') plt.xlabel('Orbital Phase') plt.ylabel('Flux') plt.legend(loc='best') # PLOT 2: position-flux plt.figure(502) plt.subplot(2, 1, 1) plt.title(event.planetname + ' Position vs. Binned Flux') plt.errorbar(binyy, binyaplev, binyapstd, fmt=fmt1[pos], label=('pos %i y' % (pos))) plt.ylabel('Flux') plt.legend(loc='best') plt.subplot(2, 1, 2) plt.errorbar(binxx, binxaplev, binxapstd, fmt=fmt1[pos], label=('pos %i x' % (pos))) plt.xlabel('Pixel Postion') plt.ylabel('Flux') plt.legend(loc='best') #PLOT 3: position-phase plt.figure(503) plt.plot(phase, x, 'b,') plt.plot(phase, y, 'r,') plt.title(event.planetname + ' Phase vs. Position') plt.xlabel('Orbital Phase') plt.ylabel('Pixel Position') plt.legend('xy') #PLOT 4: flux-radial distance plt.figure(504) plt.errorbar(binrr, binraplev, binrapstd, fmt=fmt1[pos], label=('pos %i' % (pos))) plt.title(event.planetname + ' Radial Distance vs. Flux') plt.xlabel('Distance From Center of Pixel') plt.ylabel('Flux') plt.legend(loc='best') # ::::::::::: Background setting ::::::::::::::::: if np.size(background) != 0: # number of points per bin: npoints = 42 nbins = int(np.size(background) / npoints) medianbg = np.zeros(nbins) bphase = np.zeros(nbins) # background bin phase bintime = np.zeros(nbins) # background bin JD time for i in range(nbins): start = int(1.0 * i * npoints) end = int(1.0 * (i + 1) * npoints) medianbg[i] = np.median(background[start:end]) bphase[i] = np.mean(phase[start:end]) bintime[i] = np.mean(jdtime[start:end]) # PLOT 5: background-phase day = int(np.floor(np.amin(jdtime))) timeunits1 = jdtime - day timeunits2 = bintime - day xlabel = 'JD - ' + str(day) if event.ecltype == 's': timeunits1 = phase timeunits2 = bphase xlabel = 'Phase' plt.figure(505) plt.plot(timeunits1, background, color='0.45', linestyle='None', marker=',') if np.size(background) > 10000: plt.plot(timeunits2, medianbg, fmt2[pos], label='median bins') plt.title(event.planetname + ' Background level') plt.xlabel(xlabel) plt.ylabel('Flux') # PLOT 6: width-flux plt.figure(506) plt.subplot(2, 1, 1) plt.title(event.planetname + ' Gaussian Width vs. Binned Flux') plt.errorbar(binsyy, binyaplev, binyapstd, fmt=fmt1[pos], label=('width %i y' % (pos))) plt.ylabel('Flux') plt.legend(loc='best') plt.subplot(2, 1, 2) plt.errorbar(binsxx, binxaplev, binxapstd, fmt=fmt1[pos], label=('width %i x' % (pos))) plt.xlabel('Gaussian Width') plt.ylabel('Flux') plt.legend(loc='best') figname1 = str(event.eventname) + "-fig501.png" figname2 = str(event.eventname) + "-fig502.png" figname3 = str(event.eventname) + "-fig503.png" figname4 = str(event.eventname) + "-fig504.png" figname5 = str(event.eventname) + "-fig505.png" figname6 = str(event.eventname) + "-fig506.png" plt.figure(501) plt.savefig(figname1) plt.figure(502) plt.savefig(figname2) plt.figure(503) plt.savefig(figname3) plt.figure(504) plt.savefig(figname4) plt.figure(505) plt.plot(timeunits1[0], background[0], color='0.45', linestyle='None', marker=',', label='all points') plt.legend(loc='best') plt.savefig(figname5) plt.figure(506) plt.savefig(figname6) # Saving me.saveevent(event, event.eventname + "_p5c") cwd = os.getcwd() + "/" # Print outputs, end-time, and close log. log.writelog("Output files:") log.writelog("Data:") log.writelog(" " + cwd + event.eventname + "_p5c.dat") log.writelog("Log:") log.writelog(" " + cwd + logname) log.writelog("Figures:") log.writelog(" " + cwd + figname1) log.writelog(" " + cwd + figname2) log.writelog(" " + cwd + figname3) log.writelog(" " + cwd + figname4) log.writelog(" " + cwd + figname5) log.writelog(" " + cwd + figname6) log.writeclose('\nEnd Checks: ' + time.ctime()) return event
def checks1(eventname, cwd, period=None, ephtime=None): owd = os.getcwd() os.chdir(cwd) # Load the Event event = me.loadevent(eventname) # Create a log oldlogname = event.logname logname = event.eventname + "_p5.log" log = le.Logedit(logname, oldlogname) log.writelog('\nStart Checks: ' + time.ctime()) # If p5 run after p3: we are using results from PSFfit: if not hasattr(event, "phottype"): event.phottype = "psffit" try: os.mkdir("psffit/") except: pass os.chdir("psffit/") # Move frame parameters to fit Kevin's syntax: # event.fp.param --> event.param event.filenames = event.fp.filename event.x = event.fp.x event.y = event.fp.y event.time = event.fp.time event.pos = event.fp.pos event.frmvis = event.fp.frmvis event.filename = event.eventname event.aplev = event.fp.aplev event.background = event.fp.skylev event.good = event.fp.good if event.phottype == "aper": event.aperr = event.fp.aperr log.writelog('Photometry method is APERTURE') elif event.phottype == "var": event.aperr = event.fp.aperr log.writelog('Photometry method is VARIABLE APERTURE') elif event.phottype == "ell": event.aperr = event.fp.aperr log.writelog('Photometry method is ELLIPTICAL APERTURE') elif event.phottype == "psffit": # FINDME: do something with aperr event.aperr = .0025 * np.mean(event.aplev) * np.ones( np.shape(event.aplev)) log.writelog('Photometry method is PSF FITTING') elif event.phottype == "optimal": event.aperr = event.fp.aperr log.writelog('Photometry method is OPTIMAL') # UPDATE period AND ephtime if period is not None: event.period = period[0] event.perioderr = period[1] if ephtime is not None: event.ephtime = ephtime[0] event.ephtimeerr = ephtime[1] log.writelog("\nCurrent event = " + event.eventname) log.writelog("Kurucz file = " + event.kuruczfile) log.writelog("Filter file = " + event.filtfile) # Light-time correction to BJD: # Julian observation date #event.juldat = event.jdjf80 + event.fp.time / 86400.0 event.juldat = event.fp.juldat = event.j2kjd + event.fp.time / 86400.0 if not event.ishorvec: log.writeclose('\nHorizon file not found!') return print("Calculating BJD correction...") event.fp.bjdcor = np.zeros(event.fp.juldat.shape) # Sometimes bad files are just missing files, in which case they have # times of 0, which causes problem in the following interpolation. So # we must mask out these files. We don't use the event.fp.good mask # because we may want to know the bjd of bad images nonzero = np.where(event.fp.time != 0.0) event.fp.bjdcor[nonzero] = stc.suntimecorr(event.ra, event.dec, event.fp.juldat[nonzero], event.horvecfile) # Get bjd times: event.bjdcor = event.fp.bjdcor #event.bjddat = event.fp.juldat + event.fp.bjdcor / 86400.0 event.bjdutc = event.fp.juldat + event.fp.bjdcor / 86400.0 # utc bjd date event.bjdtdb = np.empty(event.bjdutc.shape) for i in range(event.bjdtdb.shape[0]): event.bjdtdb[i] = utc_tt.utc_tdb(event.bjdutc[i], event.topdir + '/' + event.leapdir) # terrestial bjd date # ccampo 3/18/2011: check which units phase should be in try: if event.tep.ttrans.unit == "BJDTDB": event.timestd = "tdb" event.fp.phase = tp.time2phase(event.bjdtdb, event.ephtime, event.period, event.ecltype) else: event.timestd = "utc" event.fp.phase = tp.time2phase(event.bjdutc, event.ephtime, event.period, event.ecltype) except: event.timestd = "utc" event.fp.phase = tp.time2phase(event.bjdutc, event.ephtime, event.period, event.ecltype) # assign phase variable event.phase = event.fp.phase # ccampo 3/18/2011: moved this above # Eclipse phase, BJD #event.fp.phase = tp.time2phase(event.fp.juldat + event.fp.bjdcor / 86400.0, # event.ephtime, event.period, event.ecltype) # verify leapsecond correction hfile = event.filenames[0, 0] try: image, event.header = fits.getdata(hfile, header=True) dt = ((event.bjdtdb - event.bjdutc) * 86400.0)[0, 0] dt2 = event.header['ET_OBS'] - event.header['UTCS_OBS'] log.writelog('Leap second correction : ' + str(dt) + ' = ' + str(dt2)) except: log.writelog('Could not verify leap-second correction.') log.writelog('Min and Max light-time correction: ' + np.str(np.amin(event.fp.bjdcor)) + ', ' + np.str(np.amax(event.fp.bjdcor)) + ' seconds') # Verify light-time correction try: image, event.header = fits.getdata(hfile, header=True) try: log.writelog('BJD Light-time correction: ' + str(event.bjdcor[0, 0]) + ' = ' + str((event.header['BMJD_OBS'] - event.header['MJD_OBS']) * 86400)) except: log.writelog('HJD Light-time correction: ' + str(event.bjdcor[0, 0]) + ' = ' + str((event.header['HMJD_OBS'] - event.header['MJD_OBS']) * 86400)) except: log.writelog('Could not verify light-time correction.') # Number of good frames should be > 95% log.writelog("Good Frames = %7.3f" % (np.mean(event.good) * 100) + " %") log.writelog('\nCentering: X mean X stddev Y mean Y stddev') for pos in range(event.npos): log.writelog( 'position %2d:' % pos + ' %10.5f' % np.mean(event.x[pos, np.where(event.good[pos])]) + ' %9.5f' % np.std(event.x[pos, np.where(event.good[pos])]) + ' %10.5f' % np.mean(event.y[pos, np.where(event.good[pos])]) + ' %9.5f' % np.std(event.y[pos, np.where(event.good[pos])])) # COMPUTE RMS POSITION CONSISTENCY event.xprecision = np.sqrt(np.mean(np.ediff1d(event.x)**2)) event.yprecision = np.sqrt(np.mean(np.ediff1d(event.y)**2)) log.writelog('RMS of x precision = ' + str(np.round(event.xprecision, 4)) + ' pixels.') log.writelog('RMS of y precision = ' + str(np.round(event.yprecision, 4)) + ' pixels.') if event.phottype == "aper": log.writelog('\nCenter & photometry half-width/aperture sizes = ' + str(event.ctrim) + ', ' + str(event.photap) + ' pixels.') log.writelog('Period = ' + str(event.period) + ' +/- ' + str(event.perioderr) + ' days') log.writelog('Ephemeris = ' + str(event.ephtime) + ' +/- ' + str(event.ephtimeerr) + ' JD') # Compute elliptical area if gaussian centering if event.method == 'fgc' or event.method == 'rfgc': event.fp.ellarea = np.pi * (3 * event.fp.xsig) * (3 * event.fp.ysig) fmt1 = [ 'bo', 'go', 'yo', 'ro', 'ko', 'co', 'mo', 'bs', 'gs', 'ys', 'rs', 'ks', 'cs', 'ms' ] fmt2 = ['b,', 'g,', 'y,', 'r,'] fmt3 = ['b.', 'g.', 'y.', 'r.'] plt.figure(501) plt.clf() plt.figure(502, figsize=(8, 12)) plt.clf() plt.figure(503) plt.clf() plt.figure(504) plt.clf() plt.figure(505) plt.clf() for pos in range(event.npos): wheregood = np.where(event.good[pos, :]) # CHOOSE ONLY GOOD FRAMES FOR PLOTTING phase = event.phase[pos, :][wheregood] aplev = event.aplev[pos, :][wheregood] jdtime = event.bjdutc[pos, :][wheregood] background = event.background[pos, :][wheregood] noisepix = event.fp.noisepix[pos, :][wheregood] if event.method == "fgc" or event.method == "rfgc": ellarea = event.fp.ellarea[pos, :][wheregood] rot = event.fp.rot[pos, :][wheregood] # COMPUTE X AND Y PIXEL LOCATION RELATIVE TO ... if event.npos > 1: # CENTER OF EACH PIXEL y = (event.y[pos, :] - np.round(event.y[pos, :]))[wheregood] x = (event.x[pos, :] - np.round(event.x[pos, :]))[wheregood] else: # CENTER OF MEDIAN PIXEL y = (event.y[pos, :] - np.round(np.median(event.y)))[wheregood] x = (event.x[pos, :] - np.round(np.median(event.x)))[wheregood] # SORT aplev BY x, y AND radial POSITIONS rad = np.sqrt(x**2 + y**2) xx = np.sort(x) yy = np.sort(y) rr = np.sort(rad) xaplev = aplev[np.argsort(x)] yaplev = aplev[np.argsort(y)] raplev = aplev[np.argsort(rad)] # BIN RESULTS FOR PLOTTING POSITION SENSITIVITY EFFECT nobj = aplev.size nbins = 120 // event.npos binxx = np.zeros(nbins) binyy = np.zeros(nbins) binrr = np.zeros(nbins) binxaplev = np.zeros(nbins) binyaplev = np.zeros(nbins) binraplev = np.zeros(nbins) binxapstd = np.zeros(nbins) binyapstd = np.zeros(nbins) binrapstd = np.zeros(nbins) binphase = np.zeros(nbins) binaplev = np.zeros(nbins) binapstd = np.zeros(nbins) binnpix = np.zeros(nbins) for i in range(nbins): start = int(1. * i * nobj / nbins) end = int(1. * (i + 1) * nobj / nbins) binxx[i] = np.mean(xx[start:end]) binyy[i] = np.mean(yy[start:end]) binrr[i] = np.mean(rr[start:end]) binxaplev[i] = np.median(xaplev[start:end]) binyaplev[i] = np.median(yaplev[start:end]) binraplev[i] = np.median(raplev[start:end]) binxapstd[i] = np.std(xaplev[start:end]) / np.sqrt(end - start) binyapstd[i] = np.std(yaplev[start:end]) / np.sqrt(end - start) binrapstd[i] = np.std(raplev[start:end]) / np.sqrt(end - start) binphase[i] = np.mean(phase[start:end]) binaplev[i] = np.median(aplev[start:end]) binapstd[i] = np.std(aplev[start:end]) / np.sqrt(end - start) binnpix[i] = np.mean(noisepix[start:end]) # PLOT 1: flux plt.figure(501) plt.errorbar(binphase, binaplev, binapstd, fmt=fmt1[pos], linewidth=1, label=('pos %i' % (pos))) plt.title(event.planetname + ' Phase vs. Binned Flux') plt.xlabel('Orbital Phase') plt.ylabel('Flux') plt.legend(loc='best') # PLOT 2: position-flux plt.figure(502) plt.subplot(2, 1, 1) plt.title(event.planetname + ' Position vs. Binned Flux') plt.errorbar(binyy, binyaplev, binyapstd, fmt=fmt1[pos], label=('pos %i y' % (pos))) plt.ylabel('Flux') plt.legend(loc='best') plt.subplot(2, 1, 2) plt.errorbar(binxx, binxaplev, binxapstd, fmt=fmt1[pos], label=('pos %i x' % (pos))) plt.xlabel('Pixel Postion') plt.ylabel('Flux') plt.legend(loc='best') #PLOT 3: position-phase plt.figure(503) plt.plot(phase, x, 'b,') plt.plot(phase, y, 'r,') plt.title(event.planetname + ' Phase vs. Position') plt.xlabel('Orbital Phase') plt.ylabel('Pixel Position') plt.legend('xy') #PLOT 4: flux-radial distance plt.figure(504) plt.errorbar(binrr, binraplev, binrapstd, fmt=fmt1[pos], label=('pos %i' % (pos))) plt.title(event.planetname + ' Radial Distance vs. Flux') plt.xlabel('Distance From Center of Pixel') plt.ylabel('Flux') plt.legend(loc='best') # ::::::::::: Background setting ::::::::::::::::: if np.size(background) != 0: # number of points per bin: npoints = 42 nbins = int(np.size(background) // npoints) medianbg = np.zeros(nbins) bphase = np.zeros(nbins) # background bin phase bintime = np.zeros(nbins) # background bin JD time for i in range(nbins): start = int(1.0 * i * npoints) end = int(1.0 * (i + 1) * npoints) medianbg[i] = np.median(background[start:end]) bphase[i] = np.mean(phase[start:end]) bintime[i] = np.mean(jdtime[start:end]) # PLOT 5: background-phase day = int(np.floor(np.amin(jdtime))) timeunits1 = jdtime - day timeunits2 = bintime - day xlabel = 'JD - ' + str(day) if event.ecltype == 's': timeunits1 = phase timeunits2 = bphase xlabel = 'Phase' plt.figure(505) plt.plot(timeunits1, background, color='0.45', linestyle='None', marker=',') if np.size(background) > 10000: plt.plot(timeunits2, medianbg, fmt2[pos], label='median bins') plt.title(event.planetname + ' Background level') plt.xlabel(xlabel) plt.ylabel('Flux') plt.plot(timeunits1[0], background[0], color='0.45', linestyle='None', marker=',', label='all points') plt.legend(loc='best') else: print("WARNING: background has zero size.") #PLOT 7: Noise Pixels Binned plt.figure(507) plt.scatter(binphase, binnpix) plt.xlabel("Orbital Phase") plt.ylabel("Noise Pixels") plt.title(event.planetname + " Binned Noise Pixels") #PLOT 8: Noise Pixel Variance plt.figure(508) npixvar = bd.subarnvar(noisepix, event) subarnbinphase = bd.subarnbin(phase, event) plt.scatter(subarnbinphase, npixvar, s=1) plt.xlabel("Orbital Phase") plt.ylabel("Noise Pixel Variance") plt.title(event.planetname + " Noise Pixels Variance") #PLOT 9 and 10: Elliptical Area and Variance if event.method == 'fgc' or event.method == 'rfgc': plt.figure(509) plt.scatter(phase, ellarea, s=0.1) plt.xlabel("Orbital Phase") plt.ylabel("Elliptical Area") plt.title(event.planetname + " Gaussian Centering Elliptical Area") plt.figure(510) ellareavar = bd.subarnvar(ellarea, event) plt.scatter(subarnbinphase, ellareavar, s=1) plt.xlabel("Orbital Phase") plt.ylabel("Elliptical Area Variance") plt.title(event.planetname + " Elliptical Area Variance") if event.method == 'rfgc': plt.figure(511) plt.scatter(phase, rot % (np.pi / 2) * 180 / np.pi, s=1) plt.xlabel("Orbital Phase") plt.ylabel("Rotation (deg)") plt.title(event.planetname + " Gaussian Centering Rotation") #PLOT 6: Preflash if event.havepreflash: plt.figure(506) plt.errorbar((event.prefp.time[0] - event.prefp.time[0, 0]) / 60., event.prefp.aplev[0], yerr=event.prefp.aperr[0], fmt="o") plt.xlabel("Time since start of preflash (minutes)") plt.ylabel("Flux") plt.title(event.planetname + " Preflash") figname1 = str(event.eventname) + "-fig501.png" figname2 = str(event.eventname) + "-fig502.png" figname3 = str(event.eventname) + "-fig503.png" figname4 = str(event.eventname) + "-fig504.png" figname5 = str(event.eventname) + "-fig505.png" figname6 = str(event.eventname) + "-fig506.png" figname7 = str(event.eventname) + "-fig507.png" figname8 = str(event.eventname) + "-fig508.png" figname9 = str(event.eventname) + "-fig509.png" figname10 = str(event.eventname) + "-fig510.png" figname11 = str(event.eventname) + "-fig511.png" plt.figure(501) plt.savefig(figname1) plt.figure(502) plt.savefig(figname2) plt.figure(503) plt.savefig(figname3) plt.figure(504) plt.savefig(figname4) plt.figure(505) plt.savefig(figname5) plt.figure(506) if event.havepreflash: plt.savefig(figname6) plt.figure(507) plt.savefig(figname7) plt.figure(508) plt.savefig(figname8) if event.method == 'fgc' or event.method == 'rfgc': plt.figure(509) plt.savefig(figname9) plt.figure(510) plt.savefig(figname10) if event.method == 'rfgc': plt.figure(511) plt.savefig(figname11) # Saving me.saveevent(event, event.eventname + "_p5c") cwd += "/" # Print outputs, end-time, and close log. log.writelog("Output files:") log.writelog("Data:") log.writelog(" " + cwd + event.eventname + "_p5c.dat") log.writelog("Log:") log.writelog(" " + cwd + logname) log.writelog("Figures:") log.writelog(" " + cwd + figname1) log.writelog(" " + cwd + figname2) log.writelog(" " + cwd + figname3) log.writelog(" " + cwd + figname4) log.writelog(" " + cwd + figname5) if event.havepreflash: log.writelog(" " + cwd + figname6) log.writelog(" " + cwd + figname7) log.writelog(" " + cwd + figname8) if event.method == 'fgc' or event.method == 'rfgc': log.writelog(" " + cwd + figname9) log.writelog(" " + cwd + figname10) if event.method == 'rfgc': log.writelog(" " + cwd + figname11) log.writeclose('\nEnd Checks: ' + time.ctime()) os.chdir(owd) return event
def centering(event, pcf, centerdir): tini = time.time() # Create centering log logname = event.logname log = le.Logedit(centerdir + "/" + logname, logname) log.writelog("\nStart " + centerdir + " centering: " + time.ctime()) os.chdir(centerdir) # copy center.pcf in centerdir pcf.make_file("center.pcf") # Parse the attributes from the control file to the event: attrib = vars(pcf) keys = attrib.keys() for key in keys: setattr(event, key, attrib.get(key).get()) # Check least asym parameters work: if event.method in ['lac', 'lag']: if event.ctrim < (event.cradius + event.csize) and event.ctrim is not 0: event.ctrim = event.cradius + event.csize + 1 log.writelog('Trim radius is too small, changed to: %i'%event.ctrim) if event.psfctrim < (event.psfcrad + event.psfcsize) and event.psfctrim is not 0: event.psfctrim = event.psfcrad + event.psfcsize + 1 log.writelog('PSF Trim radius is too small, changed to: %i' %event.psfctrim) # Centering bad pixel mask: centermask = np.ones((event.ny, event.nx)) if event.ymask is not None: ymask = np.asarray(event.ymask, int) xmask = np.asarray(event.xmask, int) for i in np.arange(len(ymask)): centermask[ymask[i], xmask[i]] = 0 # PSF: # Re-evaluate if a PSF has been redefined: if event.newpsf is not None: event.ispsf = os.path.isfile(event.newpsf) if event.ispsf: event.psffile = event.newpsf log.writelog('The PSF file has been redefined!') log.writelog("PSF: " + event.psffile) # PSF Centering: if event.ispsf: event.psfim = pf.getdata(event.psffile) # Guess of the center of the PSF (center of psfim) psfctrguess = np.asarray(np.shape(event.psfim))/2 # Do not find center of PSF: if event.nopsfctr: event.psfctr = psfctrguess # Find center of PSF: else: ''' if event.method == "bpf" or event.method == "ipf": method = "fgc" else: method = event.method event.psfctr, extra = cd.centerdriver(method, event.psfim, psfctrguess, event.psfctrim, event.psfcrad, event.psfcsize) ''' # Always use 'fgc' on PSF, for testing event.psfctr, extra = cd.centerdriver("fgc", event.psfim, psfctrguess, event.psfctrim, event.psfcrad, event.psfcsize) #FINDME log.writelog('PSF center found.') print(event.psfctr) #FINDME else: event.psfim = None event.psfctr = None log.writelog('No PSF supplied.') # Find center of the mean Image: event.targpos = np.zeros((2, event.npos)) for pos in np.arange(event.npos): meanim = event.meanim[:,:,pos] guess = event.srcest[:, pos] targpos, extra = cd.centerdriver(event.method, meanim, guess, event.ctrim, event.cradius, event.csize, fitbg=event.fitbg, psf=event.psfim, psfctr=event.psfctr, expand=event.expand) event.targpos[:,pos] = targpos log.writelog("Center position(s) of the mean Image(s):\n" + str(np.transpose(event.targpos))) # Inclusion :::::::: # Multy Process set up: # Shared memory arrays allow only 1D Arrays :( event.maxnimpos = int(event.maxnimpos) event.npos = int(event.npos) x = Array("d", np.zeros(event.npos * event.maxnimpos)) y = Array("d", np.zeros(event.npos * event.maxnimpos)) sx = Array("d", np.zeros(event.npos * event.maxnimpos)) sy = Array("d", np.zeros(event.npos * event.maxnimpos)) flux = Array("d", np.zeros(event.npos * event.maxnimpos)) sky = Array("d", np.zeros(event.npos * event.maxnimpos)) goodfit = Array("d", np.zeros(event.npos * event.maxnimpos)) # Size of chunk of data each core will process: chunksize = event.maxnimpos/event.ccores + 1 print("Number of cores: " + str(event.ccores)) # Start Muti Procecess: :::::::::::::::::::::::::::::::::::::: processes = [] for nc in np.arange(event.ccores): start = nc * chunksize # Starting index to process end = (nc+1) * chunksize # Ending index to process proc = Process(target=do_center, args=(start, end, event, centermask, log, x, y, sx, sy, flux, sky, goodfit)) processes.append(proc) proc.start() # Make sure all processes finish their work: for nc in np.arange(event.ccores): processes[nc].join() # Put the results in the event. I need to reshape them: event.fp.x = np.asarray(x ).reshape(event.npos,event.maxnimpos) event.fp.y = np.asarray(y ).reshape(event.npos,event.maxnimpos) event.fp.sx = np.asarray(sx ).reshape(event.npos,event.maxnimpos) event.fp.sy = np.asarray(sy ).reshape(event.npos,event.maxnimpos) # If PSF fit: if event.method in ["ipf", "bpf"]: event.fp.flux = np.asarray(flux ).reshape(event.npos,event.maxnimpos) event.fp.psfsky = np.asarray(sky ).reshape(event.npos,event.maxnimpos) event.fp.goodfit = np.asarray(goodfit).reshape(event.npos,event.maxnimpos) # :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: # Pixel R position: event.fp.r = np.sqrt((event.fp.x % 1.0 - 0.5)**2.0 + (event.fp.y % 1.0 - 0.5)**2.0 ) log.writelog("End frames centering.") # Save print("\nSaving") if event.denoised: me.saveevent(event, event.eventname + "_ctr", save=['dendata', 'data', 'uncd', 'mask']) else: me.saveevent(event, event.eventname + "_ctr", save=['data', 'uncd', 'mask']) # Print time elapsed and close log: cwd = os.getcwd() + "/" log.writelog("Output files (" + event.centerdir + "):") log.writelog("Data:") log.writelog(" " + cwd + event.eventname + "_ctr.dat") log.writelog(" " + cwd + event.eventname + "_ctr.h5") log.writelog("Log:") log.writelog(" " + cwd + event.logname) dt = t.hms_time(time.time()-tini) log.writeclose("\nEnd Centering. Time (h:m:s): %s"%dt + " (" + event.centerdir + ")") print("------------- ------------\n") if hasattr(event, 'runp4') and event.runp4 == True: os.chdir(event.eventdir) os.system("poet.py p4 %s/"%event.centerdir)
def badpix(eventname, cwd): """ Modification History: --------------------- 2010-??-?? patricio Initial Python implementation 2014-08-13 garland switched the pyfits package to astropy.io.fits [email protected] 2017-06-20 zacchaeus Fixed None comparisons [email protected] """ owd = os.getcwd() os.chdir(cwd) tini = time.time() # Load the event event = me.loadevent(eventname) # Load the data me.updateevent(event, eventname, event.loadnext) # Create a new log starting from the old one. oldlogname = event.logname logname = event.eventname + ".log" log = le.Logedit(logname, oldlogname) event.logname = logname log.writelog('\nMARK: ' + time.ctime() + ': Starting p2badpix.') # ccampo 3/18/2011: do this in p5 # Julian observation date #event.fp.juldat = event.jdjf80 + event.fp.time / 86400.0 # ::::::::::::::::::::::: UNCERTAINTIES :::::::::::::::::::::::::::::::: # IRAC subarray data come with bogus uncertainties that are not linearly # related to photon noise. We scale them later, using the reduced chi # squared from the model fit. # ::::::::::::::::::::::: FLUX CONVERSION ::::::::::::::::::::::::::::: # Do we want flux (uJy/pix) or surface brightness (MJy/sr) units? If # doing photometry, convert to flux. Since we care about relative # numbers, it doesn't really matter. # Convert from surface brightness (MJy/sr) to flux units (uJy/pix) if event.fluxunits: log.writelog('Converting surface brightness to flux') event.data, event.uncd = btf.poet_bright2flux(event.data, event.uncd, event.posscl) if event.havepreflash: event.predata, event.preuncd = btf.poet_bright2flux( event.predata, event.preuncd, event.posscl) if event.havepostcal: event.postdata, event.postuncd = btf.poet_bright2flux( event.postdata, event.postuncd, event.posscl) else: log.writelog('Did not convert bright to flux.') # Mean Background Estimate, from zodi model event.estbg = (np.mean(event.fp.zodi[np.where(event.fp.exist)]) + np.mean(event.fp.ism[np.where(event.fp.exist)]) + np.mean(event.fp.cib[np.where(event.fp.exist)])) if event.fluxunits: event.estbg *= (event.srperas * 1e12 * np.mean(event.posscl[0, :]) * np.mean(event.posscl[1, :])) # Bad Pixel Masking log.writelog('Find and fix bad pixels') # Get permanent bad pixel mask. if not event.ispmask[0]: log.writelog('\nPermanent Bad pixel mask not found!') else: hdu = fits.open(event.pmaskfile[0]) if hdu[0].header['bitpix'] == -32: # if data type is float hdu[0].scale(type='int16') # cast it down to int16 event.pmask = hdu[0].data # IRS FIX: # IRS data contains the blue peak subarray while its pmask contains # the whole array (Hard coding) if event.photchan == 5: event.pmask = event.pmask[3:59, 86:127] # Do NOT define sigma, we have a different scheme for finding baddies # adds Spitzer rejects: fp.nsstrej & our rejects: fp.nsigrej event.mask = pbm.poet_badmask(event.data, event.uncd, event.pmask, event.inst.pcrit, event.bdmskd, event.inst.dcrit, event.fp, nimpos=event.nimpos) # User rejected pixels: if event.userrej is not None: for i in range(np.shape(event.userrej)[0]): event.mask[:, event.userrej[i, 0], event.userrej[i, 1], :] = 0 event.fp.userrej = np.sum(np.sum(1 - event.mask, axis=1), axis=1) event.fp.userrej = np.transpose(event.fp.userrej) - event.fp.nsstrej else: event.fp.userrej = np.zeros((event.npos, event.maxnimpos)) # define sigma here. # adds median sky: fp.medsky event.meanim = pcb.poet_chunkbad(event.data, event.uncd, event.mask, event.nimpos, event.sigma, event.szchunk, event.fp, event.nscyc) log.writelog('Masks combined') # Repeat procedure for preflash and postcal data: if event.havepreflash: event.premask = pbm.poet_badmask(event.predata, event.preuncd, event.pmask, event.inst.pcrit, event.prebdmskd, event.inst.dcrit, event.prefp, nimpos=event.prenimpos) if event.userrej is not None: for i in range(np.shape(event.userrej)[0]): event.premask[:, event.userrej[i, 0], event.userrej[i, 1], :] = 0 event.prefp.userrej = np.sum(np.sum(1 - event.premask, axis=1), axis=1) event.prefp.userrej = np.transpose( event.prefp.userrej) - event.prefp.nsstrej else: event.prefp.userrej = np.zeros((event.npos, event.premaxnimpos)) event.premeanim = pcb.poet_chunkbad(event.predata, event.preuncd, event.premask, event.prenimpos, event.sigma, event.szchunk, event.prefp, event.nscyc) if event.havepostcal: event.postmask = pbm.poet_badmask(event.postdata, event.postuncd, event.pmask, event.inst.pcrit, event.postbdmskd, event.inst.dcrit, event.postfp, nimpos=event.postnimpos) if event.userrej is not None: for i in range(np.shape(event.userrej)[0]): event.postmask[:, event.userrej[i, 0], event.userrej[i, 1], :] = 0 event.postfp.userrej = np.sum(np.sum(1 - event.postmask, axis=1), axis=1) event.postfp.userrej = np.transpose(event.postfp.userrej) - \ event.postfp.nsstrej else: event.postfp.userrej = np.zeros((event.npos, event.postmaxnimpos)) event.postmeanim = pcb.poet_chunkbad(event.postdata, event.postuncd, event.postmask, event.postnimpos, event.sigma, event.szchunk, event.postfp, event.nscyc) for pos in range(event.npos): fits.writeto(event.eventname + "_medpostcal.fits", event.postmeanim[:, :, pos], clobber=True) # Delete post calibration data: event.havepostcal = False del (event.postdata) del (event.postmask) del (event.postuncd) del (event.postbdmskd) # Save the data if event.instrument == 'mips': todel = ['bdmskd', 'brmskd'] # what to delete else: todel = ['bdmskd'] me.saveevent(event, event.eventname + "_bpm", save=['data', 'uncd', 'mask'], delete=todel) # Print time elapsed and close log: log.writelog("Output files:") log.writelog("Data:") log.writelog(" " + cwd + '/' + event.eventname + "_bpm.dat") log.writelog(" " + cwd + '/' + event.eventname + "_bpm.h5") log.writelog("Log:") log.writelog(" " + cwd + '/' + logname) dt = t.hms_time(time.time() - tini) log.writeclose('\nBad pixel masking time (h:m:s): %s ' % dt) os.chdir(owd) if event.runp3: #poet.p(3) os.system("python3 poet.py p3")
def badpix(eventname, control=None): tini = time.time() # Load the event event = me.loadevent(eventname) # Load the data me.updateevent(event, eventname, event.loadnext) # Create a new log starting from the old one. oldlogname = event.logname logname = event.eventname + ".log" log = le.Logedit(logname, oldlogname) event.logname = logname log.writelog('\nMARK: ' + time.ctime() + ': Starting poet_2badpix.') # ccampo 3/18/2011: do this in p5 # Julian observation date #event.fp.juldat = event.jdjf80 + event.fp.time / 86400.0 # ::::::::::::::::::::::: UNCERTAINTIES :::::::::::::::::::::::::::::::: # IRAC subarray data come with bogus uncertainties that are not linearly # related to photon noise. We scale them later, using the reduced chi # squared from the model fit. # ::::::::::::::::::::::: FLUX CONVERSION ::::::::::::::::::::::::::::: # Do we want flux (uJy/pix) or surface brightness (MJy/sr) units? If # doing photometry, convert to flux. Since we care about relative # numbers, it doesn't really matter. # Convert from surface brightness (MJy/sr) to flux units (uJy/pix) if event.fluxunits: log.writelog('Converting surface brightness to flux') event.data, event.uncd = btf.poet_bright2flux(event.data, event.uncd, event.posscl) if event.havecalaor: event.predata, event.preuncd = btf.poet_bright2flux( event.predata, event.preuncd, event.posscl) event.postdata, event.postuncd = btf.poet_bright2flux( event.postdata, event.postuncd, event.posscl) else: log.writelog('Did not convert bright to flux.') # Mean Background Estimate, from zodi model event.estbg = (np.mean(event.fp.zodi[np.where(event.fp.exist)]) + np.mean(event.fp.ism[np.where(event.fp.exist)]) + np.mean(event.fp.cib[np.where(event.fp.exist)])) if event.fluxunits: event.estbg *= (event.srperas * 1e12 * np.mean(event.posscl[0, :]) * np.mean(event.posscl[1, :])) # Bad Pixel Masking log.writelog('Find and fix bad pixels') # Get permanent bad pixel mask. if not event.ispmask[0]: log.writelog('\nPermanent Bad pixel mask not found!') else: hdu = pf.open(str(event.pmaskfile[0].decode('utf-8'))) if hdu[0].header['bitpix'] == -32: # if data type is float hdu[0].scale(type='int16') # cast it down to int16 event.pmask = hdu[0].data # IRS FIX: # IRS data contains the blue peak subarray while its pmask contains # the whole array (Hard coding) if event.photchan == 5: event.pmask = event.pmask[3:59, 86:127] # Do NOT define sigma, we have a different scheme for finding baddies # adds Spitzer rejects: fp.nsstrej & our rejects: fp.nsigrej event.mask = pbm.poet_badmask(event.data, event.uncd, event.pmask, event.inst.pcrit, event.bdmskd, event.inst.dcrit, event.fp, nimpos=event.nimpos) # User rejected pixels: if event.userrej != None: for i in np.arange(np.shape(event.userrej)[0]): event.mask[:, event.userrej[i, 0], event.userrej[i, 1], :] = 0 event.fp.userrej = np.sum(np.sum(1 - event.mask, axis=1), axis=1) event.fp.userrej = np.transpose(event.fp.userrej) - event.fp.nsstrej else: event.fp.userrej = np.zeros((int(event.npos), int(event.maxnimpos)), dtype=int) # define sigma here. # adds median sky: fp.medsky event.meanim = pcb.poet_chunkbad(event.data, event.uncd, event.mask, event.nimpos, event.sigma, event.szchunk, event.fp, event.nscyc) log.writelog('Masks combined') if event.havecalaor: event.premask = pbm.poet_badmask(event.predata, event.preuncd, event.pmask, event.inst.pcrit, event.prebdmskd, event.inst.dcrit, event.prefp, nimpos=event.calnimpos) event.premeanim = pcb.poet_chunkbad(event.predata, event.preuncd, event.premask, event.calnimpos, event.sigma, event.szchunk, event.prefp, event.nscyc) event.postmask = pbm.poet_badmask(event.postdata, event.postuncd, event.pmask, event.inst.pcrit, event.postbdmskd, event.inst.dcrit, event.postfp, nimpos=event.calnimpos) event.postmeanim = pcb.poet_chunkbad(event.postdata, event.postuncd, event.postmask, event.calnimpos, event.sigma, event.szchunk, event.postfp, event.nscyc) # Save the data if event.instrument == 'mips': todel = ['bdmskd', 'brmskd'] # what to delete else: todel = ['bdmskd'] me.saveevent(event, event.eventname + "_bpm", save=['data', 'uncd', 'mask'], delete=todel) # Print time elapsed and close log: cwd = os.getcwd() + "/" log.writelog("Output files:") log.writelog("Data:") log.writelog(" " + cwd + event.eventname + "_bpm.dat") log.writelog(" " + cwd + event.eventname + "_bpm.h5") log.writelog("Log:") log.writelog(" " + cwd + logname) dt = t.hms_time(time.time() - tini) log.writeclose('\nBad pixel masking time (h:m:s): %s ' % dt)
def photometry(event, pcf, photdir, mute, owd): tini = time.time() # Create photometry log logname = event.logname log = le.Logedit(photdir + "/" + logname, logname) log.writelog("\nStart " + photdir + " photometry: " + time.ctime()) parentdir = os.getcwd() + "/" os.chdir(photdir) # Parse the attributes from the control file to the event: attrib = vars(pcf) keys = attrib.keys() for key in keys: setattr(event, key, attrib.get(key)) maxnimpos, npos = event.maxnimpos, event.npos # allocating frame parameters: event.fp.aplev = np.zeros((npos, maxnimpos)) event.fp.aperr = np.zeros((npos, maxnimpos)) event.fp.nappix = np.zeros((npos, maxnimpos)) event.fp.skylev = np.zeros((npos, maxnimpos)) event.fp.skyerr = np.zeros((npos, maxnimpos)) event.fp.nskypix = np.zeros((npos, maxnimpos)) event.fp.nskyideal = np.zeros((npos, maxnimpos)) event.fp.status = np.zeros((npos, maxnimpos)) event.fp.good = np.zeros((npos, maxnimpos)) # For interpolated aperture photometry, we need to "interpolate" the # mask, which requires float values. Thus, we convert the mask to # floats (this needs to be done before processes are spawned or memory # usage balloons). if event.mask.dtype != float: event.mask = event.mask.astype(float) # Aperture photometry: if event.phottype == "aper": # not event.dooptimal or event.from_aper is None: # Multy Process set up: # Shared memory arrays allow only 1D Arrays :( aplev = Array("d", np.zeros(npos * maxnimpos)) # aperture flux aperr = Array("d", np.zeros(npos * maxnimpos)) # aperture error nappix = Array("d", np.zeros(npos * maxnimpos)) # number of aperture pixels skylev = Array("d", np.zeros(npos * maxnimpos)) # sky level skyerr = Array("d", np.zeros(npos * maxnimpos)) # sky error nskypix = Array("d", np.zeros(npos * maxnimpos)) # number of sky pixels nskyideal = Array("d", np.zeros( npos * maxnimpos)) # ideal number of sky pixels status = Array("d", np.zeros(npos * maxnimpos)) # apphot return status good = Array("d", np.zeros(npos * maxnimpos)) # good flag # Size of chunk of data each core will process: chunksize = maxnimpos // event.ncores + 1 event.aparr = np.ones(npos * maxnimpos) * event.photap + event.offset print("Number of cores: " + str(event.ncores)) # Start Muti Procecess: processes = [] for nc in range(event.ncores): start = nc * chunksize # Starting index to process end = (nc + 1) * chunksize # Ending index to process proc = Process(target=do_aphot, args=(start, end, event, log, mute, aplev, aperr, nappix, skylev, skyerr, nskypix, nskyideal, status, good, 0)) processes.append(proc) proc.start() # Make sure all processes finish their work: for nc in range(event.ncores): processes[nc].join() # Put the results in the event. I need to reshape them: event.fp.aplev = np.asarray(aplev).reshape(npos, maxnimpos) event.fp.aperr = np.asarray(aperr).reshape(npos, maxnimpos) event.fp.nappix = np.asarray(nappix).reshape(npos, maxnimpos) event.fp.skylev = np.asarray(skylev).reshape(npos, maxnimpos) event.fp.skyerr = np.asarray(skyerr).reshape(npos, maxnimpos) event.fp.nskypix = np.asarray(nskypix).reshape(npos, maxnimpos) event.fp.nskyideal = np.asarray(nskyideal).reshape(npos, maxnimpos) event.fp.status = np.asarray(status).reshape(npos, maxnimpos) event.fp.good = np.asarray(good).reshape(npos, maxnimpos) # raw photometry (no sky subtraction): event.fp.apraw = (event.fp.aplev + (event.fp.skylev * event.fp.nappix)) # Print results into the log if it wasn't done before: for pos in range(npos): for i in range(event.nimpos[pos]): log.writelog( '\nframe =%7d ' % i + 'pos =%5d ' % pos + 'y =%7.3f ' % event.fp.y[pos, i] + 'x =%7.3f' % event.fp.x[pos, i] + '\n' + 'aplev =%11.3f ' % event.fp.aplev[pos, i] + 'aperr =%9.3f ' % event.fp.aperr[pos, i] + 'nappix =%6.2f' % event.fp.nappix[pos, i] + '\n' + 'skylev=%11.3f ' % event.fp.skylev[pos, i] + 'skyerr=%9.3f ' % event.fp.skyerr[pos, i] + 'nskypix=%6.2f ' % event.fp.nskypix[pos, i] + 'nskyideal=%6.2f' % event.fp.nskyideal[pos, i] + '\n' + 'status=%7d ' % event.fp.status[pos, i] + 'good =%5d' % event.fp.good[pos, i], mute=True) elif event.phottype == "var": # variable aperture radius # Multy Process set up: # Shared memory arrays allow only 1D Arrays :( aplev = Array("d", np.zeros(npos * maxnimpos)) # aperture flux aperr = Array("d", np.zeros(npos * maxnimpos)) # aperture error nappix = Array("d", np.zeros(npos * maxnimpos)) # number of aperture pixels skylev = Array("d", np.zeros(npos * maxnimpos)) # sky level skyerr = Array("d", np.zeros(npos * maxnimpos)) # sky error nskypix = Array("d", np.zeros(npos * maxnimpos)) # number of sky pixels nskyideal = Array("d", np.zeros( npos * maxnimpos)) # ideal number of sky pixels status = Array("d", np.zeros(npos * maxnimpos)) # apphot return status good = Array("d", np.zeros(npos * maxnimpos)) # good flag # Size of chunk of data each core will process: chunksize = maxnimpos // event.ncores + 1 event.aparr = event.fp.noisepix[0]**.5 * event.photap + event.offset print("Number of cores: " + str(event.ncores)) # Start Muti Procecess: processes = [] for nc in range(event.ncores): start = nc * chunksize # Starting index to process end = (nc + 1) * chunksize # Ending index to process proc = Process(target=do_aphot, args=(start, end, event, log, mute, aplev, aperr, nappix, skylev, skyerr, nskypix, nskyideal, status, good, 0)) processes.append(proc) proc.start() # Make sure all processes finish their work: for nc in range(event.ncores): processes[nc].join() # Put the results in the event. I need to reshape them: event.fp.aplev = np.asarray(aplev).reshape(npos, maxnimpos) event.fp.aperr = np.asarray(aperr).reshape(npos, maxnimpos) event.fp.nappix = np.asarray(nappix).reshape(npos, maxnimpos) event.fp.skylev = np.asarray(skylev).reshape(npos, maxnimpos) event.fp.skyerr = np.asarray(skyerr).reshape(npos, maxnimpos) event.fp.nskypix = np.asarray(nskypix).reshape(npos, maxnimpos) event.fp.nskyideal = np.asarray(nskyideal).reshape(npos, maxnimpos) event.fp.status = np.asarray(status).reshape(npos, maxnimpos) event.fp.good = np.asarray(good).reshape(npos, maxnimpos) # raw photometry (no sky subtraction): event.fp.apraw = (event.fp.aplev + (event.fp.skylev * event.fp.nappix)) # Print results into the log if it wasn't done before: for pos in range(npos): for i in range(event.nimpos[pos]): log.writelog( '\nframe =%7d ' % i + 'pos =%5d ' % pos + 'y =%7.3f ' % event.fp.y[pos, i] + 'x =%7.3f' % event.fp.x[pos, i] + '\n' + 'aplev =%11.3f ' % event.fp.aplev[pos, i] + 'aperr =%9.3f ' % event.fp.aperr[pos, i] + 'nappix =%6.2f' % event.fp.nappix[pos, i] + '\n' + 'skylev=%11.3f ' % event.fp.skylev[pos, i] + 'skyerr=%9.3f ' % event.fp.skyerr[pos, i] + 'nskypix=%6.2f ' % event.fp.nskypix[pos, i] + 'nskyideal=%6.2f' % event.fp.nskyideal[pos, i] + '\n' + 'status=%7d ' % event.fp.status[pos, i] + 'good =%5d' % event.fp.good[pos, i], mute=True) elif event.phottype == "ell": # elliptical # Multy Process set up: # Shared memory arrays allow only 1D Arrays :( aplev = Array("d", np.zeros(npos * maxnimpos)) # aperture flux aperr = Array("d", np.zeros(npos * maxnimpos)) # aperture error nappix = Array("d", np.zeros(npos * maxnimpos)) # number of aperture pixels skylev = Array("d", np.zeros(npos * maxnimpos)) # sky level skyerr = Array("d", np.zeros(npos * maxnimpos)) # sky error nskypix = Array("d", np.zeros(npos * maxnimpos)) # number of sky pixels nskyideal = Array("d", np.zeros( npos * maxnimpos)) # ideal number of sky pixels status = Array("d", np.zeros(npos * maxnimpos)) # apphot return status good = Array("d", np.zeros(npos * maxnimpos)) # good flag # Size of chunk of data each core will process: chunksize = maxnimpos // event.ncores + 1 print("Number of cores: " + str(event.ncores)) # Start Muti Procecess: processes = [] for nc in range(event.ncores): start = nc * chunksize # Starting index to process end = (nc + 1) * chunksize # Ending index to process proc = Process(target=do_aphot, args=(start, end, event, log, mute, aplev, aperr, nappix, skylev, skyerr, nskypix, nskyideal, status, good, 0)) processes.append(proc) proc.start() # Make sure all processes finish their work: for nc in range(event.ncores): processes[nc].join() # Put the results in the event. I need to reshape them: event.fp.aplev = np.asarray(aplev).reshape(npos, maxnimpos) event.fp.aperr = np.asarray(aperr).reshape(npos, maxnimpos) event.fp.nappix = np.asarray(nappix).reshape(npos, maxnimpos) event.fp.skylev = np.asarray(skylev).reshape(npos, maxnimpos) event.fp.skyerr = np.asarray(skyerr).reshape(npos, maxnimpos) event.fp.nskypix = np.asarray(nskypix).reshape(npos, maxnimpos) event.fp.nskyideal = np.asarray(nskyideal).reshape(npos, maxnimpos) event.fp.status = np.asarray(status).reshape(npos, maxnimpos) event.fp.good = np.asarray(good).reshape(npos, maxnimpos) # raw photometry (no sky subtraction): event.fp.apraw = (event.fp.aplev + (event.fp.skylev * event.fp.nappix)) # Print results into the log if it wasn't done before: for pos in range(npos): for i in range(event.nimpos[pos]): log.writelog( '\nframe =%7d ' % i + 'pos =%5d ' % pos + 'y =%7.3f ' % event.fp.y[pos, i] + 'x =%7.3f' % event.fp.x[pos, i] + '\n' + 'aplev =%11.3f ' % event.fp.aplev[pos, i] + 'aperr =%9.3f ' % event.fp.aperr[pos, i] + 'nappix =%6.2f' % event.fp.nappix[pos, i] + '\n' + 'skylev=%11.3f ' % event.fp.skylev[pos, i] + 'skyerr=%9.3f ' % event.fp.skyerr[pos, i] + 'nskypix=%6.2f ' % event.fp.nskypix[pos, i] + 'nskyideal=%6.2f' % event.fp.nskyideal[pos, i] + '\n' + 'status=%7d ' % event.fp.status[pos, i] + 'good =%5d' % event.fp.good[pos, i], mute=True) elif event.phottype == "psffit": event.fp.aplev = event.fp.flux event.fp.skylev = event.fp.psfsky event.fp.good = np.zeros((event.npos, event.maxnimpos)) for pos in range(event.npos): event.fp.good[pos, 0:event.nimpos[pos]] = 1 elif event.phottype == "optimal": # utils for profile construction: pshape = np.array([2 * event.otrim + 1, 2 * event.otrim + 1]) subpsf = np.zeros(np.asarray(pshape, int) * event.expand) x = np.indices(pshape) clock = t.Timer(np.sum(event.nimpos), progress=np.array([0.05, 0.1, 0.25, 0.5, 0.75, 1.1])) for pos in range(npos): for i in range(event.nimpos[pos]): # Integer part of center of subimage: cen = np.rint([event.fp.y[pos, i], event.fp.x[pos, i]]) # Center in the trimed image: loc = (event.otrim, event.otrim) # Do the trim: img, msk, err = ie.trimimage(event.data[i, :, :, pos], *cen, *loc, mask=event.mask[i, :, :, pos], uncd=event.uncd[i, :, :, pos]) # Center of star in the subimage: ctr = (event.fp.y[pos, i] - cen[0] + event.otrim, event.fp.x[pos, i] - cen[1] + event.otrim) # Make profile: # Index of the position in the supersampled PSF: pix = pf.pos2index(ctr, event.expand) profile, pctr = pf.make_psf_binning(event.psfim, pshape, event.expand, [pix[0], pix[1], 1.0, 0.0], event.psfctr, subpsf) #subtract the sky level: img -= event.fp.psfsky[pos, i] # optimal photometry calculation: immean, uncert, good = op.optphot(img, profile, var=err**2.0, mask=msk) event.fp.aplev[pos, i] = immean event.fp.aperr[pos, i] = uncert event.fp.skylev[pos, i] = event.fp.psfsky[pos, i] event.fp.good[pos, i] = good # Report progress: clock.check(np.sum(event.nimpos[0:pos]) + i, name=event.centerdir) # START PREFLASH EDIT ::::::::::::::::::::::::::::::::::::: # Do aperture on preflash data: if event.havepreflash: print("\nStart preflash photometry:") premaxnimpos = event.premaxnimpos preaplev = Array("d", np.zeros(npos * premaxnimpos)) preaperr = Array("d", np.zeros(npos * premaxnimpos)) prenappix = Array("d", np.zeros(npos * premaxnimpos)) preskylev = Array("d", np.zeros(npos * premaxnimpos)) preskyerr = Array("d", np.zeros(npos * premaxnimpos)) preskynpix = Array("d", np.zeros(npos * premaxnimpos)) preskyideal = Array("d", np.zeros(npos * premaxnimpos)) prestatus = Array("d", np.zeros(npos * premaxnimpos)) pregood = Array("d", np.zeros(npos * premaxnimpos)) # Start Procecess: mute = False proc = Process(target=do_aphot, args=(0, event.prenimpos[0], event, log, mute, preaplev, preaperr, prenappix, preskylev, preskyerr, preskynpix, preskyideal, prestatus, pregood, 1)) proc.start() proc.join() # Put the results in the event. I need to reshape them: event.prefp.aplev = np.asarray(preaplev).reshape(npos, premaxnimpos) event.prefp.aperr = np.asarray(preaperr).reshape(npos, premaxnimpos) event.prefp.nappix = np.asarray(prenappix).reshape(npos, premaxnimpos) event.prefp.status = np.asarray(prestatus).reshape(npos, premaxnimpos) event.prefp.skylev = np.asarray(preskylev).reshape(npos, premaxnimpos) event.prefp.good = np.asarray(pregood).reshape(npos, premaxnimpos) # raw photometry (no sky subtraction): event.prefp.aplev = (event.prefp.aplev + (event.prefp.skylev * event.prefp.nappix)) # END PREFLASH EDIT ::::::::::::::::::::::::::::::::::::::: if event.method in ["bpf"]: event.ispsf = False # PSF aperture correction: if event.ispsf and event.phottype == "aper": log.writelog('Calculating PSF aperture:') event.psfim = event.psfim.astype(np.float64) imerr = np.ones(np.shape(event.psfim)) imask = np.ones(np.shape(event.psfim)) skyfrac = 0.1 event.aperfrac, ape, event.psfnappix, event.psfskylev, sle, \ event.psfnskypix, event.psfnskyideal, event.psfstatus \ = ap.apphot_c(event.psfim, imerr, imask, event.psfctr[0], event.psfctr[1], event.photap * event.psfexpand, event.skyin * event.psfexpand, event.skyout * event.psfexpand, skyfrac, event.apscale, event.skymed) event.aperfrac += event.psfskylev * event.psfnappix event.fp.aplev /= event.aperfrac event.fp.aperr /= event.aperfrac log.writelog('Aperture contains %f of PSF.' % event.aperfrac) if event.ispsf and event.phottype == "var": log.writelog('Calculating PSF aperture:') event.psfim = event.psfim.astype(np.float64) imerr = np.ones(np.shape(event.psfim)) imask = np.ones(np.shape(event.psfim)) skyfrac = 0.1 avgap = np.mean(event.aparr) event.aperfrac, ape, event.psfnappix, event.psfskylev, sle, \ event.psfnskypix, event.psfnskyideal, event.psfstatus \ = ap.apphot_c(event.psfim, imerr, imask, event.psfctr[0], event.psfctr[1], avgap * event.psfexpand, event.skyin * event.psfexpand, event.skyout * event.psfexpand, skyfrac, event.apscale, event.skymed) event.aperfrac += event.psfskylev * event.psfnappix event.fp.aplev /= event.aperfrac event.fp.aperr /= event.aperfrac log.writelog('Aperture contains %f of PSF.' % event.aperfrac) if event.ispsf and event.phottype == "ell": log.writelog('Calculating PSF aperture:') event.psfim = event.psfim.astype(np.float64) imerr = np.ones(np.shape(event.psfim)) imask = np.ones(np.shape(event.psfim)) skyfrac = 0.1 avgxwid = np.mean(event.fp.xsig * event.photap) avgywid = np.mean(event.fp.ysig * event.photap) avgrot = np.mean(event.fp.rot) event.aperfrac, ape, event.psfnappix, event.psfskylev, sle, \ event.psfnskypix, event.psfnskyideal, event.psfstatus \ = ap.elphot_c(event.psfim, imerr, imask, event.psfctr[0], event.psfctr[1], avgxwid * event.psfexpand, avgywid * event.psfexpand, avgrot, event.skyin * event.psfexpand, event.skyout * event.psfexpand, skyfrac, event.apscale, event.skymed) event.aperfrac += event.psfskylev * event.psfnappix event.fp.aplev /= event.aperfrac event.fp.aperr /= event.aperfrac log.writelog('Aperture contains %f of PSF.' % event.aperfrac) # Sadly we must do photometry for every aperture used # Possibly use a range and interpolate? Might be an option # for the future to speed this up. # This is commented out, as it seems to just remove the corrections # made by variable or elliptical photometry # if event.ispsf and (event.phottype == "var" or event.phottype == "ell"): # log.writelog('Calculating PSF aperture. This may take some time.') # event.psfim = event.psfim.astype(np.float64) # imerr = np.ones(np.shape(event.psfim)) # imask = np.ones(np.shape(event.psfim)) # skyfrac = 0.1 # aperfrac = Array("d", np.zeros(npos*maxnimpos))# psf flux # aperfracerr = Array("d", np.zeros(npos*maxnimpos))# psf flux error # psfnappix = Array("d", np.zeros(npos*maxnimpos))# psf aperture pix num # psfsky = Array("d", np.zeros(npos*maxnimpos))# psf sky level # psfskyerr = Array("d", np.zeros(npos*maxnimpos))# psf sky error # psfnskypix = Array("d", np.zeros(npos*maxnimpos))# psf sky pix num # psfnskyideal = Array("d", np.zeros(npos*maxnimpos))# psf ideal sky pix num # psfstatus = Array("d", np.zeros(npos*maxnimpos))# psf return status # psfgood = Array("d", np.zeros(npos*maxnimpos))# psf good flag # processes=[] # for nc in range(event.ncores): # start = nc * chunksize # end = (nc+1) * chunksize # proc = Process(target=do_aphot_psf, args=(start, end, event, log, mute, # aperfrac, aperfracerr, # psfnappix, # psfsky, psfskyerr, # psfnskypix, psfnskyideal, # psfstatus, psfgood)) # processes.append(proc) # proc.start() # for nc in range(event.ncores): # processes[nc].join() # # Reshape # event.aperfrac = np.asarray(aperfrac ).reshape(npos,maxnimpos) # event.aperfracerr = np.asarray(aperfracerr ).reshape(npos,maxnimpos) # event.psfnappix = np.asarray(psfnappix ).reshape(npos,maxnimpos) # event.psfsky = np.asarray(psfsky ).reshape(npos,maxnimpos) # event.psfskyerr = np.asarray(psfskyerr ).reshape(npos,maxnimpos) # event.psfnskypix = np.asarray(psfnskypix ).reshape(npos,maxnimpos) # event.psfnskyideal = np.asarray(psfnskyideal).reshape(npos,maxnimpos) # event.psfstatus = np.asarray(psfstatus ).reshape(npos,maxnimpos) # event.psfgood = np.asarray(psfgood ).reshape(npos,maxnimpos) # event.aperfrac += event.psfsky * event.psfnappix # event.fp.aplev /= event.aperfrac # event.fp.aperr /= event.aperfrac # log.writelog('Aperture contains average %f of PSF.'%np.mean(event.aperfrac)) # save print("\nSaving ...") # denoised data: if event.denphot: killdata = 'dendata' else: killdata = 'data' me.saveevent(event, event.eventname + "_pht", delete=[killdata, 'uncd', 'mask']) # Print time elapsed and close log: cwd = os.getcwd() + "/" log.writelog("Output files (" + event.photdir + "):") log.writelog("Data:") log.writelog(" " + cwd + event.eventname + "_pht.dat") log.writelog("Log:") log.writelog(" " + cwd + logname) dt = t.hms_time(time.time() - tini) log.writeclose("\nEnd Photometry. Time (h:m:s): %s " % dt + " (" + photdir + ")") print("-------------- ------------\n") os.chdir(owd) if event.runp5: os.system("python3 poet.py p5 %s/%s" % (event.centerdir, event.photdir))
def poetSave(event, directory='../'): me.saveevent(event, directory + "/" + event.eventname + "_p5c")
def denoise(pcf, denoisedir): tini = time.time() # Create denoising log logname = event.logname log = le.Logedit(denoisedir + "/" + logname, logname) log.writelog("\nStart " + denoisedir + " denoising: " + time.ctime()) os.chdir(denoisedir) # copy denoise.pcf in denoisedir pcf.make_file("denoise.pcf") # Parse the attributes from the control file to the event: attrib = vars(pcf) keys = attrib.keys() for key in keys: if key != 'srcest': setattr(event, key, attrib.get(key).get()) for pos in range(event.npos): # Plot histogram of noisy wavelet coefficients ylim = histwc(event, event.wavelet, event.numlvls + 1, pos, log=log, denoised=False) # Plot first 'length' frames of noisy lightcurve at pixel srcest plotlc(event, pos, length=200, denoised=False) ''' maxlvls = pywt.dwt_max_level(event.nimpos[pos], pywt.Wavelet(event.wavelet)) # Determine the number of levels to denoise for i in range(1,maxlvls+1): if (2**i)*event.framtime < event.maxtime: numlvls = i else: break ''' log.writelog("Denoising will occur on the lowest " + str(event.numlvls) + " levels at position " + str(pos) + ".") # Determine the time resolution of the highled denoised level timeres = 2**(event.numlvls) * event.framtime log.writelog("Time resolution for position " + str(pos) + ", level " + str(event.numlvls) + " is " + str(timeres) + " seconds.") # Assess presence of NaNs and Infs in masked data print("Checking for NaNs and Infs.") data = (event.data[:, :, :, pos])[np.where(event.mask[:, :, :, pos])] if (np.sum(np.isnan(data)) + np.sum(np.isinf(data))) > 0: log.writelog( "***WARNING: Found NaNs and/or Infs in masked data at position " + str(pos) + ".") del (data) pool = mp.Pool(event.ncpu) for i in range(event.nx): for j in range(event.ny): #res=bayesshrink((event.data[:,j,i,pos])[np.where(event.mask[:,j,i,pos])], event.wavelet, event.numlvls, [j,i,pos]) #writedata(res) exec( 'res = pool.apply_async(' + event.threshold + ',((event.data[:,j,i,pos])[np.where(event.mask[:,j,i,pos])], event.wavelet, event.numlvls, [j,i,pos]),callback=writedata)' ) #res = exec('pool.apply_async(' + event.threshold + ',((event.data[:,j,i,pos])[np.where(event.mask[:,j,i,pos])], event.wavelet, event.numlvls, [j,i,pos]),callback=writedata)') #res = pool.apply_async(event.threshold,((event.data[:,j,i,pos])[np.where(event.mask[:,j,i,pos])], event.wavelet, event.numlvls, [j,i,pos]),callback=writedata) pool.close() pool.join() #res.wait() #Plot histogram of denoised wavelet coefficients histwc(event, event.wavelet, event.numlvls + 1, pos, log=log, denoised=True, ylim=ylim) # Plot first 'length' frames of denoised lightcurve at pixel srcest plotlc(event, pos, length=200, denoised=True) # Save print("\nFinished Denoising. Saving.") me.saveevent(event, event.eventname + "_den", save=['data', 'uncd', 'mask']) # Print time elapsed and close log: cwd = os.getcwd() + "/" log.writelog("Output files (" + event.denoisedir + "):") log.writelog("Data:") log.writelog(" " + cwd + event.eventname + "_den.dat") log.writelog(" " + cwd + event.eventname + "_den.h5") log.writelog("Log:") log.writelog(" " + cwd + event.logname) dt = t.hms_time(time.time() - tini) log.writeclose("\nEnd Denoising. Time (h:m:s): %s" % dt + " (" + event.denoisedir + ")") print("------------- ------------\n") return
def lcWFC3(eventname, eventdir, nchan, wmin=1.125, wmax=1.65, expand=1, isplots=True): ''' Compute photometric flux over specified range of wavelengths Parameters ---------- eventname : Unique identifier for these data eventdir : Location of save file nchan : Number of spectrophotometric channels wmin : minimum wavelength wmax : maximum wavelength expand : expansion factor isplots : Set True to produce plots Returns ------- None History ------- Written by Kevin Stevenson June 2012 ''' # Load saved data # An event is an instance of an object # loadevent: load the saved files from storage # container for all of the data # event: small data structures (i.e. light curves) # aux: large data structures (i.e. image cubes, etc) # # i.e. event.BJDTDB : returns the time array # aux.spectra : 1D spectra per NDR # aux.specerr : 1D spectra error per NDR # aux.data_mhdr: master header per frame # print("Loading saved data...") ev = me.loadevent(eventdir + '/d-' + eventname + '-w1') aux = me.loadevent(eventdir + '/d-' + eventname + '-data') ev.spectra = aux.spectra specerr = aux.specerr data_mhdr = aux.data_mhdr # Determine wavelength bins binsize = (wmax - wmin) / nchan # width in bins wave_low = np.round([i for i in np.linspace(wmin, wmax - binsize, nchan)], 3) # Left edge of the wavelength bins wave_hi = np.round([i for i in np.linspace(wmin + binsize, wmax, nchan)], 3) # Right edge of the wavelength bins # binwave = (wave_low + wave_hi)/2. # Middle of wavelength bin # Increase resolution of spectra: uses np.zoom to oversample the image in a flux conserving interpolation if expand > 1: # note: ev.n_spec : number of spectra per frame :: hopefully just one print("Increasing spectra resolution...") # ev.spectra.shape[3] : wavelength (dispersion) dimension hdspectra = np.zeros( (ev.n_files, ev.n_spec, ev.n_reads - 1, expand * ev.spectra.shape[3])) # hdspectra : high definition spectra hdspecerr = np.zeros( (ev.n_files, ev.n_spec, ev.n_reads - 1, expand * ev.spectra.shape[3])) # hdspecerr : high definition spectra error hdwave = np.zeros((ev.n_img, ev.n_spec, expand * ev.spectra.shape[3] )) # hdwave : high definition wavelength array # This is the 'zoom' step for n in range( ev.n_spec): # per spectrum on the image (n_spec == 1 for WFC3) # This operates over all 1D stellar spectrum (over time) at once hdspectra[:, n] = spni.zoom(ev.spectra[:, n], zoom=[1, 1, expand]) hdspecerr[:, n] = spni.zoom(specerr[:, n], zoom=[1, 1, expand]) * np.sqrt(expand) for m in range(ev.n_img): # n_img : number of direct images # Some visits have a new wavelength solution per orbit for n in range(ev.n_spec): hdwave[m, n] = spni.zoom(ev.wave[m][n], zoom=expand) # Store high defition spectra ev.spectra = hdspectra specerr = hdspecerr ev.wave = hdwave # Correct for drift, if calculated if hasattr(ev, 'drift_model') and ev.drift_model != None: # Correct for drift :: map the motion of the spectrum across the detector # provides higher precision on the wavelength solution print( 'Applying drift correction... (Old stare-mode version, may not work)' ) # Staring Mode Operations # ev.drift_model is defined in `w1` nx = ev.spectra.shape[ 3] # number of pixels in the wavelength direction # FINDME: CHANGED the SHAPE[2] to a SHAPE[3] for m in range(ev.n_files): # iterate over time for n in range(ev.n_spec ): # iterate over number of spectra (ideally == 1) spline = spi.UnivariateSpline( np.arange(nx), ev.spectra[m, n], k=3, s=0) # Compute the spline for the shift ev.spectra[m, n] = spline( np.arange(nx) + ev.drift_model[n, m]) # Shifts the spectrum # finished Stare-mode operations elif ev.detector == 'IR': # This is for Scanning Mode #Calculate drift over all frames and non-destructive reads print('Applying drift correction...') # hst.drift_fit calculates the drift in the 1D spectra :: Does a cross correlation # ev : class with the data # preclip : left edge of spectrum # preclip : right edge of spectrum ev.drift, ev.drift_model, ev.goodmask = hst.drift_fit(ev, preclip=0, postclip=None, width=5 * expand, deg=2, validRange=11 * expand) # Correct for drift if ev.n_reads > 2: # Throw away the first NDR -- it's bad print('WARNING: Marking all first reads as bad.') istart = 1 else: print('Using first reads.') istart = 0 # Apply the Drift correction (fancy word for spline) nx = ev.spectra.shape[ 3] # number of pixels in the wavelength direction for m in range(ev.n_files): # iterate over time for n in range(ev.n_spec ): # iterate over number of spectra (ideally == 1) for p in range(istart, ev.n_reads - 1): # Compute the spline for the shift spline = spi.UnivariateSpline(np.arange(nx), ev.spectra[m, n, p], k=3, s=0) # Using measured drift, not model fit # `model fit` comes from the ev.drift_model # `measured drift` comes from spline of order 3 ev.spectra[m, n, p] = spline( np.arange(nx) + ev.drift[n, m, p]) # Shifts the spectrum #Apply scan height correction #print('Applying scan height correction...') #ev.spectra += ev.spectra[0,0]*(1-ev.scanHeight[:,:,np.newaxis,np.newaxis]/ev.scanHeight[0,0]) #ev.spectra /= ev.scanHeight[:,:,np.newaxis,np.newaxis] #specerr /= ev.scanHeight[:,:,np.newaxis,np.newaxis] else: # UVIS Stuff istart = 0 # Assign scan direction: 0:forward vs 1:reverse ev.scandir = np.zeros( ev.n_files) # Sets up all images as forward scan: modify later ev.n_scan0 = 0 # Number of forward scans ev.n_scan1 = 0 # Number of reverse scans try: scan0 = data_mhdr[0][ 'POSTARG2'] # POSTARG2 changes for forward and reverse scan 0: first file scan1 = data_mhdr[1][ 'POSTARG2'] # POSTARG2 changes for forward and reverse scan 1: first file for m in range(ev.n_files): # for every file file, check header if POSTARG2 == scan0 or scan1 if data_mhdr[m]['POSTARG2'] == scan0: # Sum up number of forward scan ev.n_scan0 += 1 elif data_mhdr[m]['POSTARG2'] == scan1: # Store scandir == 1 for reverse scanning ev.scandir[m] = 1 # Sum up number of reverse scan ev.n_scan1 += 1 else: # Something happened print('WARNING: Unknown scan direction for file ' + str(m) + '.') print("# of files in scan direction 0: " + str(ev.n_scan0)) print("# of files in scan direction 1: " + str(ev.n_scan1)) except: ev.n_scan0 = ev.n_files print("Unable to determine scan direction, assuming unidirectional.") print("Generating light curves...") ev.eventname2 = ev.eventname # Store old event name for i in range(nchan): ev.wave_low = wave_low[i] ev.wave_hi = wave_hi[i] print("Bandpass = %.3f - %.3f" % (ev.wave_low, ev.wave_hi)) # Calculate photometric flux for each spectrum ev.photflux = np.zeros( (ev.n_spec, ev.n_files, np.max( (1, ev.n_reads - 2)))) # This become the light curve (to be populated) ev.photfluxerr = np.zeros( (ev.n_spec, ev.n_files, np.max((1, ev.n_reads - 2)) )) # This become the light curve errorbars (to be populated) # ev.wave = [] for n in range(ev.n_spec): # hopefully == 1 if ev.detector == 'IR': # Compute common wavelength and indices to apply over all observations wave = np.zeros(len( ev.wave[0][n])) # To be the globale wavelength array for j in range(ev.n_img): # iterate over each image wave += ev.wave[j][n] wave /= ev.n_img # wave = np.mean(ev.wave, axis=0) # FINDME: TEST LATER # index == where wave meets BOTH requirement # Which indices in the mean wavelength (`wave`) are between the individal channel boundaries index = np.where( np.bitwise_and(wave >= wave_low[i], wave <= wave_hi[i]))[0] # index = np.where((wave >= wave_low[i])*(wave <= wave_hi[i]))[0] # FINDME: TEST LATER else: # UVIS: Use all pixels for aperture photometry index = range(len(ev.spectra[0, 0, 0])) for m in range(ev.n_files): ''' # This is a different way to compute the indices to associate with columns to be summed into 1D spectra # Select appropriate orbit-dependent wavelength if ev.n_img == (np.max(ev.orbitnum)+1): j = int(ev.orbitnum[m]) else: j = 0 #Method 1 ev.wave.append(np.mean(ev.wavegrid[j][n],axis=0)) index = np.where(np.bitwise_and(ev.wave[n] >= wave_low, ev.wave[n] <= wave_hi))[0] #Method 2 index = np.where(np.bitwise_and(ev.wave[j][n] >= wave_low, ev.wave[j][n] <= wave_hi))[0] ''' # This creates a light curve per NDR ev.photflux[n, m] = np.sum( ev.spectra[m, n, istart:, index], axis=0 ) # Summing in the 1D spectral plane (m == 1 for WFC3) ev.photfluxerr[n, m] = np.sqrt( np.sum(specerr[m, n, istart:, index]**2, axis=0) ) # Summing in quadrature the 1D spectral plane (m == 1 for WFC3) # Save results for individual channels into individual files ev.eventname = ev.eventname2 + '_' + str(int( ev.wave_low * 1e3)) + '_' + str(int(ev.wave_hi * 1e3)) # me.saveevent(ev, eventdir + '/d-' + ev.eventname + '-w3', delete=['data_mhdr', 'spectra', 'specerr']) # saveevent stores everything (that we want) into a pickle me.saveevent(ev, eventdir + '/d-' + ev.eventname + '-w3') # Produce plot if isplots == True: # 3XYZ: 3: w3 function # X: Plot Number # Y & Z: Spectral Channel Number # # Normalized Flux vs Time plt.figure(3000 + i, figsize=(10, 8)) plt.clf() # this clears the frame plt.suptitle('Wavelength range: ' + str(wave_low[i]) + '-' + str(wave_hi[i])) ax = plt.subplot(111) #for n in range(ev.n_spec): #plt.subplot(ev.n_spec,1,1) #plt.title('Star ' + str(n)) #igood = np.where(ev.goodmask[0])[0] iscan0 = np.where(ev.scandir == 0)[0] iscan1 = np.where(ev.scandir == 1)[0] mjd = np.floor(ev.bjdtdb[0]) flux0 = np.sum(ev.photflux[0][iscan0], axis=1) / np.sum( ev.photflux[0, [iscan0[-1]]]) #err = np.sqrt(1 / np.sum(1/ev.photfluxerr[0]**2,axis=1))/np.sum(ev.photflux[0,-1]) try: err0 = np.sqrt(np.sum(ev.photfluxerr[0][iscan0]**2, axis=1)) / np.sum( ev.photflux[0, [iscan0[-1]]]) except: err0 = 0 # err1 = 0 plt.errorbar(ev.bjdtdb[iscan0] - mjd, flux0, err0, fmt='bo') plt.text( 0.05, 0.1, "MAD = " + str(np.round(1e6 * np.median(np.abs(np.ediff1d(flux0))))) + " ppm", transform=ax.transAxes, color='b') if len(iscan1) > 0: flux1 = np.sum(ev.photflux[0][iscan1], axis=1) / np.sum( ev.photflux[0, [iscan0[-1]]]) err1 = np.sqrt(np.sum(ev.photfluxerr[0][iscan1]**2, axis=1)) / np.sum( ev.photflux[0, [iscan1[-1]]]) plt.errorbar(ev.bjdtdb[iscan1] - mjd, flux1, err1, fmt='ro') plt.text( 0.05, 0.05, "MAD = " + str(np.round(1e6 * np.median(np.abs(np.ediff1d(flux1))))) + " ppm", transform=ax.transAxes, color='r') plt.ylabel('Normalized Flux') plt.xlabel('Time [MJD + ' + str(mjd) + ']') plt.subplots_adjust(left=0.10, right=0.95, bottom=0.10, top=0.90, hspace=0.20, wspace=0.3) plt.savefig(eventdir + '/figs/' + ev.eventname + '-Fig' + str(3000 + i) + '.png') #plt.pause(0.1) if ev.detector == 'IR': # Drift: frame number vs drift in the wavelength direction plt.figure(3100 + i, figsize=(10, 8)) plt.clf() for i in range(istart, ev.n_reads - 1): plt.subplot(1, np.max((1, ev.n_reads - 2)), np.max((1, i))) plt.plot(ev.drift[0, :, i], '.') if i == istart: plt.ylabel('Spectrum Drift') if i == (ev.n_reads - 1) / 2: plt.xlabel('Frame Number') plt.savefig(eventdir + '/figs/' + ev.eventname + '-Fig' + str(3100) + '.png') if (isplots == True) and (ev.detector == 'IR'): # 2D light curve with drift correction # Plot frame number vs wavelength with color associated with value # Very cool plot that produces "image" of the entire time series (hopefully corrected) plt.figure(3200, figsize=(8, ev.n_files / 20. + 0.8)) plt.clf() vmin = 0.97 vmax = 1.03 # istart = 0 normspec = np.mean(ev.spectra[:, 0, istart:], axis=1) / np.mean( ev.spectra[-6:, 0, istart:], axis=(0, 1)) ediff = np.zeros(ev.n_files) iwmin = np.where(ev.wave[0][0] > wmin)[0][0] iwmax = np.where(ev.wave[0][0] > wmax)[0][0] for i in range(ev.n_files): ediff[i] = 1e6 * np.median( np.abs(np.ediff1d(normspec[i, iwmin:iwmax]))) plt.scatter(ev.wave[0][0], np.zeros(ev.specsize) + i, c=normspec[i], s=14, linewidths=0, vmin=vmin, vmax=vmax, marker='s', cmap=plt.cm.RdYlBu_r) plt.title("MAD = " + str(np.round(np.mean(ediff), 0)) + " ppm") plt.xlim(wmin, wmax) if nchan > 1: xticks = np.round([i for i in np.linspace(wmin, wmax, nchan + 1)], 3) plt.xticks(xticks, xticks) plt.vlines(xticks, 0, ev.n_files, 'k', 'dashed') plt.ylim(0, ev.n_files) plt.ylabel('Frame Number') plt.xlabel('Wavelength ($\mu m$)') plt.colorbar() plt.tight_layout() plt.savefig(eventdir + '/figs/fig3200-2D_LC.png')
def ld_driver(eventname, eventdir, wave_low=None, wave_hi=None, n_param=4, isplots=False, stellarmodel='phoenix'): ''' ''' # Load saved data print("Loading saved data...") ev = me.loadevent(eventdir + '/d-' + eventname + '-w3') aux = me.loadevent(eventdir + '/d-' + ev.eventname2 + '-data') ev.spectra = aux.spectra ''' #FINDME: HACK ev.file_med = ev.loc_ld + 'kelt11_med.txt' ev.file_med = ev.loc_ld + 'lte6250-4.38+0.2a+0.0CMg-0.1.BT-dusty-giant-2011.cifist.He.irf.fits' ev.file_low = ev.loc_ld + 'lte6100-4.38+0.2a+0.0CMg-0.1.BT-dusty-giant-2011.cifist.He.irf.fits' ev.file_hi = ev.loc_ld + 'lte6400-4.38+0.2a+0.0CMg-0.1.BT-dusty-giant-2011.cifist.He.irf.fits' #print(ev.file_med) ''' n = 0 m = 0 ilo = np.where(ev.wave[n][m] > wave_low)[0][0] ihi = np.where(ev.wave[n][m] < wave_hi)[0][-1] + 1 # print("Computing limb-darkening coefficients...") specwave = ev.wave[n][m][ilo:ihi] * 1e4 #Angstroms #iwave = np.argsort(specwave) #specwave = specwave[iwave]*1e4 #Angstroms wavelow = specwave[0] wavehi = specwave[-1] spectrum = np.sum(ev.spectra[n, :, ilo:ihi], axis=0) if isplots: # Optimal ev.ldcoeffs = limbDarkening(ev.file_med, wavelow, wavehi, specwave=specwave, spectrum=spectrum, n_param=n_param, n_plot=4000, stellarmodel=stellarmodel) plt.title(str(n_param) + ' parameter model, optimal fit') plt.savefig(eventdir + '/figs/' + ev.eventname + '-Fig' + str(4000) + str(n_param) + '.png') try: # Low ev.ldcoeffs_low = limbDarkening(ev.file_low, wavelow, wavehi, specwave=specwave, spectrum=spectrum, n_param=n_param, n_plot=4001, stellarmodel=stellarmodel) plt.title(str(n_param) + ' parameter model, low fit') plt.savefig(eventdir + '/figs/' + ev.eventname + '-Fig' + str(4001) + str(n_param) + '.png') # Hi ev.ldcoeffs_hi = limbDarkening(ev.file_hi, wavelow, wavehi, specwave=specwave, spectrum=spectrum, n_param=n_param, n_plot=4002, stellarmodel=stellarmodel) plt.title(str(n_param) + ' parameter model, hi fit') plt.savefig(eventdir + '/figs/' + ev.eventname + '-Fig' + str(4002) + str(n_param) + '.png') except: pass else: ev.ldcoeffs = limbDarkening(ev.file_med, wavelow, wavehi, specwave=specwave, spectrum=spectrum, n_param=n_param, n_plot=False, stellarmodel=stellarmodel) try: ev.ldcoeffs_low = limbDarkening(ev.file_low, wavelow, wavehi, specwave=specwave, spectrum=spectrum, n_param=n_param, n_plot=False, stellarmodel=stellarmodel) ev.ldcoeffs_hi = limbDarkening(ev.file_hi, wavelow, wavehi, specwave=specwave, spectrum=spectrum, n_param=n_param, n_plot=False, stellarmodel=stellarmodel) except: pass print(eventname, ev.ldcoeffs) # Save results print('Saving results...') me.saveevent(ev, eventdir + '/d-' + ev.eventname + '-w4', delete=['spectra']) return
def lcWFC3(eventname, eventdir, nchan, madVariable, madVarSet, wmin=1.125, wmax=1.65, expand=1, smooth_len=None, correctDrift=True, isplots=True): ''' Compute photometric flux over specified range of wavelengths Parameters ---------- eventname : Unique identifier for these data eventdir : Location of save file nchan : Number of spectrophotometric channels wmin : minimum wavelength wmax : maximum wavelength expand : expansion factor isplots : Set True to produce plots Returns ------- None History ------- Written by Kevin Stevenson June 2012 ''' # Load saved data print("Loading saved data...") try: ev = me.loadevent(eventdir + '/d-' + eventname + '-w2') print('W2 data loaded\n') except: ev = me.loadevent(eventdir + '/d-' + eventname + '-w0') print('W0 data loaded\n') aux = me.loadevent(eventdir + '/d-' + eventname + '-data') ev.spectra = aux.spectra specerr = aux.specerr data_mhdr = aux.data_mhdr #Replace NaNs with zero ev.spectra[np.where(np.isnan(ev.spectra))] = 0 # Determine wavelength bins binsize = (wmax - wmin) / nchan wave_low = np.round([i for i in np.linspace(wmin, wmax - binsize, nchan)], 3) wave_hi = np.round([i for i in np.linspace(wmin + binsize, wmax, nchan)], 3) #binwave = (wave_low + wave_hi)/2. # Increase resolution of spectra nx = ev.spectra.shape[-1] if expand > 1: print("Increasing spectra resolution...") #hdspectra = np.zeros((ev.n_files,ev.n_reads-1,expand*nx)) #hdspecerr = np.zeros((ev.n_files,ev.n_reads-1,expand*nx)) hdspectra = spni.zoom(ev.spectra, zoom=[1, 1, expand]) hdspecerr = spni.zoom(specerr, zoom=[1, 1, expand]) * np.sqrt(expand) hdwave = np.zeros((ev.n_img, ev.n_spec, expand * nx)) for j in range(ev.n_img): hdwave[j] = spni.zoom(ev.wave[j][0], zoom=expand) ev.spectra = hdspectra specerr = hdspecerr ev.wave = hdwave nx *= expand # Smooth spectra if smooth_len != None: for m in range(ev.n_files): for n in range(ev.n_reads - 1): ev.spectra[m, n] = smooth.smooth(ev.spectra[m, n], smooth_len, 'flat') """ # First read is bad for IMA files if ev.n_reads > 2: print('WARNING: Marking all first reads as bad.') istart = 1 else: print('Using first reads.') istart = 0 """ print('Using first reads.') istart = 0 if correctDrift == True: #Shift 1D spectra #Calculate drift over all frames and non-destructive reads print('Applying drift correction...') ev.drift, ev.goodmask = hst.drift_fit2(ev, preclip=0, postclip=None, width=5 * expand, deg=2, validRange=11 * expand, istart=istart, iref=ev.iref[0]) # Correct for drift for m in range(ev.n_files): for n in range(istart, ev.n_reads - 1): spline = spi.UnivariateSpline(np.arange(nx), ev.spectra[m, n], k=3, s=0) #ev.spectra[m,n,p] = spline(np.arange(nx)+ev.drift_model[n,m,p]) #if m==13: # ev.drift[n,m,p] -= 0.476 #Using measured drift, not model fit ev.spectra[m, n] = spline(np.arange(nx) + ev.drift[m, n]) ''' # Look for bad columns igoodcol = np.ones(nx) normspec = ev.spectra/np.mean(ev.spectra,axis=2)[:,:,np.newaxis] sumspec = np.sum(normspec,axis=1)/(ev.n_reads-istart-1) stdsumspec = np.std(sumspec, axis=0) igoodcol[np.where(stdsumspec > 0.007)] = 0 #FINDME: hard coded ''' print("Generating light curves...") ev.eventname2 = ev.eventname for i in range(nchan): ev.wave_low = wave_low[i] ev.wave_hi = wave_hi[i] print("Bandpass = %.3f - %.3f" % (ev.wave_low, ev.wave_hi)) # Calculate photometric flux for each spectrum ev.photflux = np.zeros( (ev.n_spec, ev.n_files, np.max((1, ev.n_reads - 1 - istart)))) ev.photfluxerr = np.zeros( (ev.n_spec, ev.n_files, np.max((1, ev.n_reads - 1 - istart)))) #ev.wave = [] if ev.detector == 'IR': #Compute common wavelength and indeces to apply over all observations wave = np.zeros(nx) for j in range(ev.n_img): wave += ev.wave[j][0] wave /= ev.n_img #index = np.where(np.bitwise_and(wave >= wave_low[i], wave <= wave_hi[i]))[0] index = np.where((wave >= wave_low[i]) * (wave <= wave_hi[i]))[0] #define numgoodcol, totcol else: # UVIS: Use all pixels for aperture photometry index = range(len(ev.spectra[0, 0, 0])) for m in range(ev.n_files): ''' #Select appropriate orbit-dependent wavelength if ev.n_img == (np.max(ev.orbitnum)+1): j = int(ev.orbitnum[m]) else: j = 0 #Method 1 ev.wave.append(np.mean(ev.wavegrid[j][n],axis=0)) index = np.where(np.bitwise_and(ev.wave[n] >= wave_low, ev.wave[n] <= wave_hi))[0] #Method 2 index = np.where(np.bitwise_and(ev.wave[j][n] >= wave_low, ev.wave[j][n] <= wave_hi))[0] ''' ev.photflux[0, m] = np.sum(ev.spectra[m, istart:, index], axis=0) ev.photfluxerr[0, m] = np.sqrt( np.sum(specerr[m, istart:, index]**2, axis=0)) # Save results ev.eventname = ev.eventname2 + '_' + str(int( ev.wave_low * 1e3)) + '_' + str(int(ev.wave_hi * 1e3)) #me.saveevent(ev, eventdir + '/d-' + ev.eventname + '-w3', delete=['data_mhdr', 'spectra', 'specerr']) me.saveevent(ev, eventdir + '/d-' + ev.eventname + '-w3') # Produce plot if isplots == True: plt.figure(3000 + i, figsize=(10, 8)) plt.clf() plt.suptitle('Wavelength range: ' + str(wave_low[i]) + '-' + str(wave_hi[i])) ax = plt.subplot(111) #for n in range(ev.n_spec): #plt.subplot(ev.n_spec,1,1) #plt.title('Star ' + str(n)) #igood = np.where(ev.goodmask[0])[0] iscan0 = np.where(ev.scandir == 0)[0] iscan1 = np.where(ev.scandir == 1)[0] mjd = np.floor(ev.bjdtdb[0]) flux0 = np.sum(ev.photflux[0][iscan0], axis=1) / np.sum( ev.photflux[0, [iscan0[-1]]]) # forward scan #err = np.sqrt(1 / np.sum(1/ev.photfluxerr[0]**2,axis=1))/np.sum(ev.photflux[0,-1]) try: err0 = np.sqrt(np.sum(ev.photfluxerr[0][iscan0]**2, axis=1)) / np.sum( ev.photflux[0, [iscan0[-1]]]) except: err0 = 0 #err1 = 0 plt.errorbar(ev.bjdtdb[iscan0] - mjd, flux0, err0, fmt='bo') plt.text( 0.05, 0.1, "MAD = " + str(np.round(1e6 * np.median(np.abs(np.ediff1d(flux0))))) + " ppm", transform=ax.transAxes, color='b') #print(len(iscan1)) flux1 = 0 if len(iscan1) > 0: flux1 = np.sum(ev.photflux[0][iscan1], axis=1) / np.sum( ev.photflux[0, [iscan0[-1]]]) # reverse scan err1 = np.sqrt(np.sum(ev.photfluxerr[0][iscan1]**2, axis=1)) / np.sum( ev.photflux[0, [iscan0[-1]]]) plt.errorbar(ev.bjdtdb[iscan1] - mjd, flux1, err1, fmt='ro') plt.text( 0.05, 0.05, "MAD = " + str(np.round(1e6 * np.median(np.abs(np.ediff1d(flux1))))) + " ppm", transform=ax.transAxes, color='r') plt.ylabel('Normalized Flux') plt.xlabel('Time [MJD + ' + str(mjd) + ']') plt.subplots_adjust(left=0.10, right=0.95, bottom=0.10, top=0.90, hspace=0.20, wspace=0.3) plt.savefig(eventdir + '/figs/Fig' + str(3000 + i) + '-' + ev.eventname + '.png') #plt.pause(0.1) # f = open('2017-07-15-w1_spec_width_20/W5_MAD_'+ev.madVarStr+'_1D.txt','a+') # fooTemp = getattr(ev,madVariable) # print('W5: ' + ev.madVarStr + ' = ' + str(fooTemp)) # f.write(str(fooTemp) + ',' + str(np.round(1e6*np.median(np.abs(np.ediff1d(flux0))))) + ',' + str(np.round(1e6*np.median(np.abs(np.ediff1d(flux1))))) +'\n') # f.close() # print('W5_MAD_'+ ev.madVarStr +'_1D.txt saved\n') if (isplots >= 1) and (ev.detector == 'IR'): # Drift plt.figure(3100, figsize=(10, 8)) plt.clf() plt.subplot(211) for j in range(istart, ev.n_reads - 1): plt.plot(ev.drift2D[:, j, 1], '.') plt.ylabel('Spectrum Drift Along y') plt.subplot(212) for j in range(istart, ev.n_reads - 1): plt.plot(ev.drift2D[:, j, 0] + ev.drift[:, j], '.') plt.ylabel('Spectrum Drift Along x') plt.xlabel('Frame Number') plt.savefig(eventdir + '/figs/fig3100-Drift.png') # 2D light curve with drift correction plt.figure(3200, figsize=(7.85, ev.n_files / 20. + 0.8)) plt.clf() vmin = 0.98 vmax = 1.01 #FINDME normspec = np.zeros((ev.n_files, ev.spectra.shape[2])) for p in range(2): iscan = np.where(ev.scandir == p)[0] if len(iscan) > 0: normspec[iscan] = np.mean(ev.spectra[iscan],axis=1)/ \ np.mean(ev.spectra[iscan[ev.inormspec[0]:ev.inormspec[1]]],axis=(0,1)) #normspec[iscan] = np.mean(ev.spectra[iscan],axis=1)/np.mean(ev.spectra[ev.iref[p]],axis=0) #normspec = np.mean(ev.spectra[:,istart:],axis=1)/np.mean(ev.spectra[ev.inormspec[0]:ev.inormspec[1],istart:],axis=(0,1)) ediff = np.zeros(ev.n_files) iwmin = np.where(ev.wave[0][0] > wmin)[0][0] iwmax = np.where(ev.wave[0][0] > wmax)[0][0] for m in range(ev.n_files): ediff[m] = 1e6 * np.median( np.abs(np.ediff1d(normspec[m, iwmin:iwmax]))) plt.scatter(ev.wave[0][0], np.zeros(normspec.shape[-1]) + m, c=normspec[m], s=14, linewidths=0, vmin=vmin, vmax=vmax, marker='s', cmap=plt.cm.RdYlBu_r) plt.title("MAD = " + str(np.round(np.mean(ediff), 0)) + " ppm") plt.xlim(wmin, wmax) if nchan > 1: xticks = np.round([i for i in np.linspace(wmin, wmax, nchan + 1)], 3) plt.xticks(xticks, xticks) plt.vlines(xticks, 0, ev.n_files, 'k', 'dashed') plt.ylim(0, ev.n_files) plt.ylabel('Frame Number') plt.xlabel('Wavelength ($\mu m$)') plt.xticks(rotation=30) plt.colorbar() plt.tight_layout() plt.savefig(eventdir + '/figs/fig3200-' + str(nchan) + '-2D_LC.png') #plt.savefig(eventdir+'/figs/fig3200-'+str(nchan)+'-2D_LC_'+madVariable+'_'+str(madVarSet)+'.png') #ev.mad5 = np.round(np.mean(ediff),0) # f = open('2017-07-15-w1_spec_width_20/W5_MAD_'+ev.madVarStr+'.txt','a+') # fooTemp = getattr(ev,madVariable) # print('W5: ' + ev.madVarStr + ' = ' + str(fooTemp)) # f.write(str(fooTemp) + ',' + str(np.round(np.mean(ediff),0)) + '\n') # f.close() # print('W5_MAD_'+ ev.madVarStr +'.txt saved\n') if (isplots >= 3) and (ev.detector == 'IR'): # Plot individual non-destructive reads vmin = 0.97 vmax = 1.03 iwmin = np.where(ev.wave[0][0] > wmin)[0][0] iwmax = np.where(ev.wave[0][0] > wmax)[0][0] #FINDME normspec = ev.spectra[:, istart:] / np.mean( ev.spectra[ev.inormspec[0]:ev.inormspec[1], istart:], axis=0) for n in range(ev.n_reads - 1): plt.figure(3300 + n, figsize=(8, ev.n_files / 20. + 0.8)) plt.clf() ediff = np.zeros(ev.n_files) for m in range(ev.n_files): ediff[m] = 1e6 * np.median( np.abs(np.ediff1d(normspec[m, n, iwmin:iwmax]))) plt.scatter(ev.wave[0][0], np.zeros(normspec.shape[-1]) + m, c=normspec[m, n], s=14, linewidths=0, vmin=vmin, vmax=vmax, marker='s', cmap=plt.cm.RdYlBu_r) plt.title("MAD = " + str(np.round(np.mean(ediff), 0)) + " ppm") plt.xlim(wmin, wmax) plt.ylim(0, ev.n_files) plt.ylabel('Frame Number') plt.xlabel('Wavelength ($\mu m$)') plt.colorbar() plt.tight_layout() plt.savefig(ev.eventdir + '/figs/fig' + str(3300 + n) + '-2D_LC.png') """ # Aligned 1D spectra plt.figure(3300, figsize=(8,6.5)) plt.clf() #istart=0 #normspec = ev.spectra[:,istart:]/np.mean(ev.spectra[:,istart:],axis=2)[:,:,np.newaxis] normspec = ev.spectra[:,:,1:]/np.mean(ev.spectra[:,:,1:],axis=2)[:,:,np.newaxis] wave = ev.wave[0][0][1:] sumspec = np.sum(normspec,axis=1)/(ev.n_reads-istart-1) for m in range(10,16): plt.plot(wave,sumspec[m],'r-') for m in range(7,10): plt.plot(wave,sumspec[m],'.k-') """ return ev