def remove_grism_sky(flt='ibhm46ioq_flt.fits', list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'], path_to_sky = '../CONF/', out_path='./', verbose=False, plot=False, flat_correct=True, sky_subtract=True, second_pass=True, overall=True, combine_skies=False, sky_components=True, add_constant=True): """ Process a (G141) grism exposure by dividing by the F140W imaging flat-field and then subtracting by a master sky image. v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'] testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits'] list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'] """ import threedhst.grism_sky as bg #import scipy.signal as sign # flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz' im = pyfits.open(flt) bg.set_grism_flat(grism=im[0].header['FILTER'], verbose=True) segfile = os.path.basename(flt.replace('.fits','.seg.fits')).replace('.gz','') if os.path.exists(segfile): seg = pyfits.open(segfile)[0].data use_biweight=False else: seg = np.zeros(im[1].data.shape) use_biweight=True xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight) #yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)]) if plot: plt.plot(xin, yin, color='black', linewidth=2) #### Loop through sky images and find the one whose column profile most #### closely matches the input image chi2 = 1.e10 keep = None for sky in list: xsky, ysky = bg.profile(flt=path_to_sky+sky, extension=0, flatcorr=False, biweight=True) ysky /= np.mean(ysky[np.abs(xsky-507) < 50]) # ok = np.isfinite(ysky) & np.isfinite(yin) & (yin*ysky != 0) a = np.sum((ysky*yin)[ok])/np.sum((ysky*ysky)[ok]) if plot: plt.plot(xsky, ysky*a) # chi2_i = np.sum((ysky[ok]*a-yin[ok])**2) if verbose: print sky, chi2_i # if chi2_i < chi2: chi2 = chi2_i*1 keep = sky if keep is None: keep = 'sky_goodsn_vhi.fits' #### The best sky image sk = pyfits.open(path_to_sky+keep) sk[0].data[sk[0].data == 0] = 1. sk[0].data[~np.isfinite(sk[0].data)] = 1. flat = bg.flat*1. #### Only flat correction dq_ok = (im[3].data & (4+32+16+512+2048+4096)) == 0 mask = (seg == 0) & dq_ok if plot: corr = im[1].data*flat#/sk[0].data corr -= threedhst.utils.biweight(corr[mask], mean=True) ds9.frame(1) ds9.v(corr, vmin=-0.5,vmax=0.5) if flat_correct is False: flat = flat*0+1 if sky_subtract is False: sk[0].data = sk[0].data*0+1 #### Divide by the sky flat #corr = im[1].data*flat/sk[0].data # #### Show the result # if plot: # ds9.frame(2) # ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5) ### Instead, subtract the sky flat sky_stats = threedhst.utils.biweight((im[1].data*flat/sk[0].data)[mask], both=True) corr = im[1].data*flat-sky_stats[0]*sk[0].data #### Get least-sq coeffs of multiple sky components if sky_components: from scipy.linalg import lstsq import scipy.optimize import scipy.ndimage as nd import copy #grow_mask = nd.maximum_filter((~mask)*1., size=3) == 0 ims = [] #skies = ['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits'] skies = copy.deepcopy(list) for sky in skies: ims.append(pyfits.open(path_to_sky + sky)[0].data.flatten()) if add_constant: ims.append(im[1].data.flatten()*0.+1) skies.append('Constant') ims = np.array(ims) seg_mask = nd.maximum_filter((seg > 0), size=18) == 0 #### First iteration, non-weighted least-sq mask_full = seg_mask & dq_ok & ((im[1].data*bg.flat) < np.percentile((im[1].data*bg.flat)[mask], 98)) & (im[2].data > 0) & ((im[1].data*bg.flat) > np.percentile((im[1].data*bg.flat)[mask], 1)) data = (im[1].data*bg.flat)[mask_full].flatten() xcoeff, resid, rank, ss = lstsq(ims[:, mask_full.flatten()].T, data) model = np.dot(xcoeff, ims).reshape((1014,1014)) corr = im[1].data*flat-model #### Second iteration: improved mask, weighted lstsq mask_full = seg_mask & dq_ok & (corr < np.percentile(corr[mask], 98)) & (im[2].data > 0) & (corr > np.percentile(corr[mask], 1)) data = (im[1].data*bg.flat)[mask_full].flatten() wht = 1./(im[2].data)[mask_full].flatten() p0 = np.ones(ims.shape[0]) popt = scipy.optimize.leastsq(bg.obj_lstsq, p0, args=(data, ims[:, mask_full.flatten()], wht), full_output=True, ftol=1.49e-8/1000., xtol=1.49e-8/1000.) xcoeff = popt[0] model = np.dot(xcoeff, ims).reshape((1014,1014)) corr = im[1].data*flat-model #### Use the new mask mask = mask_full #### 1D column averages if True: yres = np.zeros(1014) yfull = np.zeros(1014) ydat = np.zeros(1014) fcorr = (im[1].data*flat) xfull = yfull*0. for i in range(1014): ymsk = mask_full[:,i] > 0 ydat[i] = np.median(fcorr[ymsk,i]) yfull[i] = np.median(model[ymsk,i]) yres[i] = np.median(corr[ymsk,i]) # xmsk = mask_full[i,:] > 0 xfull[i] = np.median(model[i,xmsk]) #print i yres_sm = threedhst.utils.medfilt(yres, 41) ### Make figure from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg fig = Figure(figsize=[8,4], dpi=100) fig.subplots_adjust(wspace=0.25,hspace=0.02,left=0.1, bottom=0.08,right=0.99,top=0.92) ax = fig.add_subplot(121) ax.plot(ydat, color='black') ax.plot(yfull, color='red') ax.plot(xfull, color='green') ax.set_xlim(0,1014) ax.set_title(flt) ax = fig.add_subplot(122) ax.plot(yres, color='black') ax.plot(yres_sm, color='red', linewidth=2) ax.set_xlim(0,1014) canvas = FigureCanvasAgg(fig) canvas.print_figure(flt.split('.fits')[0] + '.multisky.png', dpi=100, transparent=False) #### Update header keywords print 'Simultaneous sky components:' for i in range(len(skies)): print ' %s %.3f' %(skies[i], xcoeff[i]) im[0].header.update('GSKY%02d' %(i+1), xcoeff[i], comment='Grism sky: %s' %(skies[i])) # #### Show the result # if plot: # ds9.frame(3) # ds9.v(corr, vmin=-0.5,vmax=0.5) #### Put the result in the FLT data extension im[1].data = corr*1. #### Need to write an output file to use `profile` im.writeto(out_path+os.path.basename(flt).replace('.gz',''), clobber=True) xin, yin = bg.profile(out_path+os.path.basename(flt).replace('.gz',''), extension=1, flatcorr=False, biweight=True) im = pyfits.open(out_path+os.path.basename(flt).replace('.gz',''), mode='update') if second_pass: #### Subtract the residual difference between the observed and master sky if sky_components: ### Use column average found earlier resid = np.dot(np.ones((1014,1)), yres_sm.reshape(1,1014)) else: resid = np.dot(np.ones((1014,1)), threedhst.utils.medfilt(yin, 41).reshape(1,1014)) im[1].data -= resid #### Subtract the overall biweight mean if overall: full_mean = threedhst.utils.biweight(im[1].data[mask], mean=True) im[1].data -= full_mean print 'overall: %.4f' %(full_mean) #### Add a header keyword and write to the output image im[0].header.update('GRISMSKY',keep,comment='Image used for sky subtraction') im[0].header.update('SKYSCALE',sky_stats[0],comment='Scale factor of sky') #### Sky flat keyword if 'SKYFLAT' in im[0].header.keys(): im[0].header['SKYFLAT'] = (flat_correct | im[0].header['SKYFLAT'], 'Direct image flat applied') else: im[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied') bad = ~np.isfinite(im[1].data) im[1].data[bad] = 1 im[3].data[bad] = im[3].data[bad] | 32 im.flush() #### Show the final result, compare to the earlier version in PREP_FLT if plot: ds9.frame(3) ds9.v(im[1].data, vmin=-0.5,vmax=0.5) chk = pyfits.open(threedhst.utils.find_fits_gz(flt.replace('RAW','PREP_FLT').replace('.gz',''))) ds9.frame(4) ds9.v(chk[1].data, vmin=-0.5,vmax=0.5)
def show_profile(): import threedhst.grism_sky as bg """ Look at images collapsed along columns to separate into groups with similar patterns. """ #### COSMOS flt_files = glob.glob('ibhm*flt.seg.fits') PATH = '/3DHST/Spectra/Work/COSMOS/RAW/' GZ = '.gz' fp = open('COSMOS.g141.list') flt_files = fp.readlines() fp.close() for i in range(len(flt_files)): flt_files[i] = flt_files[i][:-1].replace('msk','flt') N = len(flt_files) profiles = np.zeros((N, 1014)) for i,flt in enumerate(flt_files): flt = flt.replace('.seg','') if os.path.exists(flt+'.mask.reg'): continue # print flt xi, yi = bg.profile(flt=PATH+flt+GZ) profiles[i,:] = yi norm = np.zeros(N) test = norm > 0 for i in range(N): yi = profiles[i,:] norm[i] = np.mean(yi[np.abs(xi-507) < 50]) test[i] = np.median(yi[np.abs(xi-40) < 10]/norm[i]) < 1.95 if test[i]: p = plt.plot(xi,yi/norm[i],color=(norm[i]/3.3,0,0), alpha=0.1) else: norm[i] = 0 profiles_norm = profiles / np.dot(norm.reshape(N,1), np.ones((1,1014))) avg = np.mean(profiles_norm[norm != 0, :], axis=0) plt.plot(xi, avg, color='blue', alpha=0.5) # for i in range(N): # yi = profiles[i,:]*1. # if yi.sum() == 0: # continue # # # yi-=0. # nor = np.mean(yi[np.abs(xi-307) < 50]) # p = plt.plot(xi,yi/nor,color=(norm[i]/6,0,0), alpha=0.1) plt.ylim(0.92,1.04) plt.xlim(-10,1024) plt.savefig('COSMOS.png') #### GOODS-N flt_files = glob.glob('ib37*seg.fits') PATH = '/research/HST/GRISM/3DHST/GOODS-N/RAW/' Ng = len(flt_files) profiles_g = np.zeros((Ng, 1014)) for i,flt in enumerate(flt_files): flt = flt.replace('.seg','') if os.path.exists(flt+'.mask.reg'): continue # print flt xi, yi = bg.profile(flt=PATH+flt+GZ) profiles_g[i,:] = yi xi = np.arange(1014) norm_g = np.zeros(Ng) test = norm_g > 0 for i in range(Ng): yi = profiles_g[i,:] norm_g[i] = np.mean(yi[np.abs(xi-507) < 50]) # very hi test[i] = np.median(yi[np.abs(xi-200) < 20]/norm_g[i]) > 1.02 # lo #test[i] = np.median(yi[np.abs(xi-200) < 20]/norm_g[i]) < 1.01 #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_g[i]) > 0.96) # hi #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_g[i]) < 0.96) # if test[i]: p = plt.plot(xi,yi/norm_g[i],color=(0,0,norm_g[i]/1.8), alpha=0.1) else: norm_g[i]*=0 # plt.ylim(0.92,1.04) plt.xlim(-10,1024) profiles_norm_g = profiles_g / np.dot(norm_g.reshape(Ng,1), np.ones((1,1014))) avg_g = np.mean(profiles_norm_g[norm_g != 0, :], axis=0) plt.plot(xi, avg_g, color='green', alpha=0.5) plt.ylim(0.92,1.04) plt.xlim(-10,1024) #### AEGIS flt_files = glob.glob('ibhj[4]*seg.fits') PATH = '/research/HST/GRISM/3DHST/AEGIS/RAW/' Na = len(flt_files) profiles_a = np.zeros((Na, 1014)) for i,flt in enumerate(flt_files): flt = flt.replace('.seg','') if os.path.exists(flt+'.mask.reg'): continue # print flt xi, yi = bg.profile(flt=PATH+flt+GZ) profiles_a[i,:] = yi xi = np.arange(1014) norm_a = np.zeros(Na) test = norm_a > 0 for i in range(Na): yi = profiles_a[i,:] norm_a[i] = np.mean(yi[np.abs(xi-507) < 50]) # very hi test[i] = np.median(yi[np.abs(xi-200) < 20]/norm_a[i]) < 1.52 # lo #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_a[i]) > 0.96) # hi #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_a[i]) < 0.96) # if test[i]: p = plt.plot(xi,yi/norm_a[i],color=(0,0,norm_a[i]/1.8), alpha=0.1) else: norm_a[i]*=0 # plt.ylim(0.92,1.04) plt.xlim(-10,1024)
def remove_grism_sky(flt='ibhm46ioq_flt.fits', list=[ 'sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits' ], path_to_sky='../CONF/', out_path='./', verbose=False, plot=False, flat_correct=True, sky_subtract=True, second_pass=True, overall=True, combine_skies=False, sky_components=True, add_constant=True): """ Process a (G141) grism exposure by dividing by the F140W imaging flat-field and then subtracting by a master sky image. v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'] testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits'] list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'] """ import threedhst.grism_sky as bg #import scipy.signal as sign # flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz' im = pyfits.open(flt) bg.set_grism_flat(grism=im[0].header['FILTER'], verbose=True) segfile = os.path.basename(flt.replace('.fits', '.seg.fits')).replace('.gz', '') if os.path.exists(segfile): seg = pyfits.open(segfile)[0].data use_biweight = False else: seg = np.zeros(im[1].data.shape) use_biweight = True xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight) #yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)]) if plot: plt.plot(xin, yin, color='black', linewidth=2) #### Loop through sky images and find the one whose column profile most #### closely matches the input image chi2 = 1.e10 keep = None for sky in list: xsky, ysky = bg.profile(flt=path_to_sky + sky, extension=0, flatcorr=False, biweight=True) ysky /= np.mean(ysky[np.abs(xsky - 507) < 50]) # ok = np.isfinite(ysky) & np.isfinite(yin) & (yin * ysky != 0) a = np.sum((ysky * yin)[ok]) / np.sum((ysky * ysky)[ok]) if plot: plt.plot(xsky, ysky * a) # chi2_i = np.sum((ysky[ok] * a - yin[ok])**2) if verbose: print sky, chi2_i # if chi2_i < chi2: chi2 = chi2_i * 1 keep = sky if keep is None: keep = 'sky_goodsn_vhi.fits' #### The best sky image sk = pyfits.open(path_to_sky + keep) sk[0].data[sk[0].data == 0] = 1. sk[0].data[~np.isfinite(sk[0].data)] = 1. flat = bg.flat * 1. #### Only flat correction dq_ok = (im[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0 mask = (seg == 0) & dq_ok if plot: corr = im[1].data * flat #/sk[0].data corr -= threedhst.utils.biweight(corr[mask], mean=True) ds9.frame(1) ds9.v(corr, vmin=-0.5, vmax=0.5) if flat_correct is False: flat = flat * 0 + 1 if sky_subtract is False: sk[0].data = sk[0].data * 0 + 1 #### Divide by the sky flat #corr = im[1].data*flat/sk[0].data # #### Show the result # if plot: # ds9.frame(2) # ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5) ### Instead, subtract the sky flat sky_stats = threedhst.utils.biweight( (im[1].data * flat / sk[0].data)[mask], both=True) corr = im[1].data * flat - sky_stats[0] * sk[0].data #### Get least-sq coeffs of multiple sky components if sky_components: from scipy.linalg import lstsq import scipy.optimize import scipy.ndimage as nd import copy #grow_mask = nd.maximum_filter((~mask)*1., size=3) == 0 ims = [] #skies = ['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits'] skies = copy.deepcopy(list) for sky in skies: ims.append(pyfits.open(path_to_sky + sky)[0].data.flatten()) if add_constant: ims.append(im[1].data.flatten() * 0. + 1) skies.append('Constant') ims = np.array(ims) seg_mask = nd.maximum_filter((seg > 0), size=18) == 0 #### First iteration, non-weighted least-sq mask_full = seg_mask & dq_ok & ((im[1].data * bg.flat) < np.percentile( (im[1].data * bg.flat)[mask], 98)) & (im[2].data > 0) & ( (im[1].data * bg.flat) > np.percentile( (im[1].data * bg.flat)[mask], 1)) data = (im[1].data * bg.flat)[mask_full].flatten() xcoeff, resid, rank, ss = lstsq(ims[:, mask_full.flatten()].T, data) model = np.dot(xcoeff, ims).reshape((1014, 1014)) corr = im[1].data * flat - model #### Second iteration: improved mask, weighted lstsq mask_full = seg_mask & dq_ok & (corr < np.percentile( corr[mask], 98)) & (im[2].data > 0) & (corr > np.percentile( corr[mask], 1)) data = (im[1].data * bg.flat)[mask_full].flatten() wht = 1. / (im[2].data)[mask_full].flatten() p0 = np.ones(ims.shape[0]) popt = scipy.optimize.leastsq(bg.obj_lstsq, p0, args=(data, ims[:, mask_full.flatten()], wht), full_output=True, ftol=1.49e-8 / 1000., xtol=1.49e-8 / 1000.) xcoeff = popt[0] model = np.dot(xcoeff, ims).reshape((1014, 1014)) corr = im[1].data * flat - model #### Use the new mask mask = mask_full #### 1D column averages if True: yres = np.zeros(1014) yfull = np.zeros(1014) ydat = np.zeros(1014) fcorr = (im[1].data * flat) xfull = yfull * 0. for i in range(1014): ymsk = mask_full[:, i] > 0 ydat[i] = np.median(fcorr[ymsk, i]) yfull[i] = np.median(model[ymsk, i]) yres[i] = np.median(corr[ymsk, i]) # xmsk = mask_full[i, :] > 0 xfull[i] = np.median(model[i, xmsk]) #print i yres_sm = threedhst.utils.medfilt(yres, 41) ### Make figure from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg fig = Figure(figsize=[8, 4], dpi=100) fig.subplots_adjust(wspace=0.25, hspace=0.02, left=0.1, bottom=0.08, right=0.99, top=0.92) ax = fig.add_subplot(121) ax.plot(ydat, color='black') ax.plot(yfull, color='red') ax.plot(xfull, color='green') ax.set_xlim(0, 1014) ax.set_title(flt) ax = fig.add_subplot(122) ax.plot(yres, color='black') ax.plot(yres_sm, color='red', linewidth=2) ax.set_xlim(0, 1014) canvas = FigureCanvasAgg(fig) canvas.print_figure(flt.split('.fits')[0] + '.multisky.png', dpi=100, transparent=False) #### Update header keywords print 'Simultaneous sky components:' for i in range(len(skies)): print ' %s %.3f' % (skies[i], xcoeff[i]) im[0].header.update('GSKY%02d' % (i + 1), xcoeff[i], comment='Grism sky: %s' % (skies[i])) # #### Show the result # if plot: # ds9.frame(3) # ds9.v(corr, vmin=-0.5,vmax=0.5) #### Put the result in the FLT data extension im[1].data = corr * 1. #### Need to write an output file to use `profile` im.writeto(out_path + os.path.basename(flt).replace('.gz', ''), clobber=True) xin, yin = bg.profile(out_path + os.path.basename(flt).replace('.gz', ''), extension=1, flatcorr=False, biweight=True) im = pyfits.open(out_path + os.path.basename(flt).replace('.gz', ''), mode='update') if second_pass: #### Subtract the residual difference between the observed and master sky if sky_components: ### Use column average found earlier resid = np.dot(np.ones((1014, 1)), yres_sm.reshape(1, 1014)) else: resid = np.dot(np.ones((1014, 1)), threedhst.utils.medfilt(yin, 41).reshape(1, 1014)) im[1].data -= resid #### Subtract the overall biweight mean if overall: full_mean = threedhst.utils.biweight(im[1].data[mask], mean=True) im[1].data -= full_mean print 'overall: %.4f' % (full_mean) #### Add a header keyword and write to the output image im[0].header.update('GRISMSKY', keep, comment='Image used for sky subtraction') im[0].header.update('SKYSCALE', sky_stats[0], comment='Scale factor of sky') #### Sky flat keyword if 'SKYFLAT' in im[0].header.keys(): im[0].header['SKYFLAT'] = (flat_correct | im[0].header['SKYFLAT'], 'Direct image flat applied') else: im[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied') bad = ~np.isfinite(im[1].data) im[1].data[bad] = 1 im[3].data[bad] = im[3].data[bad] | 32 im.flush() #### Show the final result, compare to the earlier version in PREP_FLT if plot: ds9.frame(3) ds9.v(im[1].data, vmin=-0.5, vmax=0.5) chk = pyfits.open( threedhst.utils.find_fits_gz( flt.replace('RAW', 'PREP_FLT').replace('.gz', ''))) ds9.frame(4) ds9.v(chk[1].data, vmin=-0.5, vmax=0.5)
def remove_grism_sky( flt="ibhm46ioq_flt.fits", list=["sky_cosmos.fits", "sky_goodsn_lo.fits", "sky_goodsn_hi.fits", "sky_goodsn_vhi.fits"], path_to_sky="../CONF/", out_path="./", verbose=False, plot=False, flat_correct=True, sky_subtract=True, second_pass=True, overall=True, ): """ Process a (G141) grism exposure by dividing by the F140W imaging flat-field and then subtracting by a master sky image. v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'] testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits'] """ import threedhst.grism_sky as bg # import scipy.signal as sign # flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz' im = pyfits.open(flt) bg.set_grism_flat(grism=im[0].header["FILTER"]) segfile = os.path.basename(flt.replace(".fits", ".seg.fits")).replace(".gz", "") if os.path.exists(segfile): seg = pyfits.open(segfile)[0].data use_biweight = False else: seg = np.zeros(im[1].data.shape) use_biweight = True xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight) # yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)]) if plot: plt.plot(xin, yin, color="black", linewidth=2) #### Loop through sky images and find the one whose column profile most #### closely matches the input image chi2 = 1.0e10 keep = None for sky in list: xsky, ysky = bg.profile(flt=path_to_sky + sky, extension=0, flatcorr=False, biweight=True) ysky /= np.mean(ysky[np.abs(xsky - 507) < 50]) # ok = np.isfinite(ysky) & np.isfinite(yin) & (yin * ysky != 0) a = np.sum((ysky * yin)[ok]) / np.sum((ysky * ysky)[ok]) if plot: plt.plot(xsky, ysky * a) # chi2_i = np.sum((ysky[ok] * a - yin[ok]) ** 2) if verbose: print sky, chi2_i # if chi2_i < chi2: chi2 = chi2_i * 1 keep = sky if keep is None: keep = "sky_goodsn_vhi.fits" #### The best sky image sk = pyfits.open(path_to_sky + keep) sk[0].data[sk[0].data == 0] = 1.0 sk[0].data[~np.isfinite(sk[0].data)] = 1.0 flat = bg.flat * 1.0 #### Only flat correction dq_ok = (im[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0 mask = (seg == 0) & dq_ok if plot: corr = im[1].data * flat # /sk[0].data corr -= threedhst.utils.biweight(corr[mask], mean=True) ds9.frame(1) ds9.v(corr, vmin=-0.5, vmax=0.5) if flat_correct is False: flat = flat * 0 + 1 if sky_subtract is False: sk[0].data = sk[0].data * 0 + 1 #### Divide by the sky flat # corr = im[1].data*flat/sk[0].data # #### Show the result # if plot: # ds9.frame(2) # ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5) ### Instead, subtract the sky flat sky_stats = threedhst.utils.biweight((im[1].data * flat / sk[0].data)[mask], both=True) corr = im[1].data * flat - sky_stats[0] * sk[0].data # #### Show the result # if plot: # ds9.frame(3) # ds9.v(corr, vmin=-0.5,vmax=0.5) #### Put the result in the FLT data extension im[1].data = corr * 1.0 #### Need to write an output file to use `profile` im.writeto(out_path + os.path.basename(flt).replace(".gz", ""), clobber=True) xin, yin = bg.profile( out_path + os.path.basename(flt).replace(".gz", ""), extension=1, flatcorr=False, biweight=True ) im = pyfits.open(out_path + os.path.basename(flt).replace(".gz", ""), mode="update") #### Subtract the residual difference between the observed and master sky resid = np.dot(np.ones((1014, 1)), threedhst.utils.medfilt(yin, 41).reshape(1, 1014)) if second_pass: im[1].data -= resid #### Subtract the overall biweight mean if overall: im[1].data -= threedhst.utils.biweight(im[1].data[mask], mean=True) #### Add a header keyword and write to the output image im[0].header.update("GRISMSKY", keep, comment="Image used for sky subtraction") im[0].header.update("SKYSCALE", sky_stats[0], comment="Scale factor of sky") bad = ~np.isfinite(im[1].data) im[1].data[bad] = 1 im[3].data[bad] = im[3].data[bad] | 32 im.flush() #### Show the final result, compare to the earlier version in PREP_FLT if plot: ds9.frame(3) ds9.v(im[1].data, vmin=-0.5, vmax=0.5) chk = pyfits.open(threedhst.utils.find_fits_gz(flt.replace("RAW", "PREP_FLT").replace(".gz", ""))) ds9.frame(4) ds9.v(chk[1].data, vmin=-0.5, vmax=0.5)
def show_profile(): import threedhst.grism_sky as bg """ Look at images collapsed along columns to separate into groups with similar patterns. """ #### COSMOS flt_files = glob.glob('ibhm*flt.seg.fits') PATH = '/3DHST/Spectra/Work/COSMOS/RAW/' GZ = '.gz' fp = open('COSMOS.g141.list') flt_files = fp.readlines() fp.close() for i in range(len(flt_files)): flt_files[i] = flt_files[i][:-1].replace('msk', 'flt') N = len(flt_files) profiles = np.zeros((N, 1014)) for i, flt in enumerate(flt_files): flt = flt.replace('.seg', '') if os.path.exists(flt + '.mask.reg'): continue # print flt xi, yi = bg.profile(flt=PATH + flt + GZ) profiles[i, :] = yi norm = np.zeros(N) test = norm > 0 for i in range(N): yi = profiles[i, :] norm[i] = np.mean(yi[np.abs(xi - 507) < 50]) test[i] = np.median(yi[np.abs(xi - 40) < 10] / norm[i]) < 1.95 if test[i]: p = plt.plot(xi, yi / norm[i], color=(norm[i] / 3.3, 0, 0), alpha=0.1) else: norm[i] = 0 profiles_norm = profiles / np.dot(norm.reshape(N, 1), np.ones((1, 1014))) avg = np.mean(profiles_norm[norm != 0, :], axis=0) plt.plot(xi, avg, color='blue', alpha=0.5) # for i in range(N): # yi = profiles[i,:]*1. # if yi.sum() == 0: # continue # # # yi-=0. # nor = np.mean(yi[np.abs(xi-307) < 50]) # p = plt.plot(xi,yi/nor,color=(norm[i]/6,0,0), alpha=0.1) plt.ylim(0.92, 1.04) plt.xlim(-10, 1024) plt.savefig('COSMOS.png') #### GOODS-N flt_files = glob.glob('ib37*seg.fits') PATH = '/research/HST/GRISM/3DHST/GOODS-N/RAW/' Ng = len(flt_files) profiles_g = np.zeros((Ng, 1014)) for i, flt in enumerate(flt_files): flt = flt.replace('.seg', '') if os.path.exists(flt + '.mask.reg'): continue # print flt xi, yi = bg.profile(flt=PATH + flt + GZ) profiles_g[i, :] = yi xi = np.arange(1014) norm_g = np.zeros(Ng) test = norm_g > 0 for i in range(Ng): yi = profiles_g[i, :] norm_g[i] = np.mean(yi[np.abs(xi - 507) < 50]) # very hi test[i] = np.median(yi[np.abs(xi - 200) < 20] / norm_g[i]) > 1.02 # lo #test[i] = np.median(yi[np.abs(xi-200) < 20]/norm_g[i]) < 1.01 #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_g[i]) > 0.96) # hi #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_g[i]) < 0.96) # if test[i]: p = plt.plot(xi, yi / norm_g[i], color=(0, 0, norm_g[i] / 1.8), alpha=0.1) else: norm_g[i] *= 0 # plt.ylim(0.92, 1.04) plt.xlim(-10, 1024) profiles_norm_g = profiles_g / np.dot(norm_g.reshape(Ng, 1), np.ones((1, 1014))) avg_g = np.mean(profiles_norm_g[norm_g != 0, :], axis=0) plt.plot(xi, avg_g, color='green', alpha=0.5) plt.ylim(0.92, 1.04) plt.xlim(-10, 1024) #### AEGIS flt_files = glob.glob('ibhj[4]*seg.fits') PATH = '/research/HST/GRISM/3DHST/AEGIS/RAW/' Na = len(flt_files) profiles_a = np.zeros((Na, 1014)) for i, flt in enumerate(flt_files): flt = flt.replace('.seg', '') if os.path.exists(flt + '.mask.reg'): continue # print flt xi, yi = bg.profile(flt=PATH + flt + GZ) profiles_a[i, :] = yi xi = np.arange(1014) norm_a = np.zeros(Na) test = norm_a > 0 for i in range(Na): yi = profiles_a[i, :] norm_a[i] = np.mean(yi[np.abs(xi - 507) < 50]) # very hi test[i] = np.median(yi[np.abs(xi - 200) < 20] / norm_a[i]) < 1.52 # lo #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_a[i]) > 0.96) # hi #test[i] = test[i] & (np.median(yi[np.abs(xi-40) < 10]/norm_a[i]) < 0.96) # if test[i]: p = plt.plot(xi, yi / norm_a[i], color=(0, 0, norm_a[i] / 1.8), alpha=0.1) else: norm_a[i] *= 0 # plt.ylim(0.92, 1.04) plt.xlim(-10, 1024)