def remove_grism_sky(flt='ibhm46ioq_flt.fits', list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'], path_to_sky = '../CONF/', out_path='./', verbose=False, plot=False, flat_correct=True, sky_subtract=True, second_pass=True, overall=True, combine_skies=False, sky_components=True, add_constant=True): """ Process a (G141) grism exposure by dividing by the F140W imaging flat-field and then subtracting by a master sky image. v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'] testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits'] list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'] """ import threedhst.grism_sky as bg #import scipy.signal as sign # flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz' im = pyfits.open(flt) bg.set_grism_flat(grism=im[0].header['FILTER'], verbose=True) segfile = os.path.basename(flt.replace('.fits','.seg.fits')).replace('.gz','') if os.path.exists(segfile): seg = pyfits.open(segfile)[0].data use_biweight=False else: seg = np.zeros(im[1].data.shape) use_biweight=True xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight) #yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)]) if plot: plt.plot(xin, yin, color='black', linewidth=2) #### Loop through sky images and find the one whose column profile most #### closely matches the input image chi2 = 1.e10 keep = None for sky in list: xsky, ysky = bg.profile(flt=path_to_sky+sky, extension=0, flatcorr=False, biweight=True) ysky /= np.mean(ysky[np.abs(xsky-507) < 50]) # ok = np.isfinite(ysky) & np.isfinite(yin) & (yin*ysky != 0) a = np.sum((ysky*yin)[ok])/np.sum((ysky*ysky)[ok]) if plot: plt.plot(xsky, ysky*a) # chi2_i = np.sum((ysky[ok]*a-yin[ok])**2) if verbose: print sky, chi2_i # if chi2_i < chi2: chi2 = chi2_i*1 keep = sky if keep is None: keep = 'sky_goodsn_vhi.fits' #### The best sky image sk = pyfits.open(path_to_sky+keep) sk[0].data[sk[0].data == 0] = 1. sk[0].data[~np.isfinite(sk[0].data)] = 1. flat = bg.flat*1. #### Only flat correction dq_ok = (im[3].data & (4+32+16+512+2048+4096)) == 0 mask = (seg == 0) & dq_ok if plot: corr = im[1].data*flat#/sk[0].data corr -= threedhst.utils.biweight(corr[mask], mean=True) ds9.frame(1) ds9.v(corr, vmin=-0.5,vmax=0.5) if flat_correct is False: flat = flat*0+1 if sky_subtract is False: sk[0].data = sk[0].data*0+1 #### Divide by the sky flat #corr = im[1].data*flat/sk[0].data # #### Show the result # if plot: # ds9.frame(2) # ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5) ### Instead, subtract the sky flat sky_stats = threedhst.utils.biweight((im[1].data*flat/sk[0].data)[mask], both=True) corr = im[1].data*flat-sky_stats[0]*sk[0].data #### Get least-sq coeffs of multiple sky components if sky_components: from scipy.linalg import lstsq import scipy.optimize import scipy.ndimage as nd import copy #grow_mask = nd.maximum_filter((~mask)*1., size=3) == 0 ims = [] #skies = ['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits'] skies = copy.deepcopy(list) for sky in skies: ims.append(pyfits.open(path_to_sky + sky)[0].data.flatten()) if add_constant: ims.append(im[1].data.flatten()*0.+1) skies.append('Constant') ims = np.array(ims) seg_mask = nd.maximum_filter((seg > 0), size=18) == 0 #### First iteration, non-weighted least-sq mask_full = seg_mask & dq_ok & ((im[1].data*bg.flat) < np.percentile((im[1].data*bg.flat)[mask], 98)) & (im[2].data > 0) & ((im[1].data*bg.flat) > np.percentile((im[1].data*bg.flat)[mask], 1)) data = (im[1].data*bg.flat)[mask_full].flatten() xcoeff, resid, rank, ss = lstsq(ims[:, mask_full.flatten()].T, data) model = np.dot(xcoeff, ims).reshape((1014,1014)) corr = im[1].data*flat-model #### Second iteration: improved mask, weighted lstsq mask_full = seg_mask & dq_ok & (corr < np.percentile(corr[mask], 98)) & (im[2].data > 0) & (corr > np.percentile(corr[mask], 1)) data = (im[1].data*bg.flat)[mask_full].flatten() wht = 1./(im[2].data)[mask_full].flatten() p0 = np.ones(ims.shape[0]) popt = scipy.optimize.leastsq(bg.obj_lstsq, p0, args=(data, ims[:, mask_full.flatten()], wht), full_output=True, ftol=1.49e-8/1000., xtol=1.49e-8/1000.) xcoeff = popt[0] model = np.dot(xcoeff, ims).reshape((1014,1014)) corr = im[1].data*flat-model #### Use the new mask mask = mask_full #### 1D column averages if True: yres = np.zeros(1014) yfull = np.zeros(1014) ydat = np.zeros(1014) fcorr = (im[1].data*flat) xfull = yfull*0. for i in range(1014): ymsk = mask_full[:,i] > 0 ydat[i] = np.median(fcorr[ymsk,i]) yfull[i] = np.median(model[ymsk,i]) yres[i] = np.median(corr[ymsk,i]) # xmsk = mask_full[i,:] > 0 xfull[i] = np.median(model[i,xmsk]) #print i yres_sm = threedhst.utils.medfilt(yres, 41) ### Make figure from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg fig = Figure(figsize=[8,4], dpi=100) fig.subplots_adjust(wspace=0.25,hspace=0.02,left=0.1, bottom=0.08,right=0.99,top=0.92) ax = fig.add_subplot(121) ax.plot(ydat, color='black') ax.plot(yfull, color='red') ax.plot(xfull, color='green') ax.set_xlim(0,1014) ax.set_title(flt) ax = fig.add_subplot(122) ax.plot(yres, color='black') ax.plot(yres_sm, color='red', linewidth=2) ax.set_xlim(0,1014) canvas = FigureCanvasAgg(fig) canvas.print_figure(flt.split('.fits')[0] + '.multisky.png', dpi=100, transparent=False) #### Update header keywords print 'Simultaneous sky components:' for i in range(len(skies)): print ' %s %.3f' %(skies[i], xcoeff[i]) im[0].header.update('GSKY%02d' %(i+1), xcoeff[i], comment='Grism sky: %s' %(skies[i])) # #### Show the result # if plot: # ds9.frame(3) # ds9.v(corr, vmin=-0.5,vmax=0.5) #### Put the result in the FLT data extension im[1].data = corr*1. #### Need to write an output file to use `profile` im.writeto(out_path+os.path.basename(flt).replace('.gz',''), clobber=True) xin, yin = bg.profile(out_path+os.path.basename(flt).replace('.gz',''), extension=1, flatcorr=False, biweight=True) im = pyfits.open(out_path+os.path.basename(flt).replace('.gz',''), mode='update') if second_pass: #### Subtract the residual difference between the observed and master sky if sky_components: ### Use column average found earlier resid = np.dot(np.ones((1014,1)), yres_sm.reshape(1,1014)) else: resid = np.dot(np.ones((1014,1)), threedhst.utils.medfilt(yin, 41).reshape(1,1014)) im[1].data -= resid #### Subtract the overall biweight mean if overall: full_mean = threedhst.utils.biweight(im[1].data[mask], mean=True) im[1].data -= full_mean print 'overall: %.4f' %(full_mean) #### Add a header keyword and write to the output image im[0].header.update('GRISMSKY',keep,comment='Image used for sky subtraction') im[0].header.update('SKYSCALE',sky_stats[0],comment='Scale factor of sky') #### Sky flat keyword if 'SKYFLAT' in im[0].header.keys(): im[0].header['SKYFLAT'] = (flat_correct | im[0].header['SKYFLAT'], 'Direct image flat applied') else: im[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied') bad = ~np.isfinite(im[1].data) im[1].data[bad] = 1 im[3].data[bad] = im[3].data[bad] | 32 im.flush() #### Show the final result, compare to the earlier version in PREP_FLT if plot: ds9.frame(3) ds9.v(im[1].data, vmin=-0.5,vmax=0.5) chk = pyfits.open(threedhst.utils.find_fits_gz(flt.replace('RAW','PREP_FLT').replace('.gz',''))) ds9.frame(4) ds9.v(chk[1].data, vmin=-0.5,vmax=0.5)
def remove_visit_sky(asn_file='GDN12-G102_asn.fits', list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'], add_constant=False, column_average=True, mask_grow=18, flat_correct=True): """ Require that all exposures in a visit have the same zodi component. """ from scipy.linalg import lstsq import scipy.optimize import scipy.ndimage as nd import astropy.io.fits as pyfits import copy import threedhst.grism_sky as bg asn = threedhst.utils.ASNFile(asn_file) flt = pyfits.open('%s_flt.fits' %(asn.exposures[0])) bg.set_grism_flat(grism=flt[0].header['FILTER'], verbose=True) if flat_correct: flat = bg.flat*1. else: flat = bg.flat*0.+1 data = [] whts = [] masks = [] for exp in asn.exposures: flt = pyfits.open('%s_flt.fits' %(exp)) segfile = '%s_flt.seg.fits' %(exp) seg = pyfits.open(segfile)[0].data seg_mask = nd.maximum_filter((seg > 0), size=18) == 0 dq_ok = (flt[3].data & (4+32+16+512+2048+4096)) == 0 # flat_corr = flt[1].data*flat mask = seg_mask & dq_ok mask &= (flat_corr < np.percentile(flat_corr[mask], 98)) & (flt[2].data > 0) & (flat_corr > np.percentile(flat_corr[mask], 1)) # data.append(flat_corr.flatten()) whts.append(1/flt[2].data.flatten()**2) masks.append(mask.flatten()) data = np.array(data) whts = np.array(whts) masks = np.array(masks) #### Read in the master skies ims = [] skies = copy.deepcopy(list) for sky in skies: ims.append(pyfits.open(os.getenv('THREEDHST') + '/CONF/' + sky)[0].data.flatten()) if add_constant: ims.append(flt[1].data.flatten()*0.+1) skies.append('Constant') ims = np.array(ims) #### Do the fit tol=1.49e-8 # not sure what this controls p0 = np.ones((ims.shape[0]-1)*len(asn.exposures)+1) popt = scipy.optimize.leastsq(bg.obj_lstsq_visit, p0, args=(data, ims, whts, masks), full_output=True, ftol=tol/1000., xtol=tol/1000.) xcoeff = popt[0] sh_temp = ims.shape logstr = 'Master grism sky: %s\n\n FLT %s\n' %(asn_file, ' '.join(skies)) for i in range(len(asn.exposures)): coeff = np.zeros(sh_temp[0]) coeff[0] = xcoeff[0] coeff[1:] = xcoeff[1+i*(sh_temp[0]-1):1+(i+1)*(sh_temp[0]-1)] bg_model = np.dot(coeff, ims).reshape((1014,1014)) logstr += '%s %s\n' %(asn.exposures[i], ''.join([' %9.4f' %(c) for c in coeff])) flt = pyfits.open('%s_flt.fits' %(asn.exposures[i]), mode='update') flt[1].data = flt[1].data*flat - bg_model for j in range(sh_temp[0]): if 'GSKY%02d' %(j) in flt[0].header: flt[0].header['GSKY%02d' %(j)] += coeff[j] else: flt[0].header['GSKY%02d' %(j)] = (coeff[j], 'Master sky: %s' %(skies[j])) # flt[1].header['MDRIZSKY'] = 0. if 'SKYFLAT' in flt[0].header.keys(): flt[0].header['SKYFLAT'] = (flat_correct | flt[0].header['SKYFLAT'], 'Direct image flat applied') else: flt[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied') flt.flush() threedhst.showMessage(logstr) if column_average: #for iter in range(2): #grism_sky_column_average(asn_file=asn_file, mask_grow=mask_grow) grism_sky_column_average_GP(asn_file=asn_file, mask_grow=mask_grow)
def remove_grism_sky( flt="ibhm46ioq_flt.fits", list=["sky_cosmos.fits", "sky_goodsn_lo.fits", "sky_goodsn_hi.fits", "sky_goodsn_vhi.fits"], path_to_sky="../CONF/", out_path="./", verbose=False, plot=False, flat_correct=True, sky_subtract=True, second_pass=True, overall=True, ): """ Process a (G141) grism exposure by dividing by the F140W imaging flat-field and then subtracting by a master sky image. v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'] testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits'] """ import threedhst.grism_sky as bg # import scipy.signal as sign # flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz' im = pyfits.open(flt) bg.set_grism_flat(grism=im[0].header["FILTER"]) segfile = os.path.basename(flt.replace(".fits", ".seg.fits")).replace(".gz", "") if os.path.exists(segfile): seg = pyfits.open(segfile)[0].data use_biweight = False else: seg = np.zeros(im[1].data.shape) use_biweight = True xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight) # yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)]) if plot: plt.plot(xin, yin, color="black", linewidth=2) #### Loop through sky images and find the one whose column profile most #### closely matches the input image chi2 = 1.0e10 keep = None for sky in list: xsky, ysky = bg.profile(flt=path_to_sky + sky, extension=0, flatcorr=False, biweight=True) ysky /= np.mean(ysky[np.abs(xsky - 507) < 50]) # ok = np.isfinite(ysky) & np.isfinite(yin) & (yin * ysky != 0) a = np.sum((ysky * yin)[ok]) / np.sum((ysky * ysky)[ok]) if plot: plt.plot(xsky, ysky * a) # chi2_i = np.sum((ysky[ok] * a - yin[ok]) ** 2) if verbose: print sky, chi2_i # if chi2_i < chi2: chi2 = chi2_i * 1 keep = sky if keep is None: keep = "sky_goodsn_vhi.fits" #### The best sky image sk = pyfits.open(path_to_sky + keep) sk[0].data[sk[0].data == 0] = 1.0 sk[0].data[~np.isfinite(sk[0].data)] = 1.0 flat = bg.flat * 1.0 #### Only flat correction dq_ok = (im[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0 mask = (seg == 0) & dq_ok if plot: corr = im[1].data * flat # /sk[0].data corr -= threedhst.utils.biweight(corr[mask], mean=True) ds9.frame(1) ds9.v(corr, vmin=-0.5, vmax=0.5) if flat_correct is False: flat = flat * 0 + 1 if sky_subtract is False: sk[0].data = sk[0].data * 0 + 1 #### Divide by the sky flat # corr = im[1].data*flat/sk[0].data # #### Show the result # if plot: # ds9.frame(2) # ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5) ### Instead, subtract the sky flat sky_stats = threedhst.utils.biweight((im[1].data * flat / sk[0].data)[mask], both=True) corr = im[1].data * flat - sky_stats[0] * sk[0].data # #### Show the result # if plot: # ds9.frame(3) # ds9.v(corr, vmin=-0.5,vmax=0.5) #### Put the result in the FLT data extension im[1].data = corr * 1.0 #### Need to write an output file to use `profile` im.writeto(out_path + os.path.basename(flt).replace(".gz", ""), clobber=True) xin, yin = bg.profile( out_path + os.path.basename(flt).replace(".gz", ""), extension=1, flatcorr=False, biweight=True ) im = pyfits.open(out_path + os.path.basename(flt).replace(".gz", ""), mode="update") #### Subtract the residual difference between the observed and master sky resid = np.dot(np.ones((1014, 1)), threedhst.utils.medfilt(yin, 41).reshape(1, 1014)) if second_pass: im[1].data -= resid #### Subtract the overall biweight mean if overall: im[1].data -= threedhst.utils.biweight(im[1].data[mask], mean=True) #### Add a header keyword and write to the output image im[0].header.update("GRISMSKY", keep, comment="Image used for sky subtraction") im[0].header.update("SKYSCALE", sky_stats[0], comment="Scale factor of sky") bad = ~np.isfinite(im[1].data) im[1].data[bad] = 1 im[3].data[bad] = im[3].data[bad] | 32 im.flush() #### Show the final result, compare to the earlier version in PREP_FLT if plot: ds9.frame(3) ds9.v(im[1].data, vmin=-0.5, vmax=0.5) chk = pyfits.open(threedhst.utils.find_fits_gz(flt.replace("RAW", "PREP_FLT").replace(".gz", ""))) ds9.frame(4) ds9.v(chk[1].data, vmin=-0.5, vmax=0.5)
def remove_grism_sky(flt='ibhm46ioq_flt.fits', list=[ 'sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits' ], path_to_sky='../CONF/', out_path='./', verbose=False, plot=False, flat_correct=True, sky_subtract=True, second_pass=True, overall=True, combine_skies=False, sky_components=True, add_constant=True): """ Process a (G141) grism exposure by dividing by the F140W imaging flat-field and then subtracting by a master sky image. v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'] testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits'] list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'] """ import threedhst.grism_sky as bg #import scipy.signal as sign # flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz' im = pyfits.open(flt) bg.set_grism_flat(grism=im[0].header['FILTER'], verbose=True) segfile = os.path.basename(flt.replace('.fits', '.seg.fits')).replace('.gz', '') if os.path.exists(segfile): seg = pyfits.open(segfile)[0].data use_biweight = False else: seg = np.zeros(im[1].data.shape) use_biweight = True xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight) #yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)]) if plot: plt.plot(xin, yin, color='black', linewidth=2) #### Loop through sky images and find the one whose column profile most #### closely matches the input image chi2 = 1.e10 keep = None for sky in list: xsky, ysky = bg.profile(flt=path_to_sky + sky, extension=0, flatcorr=False, biweight=True) ysky /= np.mean(ysky[np.abs(xsky - 507) < 50]) # ok = np.isfinite(ysky) & np.isfinite(yin) & (yin * ysky != 0) a = np.sum((ysky * yin)[ok]) / np.sum((ysky * ysky)[ok]) if plot: plt.plot(xsky, ysky * a) # chi2_i = np.sum((ysky[ok] * a - yin[ok])**2) if verbose: print sky, chi2_i # if chi2_i < chi2: chi2 = chi2_i * 1 keep = sky if keep is None: keep = 'sky_goodsn_vhi.fits' #### The best sky image sk = pyfits.open(path_to_sky + keep) sk[0].data[sk[0].data == 0] = 1. sk[0].data[~np.isfinite(sk[0].data)] = 1. flat = bg.flat * 1. #### Only flat correction dq_ok = (im[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0 mask = (seg == 0) & dq_ok if plot: corr = im[1].data * flat #/sk[0].data corr -= threedhst.utils.biweight(corr[mask], mean=True) ds9.frame(1) ds9.v(corr, vmin=-0.5, vmax=0.5) if flat_correct is False: flat = flat * 0 + 1 if sky_subtract is False: sk[0].data = sk[0].data * 0 + 1 #### Divide by the sky flat #corr = im[1].data*flat/sk[0].data # #### Show the result # if plot: # ds9.frame(2) # ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5) ### Instead, subtract the sky flat sky_stats = threedhst.utils.biweight( (im[1].data * flat / sk[0].data)[mask], both=True) corr = im[1].data * flat - sky_stats[0] * sk[0].data #### Get least-sq coeffs of multiple sky components if sky_components: from scipy.linalg import lstsq import scipy.optimize import scipy.ndimage as nd import copy #grow_mask = nd.maximum_filter((~mask)*1., size=3) == 0 ims = [] #skies = ['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits'] skies = copy.deepcopy(list) for sky in skies: ims.append(pyfits.open(path_to_sky + sky)[0].data.flatten()) if add_constant: ims.append(im[1].data.flatten() * 0. + 1) skies.append('Constant') ims = np.array(ims) seg_mask = nd.maximum_filter((seg > 0), size=18) == 0 #### First iteration, non-weighted least-sq mask_full = seg_mask & dq_ok & ((im[1].data * bg.flat) < np.percentile( (im[1].data * bg.flat)[mask], 98)) & (im[2].data > 0) & ( (im[1].data * bg.flat) > np.percentile( (im[1].data * bg.flat)[mask], 1)) data = (im[1].data * bg.flat)[mask_full].flatten() xcoeff, resid, rank, ss = lstsq(ims[:, mask_full.flatten()].T, data) model = np.dot(xcoeff, ims).reshape((1014, 1014)) corr = im[1].data * flat - model #### Second iteration: improved mask, weighted lstsq mask_full = seg_mask & dq_ok & (corr < np.percentile( corr[mask], 98)) & (im[2].data > 0) & (corr > np.percentile( corr[mask], 1)) data = (im[1].data * bg.flat)[mask_full].flatten() wht = 1. / (im[2].data)[mask_full].flatten() p0 = np.ones(ims.shape[0]) popt = scipy.optimize.leastsq(bg.obj_lstsq, p0, args=(data, ims[:, mask_full.flatten()], wht), full_output=True, ftol=1.49e-8 / 1000., xtol=1.49e-8 / 1000.) xcoeff = popt[0] model = np.dot(xcoeff, ims).reshape((1014, 1014)) corr = im[1].data * flat - model #### Use the new mask mask = mask_full #### 1D column averages if True: yres = np.zeros(1014) yfull = np.zeros(1014) ydat = np.zeros(1014) fcorr = (im[1].data * flat) xfull = yfull * 0. for i in range(1014): ymsk = mask_full[:, i] > 0 ydat[i] = np.median(fcorr[ymsk, i]) yfull[i] = np.median(model[ymsk, i]) yres[i] = np.median(corr[ymsk, i]) # xmsk = mask_full[i, :] > 0 xfull[i] = np.median(model[i, xmsk]) #print i yres_sm = threedhst.utils.medfilt(yres, 41) ### Make figure from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg fig = Figure(figsize=[8, 4], dpi=100) fig.subplots_adjust(wspace=0.25, hspace=0.02, left=0.1, bottom=0.08, right=0.99, top=0.92) ax = fig.add_subplot(121) ax.plot(ydat, color='black') ax.plot(yfull, color='red') ax.plot(xfull, color='green') ax.set_xlim(0, 1014) ax.set_title(flt) ax = fig.add_subplot(122) ax.plot(yres, color='black') ax.plot(yres_sm, color='red', linewidth=2) ax.set_xlim(0, 1014) canvas = FigureCanvasAgg(fig) canvas.print_figure(flt.split('.fits')[0] + '.multisky.png', dpi=100, transparent=False) #### Update header keywords print 'Simultaneous sky components:' for i in range(len(skies)): print ' %s %.3f' % (skies[i], xcoeff[i]) im[0].header.update('GSKY%02d' % (i + 1), xcoeff[i], comment='Grism sky: %s' % (skies[i])) # #### Show the result # if plot: # ds9.frame(3) # ds9.v(corr, vmin=-0.5,vmax=0.5) #### Put the result in the FLT data extension im[1].data = corr * 1. #### Need to write an output file to use `profile` im.writeto(out_path + os.path.basename(flt).replace('.gz', ''), clobber=True) xin, yin = bg.profile(out_path + os.path.basename(flt).replace('.gz', ''), extension=1, flatcorr=False, biweight=True) im = pyfits.open(out_path + os.path.basename(flt).replace('.gz', ''), mode='update') if second_pass: #### Subtract the residual difference between the observed and master sky if sky_components: ### Use column average found earlier resid = np.dot(np.ones((1014, 1)), yres_sm.reshape(1, 1014)) else: resid = np.dot(np.ones((1014, 1)), threedhst.utils.medfilt(yin, 41).reshape(1, 1014)) im[1].data -= resid #### Subtract the overall biweight mean if overall: full_mean = threedhst.utils.biweight(im[1].data[mask], mean=True) im[1].data -= full_mean print 'overall: %.4f' % (full_mean) #### Add a header keyword and write to the output image im[0].header.update('GRISMSKY', keep, comment='Image used for sky subtraction') im[0].header.update('SKYSCALE', sky_stats[0], comment='Scale factor of sky') #### Sky flat keyword if 'SKYFLAT' in im[0].header.keys(): im[0].header['SKYFLAT'] = (flat_correct | im[0].header['SKYFLAT'], 'Direct image flat applied') else: im[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied') bad = ~np.isfinite(im[1].data) im[1].data[bad] = 1 im[3].data[bad] = im[3].data[bad] | 32 im.flush() #### Show the final result, compare to the earlier version in PREP_FLT if plot: ds9.frame(3) ds9.v(im[1].data, vmin=-0.5, vmax=0.5) chk = pyfits.open( threedhst.utils.find_fits_gz( flt.replace('RAW', 'PREP_FLT').replace('.gz', ''))) ds9.frame(4) ds9.v(chk[1].data, vmin=-0.5, vmax=0.5)
def remove_visit_sky(asn_file='GDN12-G102_asn.fits', list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'], add_constant=False, column_average=True, mask_grow=18, flat_correct=True): """ Require that all exposures in a visit have the same zodi component. """ from scipy.linalg import lstsq import scipy.optimize import scipy.ndimage as nd import astropy.io.fits as pyfits import copy import threedhst.grism_sky as bg asn = threedhst.utils.ASNFile(asn_file) flt = pyfits.open('%s_flt.fits' % (asn.exposures[0])) bg.set_grism_flat(grism=flt[0].header['FILTER'], verbose=True) if flat_correct: flat = bg.flat * 1. else: flat = bg.flat * 0. + 1 data = [] whts = [] masks = [] for exp in asn.exposures: flt = pyfits.open('%s_flt.fits' % (exp)) segfile = '%s_flt.seg.fits' % (exp) seg = pyfits.open(segfile)[0].data seg_mask = nd.maximum_filter((seg > 0), size=18) == 0 dq_ok = (flt[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0 # flat_corr = flt[1].data * flat mask = seg_mask & dq_ok mask &= (flat_corr < np.percentile(flat_corr[mask], 98)) & ( flt[2].data > 0) & (flat_corr > np.percentile(flat_corr[mask], 1)) # data.append(flat_corr.flatten()) whts.append(1 / flt[2].data.flatten()**2) masks.append(mask.flatten()) data = np.array(data) whts = np.array(whts) masks = np.array(masks) #### Read in the master skies ims = [] skies = copy.deepcopy(list) for sky in skies: ims.append( pyfits.open(os.getenv('THREEDHST') + '/CONF/' + sky)[0].data.flatten()) if add_constant: ims.append(flt[1].data.flatten() * 0. + 1) skies.append('Constant') ims = np.array(ims) #### Do the fit tol = 1.49e-8 # not sure what this controls p0 = np.ones((ims.shape[0] - 1) * len(asn.exposures) + 1) popt = scipy.optimize.leastsq(bg.obj_lstsq_visit, p0, args=(data, ims, whts, masks), full_output=True, ftol=tol / 1000., xtol=tol / 1000.) xcoeff = popt[0] sh_temp = ims.shape logstr = 'Master grism sky: %s\n\n FLT %s\n' % (asn_file, ' '.join(skies)) for i in range(len(asn.exposures)): coeff = np.zeros(sh_temp[0]) coeff[0] = xcoeff[0] coeff[1:] = xcoeff[1 + i * (sh_temp[0] - 1):1 + (i + 1) * (sh_temp[0] - 1)] bg_model = np.dot(coeff, ims).reshape((1014, 1014)) logstr += '%s %s\n' % (asn.exposures[i], ''.join( [' %9.4f' % (c) for c in coeff])) flt = pyfits.open('%s_flt.fits' % (asn.exposures[i]), mode='update') flt[1].data = flt[1].data * flat - bg_model for j in range(sh_temp[0]): if 'GSKY%02d' % (j) in flt[0].header: flt[0].header['GSKY%02d' % (j)] += coeff[j] else: flt[0].header['GSKY%02d' % (j)] = (coeff[j], 'Master sky: %s' % (skies[j])) # flt[1].header['MDRIZSKY'] = 0. if 'SKYFLAT' in flt[0].header.keys(): flt[0].header['SKYFLAT'] = (flat_correct | flt[0].header['SKYFLAT'], 'Direct image flat applied') else: flt[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied') flt.flush() threedhst.showMessage(logstr) if column_average: #for iter in range(2): grism_sky_column_average(asn_file=asn_file, mask_grow=mask_grow)
def subtract_grism_background(asn_file='GDN1-G102_asn.fits', PATH_TO_RAW='../RAW/', final_scale=0.06, visit_sky=True, column_average=True, mask_grow=18, first_run=True, sky_iter=1): """ Subtract master grism sky from FLTs """ import os import scipy.ndimage as nd import pyregion from drizzlepac import astrodrizzle import drizzlepac from stwcs import updatewcs import stwcs import threedhst.grism_sky as bg asn = threedhst.utils.ASNFile(asn_file) root = asn_file.split('_asn')[0] sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits'], 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']} # # sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light_v2.fits'], # 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']} # ### Don't use scattered light # sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits'], # 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']} # # ## Use aXe images # sky_images = {'G141':['WFC3.IR.G141.sky.V1.0.flat.fits', 'WFC3.IR.G141.sky.V1.0.flat.fits'], # 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']} if first_run: ### Rough background subtraction threedhst.process_grism.fresh_flt_files(asn_file, from_path=PATH_TO_RAW, preserve_dq=False) flt = pyfits.open('%s_flt.fits' %(asn.exposures[0])) GRISM = flt[0].header['FILTER'] bg.set_grism_flat(grism=GRISM, verbose=True) zodi = pyfits.open(os.getenv('THREEDHST')+'/CONF/%s' %(sky_images[GRISM][0]))[0].data for exp in asn.exposures: updatewcs.updatewcs('%s_flt.fits' %(exp)) flt = pyfits.open('%s_flt.fits' %(exp), mode='update') #flt = pyfits.open('%s_flt.fits' %(exp)) flt[1].data *= bg.flat # mask = (flt['DQ'].data == 0) data_range = np.percentile(flt[1].data[mask], [20, 80]) mask &= (flt[1].data >= data_range[0]) & (flt[1].data <= data_range[1]) & (flt[2].data != 0) & np.isfinite(flt[1].data) & np.isfinite(flt[2].data) ### Least-sq fit for component normalizations data = flt[1].data[mask].flatten() wht = (1./flt[2].data[mask].flatten())**2 zodi_mask = zodi[mask].flatten() coeff_zodi = np.sum(data*zodi_mask*wht)/np.sum(zodi_mask**2*wht) flt[1].data -= zodi*coeff_zodi flt.flush() threedhst.showMessage('Rough background for %s (zodi): %0.4f' %(exp, coeff_zodi)) #templates = bg_flat[:, mask.flatten()] ### Run astrodrizzle to make DRZ mosaic, grism-SExtractor mask drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, context=False, preserve=False, skysub=True, driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_combine=True, final_wcs=False, resetbits=4096, final_bits=576, driz_sep_bits=576, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7') else: flt = pyfits.open('%s_flt.fits' %(asn.exposures[0])) GRISM = flt[0].header['FILTER'] bg.set_grism_flat(grism=GRISM, verbose=True) se = threedhst.sex.SExtractor() se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' %(root) se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT' se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION' se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits' %(root) # se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True se.params['MAG_AUTO'] = True # se.options['CATALOG_NAME'] = '%s_drz_sci.cat' %(root) se.options['FILTER'] = 'Y' se.copyConvFile(grism=True) se.options['FILTER_NAME'] = 'grism.conv' se.options['DETECT_THRESH'] = '0.7' se.options['ANALYSIS_THRESH'] = '0.7' # se.sextractImage('%s_drz_sci.fits' %(root)) #### Blot segmentation map to FLT images for object mask ref = pyfits.open('%s_drz_sci.fits' %(root)) ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=0) seg = pyfits.open('%s_drz_seg.fits' %(root)) seg_data = np.cast[np.float32](seg[0].data) #### Loop through FLTs, blotting reference and segmentation threedhst.showMessage('%s: Blotting grism segmentation masks.' %(root)) for exp in asn.exposures: flt = pyfits.open('%s_flt.fits' %(exp)) flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=1) ### segmentation #print 'Segmentation image: %s_blot.fits' %(exp) blotted_seg = astrodrizzle.ablot.do_blot(seg_data, ref_wcs, flt_wcs, 1, coeffs=True, interp='nearest', sinscl=1.0, stepsize=10, wcsmap=None) seg_grow = nd.maximum_filter((blotted_seg > 0)*1, size=8) pyfits.writeto('%s_flt.seg.fits' %(exp), header=flt[1].header, data=seg_grow, clobber=True) if first_run: ### Run background subtraction scripts threedhst.process_grism.fresh_flt_files(asn_file, from_path=PATH_TO_RAW, preserve_dq=False) for exp in asn.exposures: updatewcs.updatewcs('%s_flt.fits' %(exp)) #threedhst.grism_sky.remove_grism_sky(flt=exp+'_flt.fits', list=sky_images[GRISM], path_to_sky=os.getenv('THREEDHST')+'/CONF/', verbose=True, second_pass=True, overall=True) if visit_sky: threedhst.grism_sky.remove_visit_sky(asn_file=asn_file, list=sky_images[GRISM], add_constant=False, column_average=(column_average) & (sky_iter == 1), mask_grow=mask_grow, flat_correct=first_run) if (sky_iter > 1) & (~first_run): for i in range(1, sky_iter): threedhst.grism_sky.remove_visit_sky(asn_file=asn_file, list=sky_images[GRISM], add_constant=False, column_average=column_average & (i == (sky_iter-1)), mask_grow=mask_grow, flat_correct=False) else: for exp in asn.exposures: threedhst.grism_sky.remove_grism_sky(flt='%s_flt.fits' %(exp), list=sky_images[GRISM], path_to_sky = os.getenv('THREEDHST')+'/CONF/', out_path='./', verbose=False, plot=False, flat_correct=first_run, sky_subtract=True, second_pass=column_average, overall=True, combine_skies=False, sky_components=True, add_constant=False) ### Astrodrizzle again to reflag CRs and make cleaned mosaic drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, skysub=False, skyuser='******', final_wcs=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale='2.5 0.7') # , final_wcs=True, final_rot=0)