def remove_grism_sky(flt='ibhm46ioq_flt.fits', list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'], path_to_sky = '../CONF/', out_path='./', verbose=False, plot=False, flat_correct=True, sky_subtract=True, second_pass=True, overall=True, combine_skies=False, sky_components=True, add_constant=True):
"""
Process a (G141) grism exposure by dividing by the F140W imaging flat-field
and then subtracting by a master sky image.
v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits']
testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits']
list=['zodi_G102_clean.fits', 'excess_G102_clean.fits']
"""
import threedhst.grism_sky as bg
#import scipy.signal as sign
# flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz'
im = pyfits.open(flt)
bg.set_grism_flat(grism=im[0].header['FILTER'], verbose=True)
segfile = os.path.basename(flt.replace('.fits','.seg.fits')).replace('.gz','')
if os.path.exists(segfile):
seg = pyfits.open(segfile)[0].data
use_biweight=False
else:
seg = np.zeros(im[1].data.shape)
use_biweight=True
xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight)
#yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)])
if plot:
plt.plot(xin, yin, color='black', linewidth=2)
#### Loop through sky images and find the one whose column profile most
#### closely matches the input image
chi2 = 1.e10
keep = None
for sky in list:
xsky, ysky = bg.profile(flt=path_to_sky+sky, extension=0, flatcorr=False, biweight=True)
ysky /= np.mean(ysky[np.abs(xsky-507) < 50])
#
ok = np.isfinite(ysky) & np.isfinite(yin) & (yin*ysky != 0)
a = np.sum((ysky*yin)[ok])/np.sum((ysky*ysky)[ok])
if plot:
plt.plot(xsky, ysky*a)
#
chi2_i = np.sum((ysky[ok]*a-yin[ok])**2)
if verbose:
print sky, chi2_i
#
if chi2_i < chi2:
chi2 = chi2_i*1
keep = sky
if keep is None:
keep = 'sky_goodsn_vhi.fits'
#### The best sky image
sk = pyfits.open(path_to_sky+keep)
sk[0].data[sk[0].data == 0] = 1.
sk[0].data[~np.isfinite(sk[0].data)] = 1.
flat = bg.flat*1.
#### Only flat correction
dq_ok = (im[3].data & (4+32+16+512+2048+4096)) == 0
mask = (seg == 0) & dq_ok
if plot:
corr = im[1].data*flat#/sk[0].data
corr -= threedhst.utils.biweight(corr[mask], mean=True)
ds9.frame(1)
ds9.v(corr, vmin=-0.5,vmax=0.5)
if flat_correct is False:
flat = flat*0+1
if sky_subtract is False:
sk[0].data = sk[0].data*0+1
#### Divide by the sky flat
#corr = im[1].data*flat/sk[0].data
# #### Show the result
# if plot:
# ds9.frame(2)
# ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5)
### Instead, subtract the sky flat
sky_stats = threedhst.utils.biweight((im[1].data*flat/sk[0].data)[mask], both=True)
corr = im[1].data*flat-sky_stats[0]*sk[0].data
#### Get least-sq coeffs of multiple sky components
if sky_components:
from scipy.linalg import lstsq
import scipy.optimize
import scipy.ndimage as nd
import copy
#grow_mask = nd.maximum_filter((~mask)*1., size=3) == 0
ims = []
#skies = ['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits']
skies = copy.deepcopy(list)
for sky in skies:
ims.append(pyfits.open(path_to_sky + sky)[0].data.flatten())
if add_constant:
ims.append(im[1].data.flatten()*0.+1)
skies.append('Constant')
ims = np.array(ims)
seg_mask = nd.maximum_filter((seg > 0), size=18) == 0
#### First iteration, non-weighted least-sq
mask_full = seg_mask & dq_ok & ((im[1].data*bg.flat) < np.percentile((im[1].data*bg.flat)[mask], 98)) & (im[2].data > 0) & ((im[1].data*bg.flat) > np.percentile((im[1].data*bg.flat)[mask], 1))
data = (im[1].data*bg.flat)[mask_full].flatten()
xcoeff, resid, rank, ss = lstsq(ims[:, mask_full.flatten()].T, data)
model = np.dot(xcoeff, ims).reshape((1014,1014))
corr = im[1].data*flat-model
#### Second iteration: improved mask, weighted lstsq
mask_full = seg_mask & dq_ok & (corr < np.percentile(corr[mask], 98)) & (im[2].data > 0) & (corr > np.percentile(corr[mask], 1))
data = (im[1].data*bg.flat)[mask_full].flatten()
wht = 1./(im[2].data)[mask_full].flatten()
p0 = np.ones(ims.shape[0])
popt = scipy.optimize.leastsq(bg.obj_lstsq, p0, args=(data, ims[:, mask_full.flatten()], wht), full_output=True, ftol=1.49e-8/1000., xtol=1.49e-8/1000.)
xcoeff = popt[0]
model = np.dot(xcoeff, ims).reshape((1014,1014))
corr = im[1].data*flat-model
#### Use the new mask
mask = mask_full
#### 1D column averages
if True:
yres = np.zeros(1014)
yfull = np.zeros(1014)
ydat = np.zeros(1014)
fcorr = (im[1].data*flat)
xfull = yfull*0.
for i in range(1014):
ymsk = mask_full[:,i] > 0
ydat[i] = np.median(fcorr[ymsk,i])
yfull[i] = np.median(model[ymsk,i])
yres[i] = np.median(corr[ymsk,i])
#
xmsk = mask_full[i,:] > 0
xfull[i] = np.median(model[i,xmsk])
#print i
yres_sm = threedhst.utils.medfilt(yres, 41)
### Make figure
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg
fig = Figure(figsize=[8,4], dpi=100)
fig.subplots_adjust(wspace=0.25,hspace=0.02,left=0.1,
bottom=0.08,right=0.99,top=0.92)
ax = fig.add_subplot(121)
ax.plot(ydat, color='black')
ax.plot(yfull, color='red')
ax.plot(xfull, color='green')
ax.set_xlim(0,1014)
ax.set_title(flt)
ax = fig.add_subplot(122)
ax.plot(yres, color='black')
ax.plot(yres_sm, color='red', linewidth=2)
ax.set_xlim(0,1014)
canvas = FigureCanvasAgg(fig)
canvas.print_figure(flt.split('.fits')[0] + '.multisky.png', dpi=100, transparent=False)
#### Update header keywords
print 'Simultaneous sky components:'
for i in range(len(skies)):
print ' %s %.3f' %(skies[i], xcoeff[i])
im[0].header.update('GSKY%02d' %(i+1), xcoeff[i], comment='Grism sky: %s' %(skies[i]))
# #### Show the result
# if plot:
# ds9.frame(3)
# ds9.v(corr, vmin=-0.5,vmax=0.5)
#### Put the result in the FLT data extension
im[1].data = corr*1.
#### Need to write an output file to use `profile`
im.writeto(out_path+os.path.basename(flt).replace('.gz',''), clobber=True)
xin, yin = bg.profile(out_path+os.path.basename(flt).replace('.gz',''), extension=1, flatcorr=False, biweight=True)
im = pyfits.open(out_path+os.path.basename(flt).replace('.gz',''), mode='update')
if second_pass:
#### Subtract the residual difference between the observed and master sky
if sky_components:
### Use column average found earlier
resid = np.dot(np.ones((1014,1)), yres_sm.reshape(1,1014))
else:
resid = np.dot(np.ones((1014,1)), threedhst.utils.medfilt(yin, 41).reshape(1,1014))
im[1].data -= resid
#### Subtract the overall biweight mean
if overall:
full_mean = threedhst.utils.biweight(im[1].data[mask], mean=True)
im[1].data -= full_mean
print 'overall: %.4f' %(full_mean)
#### Add a header keyword and write to the output image
im[0].header.update('GRISMSKY',keep,comment='Image used for sky subtraction')
im[0].header.update('SKYSCALE',sky_stats[0],comment='Scale factor of sky')
#### Sky flat keyword
if 'SKYFLAT' in im[0].header.keys():
im[0].header['SKYFLAT'] = (flat_correct | im[0].header['SKYFLAT'], 'Direct image flat applied')
else:
im[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied')
bad = ~np.isfinite(im[1].data)
im[1].data[bad] = 1
im[3].data[bad] = im[3].data[bad] | 32
im.flush()
#### Show the final result, compare to the earlier version in PREP_FLT
if plot:
ds9.frame(3)
ds9.v(im[1].data, vmin=-0.5,vmax=0.5)
chk = pyfits.open(threedhst.utils.find_fits_gz(flt.replace('RAW','PREP_FLT').replace('.gz','')))
ds9.frame(4)
ds9.v(chk[1].data, vmin=-0.5,vmax=0.5)
def remove_visit_sky(asn_file='GDN12-G102_asn.fits', list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'], add_constant=False, column_average=True, mask_grow=18, flat_correct=True):
"""
Require that all exposures in a visit have the same zodi component.
"""
from scipy.linalg import lstsq
import scipy.optimize
import scipy.ndimage as nd
import astropy.io.fits as pyfits
import copy
import threedhst.grism_sky as bg
asn = threedhst.utils.ASNFile(asn_file)
flt = pyfits.open('%s_flt.fits' %(asn.exposures[0]))
bg.set_grism_flat(grism=flt[0].header['FILTER'], verbose=True)
if flat_correct:
flat = bg.flat*1.
else:
flat = bg.flat*0.+1
data = []
whts = []
masks = []
for exp in asn.exposures:
flt = pyfits.open('%s_flt.fits' %(exp))
segfile = '%s_flt.seg.fits' %(exp)
seg = pyfits.open(segfile)[0].data
seg_mask = nd.maximum_filter((seg > 0), size=18) == 0
dq_ok = (flt[3].data & (4+32+16+512+2048+4096)) == 0
#
flat_corr = flt[1].data*flat
mask = seg_mask & dq_ok
mask &= (flat_corr < np.percentile(flat_corr[mask], 98)) & (flt[2].data > 0) & (flat_corr > np.percentile(flat_corr[mask], 1))
#
data.append(flat_corr.flatten())
whts.append(1/flt[2].data.flatten()**2)
masks.append(mask.flatten())
data = np.array(data)
whts = np.array(whts)
masks = np.array(masks)
#### Read in the master skies
ims = []
skies = copy.deepcopy(list)
for sky in skies:
ims.append(pyfits.open(os.getenv('THREEDHST') + '/CONF/' + sky)[0].data.flatten())
if add_constant:
ims.append(flt[1].data.flatten()*0.+1)
skies.append('Constant')
ims = np.array(ims)
#### Do the fit
tol=1.49e-8 # not sure what this controls
p0 = np.ones((ims.shape[0]-1)*len(asn.exposures)+1)
popt = scipy.optimize.leastsq(bg.obj_lstsq_visit, p0, args=(data, ims, whts, masks), full_output=True, ftol=tol/1000., xtol=tol/1000.)
xcoeff = popt[0]
sh_temp = ims.shape
logstr = 'Master grism sky: %s\n\n FLT %s\n' %(asn_file, ' '.join(skies))
for i in range(len(asn.exposures)):
coeff = np.zeros(sh_temp[0])
coeff[0] = xcoeff[0]
coeff[1:] = xcoeff[1+i*(sh_temp[0]-1):1+(i+1)*(sh_temp[0]-1)]
bg_model = np.dot(coeff, ims).reshape((1014,1014))
logstr += '%s %s\n' %(asn.exposures[i], ''.join([' %9.4f' %(c) for c in coeff]))
flt = pyfits.open('%s_flt.fits' %(asn.exposures[i]), mode='update')
flt[1].data = flt[1].data*flat - bg_model
for j in range(sh_temp[0]):
if 'GSKY%02d' %(j) in flt[0].header:
flt[0].header['GSKY%02d' %(j)] += coeff[j]
else:
flt[0].header['GSKY%02d' %(j)] = (coeff[j], 'Master sky: %s' %(skies[j]))
#
flt[1].header['MDRIZSKY'] = 0.
if 'SKYFLAT' in flt[0].header.keys():
flt[0].header['SKYFLAT'] = (flat_correct | flt[0].header['SKYFLAT'], 'Direct image flat applied')
else:
flt[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied')
flt.flush()
threedhst.showMessage(logstr)
if column_average:
#for iter in range(2):
#grism_sky_column_average(asn_file=asn_file, mask_grow=mask_grow)
grism_sky_column_average_GP(asn_file=asn_file, mask_grow=mask_grow)
def remove_grism_sky(
flt="ibhm46ioq_flt.fits",
list=["sky_cosmos.fits", "sky_goodsn_lo.fits", "sky_goodsn_hi.fits", "sky_goodsn_vhi.fits"],
path_to_sky="../CONF/",
out_path="./",
verbose=False,
plot=False,
flat_correct=True,
sky_subtract=True,
second_pass=True,
overall=True,
):
"""
Process a (G141) grism exposure by dividing by the F140W imaging flat-field
and then subtracting by a master sky image.
v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits']
testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits']
"""
import threedhst.grism_sky as bg
# import scipy.signal as sign
# flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz'
im = pyfits.open(flt)
bg.set_grism_flat(grism=im[0].header["FILTER"])
segfile = os.path.basename(flt.replace(".fits", ".seg.fits")).replace(".gz", "")
if os.path.exists(segfile):
seg = pyfits.open(segfile)[0].data
use_biweight = False
else:
seg = np.zeros(im[1].data.shape)
use_biweight = True
xin, yin = bg.profile(flt, extension=1, flatcorr=True, biweight=use_biweight)
# yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)])
if plot:
plt.plot(xin, yin, color="black", linewidth=2)
#### Loop through sky images and find the one whose column profile most
#### closely matches the input image
chi2 = 1.0e10
keep = None
for sky in list:
xsky, ysky = bg.profile(flt=path_to_sky + sky, extension=0, flatcorr=False, biweight=True)
ysky /= np.mean(ysky[np.abs(xsky - 507) < 50])
#
ok = np.isfinite(ysky) & np.isfinite(yin) & (yin * ysky != 0)
a = np.sum((ysky * yin)[ok]) / np.sum((ysky * ysky)[ok])
if plot:
plt.plot(xsky, ysky * a)
#
chi2_i = np.sum((ysky[ok] * a - yin[ok]) ** 2)
if verbose:
print sky, chi2_i
#
if chi2_i < chi2:
chi2 = chi2_i * 1
keep = sky
if keep is None:
keep = "sky_goodsn_vhi.fits"
#### The best sky image
sk = pyfits.open(path_to_sky + keep)
sk[0].data[sk[0].data == 0] = 1.0
sk[0].data[~np.isfinite(sk[0].data)] = 1.0
flat = bg.flat * 1.0
#### Only flat correction
dq_ok = (im[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0
mask = (seg == 0) & dq_ok
if plot:
corr = im[1].data * flat # /sk[0].data
corr -= threedhst.utils.biweight(corr[mask], mean=True)
ds9.frame(1)
ds9.v(corr, vmin=-0.5, vmax=0.5)
if flat_correct is False:
flat = flat * 0 + 1
if sky_subtract is False:
sk[0].data = sk[0].data * 0 + 1
#### Divide by the sky flat
# corr = im[1].data*flat/sk[0].data
# #### Show the result
# if plot:
# ds9.frame(2)
# ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5)
### Instead, subtract the sky flat
sky_stats = threedhst.utils.biweight((im[1].data * flat / sk[0].data)[mask], both=True)
corr = im[1].data * flat - sky_stats[0] * sk[0].data
# #### Show the result
# if plot:
# ds9.frame(3)
# ds9.v(corr, vmin=-0.5,vmax=0.5)
#### Put the result in the FLT data extension
im[1].data = corr * 1.0
#### Need to write an output file to use `profile`
im.writeto(out_path + os.path.basename(flt).replace(".gz", ""), clobber=True)
xin, yin = bg.profile(
out_path + os.path.basename(flt).replace(".gz", ""), extension=1, flatcorr=False, biweight=True
)
im = pyfits.open(out_path + os.path.basename(flt).replace(".gz", ""), mode="update")
#### Subtract the residual difference between the observed and master sky
resid = np.dot(np.ones((1014, 1)), threedhst.utils.medfilt(yin, 41).reshape(1, 1014))
if second_pass:
im[1].data -= resid
#### Subtract the overall biweight mean
if overall:
im[1].data -= threedhst.utils.biweight(im[1].data[mask], mean=True)
#### Add a header keyword and write to the output image
im[0].header.update("GRISMSKY", keep, comment="Image used for sky subtraction")
im[0].header.update("SKYSCALE", sky_stats[0], comment="Scale factor of sky")
bad = ~np.isfinite(im[1].data)
im[1].data[bad] = 1
im[3].data[bad] = im[3].data[bad] | 32
im.flush()
#### Show the final result, compare to the earlier version in PREP_FLT
if plot:
ds9.frame(3)
ds9.v(im[1].data, vmin=-0.5, vmax=0.5)
chk = pyfits.open(threedhst.utils.find_fits_gz(flt.replace("RAW", "PREP_FLT").replace(".gz", "")))
ds9.frame(4)
ds9.v(chk[1].data, vmin=-0.5, vmax=0.5)
def remove_grism_sky(flt='ibhm46ioq_flt.fits',
list=[
'sky_cosmos.fits', 'sky_goodsn_lo.fits',
'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits'
],
path_to_sky='../CONF/',
out_path='./',
verbose=False,
plot=False,
flat_correct=True,
sky_subtract=True,
second_pass=True,
overall=True,
combine_skies=False,
sky_components=True,
add_constant=True):
"""
Process a (G141) grism exposure by dividing by the F140W imaging flat-field
and then subtracting by a master sky image.
v1.6: list=['sky_cosmos.fits', 'sky_goodsn_lo.fits', 'sky_goodsn_hi.fits', 'sky_goodsn_vhi.fits']
testing: list=['sky.G141.set001.fits','sky.G141.set002.fits','sky.G141.set003.fits','sky.G141.set004.fits','sky.G141.set005.fits','sky.G141.set025.fits','sky.G141.set120.fits']
list=['zodi_G102_clean.fits', 'excess_G102_clean.fits']
"""
import threedhst.grism_sky as bg
#import scipy.signal as sign
# flt = '../../GOODS-N/RAW/ib3708ilq_flt.fits.gz'
im = pyfits.open(flt)
bg.set_grism_flat(grism=im[0].header['FILTER'], verbose=True)
segfile = os.path.basename(flt.replace('.fits',
'.seg.fits')).replace('.gz', '')
if os.path.exists(segfile):
seg = pyfits.open(segfile)[0].data
use_biweight = False
else:
seg = np.zeros(im[1].data.shape)
use_biweight = True
xin, yin = bg.profile(flt,
extension=1,
flatcorr=True,
biweight=use_biweight)
#yin /= threedhst.utils.biweight(yin[(np.abs(xin-507) < 50) & np.isfinite(yin)])
if plot:
plt.plot(xin, yin, color='black', linewidth=2)
#### Loop through sky images and find the one whose column profile most
#### closely matches the input image
chi2 = 1.e10
keep = None
for sky in list:
xsky, ysky = bg.profile(flt=path_to_sky + sky,
extension=0,
flatcorr=False,
biweight=True)
ysky /= np.mean(ysky[np.abs(xsky - 507) < 50])
#
ok = np.isfinite(ysky) & np.isfinite(yin) & (yin * ysky != 0)
a = np.sum((ysky * yin)[ok]) / np.sum((ysky * ysky)[ok])
if plot:
plt.plot(xsky, ysky * a)
#
chi2_i = np.sum((ysky[ok] * a - yin[ok])**2)
if verbose:
print sky, chi2_i
#
if chi2_i < chi2:
chi2 = chi2_i * 1
keep = sky
if keep is None:
keep = 'sky_goodsn_vhi.fits'
#### The best sky image
sk = pyfits.open(path_to_sky + keep)
sk[0].data[sk[0].data == 0] = 1.
sk[0].data[~np.isfinite(sk[0].data)] = 1.
flat = bg.flat * 1.
#### Only flat correction
dq_ok = (im[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0
mask = (seg == 0) & dq_ok
if plot:
corr = im[1].data * flat #/sk[0].data
corr -= threedhst.utils.biweight(corr[mask], mean=True)
ds9.frame(1)
ds9.v(corr, vmin=-0.5, vmax=0.5)
if flat_correct is False:
flat = flat * 0 + 1
if sky_subtract is False:
sk[0].data = sk[0].data * 0 + 1
#### Divide by the sky flat
#corr = im[1].data*flat/sk[0].data
# #### Show the result
# if plot:
# ds9.frame(2)
# ds9.v(corr-threedhst.utils.biweight(corr[mask], mean=True), vmin=-0.5,vmax=0.5)
### Instead, subtract the sky flat
sky_stats = threedhst.utils.biweight(
(im[1].data * flat / sk[0].data)[mask], both=True)
corr = im[1].data * flat - sky_stats[0] * sk[0].data
#### Get least-sq coeffs of multiple sky components
if sky_components:
from scipy.linalg import lstsq
import scipy.optimize
import scipy.ndimage as nd
import copy
#grow_mask = nd.maximum_filter((~mask)*1., size=3) == 0
ims = []
#skies = ['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits']
skies = copy.deepcopy(list)
for sky in skies:
ims.append(pyfits.open(path_to_sky + sky)[0].data.flatten())
if add_constant:
ims.append(im[1].data.flatten() * 0. + 1)
skies.append('Constant')
ims = np.array(ims)
seg_mask = nd.maximum_filter((seg > 0), size=18) == 0
#### First iteration, non-weighted least-sq
mask_full = seg_mask & dq_ok & ((im[1].data * bg.flat) < np.percentile(
(im[1].data * bg.flat)[mask], 98)) & (im[2].data > 0) & (
(im[1].data * bg.flat) > np.percentile(
(im[1].data * bg.flat)[mask], 1))
data = (im[1].data * bg.flat)[mask_full].flatten()
xcoeff, resid, rank, ss = lstsq(ims[:, mask_full.flatten()].T, data)
model = np.dot(xcoeff, ims).reshape((1014, 1014))
corr = im[1].data * flat - model
#### Second iteration: improved mask, weighted lstsq
mask_full = seg_mask & dq_ok & (corr < np.percentile(
corr[mask], 98)) & (im[2].data > 0) & (corr > np.percentile(
corr[mask], 1))
data = (im[1].data * bg.flat)[mask_full].flatten()
wht = 1. / (im[2].data)[mask_full].flatten()
p0 = np.ones(ims.shape[0])
popt = scipy.optimize.leastsq(bg.obj_lstsq,
p0,
args=(data, ims[:,
mask_full.flatten()],
wht),
full_output=True,
ftol=1.49e-8 / 1000.,
xtol=1.49e-8 / 1000.)
xcoeff = popt[0]
model = np.dot(xcoeff, ims).reshape((1014, 1014))
corr = im[1].data * flat - model
#### Use the new mask
mask = mask_full
#### 1D column averages
if True:
yres = np.zeros(1014)
yfull = np.zeros(1014)
ydat = np.zeros(1014)
fcorr = (im[1].data * flat)
xfull = yfull * 0.
for i in range(1014):
ymsk = mask_full[:, i] > 0
ydat[i] = np.median(fcorr[ymsk, i])
yfull[i] = np.median(model[ymsk, i])
yres[i] = np.median(corr[ymsk, i])
#
xmsk = mask_full[i, :] > 0
xfull[i] = np.median(model[i, xmsk])
#print i
yres_sm = threedhst.utils.medfilt(yres, 41)
### Make figure
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg
fig = Figure(figsize=[8, 4], dpi=100)
fig.subplots_adjust(wspace=0.25,
hspace=0.02,
left=0.1,
bottom=0.08,
right=0.99,
top=0.92)
ax = fig.add_subplot(121)
ax.plot(ydat, color='black')
ax.plot(yfull, color='red')
ax.plot(xfull, color='green')
ax.set_xlim(0, 1014)
ax.set_title(flt)
ax = fig.add_subplot(122)
ax.plot(yres, color='black')
ax.plot(yres_sm, color='red', linewidth=2)
ax.set_xlim(0, 1014)
canvas = FigureCanvasAgg(fig)
canvas.print_figure(flt.split('.fits')[0] + '.multisky.png',
dpi=100,
transparent=False)
#### Update header keywords
print 'Simultaneous sky components:'
for i in range(len(skies)):
print ' %s %.3f' % (skies[i], xcoeff[i])
im[0].header.update('GSKY%02d' % (i + 1),
xcoeff[i],
comment='Grism sky: %s' % (skies[i]))
# #### Show the result
# if plot:
# ds9.frame(3)
# ds9.v(corr, vmin=-0.5,vmax=0.5)
#### Put the result in the FLT data extension
im[1].data = corr * 1.
#### Need to write an output file to use `profile`
im.writeto(out_path + os.path.basename(flt).replace('.gz', ''),
clobber=True)
xin, yin = bg.profile(out_path + os.path.basename(flt).replace('.gz', ''),
extension=1,
flatcorr=False,
biweight=True)
im = pyfits.open(out_path + os.path.basename(flt).replace('.gz', ''),
mode='update')
if second_pass:
#### Subtract the residual difference between the observed and master sky
if sky_components:
### Use column average found earlier
resid = np.dot(np.ones((1014, 1)), yres_sm.reshape(1, 1014))
else:
resid = np.dot(np.ones((1014, 1)),
threedhst.utils.medfilt(yin, 41).reshape(1, 1014))
im[1].data -= resid
#### Subtract the overall biweight mean
if overall:
full_mean = threedhst.utils.biweight(im[1].data[mask], mean=True)
im[1].data -= full_mean
print 'overall: %.4f' % (full_mean)
#### Add a header keyword and write to the output image
im[0].header.update('GRISMSKY',
keep,
comment='Image used for sky subtraction')
im[0].header.update('SKYSCALE',
sky_stats[0],
comment='Scale factor of sky')
#### Sky flat keyword
if 'SKYFLAT' in im[0].header.keys():
im[0].header['SKYFLAT'] = (flat_correct | im[0].header['SKYFLAT'],
'Direct image flat applied')
else:
im[0].header['SKYFLAT'] = (flat_correct, 'Direct image flat applied')
bad = ~np.isfinite(im[1].data)
im[1].data[bad] = 1
im[3].data[bad] = im[3].data[bad] | 32
im.flush()
#### Show the final result, compare to the earlier version in PREP_FLT
if plot:
ds9.frame(3)
ds9.v(im[1].data, vmin=-0.5, vmax=0.5)
chk = pyfits.open(
threedhst.utils.find_fits_gz(
flt.replace('RAW', 'PREP_FLT').replace('.gz', '')))
ds9.frame(4)
ds9.v(chk[1].data, vmin=-0.5, vmax=0.5)
def remove_visit_sky(asn_file='GDN12-G102_asn.fits',
list=['zodi_G102_clean.fits', 'excess_G102_clean.fits'],
add_constant=False,
column_average=True,
mask_grow=18,
flat_correct=True):
"""
Require that all exposures in a visit have the same zodi component.
"""
from scipy.linalg import lstsq
import scipy.optimize
import scipy.ndimage as nd
import astropy.io.fits as pyfits
import copy
import threedhst.grism_sky as bg
asn = threedhst.utils.ASNFile(asn_file)
flt = pyfits.open('%s_flt.fits' % (asn.exposures[0]))
bg.set_grism_flat(grism=flt[0].header['FILTER'], verbose=True)
if flat_correct:
flat = bg.flat * 1.
else:
flat = bg.flat * 0. + 1
data = []
whts = []
masks = []
for exp in asn.exposures:
flt = pyfits.open('%s_flt.fits' % (exp))
segfile = '%s_flt.seg.fits' % (exp)
seg = pyfits.open(segfile)[0].data
seg_mask = nd.maximum_filter((seg > 0), size=18) == 0
dq_ok = (flt[3].data & (4 + 32 + 16 + 512 + 2048 + 4096)) == 0
#
flat_corr = flt[1].data * flat
mask = seg_mask & dq_ok
mask &= (flat_corr < np.percentile(flat_corr[mask], 98)) & (
flt[2].data > 0) & (flat_corr > np.percentile(flat_corr[mask], 1))
#
data.append(flat_corr.flatten())
whts.append(1 / flt[2].data.flatten()**2)
masks.append(mask.flatten())
data = np.array(data)
whts = np.array(whts)
masks = np.array(masks)
#### Read in the master skies
ims = []
skies = copy.deepcopy(list)
for sky in skies:
ims.append(
pyfits.open(os.getenv('THREEDHST') + '/CONF/' +
sky)[0].data.flatten())
if add_constant:
ims.append(flt[1].data.flatten() * 0. + 1)
skies.append('Constant')
ims = np.array(ims)
#### Do the fit
tol = 1.49e-8 # not sure what this controls
p0 = np.ones((ims.shape[0] - 1) * len(asn.exposures) + 1)
popt = scipy.optimize.leastsq(bg.obj_lstsq_visit,
p0,
args=(data, ims, whts, masks),
full_output=True,
ftol=tol / 1000.,
xtol=tol / 1000.)
xcoeff = popt[0]
sh_temp = ims.shape
logstr = 'Master grism sky: %s\n\n FLT %s\n' % (asn_file,
' '.join(skies))
for i in range(len(asn.exposures)):
coeff = np.zeros(sh_temp[0])
coeff[0] = xcoeff[0]
coeff[1:] = xcoeff[1 + i * (sh_temp[0] - 1):1 + (i + 1) *
(sh_temp[0] - 1)]
bg_model = np.dot(coeff, ims).reshape((1014, 1014))
logstr += '%s %s\n' % (asn.exposures[i], ''.join(
[' %9.4f' % (c) for c in coeff]))
flt = pyfits.open('%s_flt.fits' % (asn.exposures[i]), mode='update')
flt[1].data = flt[1].data * flat - bg_model
for j in range(sh_temp[0]):
if 'GSKY%02d' % (j) in flt[0].header:
flt[0].header['GSKY%02d' % (j)] += coeff[j]
else:
flt[0].header['GSKY%02d' % (j)] = (coeff[j], 'Master sky: %s' %
(skies[j]))
#
flt[1].header['MDRIZSKY'] = 0.
if 'SKYFLAT' in flt[0].header.keys():
flt[0].header['SKYFLAT'] = (flat_correct
| flt[0].header['SKYFLAT'],
'Direct image flat applied')
else:
flt[0].header['SKYFLAT'] = (flat_correct,
'Direct image flat applied')
flt.flush()
threedhst.showMessage(logstr)
if column_average:
#for iter in range(2):
grism_sky_column_average(asn_file=asn_file, mask_grow=mask_grow)
def subtract_grism_background(asn_file='GDN1-G102_asn.fits', PATH_TO_RAW='../RAW/', final_scale=0.06, visit_sky=True, column_average=True, mask_grow=18, first_run=True, sky_iter=1):
"""
Subtract master grism sky from FLTs
"""
import os
import scipy.ndimage as nd
import pyregion
from drizzlepac import astrodrizzle
import drizzlepac
from stwcs import updatewcs
import stwcs
import threedhst.grism_sky as bg
asn = threedhst.utils.ASNFile(asn_file)
root = asn_file.split('_asn')[0]
sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits'],
'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
#
# sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light_v2.fits'],
# 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
# ### Don't use scattered light
# sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits'],
# 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
#
# ## Use aXe images
# sky_images = {'G141':['WFC3.IR.G141.sky.V1.0.flat.fits', 'WFC3.IR.G141.sky.V1.0.flat.fits'],
# 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
if first_run:
### Rough background subtraction
threedhst.process_grism.fresh_flt_files(asn_file, from_path=PATH_TO_RAW, preserve_dq=False)
flt = pyfits.open('%s_flt.fits' %(asn.exposures[0]))
GRISM = flt[0].header['FILTER']
bg.set_grism_flat(grism=GRISM, verbose=True)
zodi = pyfits.open(os.getenv('THREEDHST')+'/CONF/%s' %(sky_images[GRISM][0]))[0].data
for exp in asn.exposures:
updatewcs.updatewcs('%s_flt.fits' %(exp))
flt = pyfits.open('%s_flt.fits' %(exp), mode='update')
#flt = pyfits.open('%s_flt.fits' %(exp))
flt[1].data *= bg.flat
#
mask = (flt['DQ'].data == 0)
data_range = np.percentile(flt[1].data[mask], [20, 80])
mask &= (flt[1].data >= data_range[0]) & (flt[1].data <= data_range[1]) & (flt[2].data != 0) & np.isfinite(flt[1].data) & np.isfinite(flt[2].data)
### Least-sq fit for component normalizations
data = flt[1].data[mask].flatten()
wht = (1./flt[2].data[mask].flatten())**2
zodi_mask = zodi[mask].flatten()
coeff_zodi = np.sum(data*zodi_mask*wht)/np.sum(zodi_mask**2*wht)
flt[1].data -= zodi*coeff_zodi
flt.flush()
threedhst.showMessage('Rough background for %s (zodi): %0.4f' %(exp, coeff_zodi))
#templates = bg_flat[:, mask.flatten()]
### Run astrodrizzle to make DRZ mosaic, grism-SExtractor mask
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, context=False, preserve=False, skysub=True, driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_combine=True, final_wcs=False, resetbits=4096, final_bits=576, driz_sep_bits=576, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7')
else:
flt = pyfits.open('%s_flt.fits' %(asn.exposures[0]))
GRISM = flt[0].header['FILTER']
bg.set_grism_flat(grism=GRISM, verbose=True)
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' %(root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION'
se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits' %(root)
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_drz_sci.cat' %(root)
se.options['FILTER'] = 'Y'
se.copyConvFile(grism=True)
se.options['FILTER_NAME'] = 'grism.conv'
se.options['DETECT_THRESH'] = '0.7'
se.options['ANALYSIS_THRESH'] = '0.7'
#
se.sextractImage('%s_drz_sci.fits' %(root))
#### Blot segmentation map to FLT images for object mask
ref = pyfits.open('%s_drz_sci.fits' %(root))
ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=0)
seg = pyfits.open('%s_drz_seg.fits' %(root))
seg_data = np.cast[np.float32](seg[0].data)
#### Loop through FLTs, blotting reference and segmentation
threedhst.showMessage('%s: Blotting grism segmentation masks.' %(root))
for exp in asn.exposures:
flt = pyfits.open('%s_flt.fits' %(exp))
flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=1)
### segmentation
#print 'Segmentation image: %s_blot.fits' %(exp)
blotted_seg = astrodrizzle.ablot.do_blot(seg_data, ref_wcs, flt_wcs, 1, coeffs=True, interp='nearest', sinscl=1.0, stepsize=10, wcsmap=None)
seg_grow = nd.maximum_filter((blotted_seg > 0)*1, size=8)
pyfits.writeto('%s_flt.seg.fits' %(exp), header=flt[1].header, data=seg_grow, clobber=True)
if first_run:
### Run background subtraction scripts
threedhst.process_grism.fresh_flt_files(asn_file, from_path=PATH_TO_RAW, preserve_dq=False)
for exp in asn.exposures:
updatewcs.updatewcs('%s_flt.fits' %(exp))
#threedhst.grism_sky.remove_grism_sky(flt=exp+'_flt.fits', list=sky_images[GRISM], path_to_sky=os.getenv('THREEDHST')+'/CONF/', verbose=True, second_pass=True, overall=True)
if visit_sky:
threedhst.grism_sky.remove_visit_sky(asn_file=asn_file, list=sky_images[GRISM], add_constant=False, column_average=(column_average) & (sky_iter == 1), mask_grow=mask_grow, flat_correct=first_run)
if (sky_iter > 1) & (~first_run):
for i in range(1, sky_iter):
threedhst.grism_sky.remove_visit_sky(asn_file=asn_file, list=sky_images[GRISM], add_constant=False, column_average=column_average & (i == (sky_iter-1)), mask_grow=mask_grow, flat_correct=False)
else:
for exp in asn.exposures:
threedhst.grism_sky.remove_grism_sky(flt='%s_flt.fits' %(exp), list=sky_images[GRISM], path_to_sky = os.getenv('THREEDHST')+'/CONF/', out_path='./', verbose=False, plot=False, flat_correct=first_run, sky_subtract=True, second_pass=column_average, overall=True, combine_skies=False, sky_components=True, add_constant=False)
### Astrodrizzle again to reflag CRs and make cleaned mosaic
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, skysub=False, skyuser='******', final_wcs=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale='2.5 0.7') # , final_wcs=True, final_rot=0)