def SN_MARSHALL():
####********************************************####
#### SN-MARSHALL (UDS)
####********************************************####
os.chdir(unicorn.GRISM_HOME+'SN-MARSHALL/PREP_FLT')
ALIGN_IMAGE = '/3DHST/Ancillary/UDS/CANDELS/hlsp_candels_hst_wfc3_uds01_f160w_v0.5_drz.fits'
#### NEW process all direct imaging
unicorn.candels.make_asn_files()
files=glob.glob('[UM]*-F1[26]*asn.fits')
FORCE=False
#### First processing step
for file in files:
if ((not os.path.exists(file.replace('asn','drz'))) & (not os.path.exists(file.replace('asn','drz')+'.gz'))) | FORCE:
#threedhst.process_grism.fresh_flt_files(file)
#threedhst.prep_flt_files.threedhst.shifts.run_tweakshifts(file)
unicorn.candels.prep_candels(asn_file=file,
ALIGN_IMAGE = ALIGN_IMAGE, ALIGN_EXTENSION=0,
GET_SHIFT=False, DIRECT_HIGHER_ORDER=1,
SCALE=0.06, geometry='rotate,shift')
### DQ mask
bad = ['MARSHALL-B6-F125W', 'MARSHALL-1-G141', 'MARSHALL-3-G141', 'MARSHALL-4-G141', 'MARSHALL-6-G141', 'MARSHALL-6-F125W']
for b in bad[1:]:
threedhst.dq.checkDQ(asn_direct_file='%s_asn.fits' %(b), asn_grism_file='%s_asn.fits' %(b), path_to_flt='../RAW/', SIMPLE_DS9=False)
for b in bad:
unicorn.candels.prep_candels(asn_file='%s_asn.fits' %(b),
ALIGN_IMAGE = ALIGN_IMAGE, ALIGN_EXTENSION=0,
GET_SHIFT=False, DIRECT_HIGHER_ORDER=1,
SCALE=0.06, geometry='rotate,shift')
#### Align to CANDELS reference
files=glob.glob('[UM]*-F1*drz.fits')
ALIGN_IMAGE = '/3DHST/Ancillary/UDS/CANDELS/hlsp_candels_hst_wfc3_uds-tot_f160w_v1.0_drz.fits'
for file in files:
root=file.split('_drz')[0]
if (not os.path.exists(root+'_align.map')) | FORCE:
threedhst.prep_flt_files.startMultidrizzle('%s_asn.fits' %(root),
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.6, driz_cr=False,
updatewcs=False, clean=True, median=False)
#
for geom in ['rotate', 'shift']:
threedhst.shifts.refine_shifts(ROOT_DIRECT=root,
ALIGN_IMAGE=ALIGN_IMAGE, ALIGN_EXTENSION=0,
fitgeometry=geom, clean=True)
#
threedhst.prep_flt_files.startMultidrizzle('%s_asn.fits' %(root),
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.6, driz_cr=False,
updatewcs=False, clean=True, median=False)
###
files=glob.glob('*-F1[26]*asn.fits')
for file in files:
asn = threedhst.utils.ASNFile(file)
for exp in asn.exposures:
flt = pyfits.open(exp+'_flt.fits', mode='update')
bad = (flt['SCI'].data > 1.4e4) & ((flt['DQ'].data & 4096) == 0)
print exp, bad.sum()
if bad.sum() == 0:
continue
#
flt['DQ'].data[bad] |= 4096
flt.flush()
#
import shutil
for filter in ['F125W','F160W'][::-1]:
threedhst.utils.combine_asn_shifts(glob.glob('[MU]*-%s_asn.fits' %(filter)),
out_root='SN-MARSHALL-%s' %(filter), path_to_FLT='./', run_multidrizzle=False)
#
threedhst.prep_flt_files.startMultidrizzle('SN-MARSHALL-%s_asn.fits' %(filter),
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False, ra=34.439064, dec=-5.2595809,
final_outnx=3300, final_outny=3300, ivar_weights=True, build_drz=False)
#
shutil.move('SN-MARSHALL-%s_drz_weight.fits' %(filter), 'SN-MARSHALL-%s_drz_ivar.fits' %(filter))
#
threedhst.prep_flt_files.startMultidrizzle('SN-MARSHALL-%s_asn.fits' %(filter),
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False, ra=34.439064, dec=-5.2595809,
final_outnx=3300, final_outny=3300, ivar_weights=False, build_drz=False)
#
shutil.move('SN-MARSHALL-%s_drz_weight.fits' %(filter), 'SN-MARSHALL-%s_drz_exp.fits' %(filter))
#### Process the grism exposures
direct = glob.glob('../RAW/ib*80_asn.fits')
grism = glob.glob('../RAW/ib*70_asn.fits')
direct = glob.glob('MARSHALL-?-F*_asn.fits')
grism = glob.glob('MARSHALL-?-G141_asn.fits')
#for i in range(0,6):
for i in [2,3,5]:
threedhst.shifts.make_grism_shiftfile(direct[i], grism[i])
pair(direct[i], grism[i], ALIGN_IMAGE = None, SKIP_GRISM=False, GET_SHIFT=False, SKIP_DIRECT=True)
# #### Direct images
# asn_list = glob.glob('MARSHALL-?-F160W_asn.fits')
# threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-F160W',
# path_to_FLT='./', run_multidrizzle=False)
# threedhst.prep_flt_files.startMultidrizzle('MARSHALL-F160W_asn.fits',
# use_shiftfile=True, skysub=False,
# final_scale=0.06, pixfrac=0.8, driz_cr=False,
# updatewcs=False, clean=True, median=False)
#
# asn_list = glob.glob('MARSHALL-?-F125W_asn.fits')
# threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-F125W',
# path_to_FLT='./', run_multidrizzle=False)
# threedhst.prep_flt_files.startMultidrizzle('MARSHALL-F125W_asn.fits',
# use_shiftfile=True, skysub=False,
# final_scale=0.06, pixfrac=0.8, driz_cr=False,
# updatewcs=False, clean=True, median=False)
#### First orientation
asn_list = glob.glob('MARSHALL-[135]-G141_asn.fits')
threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-225-G141',
path_to_FLT='./', run_multidrizzle=False)
threedhst.prep_flt_files.startMultidrizzle('MARSHALL-225-G141_asn.fits',
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False)
#### Second orientation
asn_list = glob.glob('MARSHALL-[246]-G141_asn.fits')
threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-245-G141',
path_to_FLT='./', run_multidrizzle=False)
threedhst.prep_flt_files.startMultidrizzle('MARSHALL-245-G141_asn.fits',
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False)
#
#### Placeholder direct image for each orientation
asn_list = glob.glob('MARSHALL-[1]-F*_asn.fits')
threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-225-F140W',
path_to_FLT='./', run_multidrizzle=False)
threedhst.prep_flt_files.startMultidrizzle('MARSHALL-225-F140W_asn.fits',
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False)
#
asn_list = glob.glob('MARSHALL-[2]-F*_asn.fits')
threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-245-F140W',
path_to_FLT='./', run_multidrizzle=False)
threedhst.prep_flt_files.startMultidrizzle('MARSHALL-245-F140W_asn.fits',
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False)
def udf_prepare():
"""
Make images and catalogs for extracting UDF spectra
"""
import os
import threedhst
import unicorn
import threedhst.prep_flt_files
from threedhst.prep_flt_files import process_3dhst_pair as pair
import threedhst.catIO as catIO
os.chdir(unicorn.GRISM_HOME+'UDF/PREP_FLT')
ALIGN = '../XDF/xdfh_sci.fits'
ALIGN_EXT=0
info = catIO.Readfile('files.info')
#### Make ASN files for different grisms / orientations
asn = threedhst.utils.ASNFile(glob.glob('../RAW/i*asn.fits')[0])
## 10-26
for d in ['10-26','11-01']:
match = (info.targname == 'PRIMO') & (info.filter == 'G141') & (info.date_obs == '2010-%s' %(d))
asn.exposures = []
for exp in info.file[match]:
asn.exposures.append(exp.split('_flt')[0])
#
asn.product = 'PRIMO-%s-G141' %(d.replace('-',''))
asn.write(asn.product+'_asn.fits', clobber=True)
#
match = (info.targname == 'PRIMO') & (info.filter != 'G141') & (info.date_obs == '2010-%s' %(d))
filt = info.filter[match][0]
asn.exposures = []
for exp in info.file[match]:
asn.exposures.append(exp.split('_flt')[0])
#
asn.product = 'PRIMO-%s-%s' %(d.replace('-',''), filt)
asn.write(asn.product+'_asn.fits', clobber=True)
for pointing in [34,36,37,38]:
for filt in ['F140W','G141']:
match = (info.targname == 'GOODS-SOUTH-%d' %(pointing)) & (info.filter == filt)
asn.exposures = []
for exp in info.file[match]:
asn.exposures.append(exp.split('_flt')[0])
#
asn.product = 'GOODS-SOUTH-%d-%s' %(pointing, filt)
asn.write(asn.product+'_asn.fits', clobber=True)
##### Run background subtraction on all images
direct = glob.glob('*[0-9]-F*asn.fits')
grism = glob.glob('*[0-9]-G141_asn.fits')
for i in range(len(direct)):
if not os.path.exists(grism[i].replace('asn','drz')):
pair(direct[i], grism[i], adjust_targname=False, ALIGN_IMAGE = ALIGN, ALIGN_EXTENSION=ALIGN_EXT, SKIP_GRISM=False, GET_SHIFT=True, SKIP_DIRECT=False, align_geometry='rotate,shift')
### Fix offsets for 1026 since aperture combination was different
files=['ibfup1myq_flt.fits','ibfup1n1q_flt.fits']
for file in files:
im = pyfits.open(file, mode='update')
im[0].header.update('POSTARG1', im[0].header['POSTARG1']+8.814150)
im[0].header.update('POSTARG2', im[0].header['POSTARG2']+0.025025)
im.flush()
#### Interlaced combinations
## Need to fake a combination for the interlaced direct image for PRIMO
## The first image is the direct image and the rest are G141 exposures
## to fill a 2x2 interlaced array
##
## Give them "F140W" filenames to work with interlacing code
for d, f in zip(['1026', '1101'], ['F160W', 'F125W']):
asn_im = threedhst.utils.ASNFile('PRIMO-'+d+'-%s_asn.fits' %(f))
asn = threedhst.utils.ASNFile('PRIMO-'+d+'-G141_asn.fits')
sf = threedhst.shifts.ShiftFile('PRIMO-'+d+'-G141_shifts.txt')
#
asn.exposures[0] = asn_im.exposures[0]
sf.images[0] = asn.exposures[0]+'_flt.fits'
#
### Enough images to fill 2x2 grid: (Even-Even, Odd-Odd, OE, EO)
# xo, yo = unicorn.reduce.get_interlace_offsets('PRIMO-'+d+'-G141_asn.fits', verbose=1, path_to_flt='./')
keep = [0,1,3,5]
for i in range(len(asn.exposures))[::-1]:
if i not in keep:
p = asn.exposures.pop(i)
p = sf.images.pop(i)
p = sf.xshift.pop(i)
p = sf.yshift.pop(i)
p = sf.scale.pop(i)
p = sf.rotate.pop(i)
#
### Image is combination of G141, F140W but need Multidrizzle outputs
asn.product = 'PRIMO-'+d+'-F140W'
sf.nrows = 4
asn.write('%s_asn.fits' %(asn.product), clobber=True)
sf.write('%s_shifts.txt' %(asn.product))
threedhst.prep_flt_files.startMultidrizzle(asn.product + '_asn.fits',
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False)
#####################
#### Deep F160W reference for interlaced reductions
#####################
wht = pyfits.open('xdfh_wht.fits')
sci = pyfits.open('xdfh_sci.fits')
texp = 3.e3
f = 10**(-0.4*(33.4549980163574-25.96))
#wht[0].data = wht[0].data*1000.-
wht[0].data = (1.e5*wht[0].data**2+1./((sci[0].data*f+0.5)/texp))*f**2
wht.writeto('xdfh_VAR.fits', clobber=True)
wht[0].data = 1./np.sqrt(wht[0].data)
wht.writeto('xdfh_SIG.fits', clobber=True)
sci[0].data *= f
sci.writeto('xdfh_sci_scaled.fits')
## Make catalog
os.chdir("/research/HST/GRISM/3DHST/UDF/XDF")
se = threedhst.sex.SExtractor()
se.aXeParams()
se.copyConvFile()
se.overwrite = True
se.options['CATALOG_NAME'] = 'xdf.cat'
se.options['CHECKIMAGE_NAME'] = 'xdf_seg.fits'
se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION'
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['WEIGHT_IMAGE'] = 'xdfh_VAR.fits'
se.options['FILTER'] = 'Y'
se.options['DETECT_THRESH'] = '1.5'
se.options['ANALYSIS_THRESH'] = '1.5'
se.options['MAG_ZEROPOINT'] = '33.45499801'
se.options['DEBLEND_NTHRESH'] = '64'
se.options['DEBLEND_MINCONT'] = '0.00005'
status = se.sextractImage('xdfh_sci.fits')
threedhst.sex.sexcatRegions('xdf.cat', 'xdf.reg', format=2)
## Prep blot drizzle images
REF_ROOT = 'XDF-F160W'
CATALOG = '../XDF/xdf.cat'
unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F160W', REFERENCE = '../XDF/xdfh_sci_scaled.fits', SEGM = '../XDF/xdf_seg.fits', sci_extension=0)
REF_ROOT = 'HUDF12-F160W'
CATALOG = '../HUDF12/hudf12.cat'
unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F160W', REFERENCE = '../HUDF12/hlsp_hudf12_hst_wfc3ir_udfmain_f160w_v1.0_drz.fits', SEGM = '../HUDF12/hudf12_seg.fits', sci_extension=0)
## Use new F140W image
REF_ROOT = 'HUDF12-F140W'
CATALOG = '../F140W/HUDF12-F140W.cat'
unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F140W', REFERENCE = '../F140W/HUDF12-F140W_drz_sci.fits', SEGM = '../F140W/HUDF12-F140W_seg.fits', sci_extension=0)
### Generate DRZ images
files=glob.glob('*F*asn.fits')
for file in files:
threedhst.prep_flt_files.startMultidrizzle(file,
use_shiftfile=True, skysub=False,
final_scale=0.06, pixfrac=0.8, driz_cr=False,
updatewcs=False, clean=True, median=False)
NGROW=125
ROOT = 'PRIMO-1101'
ROOT = 'GOODS-SOUTH-34'
for p in [34, 36, 37, 38]:
ROOT = 'GOODS-SOUTH-%d' %(p)
unicorn.reduce.blot_from_reference(REF_ROOT=REF_ROOT, DRZ_ROOT = ROOT+'-F140W', NGROW=NGROW, verbose=True)
unicorn.reduce.interlace_combine_blot(root=ROOT+'-F140W', view=False, pad=60+200*(ROOT=='PRIMO-1026'), REF_ROOT=REF_ROOT, CATALOG=CATALOG, NGROW=NGROW, verbose=True, auto_offsets=True, NSEGPIX=3)
# seg = pyfits.open(ROOT+'_inter_seg.fits', mode='update')
# seg[0].data[seg[0].data < 0] = 0
# seg.flush()
#
unicorn.reduce.interlace_combine(root=ROOT+'-G141', view=False, pad=60+200*(ROOT=='PRIMO-1026'), NGROW=NGROW, auto_offsets=True)
unicorn.reduce.interlace_combine(root=ROOT+'-F140W', view=False, pad=60+200*(ROOT=='PRIMO-1026'), NGROW=NGROW, auto_offsets=True)
#
ref = pyfits.open(ROOT+'_ref_inter.fits', mode='update')
ref[0].header['FILTER'] = 'F140W'
ref.flush()
### Shift 1026-G141 image right by 130 pixels
#xo, yo = unicorn.reduce.get_interlace_offsets('PRIMO-1026-F140W_asn.fits', verbose=1, path_to_flt='./')
im = pyfits.open('PRIMO-1026-G141_inter.fits', mode='update')
fill = im[1].data*0
fill[:,480:2580] = im[1].data[:,350:2450]
im[1].data = fill*1
fill = im[2].data*0
fill[:,480:2580] = im[2].data[:,350:2450]
im[2].data = fill*1
im.flush()
#### Shift 1101-G141 image down by 1 pixel
im = pyfits.open('PRIMO-1101-G141_inter.fits', mode='update')
fill = im[1].data*0
fill[100:-100,:] = im[1].data[101:-99,:]
im[1].data = fill*1
fill = im[2].data*0
fill[100:-100,:] = im[2].data[101:-99,:]
im[2].data = fill*1
im.flush()
if ROOT.startswith('PRIMO'):
ref = pyfits.open(ROOT+'_ref_inter.fits')
im140 = pyfits.open(ROOT+'-F140W_inter.fits', mode='update')
im140[0].header['FILTER'] = 'F140W'
im140[1].data = ref[1].data #/ 10**(-0.4*(26.46-25.96))
#im140[2].data = im140[2].data # / 10**(-0.4*(26.46-25.96))
im140.flush()
#
files = glob.glob('*ref_inter.fits')
for file in files:
ROOT=file.split('_ref_inter')[0]
model = unicorn.reduce.process_GrismModel(root=ROOT, MAG_LIMIT=28, REFINE_MAG_LIMIT=23, make_zeroth_model=False, grism='G141')
model.make_wcs_region_file()
#model = unicorn.reduce.GrismModel(root=ROOT, MAG_LIMIT=30, grism='G141')
### Force use F160W as detection image
#if ROOT.startswith('PRIMO'):
#model.direct[1].data = model.im[1].data*1./10**(-0.4*(26.46-25.96))
model.get_corrected_wcs()
model.make_wcs_region_file()
#for p in [34,36,37,38]:
# ROOT = 'GOODS-SOUTH-%d' %(p)
##### Extract all objects
c = threedhst.sex.mySexCat('../F140W/HUDF12-F140W.cat')
c.write(c.filename.replace('.cat','.reform.cat'), reformat_header=True)
c = catIO.Readfile('../F140W/HUDF12-F140W.reform.cat')
ok = c.mag_auto < 26
hudf.extract_all(c.number[ok], miny=-80)
### FOr some reason, some 2D files weren't extracted with 80 pix. Redo those
bad = []
files=glob.glob('*2D.fits')
for
