def SN_MARSHALL(): ####********************************************#### #### SN-MARSHALL (UDS) ####********************************************#### os.chdir(unicorn.GRISM_HOME+'SN-MARSHALL/PREP_FLT') ALIGN_IMAGE = '/3DHST/Ancillary/UDS/CANDELS/hlsp_candels_hst_wfc3_uds01_f160w_v0.5_drz.fits' #### NEW process all direct imaging unicorn.candels.make_asn_files() files=glob.glob('[UM]*-F1[26]*asn.fits') FORCE=False #### First processing step for file in files: if ((not os.path.exists(file.replace('asn','drz'))) & (not os.path.exists(file.replace('asn','drz')+'.gz'))) | FORCE: #threedhst.process_grism.fresh_flt_files(file) #threedhst.prep_flt_files.threedhst.shifts.run_tweakshifts(file) unicorn.candels.prep_candels(asn_file=file, ALIGN_IMAGE = ALIGN_IMAGE, ALIGN_EXTENSION=0, GET_SHIFT=False, DIRECT_HIGHER_ORDER=1, SCALE=0.06, geometry='rotate,shift') ### DQ mask bad = ['MARSHALL-B6-F125W', 'MARSHALL-1-G141', 'MARSHALL-3-G141', 'MARSHALL-4-G141', 'MARSHALL-6-G141', 'MARSHALL-6-F125W'] for b in bad[1:]: threedhst.dq.checkDQ(asn_direct_file='%s_asn.fits' %(b), asn_grism_file='%s_asn.fits' %(b), path_to_flt='../RAW/', SIMPLE_DS9=False) for b in bad: unicorn.candels.prep_candels(asn_file='%s_asn.fits' %(b), ALIGN_IMAGE = ALIGN_IMAGE, ALIGN_EXTENSION=0, GET_SHIFT=False, DIRECT_HIGHER_ORDER=1, SCALE=0.06, geometry='rotate,shift') #### Align to CANDELS reference files=glob.glob('[UM]*-F1*drz.fits') ALIGN_IMAGE = '/3DHST/Ancillary/UDS/CANDELS/hlsp_candels_hst_wfc3_uds-tot_f160w_v1.0_drz.fits' for file in files: root=file.split('_drz')[0] if (not os.path.exists(root+'_align.map')) | FORCE: threedhst.prep_flt_files.startMultidrizzle('%s_asn.fits' %(root), use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.6, driz_cr=False, updatewcs=False, clean=True, median=False) # for geom in ['rotate', 'shift']: threedhst.shifts.refine_shifts(ROOT_DIRECT=root, ALIGN_IMAGE=ALIGN_IMAGE, ALIGN_EXTENSION=0, fitgeometry=geom, clean=True) # threedhst.prep_flt_files.startMultidrizzle('%s_asn.fits' %(root), use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.6, driz_cr=False, updatewcs=False, clean=True, median=False) ### files=glob.glob('*-F1[26]*asn.fits') for file in files: asn = threedhst.utils.ASNFile(file) for exp in asn.exposures: flt = pyfits.open(exp+'_flt.fits', mode='update') bad = (flt['SCI'].data > 1.4e4) & ((flt['DQ'].data & 4096) == 0) print exp, bad.sum() if bad.sum() == 0: continue # flt['DQ'].data[bad] |= 4096 flt.flush() # import shutil for filter in ['F125W','F160W'][::-1]: threedhst.utils.combine_asn_shifts(glob.glob('[MU]*-%s_asn.fits' %(filter)), out_root='SN-MARSHALL-%s' %(filter), path_to_FLT='./', run_multidrizzle=False) # threedhst.prep_flt_files.startMultidrizzle('SN-MARSHALL-%s_asn.fits' %(filter), use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False, ra=34.439064, dec=-5.2595809, final_outnx=3300, final_outny=3300, ivar_weights=True, build_drz=False) # shutil.move('SN-MARSHALL-%s_drz_weight.fits' %(filter), 'SN-MARSHALL-%s_drz_ivar.fits' %(filter)) # threedhst.prep_flt_files.startMultidrizzle('SN-MARSHALL-%s_asn.fits' %(filter), use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False, ra=34.439064, dec=-5.2595809, final_outnx=3300, final_outny=3300, ivar_weights=False, build_drz=False) # shutil.move('SN-MARSHALL-%s_drz_weight.fits' %(filter), 'SN-MARSHALL-%s_drz_exp.fits' %(filter)) #### Process the grism exposures direct = glob.glob('../RAW/ib*80_asn.fits') grism = glob.glob('../RAW/ib*70_asn.fits') direct = glob.glob('MARSHALL-?-F*_asn.fits') grism = glob.glob('MARSHALL-?-G141_asn.fits') #for i in range(0,6): for i in [2,3,5]: threedhst.shifts.make_grism_shiftfile(direct[i], grism[i]) pair(direct[i], grism[i], ALIGN_IMAGE = None, SKIP_GRISM=False, GET_SHIFT=False, SKIP_DIRECT=True) # #### Direct images # asn_list = glob.glob('MARSHALL-?-F160W_asn.fits') # threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-F160W', # path_to_FLT='./', run_multidrizzle=False) # threedhst.prep_flt_files.startMultidrizzle('MARSHALL-F160W_asn.fits', # use_shiftfile=True, skysub=False, # final_scale=0.06, pixfrac=0.8, driz_cr=False, # updatewcs=False, clean=True, median=False) # # asn_list = glob.glob('MARSHALL-?-F125W_asn.fits') # threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-F125W', # path_to_FLT='./', run_multidrizzle=False) # threedhst.prep_flt_files.startMultidrizzle('MARSHALL-F125W_asn.fits', # use_shiftfile=True, skysub=False, # final_scale=0.06, pixfrac=0.8, driz_cr=False, # updatewcs=False, clean=True, median=False) #### First orientation asn_list = glob.glob('MARSHALL-[135]-G141_asn.fits') threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-225-G141', path_to_FLT='./', run_multidrizzle=False) threedhst.prep_flt_files.startMultidrizzle('MARSHALL-225-G141_asn.fits', use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False) #### Second orientation asn_list = glob.glob('MARSHALL-[246]-G141_asn.fits') threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-245-G141', path_to_FLT='./', run_multidrizzle=False) threedhst.prep_flt_files.startMultidrizzle('MARSHALL-245-G141_asn.fits', use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False) # #### Placeholder direct image for each orientation asn_list = glob.glob('MARSHALL-[1]-F*_asn.fits') threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-225-F140W', path_to_FLT='./', run_multidrizzle=False) threedhst.prep_flt_files.startMultidrizzle('MARSHALL-225-F140W_asn.fits', use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False) # asn_list = glob.glob('MARSHALL-[2]-F*_asn.fits') threedhst.utils.combine_asn_shifts(asn_list, out_root='MARSHALL-245-F140W', path_to_FLT='./', run_multidrizzle=False) threedhst.prep_flt_files.startMultidrizzle('MARSHALL-245-F140W_asn.fits', use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False)
def udf_prepare(): """ Make images and catalogs for extracting UDF spectra """ import os import threedhst import unicorn import threedhst.prep_flt_files from threedhst.prep_flt_files import process_3dhst_pair as pair import threedhst.catIO as catIO os.chdir(unicorn.GRISM_HOME+'UDF/PREP_FLT') ALIGN = '../XDF/xdfh_sci.fits' ALIGN_EXT=0 info = catIO.Readfile('files.info') #### Make ASN files for different grisms / orientations asn = threedhst.utils.ASNFile(glob.glob('../RAW/i*asn.fits')[0]) ## 10-26 for d in ['10-26','11-01']: match = (info.targname == 'PRIMO') & (info.filter == 'G141') & (info.date_obs == '2010-%s' %(d)) asn.exposures = [] for exp in info.file[match]: asn.exposures.append(exp.split('_flt')[0]) # asn.product = 'PRIMO-%s-G141' %(d.replace('-','')) asn.write(asn.product+'_asn.fits', clobber=True) # match = (info.targname == 'PRIMO') & (info.filter != 'G141') & (info.date_obs == '2010-%s' %(d)) filt = info.filter[match][0] asn.exposures = [] for exp in info.file[match]: asn.exposures.append(exp.split('_flt')[0]) # asn.product = 'PRIMO-%s-%s' %(d.replace('-',''), filt) asn.write(asn.product+'_asn.fits', clobber=True) for pointing in [34,36,37,38]: for filt in ['F140W','G141']: match = (info.targname == 'GOODS-SOUTH-%d' %(pointing)) & (info.filter == filt) asn.exposures = [] for exp in info.file[match]: asn.exposures.append(exp.split('_flt')[0]) # asn.product = 'GOODS-SOUTH-%d-%s' %(pointing, filt) asn.write(asn.product+'_asn.fits', clobber=True) ##### Run background subtraction on all images direct = glob.glob('*[0-9]-F*asn.fits') grism = glob.glob('*[0-9]-G141_asn.fits') for i in range(len(direct)): if not os.path.exists(grism[i].replace('asn','drz')): pair(direct[i], grism[i], adjust_targname=False, ALIGN_IMAGE = ALIGN, ALIGN_EXTENSION=ALIGN_EXT, SKIP_GRISM=False, GET_SHIFT=True, SKIP_DIRECT=False, align_geometry='rotate,shift') ### Fix offsets for 1026 since aperture combination was different files=['ibfup1myq_flt.fits','ibfup1n1q_flt.fits'] for file in files: im = pyfits.open(file, mode='update') im[0].header.update('POSTARG1', im[0].header['POSTARG1']+8.814150) im[0].header.update('POSTARG2', im[0].header['POSTARG2']+0.025025) im.flush() #### Interlaced combinations ## Need to fake a combination for the interlaced direct image for PRIMO ## The first image is the direct image and the rest are G141 exposures ## to fill a 2x2 interlaced array ## ## Give them "F140W" filenames to work with interlacing code for d, f in zip(['1026', '1101'], ['F160W', 'F125W']): asn_im = threedhst.utils.ASNFile('PRIMO-'+d+'-%s_asn.fits' %(f)) asn = threedhst.utils.ASNFile('PRIMO-'+d+'-G141_asn.fits') sf = threedhst.shifts.ShiftFile('PRIMO-'+d+'-G141_shifts.txt') # asn.exposures[0] = asn_im.exposures[0] sf.images[0] = asn.exposures[0]+'_flt.fits' # ### Enough images to fill 2x2 grid: (Even-Even, Odd-Odd, OE, EO) # xo, yo = unicorn.reduce.get_interlace_offsets('PRIMO-'+d+'-G141_asn.fits', verbose=1, path_to_flt='./') keep = [0,1,3,5] for i in range(len(asn.exposures))[::-1]: if i not in keep: p = asn.exposures.pop(i) p = sf.images.pop(i) p = sf.xshift.pop(i) p = sf.yshift.pop(i) p = sf.scale.pop(i) p = sf.rotate.pop(i) # ### Image is combination of G141, F140W but need Multidrizzle outputs asn.product = 'PRIMO-'+d+'-F140W' sf.nrows = 4 asn.write('%s_asn.fits' %(asn.product), clobber=True) sf.write('%s_shifts.txt' %(asn.product)) threedhst.prep_flt_files.startMultidrizzle(asn.product + '_asn.fits', use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False) ##################### #### Deep F160W reference for interlaced reductions ##################### wht = pyfits.open('xdfh_wht.fits') sci = pyfits.open('xdfh_sci.fits') texp = 3.e3 f = 10**(-0.4*(33.4549980163574-25.96)) #wht[0].data = wht[0].data*1000.- wht[0].data = (1.e5*wht[0].data**2+1./((sci[0].data*f+0.5)/texp))*f**2 wht.writeto('xdfh_VAR.fits', clobber=True) wht[0].data = 1./np.sqrt(wht[0].data) wht.writeto('xdfh_SIG.fits', clobber=True) sci[0].data *= f sci.writeto('xdfh_sci_scaled.fits') ## Make catalog os.chdir("/research/HST/GRISM/3DHST/UDF/XDF") se = threedhst.sex.SExtractor() se.aXeParams() se.copyConvFile() se.overwrite = True se.options['CATALOG_NAME'] = 'xdf.cat' se.options['CHECKIMAGE_NAME'] = 'xdf_seg.fits' se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION' se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT' se.options['WEIGHT_IMAGE'] = 'xdfh_VAR.fits' se.options['FILTER'] = 'Y' se.options['DETECT_THRESH'] = '1.5' se.options['ANALYSIS_THRESH'] = '1.5' se.options['MAG_ZEROPOINT'] = '33.45499801' se.options['DEBLEND_NTHRESH'] = '64' se.options['DEBLEND_MINCONT'] = '0.00005' status = se.sextractImage('xdfh_sci.fits') threedhst.sex.sexcatRegions('xdf.cat', 'xdf.reg', format=2) ## Prep blot drizzle images REF_ROOT = 'XDF-F160W' CATALOG = '../XDF/xdf.cat' unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F160W', REFERENCE = '../XDF/xdfh_sci_scaled.fits', SEGM = '../XDF/xdf_seg.fits', sci_extension=0) REF_ROOT = 'HUDF12-F160W' CATALOG = '../HUDF12/hudf12.cat' unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F160W', REFERENCE = '../HUDF12/hlsp_hudf12_hst_wfc3ir_udfmain_f160w_v1.0_drz.fits', SEGM = '../HUDF12/hudf12_seg.fits', sci_extension=0) ## Use new F140W image REF_ROOT = 'HUDF12-F140W' CATALOG = '../F140W/HUDF12-F140W.cat' unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F140W', REFERENCE = '../F140W/HUDF12-F140W_drz_sci.fits', SEGM = '../F140W/HUDF12-F140W_seg.fits', sci_extension=0) ### Generate DRZ images files=glob.glob('*F*asn.fits') for file in files: threedhst.prep_flt_files.startMultidrizzle(file, use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False) NGROW=125 ROOT = 'PRIMO-1101' ROOT = 'GOODS-SOUTH-34' for p in [34, 36, 37, 38]: ROOT = 'GOODS-SOUTH-%d' %(p) unicorn.reduce.blot_from_reference(REF_ROOT=REF_ROOT, DRZ_ROOT = ROOT+'-F140W', NGROW=NGROW, verbose=True) unicorn.reduce.interlace_combine_blot(root=ROOT+'-F140W', view=False, pad=60+200*(ROOT=='PRIMO-1026'), REF_ROOT=REF_ROOT, CATALOG=CATALOG, NGROW=NGROW, verbose=True, auto_offsets=True, NSEGPIX=3) # seg = pyfits.open(ROOT+'_inter_seg.fits', mode='update') # seg[0].data[seg[0].data < 0] = 0 # seg.flush() # unicorn.reduce.interlace_combine(root=ROOT+'-G141', view=False, pad=60+200*(ROOT=='PRIMO-1026'), NGROW=NGROW, auto_offsets=True) unicorn.reduce.interlace_combine(root=ROOT+'-F140W', view=False, pad=60+200*(ROOT=='PRIMO-1026'), NGROW=NGROW, auto_offsets=True) # ref = pyfits.open(ROOT+'_ref_inter.fits', mode='update') ref[0].header['FILTER'] = 'F140W' ref.flush() ### Shift 1026-G141 image right by 130 pixels #xo, yo = unicorn.reduce.get_interlace_offsets('PRIMO-1026-F140W_asn.fits', verbose=1, path_to_flt='./') im = pyfits.open('PRIMO-1026-G141_inter.fits', mode='update') fill = im[1].data*0 fill[:,480:2580] = im[1].data[:,350:2450] im[1].data = fill*1 fill = im[2].data*0 fill[:,480:2580] = im[2].data[:,350:2450] im[2].data = fill*1 im.flush() #### Shift 1101-G141 image down by 1 pixel im = pyfits.open('PRIMO-1101-G141_inter.fits', mode='update') fill = im[1].data*0 fill[100:-100,:] = im[1].data[101:-99,:] im[1].data = fill*1 fill = im[2].data*0 fill[100:-100,:] = im[2].data[101:-99,:] im[2].data = fill*1 im.flush() if ROOT.startswith('PRIMO'): ref = pyfits.open(ROOT+'_ref_inter.fits') im140 = pyfits.open(ROOT+'-F140W_inter.fits', mode='update') im140[0].header['FILTER'] = 'F140W' im140[1].data = ref[1].data #/ 10**(-0.4*(26.46-25.96)) #im140[2].data = im140[2].data # / 10**(-0.4*(26.46-25.96)) im140.flush() # files = glob.glob('*ref_inter.fits') for file in files: ROOT=file.split('_ref_inter')[0] model = unicorn.reduce.process_GrismModel(root=ROOT, MAG_LIMIT=28, REFINE_MAG_LIMIT=23, make_zeroth_model=False, grism='G141') model.make_wcs_region_file() #model = unicorn.reduce.GrismModel(root=ROOT, MAG_LIMIT=30, grism='G141') ### Force use F160W as detection image #if ROOT.startswith('PRIMO'): #model.direct[1].data = model.im[1].data*1./10**(-0.4*(26.46-25.96)) model.get_corrected_wcs() model.make_wcs_region_file() #for p in [34,36,37,38]: # ROOT = 'GOODS-SOUTH-%d' %(p) ##### Extract all objects c = threedhst.sex.mySexCat('../F140W/HUDF12-F140W.cat') c.write(c.filename.replace('.cat','.reform.cat'), reformat_header=True) c = catIO.Readfile('../F140W/HUDF12-F140W.reform.cat') ok = c.mag_auto < 26 hudf.extract_all(c.number[ok], miny=-80) ### FOr some reason, some 2D files weren't extracted with 80 pix. Redo those bad = [] files=glob.glob('*2D.fits') for