def check_backgrounds(): """ UDF Plot the automatically-determined backgrounds as a function of position in the UDF frame """ c = catIO.Readfile('../F140W/HUDF12-F140W.reform.cat') ok = c.mag_auto < 27.5 fp = open('udf_backgrounds.dat','w') fp.write('# id c0 cx cy x y mag\n') for i in np.arange(c.N)[ok]: id = c.number[i] bgfile = 'UDF_FIT/UDF_%05d.bg.dat' %(id) if os.path.exists(bgfile): line = open(bgfile).readlines()[1][:-1] fp.write('%s %7.1f %7.1f %.2f\n' %(line, c.x_image[i], c.y_image[i], c.mag_auto[i])) fp.close() bg = catIO.Readfile('udf_backgrounds.dat') ok = (bg.mag > 24) & (bg.c0+0.003 > 0) & (bg.c0+0.003 < 0.004) plt.scatter(bg.x[ok], bg.y[ok], c=bg.c0[ok], s=30, vmin=-0.004, vmax=0.002) #### Try 2D as in sciypy example (need newer version) # from scipy.interpolate import griddata # # points = [bg.x[ok], bg.y[ok]] # values = bg.c0[ok] # # grid_z2 = griddata(points, values, (bg.x, bg.y), method='cubic') from scipy import interpolate bg_spline = interpolate.SmoothBivariateSpline(bg.x[ok], bg.y[ok], bg.c0[ok], kx=4, ky=4) test = bg.c0*0. for i in range(bg.N): test[i] = bg_spline(bg.x[i], bg.y[i]) # plt.scatter(bg.x+3600, bg.y, c=test, s=30, vmin=-0.004, vmax=0.002) plt.text(2000,3800,'Data (mag > 24)', ha='center', va='center') plt.text(5600,3800,'Interpolated', ha='center', va='center') plt.savefig('background_spline.pdf') import cPickle as pickle fp = open('background_spline.pkl','wb') pickle.dump(bg_spline, fp) fp.close()
def fit_goodsn(): import unicorn.galfit import threedhst tar_path = '/research/HST/GRISM/3DHST/GOODS-N/HTML_TAR/' catalogs = glob.glob( '/research/HST/GRISM/3DHST/GOODS-N/HTML_v1.0/*drz.cat') psfs = glob.glob('star_ib*fits') matches = glob.glob( '/research/HST/GRISM/3DHST/GOODS-N/HTML_v1.0/SED/*match.cat') for i, catalog in enumerate(catalogs): cat = threedhst.sex.mySexCat(catalog) mag = np.cast[float](cat.MAG_F1392W) match = catIO.Readfile(matches[i]) q = [] for id in match.id_f140w: mat = np.where(cat.id == id)[0][0] q.append(mat) q = np.array(q) use = q[(match.logm > 10.6) & (match.rmatch < 1)] root = os.path.basename(catalog).split('_drz')[0] for id in cat.id[use]: status = unicorn.galfit.go_fit(id=id, fix_n=False, tar_root=root, tar_path=tar_path, PSF_IMAGE=psfs[i])
def fit_uds_marshall(): import unicorn.galfit import threedhst tar_path = '/research/HST/GRISM/3DHST/SN-MARSHALL/HTML/images/' catalogs = glob.glob('/research/HST/GRISM/3DHST/SN-MARSHALL/HTML/*drz.cat') PSF_IMAGE = 'star_UDS_direct1_00360.thumb.fits' matches = glob.glob( '/research/HST/GRISM/3DHST/SN-MARSHALL/HTML/SED/*match.cat') for i, catalog in enumerate(catalogs): cat = threedhst.sex.mySexCat(catalog) mag = np.cast[float](cat.MAG_F1249W) use = mag < 24.5 root = os.path.basename(catalog).split('_drz')[0] match = catIO.Readfile(matches[i]) q = [] for id in match.id_f140w: mat = np.where(cat.id == id)[0][0] q.append(mat) q = np.array(q) use = q[(match.logm >= 10.9) & (match.rmatch < 1)] for id in cat.id[use]: if not os.path.exists('%s_%05d_galfit.png' % (root, id)): status = unicorn.galfit.go_fit(id=id, fix_n=False, tar_root=root, fit_sky=False, tar_path=tar_path, PSF_IMAGE=PSF_IMAGE)
def fit_cosmos(): import unicorn.galfit import threedhst sexCat = threedhst.sex.mySexCat('orient1_drz.cat') mag = np.cast[float](sexCat.MAG_F1392W) use = mag < 23.5 root = 'orient1' #### matched catalog, do all objects that have NMBS matches to some K limit match = catIO.Readfile('../../../../COSMOS/HTML_v1.0/SED/' + root + '_match.cat') q = [] for id in match.id_f140w: mat = np.where(sexCat.id == id)[0][0] q.append(mat) q = np.array(q) use = q[(match.mag_ktot < 23) & (match.rmatch < 1) & (match.logm >= 10.9)] for id in sexCat.id[use]: try: # status = unicorn.galfit.go_fit(id=id, fix_n=False, fit_sky=False, tar_root=root, PSF_IMAGE='star_' + root + '_thumb.fits') # status = unicorn.galfit.go_fit(id=id, fix_n=False, # tar_root='orient2', # PSF_IMAGE='star_orient2_01732_thumb.fits') except: pass
def extract_new_redshifts(PATH='./'): """ UDF The fitting code saves a FITS file with p(z) but doesn't print the results. Pull it out and make a catalog """ import unicorn.interlace_test as test import unicorn.catalogs2 as cat2 c = catIO.Readfile('../F140W/HUDF12-F140W.reform.cat') zsp = cat2.SpeczCatalog() dr, idx = zsp.match_list(c.x_world, c.y_world) c.z_spec = zsp.zspec[idx] #[dr < 0.3] c.z_spec[dr > 0.3] = -1 ok = c.mag_auto < 26 original_path = os.getcwd() fp = open('udf_redshifts.dat','w') fp.write('# id mag flag z_max z_peak l68 u68 l95 u95 z_spec z_source\n') fp.write('# %s' %(PATH)) os.chdir(PATH) for i in range(ok.sum()): id = c.number[ok][i] z_spec = c.z_spec[ok][i] if z_spec > 0: z_source = zsp.source[idx][ok][i] else: z_source = '-' # logstr = 'UDS_%05d %.2f ' %(id, c.mag_auto[ok][i]) if os.path.exists('UDF_%05d.new_zfit.pz.fits' %(id)): self = test.SimultaneousFit('UDF_%05d' %(id)) self.read_master_templates() status = self.new_load_fits() if status: logstr += ' 1 %.4f %.4f %.4f %.4f %.4f %.4f %.4f %s' %(self.z_max_spec, self.z_peak_spec, self.c68[0], self.c68[1], self.c95[0], self.c95[1], z_spec, z_source) else: logstr += '-1 %.4f %.4f %.4f %.4f %.4f %.4f %.4f %s' %(-1, -1, -1, -1, -1, -1, -1, '-') else: logstr += ' 0 %.4f %.4f %.4f %.4f %.4f %.4f %.4f %s' %(-1, -1, -1, -1, -1, -1, -1, '-') print logstr fp.write(logstr+'\n') fp.close() os.chdir(original_path)
def compare_zodi(): """ Compare minimum background flux in a visit to the computed zodi """ import mywfc3.zodi from threedhst import catIO import glob from mywfc3.utils import gzfile filter = 'G141' asns = glob.glob('*%s_orbit.png' % (filter)) colors = np.array(['blue', 'red', 'orange']) #colors = np.array(['blue', 'white', 'orange']) # fig = unicorn.plotting.plot_init(xs=6, aspect=0.7, left=0.12, right=0.12) # ax = fig.add_subplot(111) bg_min = np.ones(len(asns)) zodi_predicted = bg_min * 1. nor = bg_min * 1. for i, asn in enumerate(asns): root = asn.split('_')[0] print root os.system('cat %s*%s_orbit.dat > /tmp/%s.dat' % (root[:6], filter, root)) bg1 = catIO.Readfile('/tmp/%s.dat' % (root), save_fits=False, force_lowercase=False) bg_min[i] = bg1.bg.min() files = glob.glob('%s*raw.fits*' % (root[:6])) zodi_predicted[i] = mywfc3.zodi.flt_zodi(gzfile(files[0]), verbose=False) nor[i] = mywfc3.zodi.flt_zodi(gzfile(files[0]), verbose=False, pirzkal=True) plt.scatter(bg_min, zodi_predicted, alpha=0.4, color='black', label='Synphot') plt.scatter(bg_min, nor, alpha=0.4, color='orange', label='Nor') plt.plot([0, 4], [0, 4], color='red', alpha=0.2, linewidth=3) plt.ylim(0.5, 3.5) plt.xlim(0.5, 3.5) plt.xlabel('Background, visit minimum') plt.ylabel('Predicted zodi') plt.legend(loc='lower right', prop={'size': 10}) plt.savefig('PredictedZodi_%s.pdf' % (filter))
def process_raw_all(field='AEGIS'): #### Reprocess *all* of the FLTs with variable backgrounds that #### weren't already refit above import glob import os import numpy as np import unicorn import threedhst from threedhst import catIO files = glob.glob('/3DHST/Spectra/Work/BACKGROUND/%s/*G141_orbit.dat' % (field)) redo_list = [] for file in files: bg = catIO.Readfile(file, save_fits=False, force_lowercase=True) var_bg = np.ptp(bg.bg[1:]) > 0.15 no_skip = True if os.path.exists('%sq_flt.fits' % (os.path.split(file)[-1].split('j_')[0])): im2flt_key = threedhst.utils.gethead( '%sq_flt.fits' % (os.path.split(file)[-1].split('j_')[0]), keys=['IMA2FLT']) if im2flt_key[0] == '': no_skip = True else: no_skip = False rawfile = '%sq_raw.fits' % (os.path.split(file)[-1].split('j_')[0]) print rawfile, np.ptp(bg.bg[1:]), var_bg, no_skip, var_bg & no_skip # if var_bg & no_skip: redo_list.append(rawfile) if not os.path.exists(rawfile): print '%s does not exist!' % (rawfile) continue # unicorn.prepare.make_IMA_FLT(raw=rawfile, pop_reads=[])
def make_cat(symbol_color='red', masslim=(11, 13), CIRCULARIZE=True): import unicorn.galfit import threedhst import numpy as np sexCat = threedhst.sex.mySexCat('orient1_drz.cat') sexCat.re = sexCat.id * 0. - 99 sexCat.n = sexCat.id * 0. - 99 sexCat.bovera = sexCat.id * 0. - 99 sexCat.chi2 = sexCat.id * 0. + 100 for i, id in enumerate(sexCat.id): #print id try: root = 'orient1_%05d_galfit' % (id) params = unicorn.galfit.read_log(root + '.log') x0, y0, mag, re, n, bovera, chi2 = params sexCat.re[i] = np.float(re) sexCat.n[i] = np.float(n) sexCat.bovera[i] = np.float(bovera) sexCat.chi2[i] = np.float(chi2) #print re except: pass # try: root = 'orient2_%05d_galfit' % (id) params = unicorn.galfit.read_log(root + '.log') x0, y0, mag, re, n, bovera, chi2 = params sexCat.re[i] = np.float(re) sexCat.n[i] = np.float(n) sexCat.bovera[i] = np.float(bovera) sexCat.chi2[i] = np.float(chi2) #print re except: pass ##### Make plot of sizes vs. z match = catIO.Readfile( '../../../../COSMOS/HTML_v1.0/SED/orient1_match.cat') match1 = catIO.Readfile( '../../../../COSMOS/HTML_v1.0/SED/orient1_match.cat') q = [] for id in match1.id_f140w: mat = np.where(sexCat.id == id)[0][0] q.append(mat) # match2 = catIO.Readfile( '../../../../COSMOS/HTML_v1.0/SED/orient2_match.cat') for id in match2.id_f140w: mat = np.where(sexCat.id == id)[0][0] q.append(mat) for col in match.columns: match[col] = np.append(match1[col], match2[col]) q = np.array(q) fp = open('orient1_galfit.cat', 'w') fp.write( '# id_f140w re n bovera chi2 id_nmbs46 z_peak logm star_flag r_match \n' ) for i, qi in enumerate(q): fp.write( '%-5d %7.2f %7.2f %7.2f %9.1e %-5d %8.3f %6.2f %1d %5.2f\n' % (sexCat.id[qi], sexCat.re[qi], sexCat.n[qi], sexCat.bovera[qi], sexCat.chi2[qi], match.id_phot[i], match.z_peak[i], match.logm[i], match.star_flag[i], match.rmatch[i])) fp.close() zgrid = np.array([ 0.2, 0.3, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3, 2.4, 2.5 ]) scale = np.array([ 3.268, 4.421, 5.343, 5.733, 6.082, 6.394, 6.673, 6.922, 7.144, 7.518, 7.812, 8.041, 8.216, 8.346, 8.439, 8.502, 8.539, 8.556, 8.555, 8.540, 8.512, 8.475, 8.430, 8.377, 8.320, 8.257, 8.192 ]) use = (match.star_flag == 0) & (match.rmatch < 1) use = (match.logm > masslim[0]) & (match.logm < masslim[1]) use = use & (sexCat.chi2[q] < 2) sint = np.interp(match.z_peak[use], zgrid, scale) xvd = np.array([0, 0.6, 1.1, 1.6, 2.0]) yvd = np.array([12.4, 8.0, 5.3, 4.1, 3.0]) if CIRCULARIZE: sexCat.re *= np.sqrt(1. / np.abs(sexCat.bovera)) plt.semilogy(match.z_peak[use], sexCat.re[q][use] * sint * 0.06, marker='o', linestyle='None', color=symbol_color, markersize=12, alpha=0.7) print symbol_color plt.plot(xvd, yvd, marker='None', linewidth=10, alpha=0.3, color='black') #### Add GOODS-N # catalogs = glob.glob('/research/HST/GRISM/3DHST/GOODS-N/HTML_v1.0/*drz.cat') # tar_path = '/research/HST/GRISM/3DHST/GOODS-N/HTML_TAR/' # for catalog in catalogs: # print catalog # cat = threedhst.sex.mySexCat(catalog) # mag = np.cast[float](cat.MAG_F1392W) # use = mag < 23.5 # root=os.path.basename(catalog).split('_drz')[0] # cat.re = cat.id*0.-99 # cat.n = cat.id*0.-99 # cat.bovera = cat.id*0.-99 # cat.chi2 = cat.id*0.+100 # for i,id in enumerate(cat.id): # #print id # try: # rooti=root+'_%05d_galfit' %(id) # params = unicorn.galfit.read_log(rooti+'.log') # x0, y0, mag, re, n, bovera, chi2 = params # cat.re[i] = np.float(re) # cat.n[i] = np.float(n) # cat.bovera[i] = np.float(bovera) # cat.chi2[i] = np.float(chi2) # #print re # except: # pass # # # match = catIO.Readfile('/research/HST/GRISM/3DHST/GOODS-N/HTML_v1.0/SED/'+root+'_match.cat') # q = [] # for id in match.id_f140w: # mat = np.where(cat.id == id)[0][0] # q.append(mat) # # # q = np.array(q) # # # use = (match.rmatch < 1) # use = (match.logm >= masslim[0]) & (match.logm <= masslim[1]) # use = use & (cat.chi2[q] < 2) # sint = np.interp(match.z_peak[use], zgrid, scale) # # # if CIRCULARIZE: # cat.re *= np.sqrt(1./np.abs(cat.bovera)) # # print match.z_peak[use], cat.re[q][use]*sint*0.06 # # plt.semilogy(match.z_peak[use], cat.re[q][use]*sint*0.06, marker='o', linestyle='None', color='green', markersize=10, alpha=0.6) # #### Add UDS catalogs = glob.glob('/research/HST/GRISM/3DHST/SN-MARSHALL/HTML/*drz.cat') tar_path = '/research/HST/GRISM/3DHST/SN-MARSHALL/HTML/images/' for catalog in catalogs[0:1]: cat = threedhst.sex.mySexCat(catalog) mag = np.cast[float](cat.MAG_F1249W) use = mag < 23.5 root = os.path.basename(catalog).split('_drz')[0] cat.re = cat.id * 0. - 99 cat.n = cat.id * 0. - 99 cat.bovera = cat.id * 0. - 99 cat.chi2 = cat.id * 0. + 100 for i, id in enumerate(cat.id): #print id try: rooti = root + '_%05d_galfit' % (id) params = unicorn.galfit.read_log(rooti + '.log') x0, y0, mag, re, n, bovera, chi2 = params cat.re[i] = np.float(re) cat.n[i] = np.float(n) cat.bovera[i] = np.float(bovera) cat.chi2[i] = np.float(chi2) #print re except: pass # match = catIO.Readfile( '/research/HST/GRISM/3DHST/SN-MARSHALL/HTML/SED/' + root + '_match.cat') q = [] for id in match.id_f140w: mat = np.where(cat.id == id)[0][0] q.append(mat) # q = np.array(q) # use = (match.star_flag == 0) & (match.rmatch < 1) use = (match.logm >= masslim[0]) & (match.logm <= masslim[1]) use = use & (cat.chi2[q] < 2) sint = np.interp(match.z_peak[use], zgrid, scale) print match.z_peak[use] if CIRCULARIZE: cat.re *= np.sqrt(1. / np.abs(cat.bovera)) # plt.plot(match.z_peak[use], cat.re[q][use] * sint * 0.06, marker='o', linestyle='None', color='orange', markersize=10, alpha=0.6) plt.ylim(0.5, 30) #plt.ylim(-5,25) plt.xlim(0, 3.5) plt.xlabel(r'$z_\mathrm{phot}$') plt.ylabel(r'$r_e$ [kpc]') plt.text( 2.0, 13, r'F140W, $%4.1f\ <\ \log\ M/M_\odot\ <\ %4.1f$' % (masslim[0], masslim[1])) plt.savefig('3dhst_cosmos_sizes.png')
def show_results(use_tex=False): import threedhst.catIO as catIO stats = catIO.Readfile('all_simspec.dat') ha_model, s2_model = unicorn.intersim.get_line_fluxes(z0=1.0, mag=stats.mag) xstar = [14.5, 24.1] ystar = [3.00, 2.13] yi = np.interp(stats.mag, xstar, ystar) #plt.scatter(stats.mag, yi, s=0.1, color='black') is_star = stats.r50 < yi plt.scatter(stats.mag[is_star], stats.r50[is_star], alpha=0.5) plt.scatter(stats.mag[~is_star], stats.r50[~is_star], alpha=0.2, color='red') #### Color by r50/r90 concentration concentration = stats.r50 / stats.r90 msize = np.maximum((concentration / 0.2)**4, 4) mcol = np.minimum((np.maximum(concentration, 0.3) - 0.3) / 0.2, 1) plt.scatter(stats.mag, concentration, c=mcol, alpha=0.5) mcol = np.minimum(np.log10(stats.r50 - 1.1), 1) stats.sky_avg += np.random.normal(size=stats.sky_avg.shape) * 0.01 sky_col = np.minimum((stats.sky_avg - 0.8) / 0.8, 1) plt.scatter(stats.mag, stats.sky_avg, c=sky_col, alpha=0.5) #### Continuum depth BINWIDTH = 92 bin_sn = np.sqrt(BINWIDTH / 22) binned = stats.continuum_sn * bin_sn #### Get correction functions xm, ym, ys, nn = threedhst.utils.runmed(stats.mag, binned, NBIN=80) ymag = np.interp(stats.mag, xm, ym) sub = (stats.mag > 19) & (stats.mag < 22.5) & (stats.continuum_sn > 0) & ( stats.ha_flux > 0) #& (~is_star) xm, ym, ys, nn = threedhst.utils.runmed(stats.r50[sub], (binned / ymag)[sub], NBIN=20) ysize = np.interp(stats.r50, xm, ym) xm, ym, ys, nn = threedhst.utils.runmed(stats.sky_avg[sub], (binned / ymag / ysize)[sub], NBIN=25) ysky = np.interp(stats.sky_avg, xm, ym) xm, ym, ys, nn = threedhst.utils.runmed( concentration[sub], (binned / ymag / ysize / ysky)[sub], NBIN=25) ycons = np.interp(concentration, xm, ym) fig = unicorn.catalogs.plot_init(xs=8, aspect=1. / 4, left=0.07, use_tex=use_tex) #fig.subplots_adjust(wspace=0.27, hspace=0.25, left=0.12) # 2x2 fig.subplots_adjust(wspace=0.38, hspace=0.25, left=0.074, bottom=0.22) si = 4 mark = 'o' cmap = cm.jet bins = [80, 80] ax = fig.add_subplot(141) #plt.scatter(stats.mag, stats.continuum_sn*bin_sn, alpha=0.5, c=mcol) use = np.isfinite(binned) & (binned > 0) #plt.scatter(stats.mag[use], (binned/ysize/ysky)[use], alpha=0.5, c=mcol[use], s=si, marker=mark) unicorn.intersim.show_hist_contour(stats.mag[use], (binned / ysize / ysky)[use], axrange=[[20, 24], [0.5, 100]], ylog=True, cmap=cmap, bins=bins) xm, ym, ys, nn = threedhst.utils.runmed(stats.mag[use], (binned / ysize / ysky)[use], NBIN=80) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.ylim(0.5, 100) plt.plot([20, 24], [5, 5], color='black', alpha=0.4) plt.xlim(20, 24) plt.semilogy() if use_tex: plt.xlabel(r'MAG\_AUTO $m_{140}$') else: plt.xlabel(r'MAG_AUTO $m_{140}$') plt.ylabel('continuum S/N') ax.xaxis.set_major_locator(unicorn.analysis.MyLocator(6, integer=True)) ax.xaxis.set_minor_locator(MultipleLocator(0.5)) ax.set_yticks([1, 10, 100]) ax.set_yticklabels(['1', '10', '100']) sn5_limit = np.interp(5, ym[::-1], xm[::-1]) print 'Continuum, S/N=5 @ %.3f' % (sn5_limit) print threedhst.utils.biweight(stats.r50[sub], both=True) ax = fig.add_subplot(142) #plt.scatter(stats.r50[sub], (binned/ymag/ysky)[sub], c=mcol[sub], alpha=0.5, s=si) unicorn.intersim.show_hist_contour(stats.r50[sub] * 0.06, (binned / ymag / ysky)[sub], axrange=[[0, 20 * 0.06], [0.3, 1.7]], bins=bins, cmap=cmap) xm, ym, ys, nn = threedhst.utils.runmed(stats.r50[sub] * 0.06, (binned / ymag / ysky)[sub], NBIN=20) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.fill_betweenx([0, 10], [1.7 * 0.06, 1.7 * 0.06], [2.5 * 0.06, 2.5 * 0.06], alpha=0.15, color='black') #plt.xlabel(r'$R_{50}$ [$0.\!\!^{\prime\prime}06$ pix]') plt.xlabel(r'$R_{50}$ [arcsec]') plt.ylabel(r'$\delta$ cont. S/N') plt.ylim(0.3, 1.7) #plt.ylim(0.3,2.5) plt.xlim(0, 15 * 0.06) majorLocator = MultipleLocator(0.2) minorLocator = MultipleLocator(0.1) ax.xaxis.set_major_locator(majorLocator) ax.xaxis.set_minor_locator(minorLocator) # x0 = np.interp(1,ym[::-1],xm[::-1]) # plt.plot(xm,(x0/xm), color='red') # plt.plot(xm,(x0/xm)**0.5, color='red') x0 = np.interp(1, ym[::-1], xm[::-1]) plt.plot(xm, (x0 / xm)**(0.5), color='white', alpha=0.5, linewidth=2) plt.plot(xm, (x0 / xm)**(0.5), color='red', alpha=0.8) ysize = np.interp(stats.r50 * 0.06, xm, ym) # plt.scatter(stats.r50[sub], (binned/ymag/ysize)[sub], c=sky_col[sub], alpha=0.5) # xm, ym, ys, nn = threedhst.utils.runmed(stats.r50[sub], (binned/ymag/ysize)[sub], NBIN=10) # plt.plot(xm, ym, linewidth=2, color='black', alpha=0.5) ax = fig.add_subplot(143) #plt.scatter(stats.sky_avg[sub], (binned/ymag/ysize)[sub], c=mcol[sub], alpha=0.5, s=si) unicorn.intersim.show_hist_contour(stats.sky_avg[sub], (binned / ymag / ysize)[sub], axrange=[[0.5, 3.5], [0.3, 1.7]], bins=bins, cmap=cmap) xm, ym, ys, nn = threedhst.utils.runmed(stats.sky_avg[sub], (binned / ymag / ysize)[sub], NBIN=25) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.ylim(0.3, 1.7) plt.xlim(0.5, 3.5) plt.xlabel(r'Background [e$^-$ / s]') plt.ylabel(r'$\delta$ cont S/N') ax.xaxis.set_major_locator(unicorn.analysis.MyLocator(6, integer=True)) x0 = np.interp(1, ym[::-1], xm[::-1]) plt.plot(xm, (x0 / xm)**(0.5), color='white', alpha=0.5, linewidth=2) plt.plot(xm, (x0 / xm)**(0.5), color='red', alpha=0.7) ysky = np.interp(stats.sky_avg, xm, ym) ### Very little residual trend with concentration ax = fig.add_subplot(144) #plt.scatter(concentration[sub], (binned/ymag/ysize/ysky)[sub], c=mcol[sub], s=si, alpha=0.5) unicorn.intersim.show_hist_contour(concentration[sub], (binned / ymag / ysize / ysky)[sub], axrange=[[0.25, 0.60], [0.3, 1.7]], bins=bins, cmap=cmap) xm, ym, ys, nn = threedhst.utils.runmed( concentration[sub], (binned / ymag / ysize / ysky)[sub], NBIN=25) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.xlim(0.25, 0.60) plt.ylim(0.3, 1.7) #plt.ylim(0.5,1.5) plt.xlabel(r'$C = R_{50}/R_{90}$') plt.ylabel(r'$\delta$ cont S/N') #ax.xaxis.set_major_locator(unicorn.analysis.MyLocator(5, prune=None)) ax.xaxis.set_major_locator(MultipleLocator(0.1)) ycons = np.interp(concentration, xm, ym) plt.savefig('grism_cont_sensitivity.pdf') # #### Test # plt.scatter(stats.mag, binned, alpha=0.5, c=sky_col, s=4) # xm, ym, ys, nn = threedhst.utils.runmed(stats.mag, binned, NBIN=80) # plt.errorbar(xm, ym, ys, linewidth=2, color='black', alpha=0.5) # plt.ylim(0.1,2000) # plt.plot([17,24],[5,5], color='black', alpha=0.4) # plt.xlim(17,24) # plt.semilogy() #### Line fluxes ha_sn = stats.ha_flux / stats.ha_flux_err show = np.isfinite(ha_sn) & (ha_sn > 0) & (stats.ha_flux > 0) xm, ym, ys, nn = threedhst.utils.runmed(stats.ha_flux[~is_star & show], ha_sn[~is_star & show], NBIN=25) yline_flux = np.interp(stats.ha_flux, xm, ym) #sub = (stats.ha_flux > 6) & (stats.ha_flux < 100) & (stats.mag > 18) & (np.isfinite(ha_sn)) # & (~is_star) #sub = (stats.mag > 19) & (stats.mag < 22.5) & (stats.continuum_sn > 0) & (stats.ha_flux > 0) #& (~is_star) xm, ym, ys, nn = threedhst.utils.runmed(stats.r50[sub], (ha_sn / yline_flux)[sub], NBIN=30) yline_r50 = np.interp(stats.r50, xm, ym) xm, ym, ys, nn = threedhst.utils.runmed( stats.sky_avg[sub], (ha_sn / yline_flux / yline_r50)[sub], NBIN=20) yline_sky = np.interp(stats.sky_avg, xm, ym) xm, ym, ys, nn = threedhst.utils.runmed(concentration[sub], (ha_sn / yline_flux / yline_r50 / yline_sky)[sub], NBIN=10) yline_con = np.interp(concentration, xm, ym) plt.errorbar(ha_model, stats.ha_flux, stats.ha_flux_err, marker='o', markersize=0.1, linestyle='None', color='0.5') plt.scatter(ha_model, stats.ha_flux, c=mcol, zorder=100, alpha=0.5) #plt.scatter(stats.s2_flux, s2_model, alpha=0.8, c=mc) plt.plot([0.1, 1000], [0.1, 1000], color='black', alpha=0.5) plt.xlim(0.5, 1000) plt.ylim(0.5, 1000) plt.loglog() # 2x2 #fig = unicorn.catalogs.plot_init(xs=5.5, aspect=1, left=0.08) #fig.subplots_adjust(wspace=0.27, hspace=0.25, left=0.12) fig = unicorn.catalogs.plot_init(xs=8, aspect=1. / 4, left=0.07, use_tex=use_tex) fig.subplots_adjust(wspace=0.38, hspace=0.25, left=0.074, bottom=0.22) ax = fig.add_subplot(141) si = 4 show = np.isfinite(ha_sn) & (ha_sn > 0) & (stats.ha_flux > 0) #plt.scatter(stats.ha_flux[show], ha_sn[show], c=mcol[show], s=si, zorder=100, alpha=0.3) unicorn.intersim.show_hist_contour(stats.ha_flux[show], (ha_sn / yline_r50 / yline_sky / yline_con)[show], axrange=[[0.5, 100], [0.5, 100]], bins=bins, cmap=cmap, xlog=True, ylog=True) xm, ym, ys, nn = threedhst.utils.runmed(stats.ha_flux[~is_star & show], (ha_sn / yline_r50 / yline_sky / yline_con)[~is_star & show], NBIN=25) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.plot([0.5, 100], [5, 5], color='black', alpha=0.4) plt.xlim(0.5, 100) plt.ylim(0.5, 100) plt.loglog() plt.xlabel(r'line flux [$10^{-17}$ ergs / s / cm$^2$]') plt.ylabel('line S/N') ax.set_yticks([1, 10, 100]) ax.set_yticklabels(['1', '10', '100']) ax.set_xticks([1, 10, 100]) ax.set_xticklabels(['1', '10', '100']) sn5_limit = np.interp(5, ym, xm) print 'Line, S/N=5 @ %.3e' % (sn5_limit) print threedhst.utils.biweight(stats.r50[sub], both=True) yline_flux = np.interp(stats.ha_flux, xm, ym) #plt.scatter(stats.ha_flux, ha_sn/yline_flux, c=mcol, alpha=0.2) #### Nice: line flux with respect to concentration after taking out the overall trend with #### line strength ax = fig.add_subplot(142) #plt.scatter(stats.r50[sub], (ha_sn/yline_flux)[sub], c=mcol[sub], s=si, alpha=0.3) unicorn.intersim.show_hist_contour(stats.r50[sub] * 0.06, (ha_sn / yline_flux / yline_sky / yline_con)[sub], axrange=[[0, 15 * 0.06], [0.3, 2.5]], bins=bins, cmap=cmap) xm, ym, ys, nn = threedhst.utils.runmed(stats.r50[sub] * 0.06, (ha_sn / yline_flux / yline_sky / yline_con)[sub], NBIN=30) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20 * 0.06], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.fill_betweenx([0, 10], [1.7 * 0.06, 1.7 * 0.06], [2.5 * 0.06, 2.5 * 0.06], alpha=0.15, color='black') plt.ylim(0.3, 2.5) plt.xlim(0, 15 * 0.06) #plt.xlabel(r'$R_{50}$ [$0.\!\!^{\prime\prime}06$ pix]') plt.ylabel(r'$\delta$ line S/N') #plt.semilogy() # x0 = np.interp(1,ym[::-1],xm[::-1]) # plt.plot(xm,(x0/xm), color='red') # plt.plot(xm,(x0/xm)**0.5, color='red') plt.xlabel(r'$R_{50}$ [arcsec]') ax.xaxis.set_major_locator(MultipleLocator(0.2)) ax.xaxis.set_minor_locator(MultipleLocator(0.1)) x0 = np.interp(1, ym[::-1], xm[::-1]) plt.plot(xm, (x0 / xm)**(0.5), color='red', alpha=0.7) yline_r50 = np.interp(stats.r50 * 0.06, xm, ym) ax = fig.add_subplot(143) #plt.scatter(stats.sky_avg[sub], (ha_sn/yline_flux/yline_r50)[sub], c=mcol[sub], s=si, alpha=0.3) unicorn.intersim.show_hist_contour(stats.sky_avg[sub], (ha_sn / yline_flux / yline_r50 / yline_con)[sub], axrange=[[0.5, 3.5], [0.3, 1.7]], bins=bins, cmap=cmap) xm, ym, ys, nn = threedhst.utils.runmed(stats.sky_avg[sub], (ha_sn / yline_flux / yline_r50 / yline_con)[sub], NBIN=20) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.ylim(0.3, 1.7) plt.xlim(0.5, 3.5) plt.xlabel(r'Background [e$^-$ / s]') plt.ylabel(r'$\delta$ line S/N') ax.xaxis.set_major_locator(unicorn.analysis.MyLocator(6, integer=True)) yline_sky = np.interp(stats.sky_avg, xm, ym) x0 = np.interp(1, ym[::-1], xm[::-1]) plt.plot(xm, (x0 / xm)**(0.5), color='red', alpha=0.7) ax = fig.add_subplot(144) #plt.scatter(concentration[sub], (ha_sn/yline_flux/yline_r50/yline_sky)[sub], c=mcol[sub], s=si, alpha=0.3) unicorn.intersim.show_hist_contour(concentration[sub], (ha_sn / yline_flux / yline_r50 / yline_sky)[sub], axrange=[[0.25, 0.60], [0.3, 1.7]], bins=bins, cmap=cmap) xm, ym, ys, nn = threedhst.utils.runmed(concentration[sub], (ha_sn / yline_flux / yline_r50 / yline_sky)[sub], NBIN=10) plt.plot(xm, ym, linewidth=2, color='white', alpha=0.5, zorder=100) plt.plot(xm, ym, linewidth=1, color='black', alpha=0.8, zorder=100) plt.plot([0, 20], [1, 1], linewidth=1, alpha=0.4, zorder=101, color='black') plt.xlim(0.25, 0.60) plt.ylim(0.3, 1.7) plt.xlabel(r'$C = R_{50}/R_{90}$') plt.ylabel(r'$\delta$ line S/N') ax.xaxis.set_major_locator(MultipleLocator(0.1)) yline_con = np.interp(concentration, xm, ym) plt.savefig('grism_line_sensitivity.pdf') # #### Test: # show = (np.isfinite(ha_sn)) & (stats.ha_flux > 0) # plt.scatter(stats.ha_flux[show], (ha_sn/yline_sky)[show], c=mcol[show], zorder=100, alpha=0.2) # xm, ym, ys, nn = threedhst.utils.runmed(stats.ha_flux[show], (ha_sn/yline_sky)[show], NBIN=25) # plt.plot(xm, ym, linewidth=2, color='black', alpha=0.5, zorder=100) # plt.plot([0.5,1000],[5,5], color='black', alpha=0.4) # plt.xlim(0.5,1000) # plt.ylim(0.5,300) # plt.loglog() #plt.semilogy() # plt.scatter(stats.mag, stats.ha_flux, c=mcol, zorder=100, alpha=0.5) plt.ylim(0.1, 5000) plt.semilogy() #### EQW dha = stats.ha_eqw - 130. hy, hx, hh = plt.hist(dha / stats.ha_eq_err, range=(-5, 5), bins=50, alpha=0.7) threedhst.utils.biweight(dha / stats.ha_eq_err, both=True) #### redshift dz = (stats.z_fit - 1) / 2. plt.scatter(stats.mag, dz, c=mcol, alpha=0.5) plt.scatter(stats.ha_flux, dz, c=mcol, alpha=0.5) plt.xlim(0.1, 5000) plt.semilogx() #### surface density mu = stats.mag - 2 * np.log(stats.r90 * 0.06) plt.scatter(stats.mag, mu, c=mcol)
def get_results(force_new=False): """ Collate the results from the simulated spectra and the input catalogs into single output catalogs suitable for reading and plotting. for field in ['AEGIS','COSMOS','UDS','GOODS-S']: os.chdir(unicorn.GRISM_HOME+'%s/PREP_FLT' %(field)) unicorn.intersim.get_results() os.chdir(unicorn.GRISM_HOME+'SIMULATIONS') status = os.system('cat ../AEGIS/PREP_FLT/simspec.dat ../COSMOS/PREP_FLT/simspec.dat ../GOODS-S/PREP_FLT/simspec.dat ../UDS/PREP_FLT/simspec.dat > all_simspec.dat') """ import threedhst.catIO as catIO files = glob.glob('*linefit.dat') cat = None if (not os.path.exists('simspec.dat')) | force_new: fp = open('simspec.dat', 'w') fp.write( '# object sky_avg sky_lo sky_hi mag r50 r90 z_fit continuum_sn ha_flux ha_flux_err ha_eqw ha_eq_err s2_flux s2_flux_err s2_eqw s2_eq_err\n' ) fp.write('dummy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n') fp.close() log = catIO.Readfile('simspec.dat') for ii, file in enumerate(files): root = file.split('.linefit')[0] print unicorn.noNewLine + '%s (%d/%d)' % (root, ii + 1, len(files)) if root in log.object: continue # fp = open('simspec.dat', 'a') pointing = root.split('_')[0] id = int(root.split('_')[1]) if cat is None: cat = threedhst.sex.mySexCat(pointing + '_inter.cat') ### Get sky background asn = threedhst.utils.ASNFile(pointing + '-G141_asn.fits') bg = [] for exp in asn.exposures: flt = pyfits.open(exp + '_flt.fits') bg.append(flt[0].header['SKYSCALE']) # bg_avg = np.mean(bg) bg_lo = np.min(bg) bg_hi = np.max(bg) else: if not cat.filename.startswith(pointing + '-'): cat = threedhst.sex.mySexCat(pointing + '_inter.cat') asn = threedhst.utils.ASNFile(pointing + '-G141_asn.fits') bg = [] for exp in asn.exposures: flt = pyfits.open(exp + '_flt.fits') bg.append(flt[0].header['SKYSCALE']) # bg_avg = np.mean(bg) bg_lo = np.min(bg) bg_hi = np.max(bg) # gris = unicorn.interlace_fit.GrismSpectrumFit(root, verbose=False) if not gris.status: fp.close() continue # result = gris.stats() if result is False: fp.close() continue # DIRECT_MAG, Q_Z, F_COVER, F_FLAGGED, MAX_CONTAM, INT_CONTAM, F_NEGATIVE = result # lwindow = (gris.oned.data.wave > 1.4e4) & (gris.oned.data.wave < 1.6e4) if (lwindow.sum() < 10) | (INT_CONTAM > 0.3): fp.close() continue # continuum_sn = np.median( (gris.oned.data.flux / gris.oned.data.error)[lwindow]) # lfit = catIO.Readfile(root + '.linefit.dat') if lfit.status is None: fp.close() continue # if 'Ha' in lfit.line: ix = np.arange(len(lfit.line))[lfit.line == 'Ha'][0] ha_flux, ha_flux_err, ha_eqw, ha_eqw_err = lfit.flux[ ix], lfit.error[ix], lfit.eqw_obs[ix], lfit.eqw_obs_err[ix] else: ha_flux, ha_flux_err, ha_eqw, ha_eqw_err = -1, -1, -1, -1 # if 'SII' in lfit.line: ix = np.arange(len(lfit.line))[lfit.line == 'SII'][0] s2_flux, s2_flux_err, s2_eqw, s2_eqw_err = lfit.flux[ ix], lfit.error[ix], lfit.eqw_obs[ix], lfit.eqw_obs_err[ix] else: s2_flux, s2_flux_err, s2_eqw, s2_eqw_err = -1, -1, -1, -1 # ic = np.arange(cat.nrows)[cat.id == id][0] fp.write( ' %s %5.2f %5.2f %5.2f %6.3f %6.2f %6.2f %6.4f %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f\n' % (root, bg_avg, bg_lo, bg_hi, DIRECT_MAG, float(cat.FLUX_RADIUS[ic]), float(cat.FLUX_RADIUS2[ic]), gris.z_max_spec, continuum_sn, ha_flux, ha_flux_err, ha_eqw, ha_eqw_err, s2_flux, s2_flux_err, s2_eqw, s2_eqw_err)) # fp.close()
def check_fast(object='AEGIS-1-G141_00497', wmin=2000, wmax=2.4e4, logx=False, image_type='png'): if object.startswith('GOODS-S') | object.startswith( 'WFC3') | object.startswith('GEORGE') | object.startswith('PRIMO'): abzp = 23.86 else: abzp = 25 obs_sed = catIO.Readfile('ASCII/%s_obs_sed.dat' % (object)) temp_sed = catIO.Readfile('ASCII/%s_temp_sed.dat' % (object)) lc = obs_sed.lc dlam_spec = lc[-1] - lc[-2] is_spec = np.append( np.abs(1 - np.abs(lc[1:] - lc[0:-1]) / dlam_spec) < 0.05, True) obs_convert = 10**(-0.4 * (abzp + 48.6)) * 3.e18 / lc**2 / 10.**-19 fig = unicorn.catalogs.plot_init(square=True, xs=5, aspect=2. / 3, left=0.12) ax = fig.add_subplot(111) ax.plot(lc[~is_spec], obs_sed.fnu[~is_spec] * obs_convert[~is_spec], marker='o', color='orange', linestyle='None', markersize=15, alpha=0.7) ax.plot(lc[is_spec], obs_sed.fnu[is_spec] * obs_convert[is_spec], color='blue', alpha=0.5) ax.plot(lc[~is_spec], obs_sed.obs_sed[~is_spec] * obs_convert[~is_spec], color='red', alpha=0.7, marker='o', linestyle='None', markersize=8) temp_convert = 10**(-0.4 * (abzp + 48.6)) * 3.e18 / temp_sed.lam**2 / 10.**-19 flam_temp = temp_sed.fnu_temp * temp_convert ax.plot(temp_sed.lam, flam_temp, color='red', alpha=0.3) #fast_norm = 1./np.interp(2.2e4, wfast, tfast)*np.interp(2.2e4, temp_sed.lam, flam_temp) wfast, tfast = np.loadtxt('FAST_OUTPUT/BEST_FITS/%s_threedhst_1.fit' % (object), skiprows=1, unpack=True) ax.plot(wfast, tfast, color='green', alpha=0.5) wfast, tfast = np.loadtxt('FAST_OUTPUT/BEST_FITS/%s_threedhst_2.fit' % (object), skiprows=1, unpack=True) ax.plot(wfast, tfast, color='purple', alpha=0.5) if logx: ax.semilogx() ax.set_xlim(wmin, wmax) ymax = np.max(obs_sed.fnu * obs_convert) ax.set_ylim(-0.1 * ymax, 1.3 * ymax) fout = catIO.Readfile('FAST_OUTPUT/%s_threedhst.fout' % (object)) ax.set_xlabel(r'$\lambda$') ax.set_ylabel(r'$f_\lambda\ (10^{-19}$)') xtext = 2e4 xal = 'right' ax.text(xtext, 0.4 * ymax, r'$z_\mathrm{gris}=%.4f$' % (fout.z[0]), horizontalalignment=xal) ax.text(3000, 1.1 * ymax, object) ax.text(xtext, 0.3 * ymax, r'log M: $%.1f^{\ %.1f}_{\ %.1f}$ $%.1f^{\ %.1f}_{\ %.1f}$' % (fout.lmass[0], fout.u68_lmass[0], fout.l68_lmass[0], fout.lmass[1], fout.u68_lmass[1], fout.l68_lmass[1]), horizontalalignment=xal) ax.text(xtext, 0.2 * ymax, r'$A_V$: $%.1f^{\ %.1f}_{\ %.1f}$ $%.1f^{\ %.1f}_{\ %.1f}$' % (fout.av[0], fout.u68_av[0], fout.l68_av[0], fout.av[1], fout.u68_av[1], fout.l68_av[1]), horizontalalignment=xal) ax.text( xtext, 0.1 * ymax, r'log $\tau$: $%.1f^{\ %.1f}_{\ %.1f}$ $%.1f^{\ %.1f}_{\ %.1f}$' % (fout.ltau[0], fout.u68_ltau[0], fout.l68_ltau[0], fout.ltau[1], fout.u68_ltau[1], fout.l68_ltau[1]), horizontalalignment=xal) ax.text(xtext, 0.0 * ymax, r'log Age: $%.1f^{\ %.1f}_{\ %.1f}$ $%.1f^{\ %.1f}_{\ %.1f}$' % (fout.lage[0], fout.u68_lage[0], fout.l68_lage[0], fout.lage[1], fout.u68_lage[1], fout.l68_lage[1]), horizontalalignment=xal) ### Save to PNG outfile = object + '_fast.' + image_type if USE_PLOT_GUI: fig.savefig(outfile, dpi=100, transparent=False) else: canvas = FigureCanvasAgg(fig) canvas.print_figure(outfile, dpi=100, transparent=False) plt.close()
def make_imaging_flat(): """ Make average background images with object masks """ from pyraf import iraf #files = glob.glob('ibhm*flt.seg.fits') #PATH = ('/3DHST/Spectra/Work/%s/RAW/' %(field))*len(files) ###################### Grism sky backgrounds filter, flat_file = 'G141', 'u4m1335mi_pfl.fits' flat = pyfits.open(IREF + '/' + flat_file)[1].data[ 5:-5, 5:-5] / pyfits.open(IREF + '/flat.IR_avg.fits')[1].data[5:-5, 5:-5] flat[flat <= 0] = 5 flat[flat > 5] = 5 ##################### Direct flat-field filter, flat_file = 'F140W', 'uc721143i_pfl.fits' filter, flat_file = 'F125W', 'uc72113qi_pfl.fits' filter, flat_file = 'F160W', 'uc721145i_pfl.fits' filter, flat_file = 'F105W', 'uc72113oi_pfl.fits' flat = pyfits.open(IREF + '/' + flat_file)[1].data[5:-5, 5:-5] flat[flat <= 0] = 5 flat[flat > 5] = 5 ############### 3D-HST os.chdir("/3DHST/Spectra/Work/Background") fields = ['COSMOS', 'GOODS-N', 'GOODS-S', 'AEGIS', 'UDS'] PREP_FLT = '/3DHST/Spectra/Work/xxx/PREP_FLT/' RAW = '/3DHST/Spectra/Work/xxx/RAW/' ############### CANDELS os.chdir('/Users/gbrammer/CANDELS/Flats/') fields = ['GOODS-S', 'EGS', 'UDS'] PREP_FLT = '/Users/gbrammer/CANDELS/xxx/PREP_FLT/' RAW = '/Users/gbrammer/CANDELS/xxx/RAW/' PATHS = [] files = [] file_field = [] for field in fields: info = catIO.Readfile(PREP_FLT.replace('xxx', field) + 'files.info') field_files = info.file[info.filter == filter] files.extend(field_files) PATHS.extend([RAW.replace('xxx', field)] * len(field_files)) file_field.extend([field] * len(field_files)) ################## NF = len(files) idx = np.arange(NF) ## Otherwise get it from "show_profile" above test = idx > -10 fp = open('background.%s.dat' % (filter), 'w') for j, i in enumerate(idx): if ~test[i]: continue # fi = files[i] if not os.path.exists(fi.replace('flt', 'flt.seg')): continue # if os.path.exists(fi.replace('.gz', '') + '.mask.reg'): continue # flt = pyfits.open(PATHS[i] + files[i]) flt[1].data *= flat print unicorn.noNewLine + '%d %s %s' % (i, files[i], flt[0].header['PFLTFILE']) # ### Segmentation mask masked = pyfits.open(fi.replace('flt', 'flt.seg'))[0].data == 0 ### DQ mask, hot pixels and the "death star" dq_ok = (flt[3].data & (4 + 32 + 16)) == 0 # ok = masked & np.isfinite(flt[1].data) & (dq_ok) #flt[1].data /= np.median(flt[1].data[ok]) level = threedhst.utils.biweight(flt[1].data[ok], mean=True) fp.write('%s %s %.3f\n' % (files[i].replace('flt', 'msk').replace( '.gz', ''), file_field[i], level)) # #flt[1].data /= level #flt[1].data[(ok == False)] = 0 #pyfits.writeto(files[i].replace('flt','msk').replace('.gz',''), flt[1].data, clobber=True, header=flt[1].header) fp.close() ## background.dat # # nsum = np.sum(X != 0, axis=0).reshape(1014,1014) # avg = np.sum(X, axis=0).reshape(1014,1014)/nsum # sky = avg #### Use iraf.imcombine for field in fields: info = catIO.Readfile(PREP_FLT.replace('xxx', field) + 'files.info') field_files = info.file[info.filter == filter] if len(field_files) < 10: continue # fp = open('%s.%s.list' % (field, filter), 'w') for ff in field_files: msk = ff.replace('flt.fits.gz', 'msk.fits') if os.path.exists(msk): fp.write('%s\n' % (msk)) fp.close() # iraf.imcombine(input='@%s.%s.list' % (field, filter), output='combine.%s.%s' % (field, filter), headers='', bpmasks='', rejmasks='', nrejmasks='', expmasks='', sigmas='', logfile='STDOUT', combine='average', reject='minmax', project=iraf.no, outtype='real', outlimits='', offsets='none', masktype='none', maskvalue='0', blank=0.0, scale='none', zero='none', weight='none', statsec='', expname='', lthreshold=1e-06, hthreshold=100.0, nlow=5, nhigh=5, nkeep=1, mclip=iraf.yes, lsigma=3.0, hsigma=3.0, rdnoise='0.', gain='1.', snoise='0.', sigscale=0.1, pclip=-0.5) ##### Weight by the square of the background level (more flat signal for higher bg!) bg_flt, bg_field, bg = np.loadtxt('background.%s.dat' % (filter), dtype=np.str, unpack=True) weights = np.cast[float](bg)**2 fp = open('%s.list' % (filter), 'w') fpw = open('%s.weight' % (filter), 'w') for msk, wht in zip(bg_flt, weights): if os.path.exists(msk): fp.write('%s\n' % (msk)) fpw.write('%.2f\n' % (wht)) fp.close() fpw.close() iraf.imcombine(input='@%s.list' % (filter), output='combine.%s' % (filter), headers='', bpmasks='', rejmasks='', nrejmasks='', expmasks='', sigmas='', logfile='STDOUT', combine='average', reject='minmax', project=iraf.no, outtype='real', outlimits='', offsets='none', masktype='none', maskvalue='0', blank=0.0, scale='none', zero='none', weight='@%s.weight' % (filter), statsec='', expname='', lthreshold=1e-06, hthreshold=100.0, nlow=5, nhigh=5, nkeep=1, mclip=iraf.yes, lsigma=3.0, hsigma=3.0, rdnoise='0.', gain='1.', snoise='0.', sigscale=0.1, pclip=-0.5) ##### Final processing combined_files = glob.glob('combine*%s*fits' % (filter)) for file in combined_files: sky = pyfits.open(file)[0].data # ##### Fix bad pixels if filter != 'G141': ratio = sky / flat stats = threedhst.utils.biweight(ratio[np.isfinite(ratio)], both=True) sky = sky / stats[0] max = stats[1] * 5 else: max = 10 # x, y = np.where((np.isfinite(sky) == False) | (sky / flat > (1 + max)) | (sky == 0)) NX = len(x) print '%s: N_fix = %d' % (file, NX) pad = 1 for i in range(NX): xi = x[i] yi = y[i] sub = sky[xi - pad:xi + pad + 2, yi - pad:yi + pad + 2] if (np.sum(sub) != 0.0): sky[xi, yi] = np.median(sub[np.isfinite(sub)]) # still_bad = (np.isfinite(sky) == False) | (sky <= 0.01) sky[still_bad] = flat[still_bad] # #### for DIRECT flat if filter == 'G141': flatim = pyfits.open(unicorn.GRISM_HOME + 'CONF/sky_cosmos.fits') flatim[0].data = sky #flatim[3].data[5:-5,5:-5] = nsum flatim.writeto(file.replace('combine', 'sky'), clobber=True) else: flatim = pyfits.open(IREF + '/' + flat_file) flatim[1].data[5:-5, 5:-5] = sky #flatim[3].data[5:-5,5:-5] = nsum flatim.writeto(file.replace('combine', 'flat'), clobber=True)
def fit(self, ascii_file='AEGIS-3-G141_00177.dat', chi2_limit=1.5, trim_mtype=True, max_contam=0.05, xrange=(1.1e4, 1.65e4), img_type='png', flux_min=0): import threedhst.catIO as catIO import numpy as np ### handle v2.0 format spectra spec = catIO.Readfile(ascii_file) if 'trace' in spec.columns: spec.lam = spec.wave * 1. spec.flux /= spec.sensitivity spec.error /= spec.sensitivity spec.contam /= spec.sensitivity ascii_file = ascii_file.replace('.1D', '') else: spec.error /= 2.5 spec.flux -= spec.contam self.spec = spec self.img_type = img_type chi2 = np.zeros(self.NTEMP) types = [] anorm = chi2 * 0. use = (spec.lam > xrange[0]) & (spec.lam < xrange[1]) & ( spec.contam / spec.flux < max_contam) & (np.isfinite( spec.flux)) & (spec.flux > flux_min) self.use = use if len(spec.lam[use]) < 50: return False for i in range(self.NTEMP): temp = self.templates[i] types.append(temp.type.strip()) yint = np.interp(spec.lam[use], temp.wave, temp.flux) # anorm[i] = np.sum(yint * spec.flux[use] * spec.error[use]** 2) / np.sum(yint**2 * spec.error[use]**2) # chi2[i] = np.sum( (anorm[i] * yint - spec.flux[use])**2 / spec.error[use]**2) types = np.cast[str](types) DOF = len(yint) - 1 chi2 /= DOF min = np.where(chi2 == chi2.min())[0][0] if trim_mtype: trim_mtype = types[min].startswith('M') if (chi2.min() < chi2_limit) & (~trim_mtype): print unicorn.noNewLine + ascii_file + ' * ' + ' %s %0.2f' % ( types[min], chi2.min()) self.spec = spec self.types = types self.chi2 = chi2 self.anorm = anorm self.ascii_file = ascii_file self.make_plot() else: print unicorn.noNewLine + ascii_file + ' %s %0.2f' % (types[min], chi2.min())
def udf_prepare(): """ Make images and catalogs for extracting UDF spectra """ import os import threedhst import unicorn import threedhst.prep_flt_files from threedhst.prep_flt_files import process_3dhst_pair as pair import threedhst.catIO as catIO os.chdir(unicorn.GRISM_HOME+'UDF/PREP_FLT') ALIGN = '../XDF/xdfh_sci.fits' ALIGN_EXT=0 info = catIO.Readfile('files.info') #### Make ASN files for different grisms / orientations asn = threedhst.utils.ASNFile(glob.glob('../RAW/i*asn.fits')[0]) ## 10-26 for d in ['10-26','11-01']: match = (info.targname == 'PRIMO') & (info.filter == 'G141') & (info.date_obs == '2010-%s' %(d)) asn.exposures = [] for exp in info.file[match]: asn.exposures.append(exp.split('_flt')[0]) # asn.product = 'PRIMO-%s-G141' %(d.replace('-','')) asn.write(asn.product+'_asn.fits', clobber=True) # match = (info.targname == 'PRIMO') & (info.filter != 'G141') & (info.date_obs == '2010-%s' %(d)) filt = info.filter[match][0] asn.exposures = [] for exp in info.file[match]: asn.exposures.append(exp.split('_flt')[0]) # asn.product = 'PRIMO-%s-%s' %(d.replace('-',''), filt) asn.write(asn.product+'_asn.fits', clobber=True) for pointing in [34,36,37,38]: for filt in ['F140W','G141']: match = (info.targname == 'GOODS-SOUTH-%d' %(pointing)) & (info.filter == filt) asn.exposures = [] for exp in info.file[match]: asn.exposures.append(exp.split('_flt')[0]) # asn.product = 'GOODS-SOUTH-%d-%s' %(pointing, filt) asn.write(asn.product+'_asn.fits', clobber=True) ##### Run background subtraction on all images direct = glob.glob('*[0-9]-F*asn.fits') grism = glob.glob('*[0-9]-G141_asn.fits') for i in range(len(direct)): if not os.path.exists(grism[i].replace('asn','drz')): pair(direct[i], grism[i], adjust_targname=False, ALIGN_IMAGE = ALIGN, ALIGN_EXTENSION=ALIGN_EXT, SKIP_GRISM=False, GET_SHIFT=True, SKIP_DIRECT=False, align_geometry='rotate,shift') ### Fix offsets for 1026 since aperture combination was different files=['ibfup1myq_flt.fits','ibfup1n1q_flt.fits'] for file in files: im = pyfits.open(file, mode='update') im[0].header.update('POSTARG1', im[0].header['POSTARG1']+8.814150) im[0].header.update('POSTARG2', im[0].header['POSTARG2']+0.025025) im.flush() #### Interlaced combinations ## Need to fake a combination for the interlaced direct image for PRIMO ## The first image is the direct image and the rest are G141 exposures ## to fill a 2x2 interlaced array ## ## Give them "F140W" filenames to work with interlacing code for d, f in zip(['1026', '1101'], ['F160W', 'F125W']): asn_im = threedhst.utils.ASNFile('PRIMO-'+d+'-%s_asn.fits' %(f)) asn = threedhst.utils.ASNFile('PRIMO-'+d+'-G141_asn.fits') sf = threedhst.shifts.ShiftFile('PRIMO-'+d+'-G141_shifts.txt') # asn.exposures[0] = asn_im.exposures[0] sf.images[0] = asn.exposures[0]+'_flt.fits' # ### Enough images to fill 2x2 grid: (Even-Even, Odd-Odd, OE, EO) # xo, yo = unicorn.reduce.get_interlace_offsets('PRIMO-'+d+'-G141_asn.fits', verbose=1, path_to_flt='./') keep = [0,1,3,5] for i in range(len(asn.exposures))[::-1]: if i not in keep: p = asn.exposures.pop(i) p = sf.images.pop(i) p = sf.xshift.pop(i) p = sf.yshift.pop(i) p = sf.scale.pop(i) p = sf.rotate.pop(i) # ### Image is combination of G141, F140W but need Multidrizzle outputs asn.product = 'PRIMO-'+d+'-F140W' sf.nrows = 4 asn.write('%s_asn.fits' %(asn.product), clobber=True) sf.write('%s_shifts.txt' %(asn.product)) threedhst.prep_flt_files.startMultidrizzle(asn.product + '_asn.fits', use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False) ##################### #### Deep F160W reference for interlaced reductions ##################### wht = pyfits.open('xdfh_wht.fits') sci = pyfits.open('xdfh_sci.fits') texp = 3.e3 f = 10**(-0.4*(33.4549980163574-25.96)) #wht[0].data = wht[0].data*1000.- wht[0].data = (1.e5*wht[0].data**2+1./((sci[0].data*f+0.5)/texp))*f**2 wht.writeto('xdfh_VAR.fits', clobber=True) wht[0].data = 1./np.sqrt(wht[0].data) wht.writeto('xdfh_SIG.fits', clobber=True) sci[0].data *= f sci.writeto('xdfh_sci_scaled.fits') ## Make catalog os.chdir("/research/HST/GRISM/3DHST/UDF/XDF") se = threedhst.sex.SExtractor() se.aXeParams() se.copyConvFile() se.overwrite = True se.options['CATALOG_NAME'] = 'xdf.cat' se.options['CHECKIMAGE_NAME'] = 'xdf_seg.fits' se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION' se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT' se.options['WEIGHT_IMAGE'] = 'xdfh_VAR.fits' se.options['FILTER'] = 'Y' se.options['DETECT_THRESH'] = '1.5' se.options['ANALYSIS_THRESH'] = '1.5' se.options['MAG_ZEROPOINT'] = '33.45499801' se.options['DEBLEND_NTHRESH'] = '64' se.options['DEBLEND_MINCONT'] = '0.00005' status = se.sextractImage('xdfh_sci.fits') threedhst.sex.sexcatRegions('xdf.cat', 'xdf.reg', format=2) ## Prep blot drizzle images REF_ROOT = 'XDF-F160W' CATALOG = '../XDF/xdf.cat' unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F160W', REFERENCE = '../XDF/xdfh_sci_scaled.fits', SEGM = '../XDF/xdf_seg.fits', sci_extension=0) REF_ROOT = 'HUDF12-F160W' CATALOG = '../HUDF12/hudf12.cat' unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F160W', REFERENCE = '../HUDF12/hlsp_hudf12_hst_wfc3ir_udfmain_f160w_v1.0_drz.fits', SEGM = '../HUDF12/hudf12_seg.fits', sci_extension=0) ## Use new F140W image REF_ROOT = 'HUDF12-F140W' CATALOG = '../F140W/HUDF12-F140W.cat' unicorn.reduce.prepare_blot_reference(REF_ROOT=REF_ROOT, filter='F140W', REFERENCE = '../F140W/HUDF12-F140W_drz_sci.fits', SEGM = '../F140W/HUDF12-F140W_seg.fits', sci_extension=0) ### Generate DRZ images files=glob.glob('*F*asn.fits') for file in files: threedhst.prep_flt_files.startMultidrizzle(file, use_shiftfile=True, skysub=False, final_scale=0.06, pixfrac=0.8, driz_cr=False, updatewcs=False, clean=True, median=False) NGROW=125 ROOT = 'PRIMO-1101' ROOT = 'GOODS-SOUTH-34' for p in [34, 36, 37, 38]: ROOT = 'GOODS-SOUTH-%d' %(p) unicorn.reduce.blot_from_reference(REF_ROOT=REF_ROOT, DRZ_ROOT = ROOT+'-F140W', NGROW=NGROW, verbose=True) unicorn.reduce.interlace_combine_blot(root=ROOT+'-F140W', view=False, pad=60+200*(ROOT=='PRIMO-1026'), REF_ROOT=REF_ROOT, CATALOG=CATALOG, NGROW=NGROW, verbose=True, auto_offsets=True, NSEGPIX=3) # seg = pyfits.open(ROOT+'_inter_seg.fits', mode='update') # seg[0].data[seg[0].data < 0] = 0 # seg.flush() # unicorn.reduce.interlace_combine(root=ROOT+'-G141', view=False, pad=60+200*(ROOT=='PRIMO-1026'), NGROW=NGROW, auto_offsets=True) unicorn.reduce.interlace_combine(root=ROOT+'-F140W', view=False, pad=60+200*(ROOT=='PRIMO-1026'), NGROW=NGROW, auto_offsets=True) # ref = pyfits.open(ROOT+'_ref_inter.fits', mode='update') ref[0].header['FILTER'] = 'F140W' ref.flush() ### Shift 1026-G141 image right by 130 pixels #xo, yo = unicorn.reduce.get_interlace_offsets('PRIMO-1026-F140W_asn.fits', verbose=1, path_to_flt='./') im = pyfits.open('PRIMO-1026-G141_inter.fits', mode='update') fill = im[1].data*0 fill[:,480:2580] = im[1].data[:,350:2450] im[1].data = fill*1 fill = im[2].data*0 fill[:,480:2580] = im[2].data[:,350:2450] im[2].data = fill*1 im.flush() #### Shift 1101-G141 image down by 1 pixel im = pyfits.open('PRIMO-1101-G141_inter.fits', mode='update') fill = im[1].data*0 fill[100:-100,:] = im[1].data[101:-99,:] im[1].data = fill*1 fill = im[2].data*0 fill[100:-100,:] = im[2].data[101:-99,:] im[2].data = fill*1 im.flush() if ROOT.startswith('PRIMO'): ref = pyfits.open(ROOT+'_ref_inter.fits') im140 = pyfits.open(ROOT+'-F140W_inter.fits', mode='update') im140[0].header['FILTER'] = 'F140W' im140[1].data = ref[1].data #/ 10**(-0.4*(26.46-25.96)) #im140[2].data = im140[2].data # / 10**(-0.4*(26.46-25.96)) im140.flush() # files = glob.glob('*ref_inter.fits') for file in files: ROOT=file.split('_ref_inter')[0] model = unicorn.reduce.process_GrismModel(root=ROOT, MAG_LIMIT=28, REFINE_MAG_LIMIT=23, make_zeroth_model=False, grism='G141') model.make_wcs_region_file() #model = unicorn.reduce.GrismModel(root=ROOT, MAG_LIMIT=30, grism='G141') ### Force use F160W as detection image #if ROOT.startswith('PRIMO'): #model.direct[1].data = model.im[1].data*1./10**(-0.4*(26.46-25.96)) model.get_corrected_wcs() model.make_wcs_region_file() #for p in [34,36,37,38]: # ROOT = 'GOODS-SOUTH-%d' %(p) ##### Extract all objects c = threedhst.sex.mySexCat('../F140W/HUDF12-F140W.cat') c.write(c.filename.replace('.cat','.reform.cat'), reformat_header=True) c = catIO.Readfile('../F140W/HUDF12-F140W.reform.cat') ok = c.mag_auto < 26 hudf.extract_all(c.number[ok], miny=-80) ### FOr some reason, some 2D files weren't extracted with 80 pix. Redo those bad = [] files=glob.glob('*2D.fits') for
def orbit_tracks(FILTER='F105W', PATH='/Users/brammer/WFC3/Backgrounds/BroadBand/F105W', exposures=['ibp329iq', 'ibp329is'], axes=None, axlabels=[1, 1, 1]): """ Make a figure showing the orbit tracks / illuminated or not and the Term angle and compare to the observed background level (and predicted zodi). """ ### F105W (all HUDF-DEEP-WFC3) # FILTER = 'F105W' # PATH = '/Users/brammer/WFC3/Backgrounds/BroadBand/F105W' # exposures = ['ibp329iq', 'ibp329is'] # FILTER = 'F125W' # PATH = '/Users/brammer/WFC3/Backgrounds/BroadBand/Others' # exposures = ['ib5x19tm', 'ib5x19tq'] # # FILTER = 'F160W' # PATH = '/Users/brammer/WFC3/Backgrounds/BroadBand/Others' # exposures = ['ib5x21ht', 'ib5x21hw'] # FILTER = 'G102' # PATH = '/Users/brammer/WFC3/GrismPrograms/Koekemoer/RAW/' # exposures = ['ibl003ac', 'ibl003ae', 'ibl003af', 'ibl003ag'] # PATH = '/Users/brammer/WFC3/GrismPrograms/Stanford/RAW/' # exposures = ['ibkn06dh', 'ibkn06dm', 'ibkn06dp', 'ibkn06dt'] # FILTER = 'G141' # PATH = '/Users/brammer/WFC3/GrismPrograms/Koekemoer/RAW/' # exposures = ['ibl003a7', 'ibl003a9', 'ibl003aa', 'ibl003ab'] # PATH = '/Users/brammer/WFC3/GrismPrograms/CANDELS-SNe/RAW/' # exposures = ['ibfug1je', 'ibfug1jh'] if axes is None: fig = unicorn.plotting.plot_init(xs=3, aspect=1 / 0.5, left=0.1, top=0.05, bottom=0.1, hspace=0.02, use_tex=True, NO_GUI=True) ax = fig.add_subplot(212) ax_limb = fig.add_subplot(211) else: ax, ax_limb = axes N = len(exposures) ax.plot([-10, 100], [200, 200], linestyle='--', label='LimbAng = 20', color='black') ## dummy for label for i, exp in enumerate(exposures): dat = catIO.Readfile('%s/%sj_%s_orbit.dat' % (PATH, exp, FILTER), save_fits=False) raw = pyfits.open(gzfile('%s/%sq_raw.fits' % (PATH, exp))) #bg_min = mywfc3.zodi.flt_zodi(raw.filename(), verbose=False) bg_min = dat.zodi # spt = pyfits.getheader(gzfile('%s/%sq_spt.fits' % (PATH, exp)), 0) #### Start/stop times pstr = astropy.time.Time(spt['PSTRTIME'].replace('.', ':'), format='yday', in_subfmt='date_hms', scale='utc') if i == 0: tstr = pstr NSAMP = raw[0].header['NSAMP'] times = np.zeros(NSAMP - 1) for j in range(NSAMP - 1): times[j] = raw['SCI', j + 1].header['SAMPTIME'] times = (times[::-1][1:] + (pstr - tstr).sec) / 60. ax.plot(times, dat.bg, color='black', linewidth=2) l40 = dat.limbang > 40 ax.plot(times[l40], dat.bg[l40], color='red', alpha=0.4, linewidth=4, zorder=-10, label=r'LimbAng $>$ 40' * (i == 0)) shadow = dat.shadow == 1 ax.plot(times[shadow], dat.bg[shadow], color='black', alpha=0.2, linewidth=7, zorder=-20, label=r'SHADOW' * (i == 0)) ax_limb.plot(times[shadow], dat.limbang[shadow], color='black', alpha=0.2, linewidth=7, zorder=-20, label=r'SHADOW' * (i == 0)) day = dat.brightlimb > 0 ax.plot(times[day], dat.bg[day], color='blue', alpha=0.4, linewidth=7, zorder=-10, label='BrightLimb = 1' * (i == 0)) # ax.plot(times, dat.bg * 0. + bg_min, color='black', linestyle=':', linewidth=2, label='Predicted zodi' * (i == 0)) ax.text(times[0] + 0.2, 0.04 + (N == 4) * ((i % 2) * 0.2 - 0.1), exp + 'q', ha='left', va='bottom', fontsize=6) # ax_limb.plot(times, dat.limbang, color='black', linewidth=2) ax_limb.plot(times[day], dat.limbang[day], color='blue', alpha=0.4, linewidth=7, zorder=-10) ax_limb.plot(times[l40], dat.limbang[l40], color='red', alpha=0.4, linewidth=4, zorder=-10) #ax_limb.plot(times, dat.termang, color='green', alpha=0.5, linewidth=2, label=['TermAng','','',''][i]) ax.set_ylim(-0.3, 3.4) ax.set_xlim(-5, 55) ax.set_xlabel(r'$\Delta\,t$ (minutes)') ax_limb.plot([-10, 100], [20, 20], linestyle='--', color='black') ax_limb.set_ylim(2, 100) ax_limb.set_xticklabels([]) ax_limb.set_xlim(-5, 55) ax_limb.set_title(FILTER, size=10) if axlabels[0] == 0: ax_limb.set_xticklabels([]) if axlabels[1] == 0: ax_limb.set_yticklabels([]) ax.set_yticklabels([]) else: ax_limb.set_ylabel('LimbAng') ax.set_ylabel('Background (e- / s)') if axlabels[2]: ax.legend(loc='upper left', prop={'size': 7}) #ax_limb.legend(loc='lower left', prop={'size':8}) if axes is None: unicorn.plotting.savefig(fig, 'track_%s.pdf' % (FILTER))
def excess_statistics(): """ Show cumulative distribution of (read) excess backgrounds """ master = {} master['F105W'] = catIO.Readfile( '/user/brammer/WFC3_Backgrounds/F105W/master.dat') master['G141'] = catIO.Readfile( '/user/brammer/WFC3_Backgrounds/GrismPrograms/master_G141.dat') master['G102'] = catIO.Readfile( '/user/brammer/WFC3_Backgrounds/GrismPrograms/master_G102.dat') la, ra, ta, ba = 0.06, 0.06, 0.07, 0.13 NX, NY = 1, 1 aspect = (NY * (1 + ta + ba)) / ((3. * NX) * (1 + la + ra)) dx = (1 - la - ra) / 3. fig = unicorn.plotting.plot_init(xs=8, aspect=aspect, left=0, right=0, top=0, bottom=0, hspace=0.0, wspace=0, use_tex=True, NO_GUI=True) for i in range(3): bgf = master[master.keys()[i]] ax = fig.add_axes((la + dx * i, ba, dx, 1 - ba - ta)) # bg_ratio, xr = bgf.bg - bgf.zodi, (-0.5, 6) bg_ratio, xr = bgf.bg / bgf.zodi, (0.5, 6.8) # yh1, xh1, nn = ax.hist(bg_ratio[bgf.shadow == 1], range=xr, bins=100, alpha=0.2, color='black', histtype='stepfilled', log=True) yhc1 = np.cumsum(yh1[::-1]) # yh0, xh0, nn = ax.hist(bg_ratio[bgf.shadow == 0], range=xr, bins=100, alpha=0.2, color='red', histtype='stepfilled', log=True) yhc0 = np.cumsum(yh0[::-1]) # axr = ax.twinx() # if i == 0: ax.plot([100, 110], [1.e4, 1.e4], linewidth=8, color='black', alpha=0.2, label='SHADOW = True') ax.plot([100, 110], [1.e4, 1.e4], linewidth=8, color='red', alpha=0.2, label='SHADOW = False') ax.plot([100, 110], [1.e4, 1.e4], linewidth=1.2, color='red', alpha=1, label=r'$f(>X)$') ax.legend(loc='right', prop={'size': 9}, scatterpoints=1, frameon=True) # yi = np.interp(2, xh0[1:], yhc0[::-1] * 1. / bgf.N) axr.scatter([2], [yi], color='red', marker='o', label=r'$f(X>2)$ = %.1f' % (yi * 100) + '\%') axr.plot(xh1[1:][::-1], yhc1 * 1. / bgf.N, color='black', linewidth=1.2) axr.plot(xh0[1:][::-1], yhc0 * 1. / bgf.N, color='red', linewidth=1.2) # ### efficiency = 1/X eff = 1 / (np.clip(bgf.bg / bgf.zodi, 1, 10)) axr.scatter([100, 110], [-1, -1], linewidth=1, color='white', alpha=0.2, label='eff. = %d' % (np.sum(eff) / bgf.N * 100) + '\%') axr.legend(loc='upper right', prop={'size': 9}, scatterpoints=1, frameon=False) ax.set_xlim(xr) #ax.set_ylim(5,yh1.max()*1.2) ax.set_ylim(5, 3000) axr.set_ylim(0, 0.65) if i > 0: ax.set_yticklabels([]) else: ax.set_ylabel(r'$N_\mathrm{read}$') if i < 2: axr.set_yticklabels([]) else: axr.set_ylabel(r'$f(>X)$') ax.set_title(master.keys()[i], size=10) if i == 1: ax.set_xlabel(r'$X$ = background observed / predicted zodi') # unicorn.plotting.savefig(fig, '/tmp/excess_statistics.pdf')
def z2_galaxies(zrange=(1.6, 2.4)): os.chdir('/research/HST/GRISM/3DHST/ANALYSIS/GALFIT/TEST/COSMOS/Z2_COSMOS') root_path = '/research/HST/GRISM/3DHST/COSMOS/HTML_v1.0/' root = 'orient1' co = catIO.Readfile('../orient1_galfit.cat') co.re *= np.sqrt(1. / co.bovera) #### Select z~2 use = (co.z_peak >= zrange[0]) & (co.z_peak <= zrange[1]) & ( co.logm > 10.9) & (co.r_match < 1) & (co.re > 0) idx = np.arange(len(use)) use = idx[use] use = use[np.argsort(co.re[use])] #### Angular scale zgrid = np.array([ 0.2, 0.3, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3, 2.4, 2.5 ]) scale = np.array([ 3.268, 4.421, 5.343, 5.733, 6.082, 6.394, 6.673, 6.922, 7.144, 7.518, 7.812, 8.041, 8.216, 8.346, 8.439, 8.502, 8.539, 8.556, 8.555, 8.540, 8.512, 8.475, 8.430, 8.377, 8.320, 8.257, 8.192 ]) sint = np.interp(co.z_peak, zgrid, scale) co.re_kpc = co.re * 0.06 * sint fp = open('z2.html', 'w') fp.write(""" <html> <head> <link rel="stylesheet" href="http://localhost/~gbrammer/COSMOS/scripts/style.css" type="text/css" id="" media="print, projection, screen" /> <script type="text/javascript" src="http://localhost/~gbrammer/COSMOS/scripts/jquery-1.4.2.min.js"></script> <script type="text/javascript" src="http://localhost/~gbrammer/COSMOS/scripts/jquery.tablesorter.min.js"></script> <script type="text/javascript" id="js"> // Add ability to sort the table $(document).ready(function() { $.tablesorter.defaults.sortList = [[4,4]]; $("table").tablesorter({ // pass the headers argument and assing a object headers: { // assign the secound column (we start counting zero) 4: { sorter: false }, 5: { sorter: false }, } }); }); </script> </head> <body> <table id="myTable" cellspacing="1" class="tablesorter"> <thead> <th> Grism id </th> <th> z </th> <th> logM </th> <th> r_e </th> <th> Thumb </th> <th> SED </th> </thead> <tbody> """) old = 0 for i in use: if co.id_f140w[i] == old: root = 'orient2' else: root = 'orient1' old = co.id_f140w[i] file = "%s_%05d" % (root, co.id_f140w[i]) if not os.path.exists('%s/SED/%s_SED.png' % (root_path, file)): root = 'orient2' file = "%s_%05d" % (root, co.id_f140w[i]) fp.write(""" <tr> <td> %s </td> <td> %5.2f </td> <td> %5.2f </td> <td> %5.2f </td> <td> <img src=../%s_galfit.png height=180px> </td> <td> <img src=%s/SED/%s_SED.png height=180px> </td> </tr> """ % (file, co.z_peak[i], co.logm[i], co.re_kpc[i], file, root_path, file)) fp.write("</tbody></table></body></html>") fp.close()
def check_redshifts(): import astropy.cosmology as cc cosmo = cc.LambdaCDM(H0=70, Om0=0.3, Ode0=0.7) c = catIO.Readfile('../Catalog/SN-MARSHALL-F160W.reform.cat') c*k = c.number < -100 zc = catIO.Readfile('marshall_redshifts.dat') for id in zc.id: idi = int(id[-5:]) #print idi, c*k.sum() c*k = c*k | (c.number == idi) # star = (c.mag_auto[c*k] < 24) & (c.flux_radius[c*k] < 2.6) cat, zout, fout = unicorn.analysis.read_catalogs('UDS') mat = catIO.CoordinateMatcher(cat) dr, idx = mat.match_list(c.x_world, c.y_world) logm, logm_z, z_phot = fout.lmass[idx][c*k], fout.z[idx][c*k], zout.z_peak[idx][c*k] ok = (zc.z_max > 0) & ~star dz95 = (zc.u95-zc.l95)/(1+zc.z_max) dz68 = (zc.u68-zc.l68)/(1+zc.z_max) best = ok & (dz68 < 0.015) #plt.scatter(zc.z_max[best], zc.mag[best], alpha=0.5) #plt.scatter(zc.z_max[best], logm[best], alpha=0.5) zr = [0,4] zrange = np.log(1+np.array(zr)) dz = 0.01 nbins = zrange[1]/dz h = np.histogram(np.log(1+zc.z_max[best]), bins=nbins, range=zrange) h_idx = np.digitize(np.log(1+zc.z_max), h[1]) z = np.exp(h[1][:-1]*0.5+h[1][1:]*0.5)-1 cmv = h[1]*0. for i in range(len(h[1])): cmv[i] = cosmo.comoving_volume(h[1][i])/(4*np.pi*(360./2/np.pi)**2) survey_area = (2.2/60.)**2 #sq deg dcmv = np.diff(cmv)*survey_area nh = np.maximum(h[0], 0.01) plt.plot(z, nh, linestyle='steps') peak = np.abs(np.log(1+zc.z_max)-np.log(1+0.904)) < 0.005 peak = np.abs(np.log(1+zc.z_max)-np.log(1+1.912)) < 0.005 peak = np.abs(np.log(1+zc.z_max)-np.log(1+2.310)) < 0.005 zbin = 0.904; bin_id = np.arange(len(h[1]))[z >= zbin][0]+1 zbin = 1.912; bin_id = np.arange(len(h[1]))[z >= zbin][0]+1 zbin = 1.522; bin_id = np.arange(len(h[1]))[z >= zbin][0]+1 zbin = 2.310; bin_id = np.arange(len(h[1]))[z >= zbin][0]+1 sel = (h_idx == bin_id) & best fp = open('/tmp/view_list','w') for id in zc.id[sel]: fp.write('%s.new_zfit.png\n' %(id)) fp.write('%s.new_zfit.2D.png\n' %(id)) fp.write('%s_stack.png\n' %(id)) fp.close() os.system('open `cat /tmp/view_list`') plt.scatter(c.x_image, c.y_image, color='black', alpha=0.1) plt.scatter(c.x_image[c*k][sel], c.y_image[c*k][sel], color='red', alpha=0.8)
def make_g141_bg(): """ Make average background images with object masks """ from pyraf import iraf os.chdir("/3DHST/Spectra/Work/Background") field = 'COSMOS' PATHS = [] files = [] for field in ['COSMOS', 'GOODS-N', 'GOODS-S', 'AEGIS', 'UDS']: info = catIO.Readfile('/3DHST/Spectra/Work/%s/PREP_FLT/files.info' % (field)) field_files = info.file[info.filter == 'G141'] files.extend(field_files) PATHS.extend(['/3DHST/Spectra/Work/%s/RAW/' % (field)] * len(info.file[info.filter == 'G141'])) field = 'ALL' #files = glob.glob('ibhm*flt.seg.fits') #PATH = ('/3DHST/Spectra/Work/%s/RAW/' %(field))*len(files) # #### Direct flat-field flat = flat_g141[1].data[5:1019, 5:1019] / pyfits.open( 'COSMOS_f140w_flat.fits')[1].data[5:-5, 5:-5] flat[flat <= 0] = 5 flat[flat > 5] = 5 NF = len(files) idx = np.arange(NF) nxpix, nypix = 1014, 1014 #nxpix, nypix = 507, 507 X = np.zeros((NF, nxpix * nypix)) ## Otherwise get it from "show_profile" above test = idx > -10 for j, i in enumerate(idx): if ~test[i]: continue # fi = files[i] if not os.path.exists(fi.replace('flt', 'flt.seg')): continue # if os.path.exists(fi.replace('.gz', '') + '.mask.reg'): continue # flt = pyfits.open(PATHS[i] + files[i]) flt[1].data *= flat print unicorn.noNewLine + '%d %s %s' % (i, files[i], flt[0].header['PFLTFILE']) # ### Segmentation mask masked = pyfits.open(fi.replace('flt', 'flt.seg'))[0].data == 0 ### DQ mask, hot pixels and the "death star" dq_ok = (flt[3].data & (4 + 32 + 16)) == 0 # ok = masked & np.isfinite(flt[1].data) & (dq_ok) #flt[1].data /= np.median(flt[1].data[ok]) flt[1].data /= threedhst.utils.biweight(flt[1].data[ok], mean=True) flt[1].data[(ok == False)] = 0 X[j, :] = flt[1].data[0:nypix, 0:nxpix].flatten() # #pyfits.writeto(files[i].replace('flt','msk').replace('.gz',''), flt[1].data, clobber=True, header=flt[1].header) #### Average #nsum = np.sum(X != 0, axis=0).reshape(1014,1014) #avg = np.sum(X, axis=0).reshape(1014,1014)/nsum for field in ['COSMOS', 'GOODS-N', 'GOODS-S', 'AEGIS', 'UDS']: info = catIO.Readfile('/3DHST/Spectra/Work/%s/PREP_FLT/files.info' % (field)) field_files = info.file[info.filter == 'G141'] fp = open(field + '.g141.list', 'w') for ff in field_files: msk = ff.replace('flt.fits.gz', 'msk.fits') if os.path.exists(msk): fp.write('%s\n' % (msk)) fp.close() # iraf.imcombine(input='@%s.g141.list' % (field), output='combined_g141_%s' % (field), headers='', bpmasks='', rejmasks='', nrejmasks='', expmasks='', sigmas='', logfile='STDOUT', combine='average', reject='minmax', project=iraf.no, outtype='real', outlimits='', offsets='none', masktype='none', maskvalue='0', blank=0.0, scale='none', zero='none', weight='none', statsec='', expname='', lthreshold=0.02, hthreshold=20.0, nlow=3, nhigh=3, nkeep=1, mclip=iraf.yes, lsigma=3.0, hsigma=3.0, rdnoise='0.', gain='1.', snoise='0.', sigscale=0.1, pclip=-0.5) fp = open('msk_list', 'w') for file in files: fp.write(file + '\n') fp.close() iraf.imcombine(input='@msk_list', output='combine_masked', headers='', bpmasks='', rejmasks='', nrejmasks='', expmasks='', sigmas='', logfile='STDOUT', combine='average', reject='minmax', project=iraf.no, outtype='real', outlimits='', offsets='none', masktype='none', maskvalue='0', blank=0.0, scale='none', zero='none', weight='none', statsec='', expname='', lthreshold=1e-06, hthreshold=100.0, nlow=5, nhigh=5, nkeep=1, mclip=iraf.yes, lsigma=3.0, hsigma=3.0, rdnoise='0.', gain='1.', snoise='0.', sigscale=0.1, pclip=-0.5) sky = pyfits.open('combine_COSMOS.fits')[0].data # #### Average # nsum = np.sum(X != 0, axis=0).reshape(nypix,nxpix) # avg = np.sum(X, axis=0).reshape(nypix,nxpix)/nsum # # ### Fill empty pixels with no input images # sky = avg x, y = np.where((np.isfinite(sky) == False) | (sky == 0)) NX = len(x) pad = 1 for i in range(NX): xi = x[i] yi = y[i] sub = sky[xi - pad:xi + pad + 2, yi - pad:yi + pad + 2] if (np.sum(sub) != 0.0): sky[xi, yi] = np.median(sub[np.isfinite(sub)]) still_bad = (np.isfinite(sky) == False) | (sky <= 0.01) sky[still_bad] = flat[0:nypix, 0:nxpix][still_bad] # bad_flat = (flat < 0.5) # sky[bad_flat] = flat[bad_flat] im_sky = pyfits.PrimaryHDU(data=sky) im_n = pyfits.ImageHDU(data=nsum) im = pyfits.HDUList([im_sky, im_n]) im.writeto('sky.fits', clobber=True) #### for DIRECT flat flatim = pyfits.open('/3DHST/Spectra/Work/CONF/sky_cosmos.fits') flatim[0].data = sky flatim[1].data = sky #flatim[3].data[5:-5,5:-5] = nsum flatim.writeto('%s_g141_flat.fits' % (field), clobber=True)