from astropy.table import Table, Column from fermipy.roi_model import ROIModel from haloanalysis.utils import collect_dirs usage = "usage: %(prog)s" description = "Aggregate analysis output." parser = argparse.ArgumentParser(usage=usage,description=description) parser.add_argument('--output', default = 'test.fits') parser.add_argument('dirs', nargs='+', default = None, help='Run analyses in all subdirectories of this ' 'directory.') args = parser.parse_args() dirs = sorted(collect_dirs(args.dirs)) cols = [Column([], name='name', dtype='S20', format='%s'), Column([], name='ra', dtype='f8', format='%.3f',unit='deg'), Column([], name='dec', dtype='f8', format='%.3f',unit='deg'), Column([], name='glon', dtype='f8', format='%.3f',unit='deg'), Column([], name='glat', dtype='f8', format='%.3f',unit='deg'), Column([], name='index', dtype='f8', format='%.3f',unit='deg'), Column([], name='prefactor', dtype='f8', format='%.4g',unit='deg'), Column([], name='eflux', dtype='f8', format='%.4g',unit='deg'), Column([], name='ts', dtype='f8', format='%.2f'), Column([], name='npred', dtype='f8', format='%.2f'), Column([], name='ext_ts', dtype='f8', format='%.2f'), Column([], name='ext_mle', dtype='f8', format='%.3f',unit='deg'), Column([], name='ext_err', dtype='f8', format='%.3f',unit='deg'), Column([], name='ext_ul95', dtype='f8', format='%.3f',unit='deg'),
def main(): usage = "usage: %(prog)s" description = "Aggregate analysis output." parser = argparse.ArgumentParser(usage=usage,description=description) parser.add_argument('--output', default = 'test.fits') parser.add_argument('--suffix', default = '') parser.add_argument('dirs', nargs='+', default = None, help='Run analyses in all subdirectories of this ' 'directory.') args = parser.parse_args() dirs = sorted(collect_dirs(args.dirs)) nfit = 6 nscan_pts = 9 nebins = 20 halo_scan_shape = (13,9) eflux_scan_pts = np.logspace(-9,-5,41) cols_dict_srcs = OrderedDict() cols_dict_srcs['name'] = dict(dtype='S20', format='%s',description='Source Name') cols_dict_srcs['assoc'] = dict(dtype='S20', format='%s') cols_dict_srcs['class'] = dict(dtype='S20', format='%s') cols_dict_srcs['ra'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict_srcs['dec'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict_srcs['glon'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict_srcs['glat'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict_srcs['ts'] = dict(dtype='f8', format='%.2f') cols_dict_srcs['npred'] = dict(dtype='f8', format='%.2f') cols_dict_srcs['offset'] = dict(dtype='f8', format='%.2f') cols_dict_srcs['offset_roi'] = dict(dtype='f8', format='%.2f') cols_dict_srcs['in3fgl'] = dict(dtype='i8', format='%i') cols_dict_lnl = OrderedDict() cols_dict_lnl['name'] = dict(dtype='S20', format='%s',description='Source Name') cols_dict_lnl['fit_ext_scan_dlnl'] = dict(dtype='f8', format='%.3f',shape=(27)) cols_dict_lnl['fit1_ext_scan_dlnl'] = dict(dtype='f8', format='%.3f',shape=(27)) cols_dict_lnl['fit_halo_scan_dlnl'] = dict(dtype='f8', format='%.3f', shape=halo_scan_shape + (len(eflux_scan_pts),)) cols_dict_lnl['fit1_halo_scan_dlnl'] = dict(dtype='f8', format='%.3f', shape=halo_scan_shape + (len(eflux_scan_pts),)) cols_dict_lnl['fit_halo_sed_scan_dlnl'] = dict(dtype='f8', format='%.3f', shape=(halo_scan_shape[0],nebins,len(eflux_scan_pts))) cols_dict_lnl['fit_src_sed_scan_dlnl'] = dict(dtype='f8', format='%.3f', shape=(nebins,nscan_pts)) cols_dict_lnl['fit_src_sed_scan_eflux'] = dict(dtype='f8', format='%.4g', shape=(nebins,nscan_pts)) cols_dict = OrderedDict() cols_dict['name'] = dict(dtype='S20', format='%s',description='Source Name') cols_dict['codename'] = dict(dtype='S20', format='%s') cols_dict['linkname'] = dict(dtype='S20', format='%s') cols_dict['assoc'] = dict(dtype='S20', format='%s') cols_dict['class'] = dict(dtype='S20', format='%s') cols_dict['ra'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict['dec'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict['glon'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict['glat'] = dict(dtype='f8', format='%.3f',unit='deg') cols_dict['ts'] = dict(dtype='f8', format='%.2f') cols_dict['npred'] = dict(dtype='f8', format='%.2f') cols_dict['fitn_ts'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fit_ts'] = dict(dtype='f8', format='%.2f') cols_dict['fitn_offset'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fit_offset'] = dict(dtype='f8', format='%.2f') cols_dict['dfde1000'] = dict(dtype='f8', format='%.4g') cols_dict['dfde1000_err'] = dict(dtype='f8', format='%.4g') cols_dict['dfde1000_index'] = dict(dtype='f8', format='%.2f') cols_dict['dfde1000_index_err'] = dict(dtype='f8', format='%.2f') cols_dict['flux1000'] = dict(dtype='f8', format='%.4g') cols_dict['flux1000_err'] = dict(dtype='f8', format='%.4g') cols_dict['eflux1000'] = dict(dtype='f8', format='%.4g') cols_dict['eflux1000_err'] = dict(dtype='f8', format='%.4g') cols_dict['dfde100'] = dict(dtype='f8', format='%.4g') cols_dict['dfde100_err'] = dict(dtype='f8', format='%.4g') cols_dict['flux100'] = dict(dtype='f8', format='%.4g') cols_dict['flux100_err'] = dict(dtype='f8', format='%.4g') cols_dict['eflux100'] = dict(dtype='f8', format='%.4g') cols_dict['eflux100_err'] = dict(dtype='f8', format='%.4g') cols_dict['spectrum_type'] = dict(dtype='S20', format='%s') cols_dict['fitn_halo_ts'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_halo_eflux_ul95'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_halo_width'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_halo_index'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fit_halo_ts'] = dict(dtype='f8', format='%.2f') cols_dict['fit_halo_eflux_ul95'] = dict(dtype='f8', format='%.2f') cols_dict['fit_halo_width'] = dict(dtype='f8', format='%.2f') cols_dict['fit_halo_index'] = dict(dtype='f8', format='%.2f') cols_dict['fitn_ext_ts'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_ext_mle'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_ext_err'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_ext_ul95'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fit_ext_ts'] = dict(dtype='f8', format='%.2f') cols_dict['fit_ext_mle'] = dict(dtype='f8', format='%.2f') cols_dict['fit_ext_err'] = dict(dtype='f8', format='%.2f') cols_dict['fit_ext_ul95'] = dict(dtype='f8', format='%.2f') cols_dict['fitn_dlike'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_dlike_ext'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_dlike_halo'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fit_dlike'] = dict(dtype='f8', format='%.2f') cols_dict['fit_dlike_ext'] = dict(dtype='f8', format='%.2f') cols_dict['fit_dlike_halo'] = dict(dtype='f8', format='%.2f') cols_dict['fitn_dlike1'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_dlike1_ext'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fitn_dlike1_halo'] = dict(dtype='f8', format='%.2f',shape=(nfit,)) cols_dict['fit_dlike1'] = dict(dtype='f8', format='%.2f') cols_dict['fit_dlike1_ext'] = dict(dtype='f8', format='%.2f') cols_dict['fit_dlike1_halo'] = dict(dtype='f8', format='%.2f') cols_dict['fit1_halo_scan_ts'] = dict(dtype='f8', format='%.2f',shape=halo_scan_shape) cols_dict['fit1_halo_scan_eflux_ul95'] = dict(dtype='f8', format='%.4g',shape=halo_scan_shape) cols_dict['fit_halo_scan_ts'] = dict(dtype='f8', format='%.2f',shape=halo_scan_shape) cols_dict['fit_halo_scan_eflux_ul95'] = dict(dtype='f8', format='%.4g',shape=halo_scan_shape) cols_dict['fit_nsrc'] = dict(dtype='i8') cols_dict['fit_mean_sep'] = dict(dtype='f8', format='%.3f') cols = [Column(name=k, **v) for k,v in cols_dict.items()] cols_lnl = [Column(name=k, **v) for k,v in cols_dict_lnl.items()] cols_srcs = [Column(name=k, **v) for k,v in cols_dict_srcs.items()] tab = Table(cols) tab_lnl = Table(cols_lnl) tab_srcs = Table(cols_srcs) row_dict = {} row_dict_lnl = {} row_dict_srcs = {} halo_name = 'halo_RadialGaussian' for d in dirs: logfile = os.path.join(d,'run-halo-analysis.log') if not os.path.isfile(logfile): continue if not check_log(logfile)=='Successful': continue print d fit_data = [] halo_data = [] halo_data_sed = [] halo_fit = [] new_src_data = [] if not os.path.isfile(os.path.join(d,'new_source_data.npy')): print 'skipping' continue new_src_data = np.load(os.path.join(d,'new_source_data.npy')) for i in range(nfit): file1 = os.path.join(d,'fit%i%s.npy'%(i,args.suffix)) file2 = os.path.join(d,'fit%i%s_%s_data.npy'%(i,args.suffix,halo_name)) file3 = os.path.join(d,'fit%i%s_%s.npy'%(i,args.suffix,halo_name)) file4 = os.path.join(d,'fit%i%s_%s_data_idx_free.npy'%(i,args.suffix,halo_name)) if os.path.isfile(file1): fit_data += [np.load(file1).flat[0]] if os.path.isfile(file2): halo_data += [np.load(file2)] if os.path.isfile(file4): halo_data_sed += [np.load(file4)] if os.path.isfile(file3): halo_fit += [np.load(file3).flat[0]] if len(fit_data) == 0: print 'skipping' continue src = find_source(fit_data[0]) src_name = src['name'] src_data = [] ext_data = [] new_srcs = [] src_ts = [np.nan]*nfit src_offset = [np.nan]*nfit fit_dlike = [np.nan]*nfit fit_dlike_ext = [np.nan]*nfit fit_dlike_halo = [np.nan]*nfit fit_dlike1 = [np.nan]*nfit fit_dlike1_ext = [np.nan]*nfit fit_dlike1_halo = [np.nan]*nfit extn_ts = [np.nan]*nfit extn_mle = [np.nan]*nfit extn_err = [np.nan]*nfit extn_ul95 = [np.nan]*nfit halon_ts = [np.nan]*nfit halon_width = [np.nan]*nfit halon_index = [np.nan]*nfit halon_eflux_ul95 = [np.nan]*nfit halo_pars = [] for hd in halo_data: halo_pars += [extract_halo_data(hd,halo_scan_shape)] halo_seds = [] for hd in halo_data_sed: halo_seds += [extract_halo_sed(hd,halo_scan_shape)] ext_width = [] for i, fd in enumerate(fit_data): src = fd['sources'][src_name] ext = src['extension'] src_data += [src] ext_data += [src['extension']] if src['extension'] is not None: extn_ts[i] = max(ext['ts_ext'],0) extn_mle[i] = ext['ext'] extn_err[i] = ext['ext_err'] extn_ul95[i] = ext['ext_ul95'] ext_width += [ext['width']] for i, fd in enumerate(halo_fit): halon_ts[i+1] = fd['sources'][halo_name]['ts'] halon_eflux_ul95[i+1] = fd['sources'][halo_name]['eflux_ul95'] halon_width[i+1] = fd['sources'][halo_name]['SpatialWidth'] halon_index[i+1] = np.abs(fd['sources'][halo_name]['params']['Index'][0]) for i in range(len(src_data)): src_ts[i] = src_data[i]['ts'] src_offset[i] = src_data[i]['offset'] fit_nsrc = len(fit_data)-1 for i in range(len(fit_data)): loglike = fit_data[i]['roi']['loglike'] loglike1 = fit_data[1]['roi']['loglike'] if i >= 1: fit_dlike[i] = loglike - fit_data[i-1]['roi']['loglike'] fit_dlike1[i] = loglike - loglike1 fit_dlike1_ext[i] = (loglike+extn_ts[i]/2.) - loglike1 fit_dlike1_halo[i] = (loglike+halon_ts[i]/2.) - loglike1 if i >= 1 and i < len(fit_data)-1: fit_dlike_ext[i+1] = (fit_data[i]['roi']['loglike']+extn_ts[i]/2.) - fit_data[i+1]['roi']['loglike'] fit_dlike_halo[i+1] = (fit_data[i]['roi']['loglike']+halon_ts[i]/2.) - fit_data[i+1]['roi']['loglike'] x = [src_data[-1]['offset_glon']] y = [src_data[-1]['offset_glat']] for j in range(len(new_src_data)): x += [new_src_data[j]['offset_glon']] y += [new_src_data[j]['offset_glat']] xc, yc = centroid(x,y) fit_mean_sep = mean_separation(x,y,xc,yc) if fit_nsrc == 1: fit_mean_sep = np.nan codename = os.path.basename(d) linkname = '{%s}'%codename.replace('+','p').replace('.','_') row_dict['fit1_halo_scan_ts'] = np.array(halo_pars[0]['ts']).reshape(halo_scan_shape) row_dict['fit1_halo_scan_eflux_ul95'] = np.array(halo_pars[0]['eflux']).reshape(halo_scan_shape) row_dict['fit_halo_scan_ts'] = np.array(halo_pars[-1]['ts']).reshape(halo_scan_shape) row_dict['fit_halo_scan_eflux_ul95'] = np.array(halo_pars[-1]['eflux']).reshape(halo_scan_shape) row_dict['name'] = src_data[0]['name'] row_dict['codename'] = codename row_dict['linkname'] = linkname row_dict['assoc'] = src_data[0]['assoc']['ASSOC1'] row_dict['class'] = src_data[0]['class'] row_dict['ra'] = src_data[1]['ra'] row_dict['dec'] = src_data[1]['dec'] row_dict['glon'] = src_data[1]['glon'] row_dict['glat'] = src_data[1]['glat'] row_dict['ts'] = src_data[1]['ts'] row_dict['npred'] = src_data[1]['npred'] row_dict['fitn_ts'] = src_ts row_dict['fit_ts'] = src_data[-1]['ts'] row_dict['fitn_offset'] = src_offset row_dict['fit_offset'] = src_data[-1]['offset'] for t in ['dfde','flux','eflux']: row_dict['%s1000'%t] = src_data[1]['%s1000'%t][0] row_dict['%s1000_err'%t] = src_data[1]['%s1000'%t][1] row_dict['%s100'%t] = src_data[1]['%s100'%t][0] row_dict['%s100_err'%t] = src_data[1]['%s100'%t][1] row_dict['dfde1000_index'] = src_data[1]['dfde1000_index'][0] row_dict['dfde1000_index_err'] = src_data[1]['dfde1000_index'][1] row_dict['spectrum_type'] = src_data[1]['SpectrumType'] row_dict['fitn_dlike'] = fit_dlike row_dict['fitn_dlike_ext'] = fit_dlike_ext row_dict['fitn_dlike_halo'] = fit_dlike_halo row_dict['fit_dlike'] = fit_dlike[fit_nsrc] row_dict['fit_dlike_ext'] = fit_dlike_ext[fit_nsrc] row_dict['fit_dlike_halo'] = fit_dlike_halo[fit_nsrc] row_dict['fitn_dlike1'] = fit_dlike1 row_dict['fitn_dlike1_ext'] = fit_dlike1_ext row_dict['fitn_dlike1_halo'] = fit_dlike1_halo row_dict['fit_dlike1'] = fit_dlike1[fit_nsrc] row_dict['fit_dlike1_ext'] = fit_dlike1_ext[fit_nsrc] row_dict['fit_dlike1_halo'] = fit_dlike1_halo[fit_nsrc] row_dict['fit_mean_sep'] = fit_mean_sep row_dict['fit_nsrc'] = fit_nsrc row_dict['fitn_halo_ts'] = halon_ts row_dict['fitn_halo_width'] = halon_width row_dict['fitn_halo_index'] = halon_index row_dict['fitn_halo_eflux_ul95'] = halon_eflux_ul95 row_dict['fit_halo_ts'] = halon_ts[fit_nsrc] row_dict['fit_halo_width'] = halon_width[fit_nsrc] row_dict['fit_halo_index'] = halon_index[fit_nsrc] row_dict['fit_halo_eflux_ul95'] = halon_eflux_ul95[fit_nsrc] row_dict['fitn_ext_ts'] = extn_ts row_dict['fitn_ext_mle'] = extn_mle row_dict['fitn_ext_err'] = extn_err row_dict['fitn_ext_ul95'] = extn_ul95 row_dict['fit_ext_ts'] = extn_ts[fit_nsrc] row_dict['fit_ext_mle'] = extn_mle[fit_nsrc] row_dict['fit_ext_err'] = extn_err[fit_nsrc] row_dict['fit_ext_ul95'] = extn_ul95[fit_nsrc] row = [] for k in cols_dict.keys(): row += [row_dict[k]] tab.add_row(row) from scipy.interpolate import UnivariateSpline fit1_dlnl_interp = np.zeros(halo_scan_shape + (len(eflux_scan_pts),)) fit_dlnl_interp = np.zeros(halo_scan_shape + (len(eflux_scan_pts),)) fit_sed_dlnl_interp = np.zeros((halo_scan_shape[0],nebins,len(eflux_scan_pts))) for i in range(halo_scan_shape[0]): for j in range(nebins): sp = UnivariateSpline(halo_seds[-1]['dlnl_eflux'][i,j], halo_seds[-1]['dlnl'][i,j],k=2,s=0.001) fit_sed_dlnl_interp[i,j] = sp(eflux_scan_pts) for j in range(halo_scan_shape[1]): sp = UnivariateSpline(halo_pars[0]['dlnl_eflux'][i,j],halo_pars[0]['dlnl'][i,j],k=2,s=0.001) fit1_dlnl_interp[i,j] = sp(eflux_scan_pts) sp = UnivariateSpline(halo_pars[-1]['dlnl_eflux'][i,j],halo_pars[-1]['dlnl'][i,j],k=2,s=0.001) fit_dlnl_interp[i,j] = sp(eflux_scan_pts) row_dict_lnl['fit_src_sed_scan_dlnl'] = np.zeros((nebins,nscan_pts)) row_dict_lnl['fit_src_sed_scan_eflux'] = np.zeros((nebins,nscan_pts)) for i in range(nebins): row_dict_lnl['fit_src_sed_scan_dlnl'][i,:] = src_data[-1]['sed']['lnlprofile'][i]['dloglike'] row_dict_lnl['fit_src_sed_scan_eflux'][i,:] = src_data[-1]['sed']['lnlprofile'][i]['eflux'] # row_dict_lnl['fit_src_sed_scan_dlnl'] row_dict_lnl['name'] = src_data[0]['name'] row_dict_lnl['fit1_ext_scan_dlnl'] = ext_data[1]['dloglike'] row_dict_lnl['fit_ext_scan_dlnl'] = ext_data[-1]['dloglike'] row_dict_lnl['fit1_halo_scan_dlnl'] = fit1_dlnl_interp row_dict_lnl['fit_halo_scan_dlnl'] = fit_dlnl_interp row_dict_lnl['fit_halo_sed_scan_dlnl'] = fit_sed_dlnl_interp row = [] for k in cols_dict_lnl.keys(): row += [row_dict_lnl[k]] tab_lnl.add_row(row) m = tab['class']=='' tab['class'][m] = 'unkn' output_lnl = os.path.splitext(args.output)[0] + '_lnl.fits' output_srcs = os.path.splitext(args.output)[0] + '_srcs.fits' tab.write(args.output,format='fits',overwrite=True) tab_lnl.write(output_lnl,format='fits',overwrite=True) #tab_srcs.write(output_srcs,format='fits',overwrite=True) col1 = pyfits.Column(name='halo_scan_width', format='D', array=halo_pars[0]['width'][:,0]) col2 = pyfits.Column(name='halo_scan_index', format='D', array=halo_pars[0]['index'][0,:]) col3 = pyfits.Column(name='halo_scan_eflux', format='D', array=eflux_scan_pts) col4 = pyfits.Column(name='ext_width', format='D', array=ext_width[0]) tbhdu1 = pyfits.BinTableHDU.from_columns([col1],name='halo_scan_width') tbhdu2 = pyfits.BinTableHDU.from_columns([col2],name='halo_scan_index') tbhdu3 = pyfits.BinTableHDU.from_columns([col3],name='halo_scan_eflux') tbhdu4 = pyfits.BinTableHDU.from_columns([col4],name='ext_width') hdulist = pyfits.open(args.output) hdulist.append(tbhdu1) hdulist.append(tbhdu2) hdulist.append(tbhdu3) hdulist.append(tbhdu4) hdulist.writeto(args.output,clobber=True) hdulist = pyfits.open(output_lnl) hdulist.append(tbhdu1) hdulist.append(tbhdu2) hdulist.writeto(output_lnl,clobber=True)