def makeInjObjs(injfile, posfiles): """ Make a list of results objects from the posfiles and injections """ getByTime = False resObjs = [] if injfile: import itertools injections = SimInspiralUtils.ReadSimInspiralFromFiles([injfile]) if (len(injections) != len(posfiles)): print 'Different numbers of injections and posterior files, attempting to recognise by time' getByTime = True i = 0 for posfile in posfiles: peparser = bppu.PEOutputParser('common') resObj = peparser.parse(open(posfile, 'r')) pos = bppu.Posterior(resObj) if not getByTime: injection = injections[i] else: time = pos['time'].mean injection = getInjByTime(time, injections) if injection is None: continue pos.set_injection(injection) resObjs.append(pos) i = i + 1 return resObjs
def makeInjObjs(injfile,event,posfiles): """ Make a list of results objects from the posfiles and injections """ getByTime=False resObjs=[] if injfile: import itertools injections = SimInspiralUtils.ReadSimInspiralFromFiles([injfile]) i=0 for posfile in posfiles: peparser=bppu.PEOutputParser('common') resObj=peparser.parse(open(posfile,'r')) pos=bppu.Posterior(resObj) if event is not None: injection=injections[event] else: time=pos['time'].mean injection=getInjByTime(time,injections) if injection is None: continue pos.set_injection(injection) resObjs.append(pos) i=i+1 return resObjs
def get_coincs_from_coire(self, files, stat='snr'): """ uses CoincInspiralUtils to get data from old-style (coire'd) coincs """ coincTrigs = CoincInspiralUtils.coincInspiralTable() inspTrigs = SnglInspiralUtils.ReadSnglInspiralFromFiles(files, \ mangle_event_id = True,verbose=None) statistic = CoincInspiralUtils.coincStatistic(stat, None, None) coincTrigs = CoincInspiralUtils.coincInspiralTable( inspTrigs, statistic) try: inspInj = SimInspiralUtils.ReadSimInspiralFromFiles(files) coincTrigs.add_sim_inspirals(inspInj) #FIXME: name the exception! except: pass #now extract the relevant information into CoincData objects for ctrig in coincTrigs: coinc = CoincData() coinc.set_ifos(ctrig.get_ifos()[1]) coinc.set_gps( dict( (trig.ifo, LIGOTimeGPS(trig.get_end())) for trig in ctrig)) coinc.set_snr( dict((trig.ifo, getattr(ctrig, trig.ifo).snr) for trig in ctrig)) coinc.set_effDs( dict((trig.ifo, getattr(ctrig, trig.ifo).eff_distance) for trig in ctrig)) coinc.set_masses(dict((trig.ifo,getattr(ctrig,trig.ifo).mass1) for trig in ctrig), \ dict((trig.ifo,getattr(ctrig,trig.ifo).mass2) for trig in ctrig)) try: effDs_inj = {} for ifo in coinc.ifo_list: if ifo == 'H1': effDs_inj[ifo] = getattr(ctrig, 'sim').eff_dist_h elif ifo == 'L1': effDs_inj[ifo] = getattr(ctrig, 'sim').eff_dist_l elif ifo == 'V1': effDs_inj[ifo] = getattr(ctrig, 'sim').eff_dist_v dist_inj = getattr(ctrig, 'sim').distance coinc.set_inj_params(getattr(ctrig,'sim').latitude,getattr(ctrig,'sim').longitude, \ getattr(ctrig,'sim').mass1,getattr(ctrig,'sim').mass2,dist_inj,effDs_inj) coinc.is_injection = True #FIXME: name the exception! except: pass self.append(coinc)
roq_force_flow = cp.getfloat('lalinference','roq_force_flow') print "WARNING: Forcing the f_low to ", str(roq_force_flow), "Hz" print "WARNING: Overwriting user choice of flow, srate, seglen, and (mc_min, mc_max and q-min) or (mass1_min, mass1_max, mass2_min, mass2_max)" if opts.gid is not None: gid=opts.gid elif opts.injections is not None or cp.has_option('input','injection-file'): print "Only 0-th event in the XML table will be considered while running with ROQ\n" # Read event 0 from Siminspiral Table from pylal import SimInspiralUtils if opts.injections is not None: inxml=opts.injections else: inxml=cp.get('input','injection-file') injTable=SimInspiralUtils.ReadSimInspiralFromFiles([inxml]) row=injTable[0] else: gid=None row=None roq_bounds = pipe_utils.Query_ROQ_Bounds_Type(path, roq_paths) if roq_bounds == 'chirp_mass_q': print 'ROQ has bounds in chirp mass and mass-ratio' mc_priors, trigger_mchirp = pipe_utils.get_roq_mchirp_priors(path, roq_paths, roq_params, key, gid=gid, sim_inspiral=row) elif roq_bounds == 'component_mass': print 'ROQ has bounds in component masses' # get component mass bounds, then compute the chirp mass that can be safely covered # further below we pass along the component mass bounds to the sampler, not the tighter chirp-mass, q bounds m1_priors, m2_priors, trigger_mchirp = pipe_utils.get_roq_component_mass_priors(path, roq_paths, roq_params, key, gid=gid, sim_inspiral=row) mc_priors = {}
def compare_bayes(outdir,names_and_pos_folders,injection_path,eventnum,username,password,reload_flag,clf,ldg_flag,contour_figsize=(4.5,4.5),contour_dpi=250,contour_figposition=[0.15,0.15,0.5,0.75],fail_on_file_err=True,covarianceMatrices=None,meanVectors=None,Npixels2D=50): injection=None if injection_path is not None and os.path.exists(injection_path) and eventnum is not None: eventnum=int(eventnum) import itertools injections = SimInspiralUtils.ReadSimInspiralFromFiles([injection_path]) if eventnum is not None: if(len(injections)<eventnum): print "Error: You asked for event %d, but %s contains only %d injections" %(eventnum,injection_path,len(injections)) sys.exit(1) else: injection=injections[eventnum] #Create analytic likelihood functions if covariance matrices and mean vectors were given analyticLikelihood = None if covarianceMatrices and meanVectors: analyticLikelihood = bppu.AnalyticLikelihood(covarianceMatrices, meanVectors) peparser=bppu.PEOutputParser('common') pos_list={} tp_list={} common_params=None working_folder=os.getcwd() for name,pos_folder in names_and_pos_folders: import urlparse pos_folder_url=urlparse.urlparse(pos_folder) pfu_scheme,pfu_netloc,pfu_path,pfu_params,pfu_query,pfu_fragment=pos_folder_url if 'http' in pfu_scheme: """ Retrieve a file over http(s). """ downloads_folder=os.path.join(os.getcwd(),"downloads") pos_folder_parse=urlparse.urlparse(pos_folder) pfp_scheme,pfp_netloc,pfp_path,pfp_params,pfp_query,pfp_fragment=pos_folder_parse head,tail=os.path.split(pfp_path) if tail is 'posplots.html' or tail: pos_file_part=head else: pos_file_part=pfp_path pos_file_url=urlparse.urlunsplit((pfp_scheme,pfp_netloc,os.path.join(pos_file_part,'posterior_samples.dat'),'','')) print pos_file_url pos_file=os.path.join(os.getcwd(),downloads_folder,"%s.dat"%name) if not os.path.exists(pos_file): reload_flag=True if reload_flag: if os.path.exists(pos_file): os.remove(pos_file) if not os.path.exists(downloads_folder): os.makedirs(downloads_folder) open_url_wget(pos_file_url,un=username,pw=password,args=["-O","%s"%pos_file]) elif pfu_scheme is '' or pfu_scheme is 'file': pos_file=os.path.join(pos_folder,'%s.dat'%name) # Try looking for posterior_samples.dat if name.dat doesn't exist if not os.path.exists(pos_file): print '%s does not exist, trying posterior_samples.dat'%(pos_file) pos_file=os.path.join(pos_folder,'posterior_samples.dat') else: print "Unknown scheme for input data url: %s\nFull URL: %s"%(pfu_scheme,str(pos_folder_url)) exit(0) print "Reading posterior samples from %s ..."%pos_file try: common_output_table_header,common_output_table_raw=peparser.parse(open(pos_file,'r')) except: print 'Unable to read file '+pos_file continue test_and_switch_param(common_output_table_header,'distance','dist') test_and_switch_param(common_output_table_header,'chirpmass','mchirp') test_and_switch_param(common_output_table_header,'mc','mchirp') test_and_switch_param(common_output_table_header,'asym_massratio','q') test_and_switch_param(common_output_table_header,'massratio', 'eta') test_and_switch_param(common_output_table_header,'RA','ra') test_and_switch_param(common_output_table_header,'rightascension','ra') test_and_switch_param(common_output_table_header,'declination','dec') test_and_switch_param(common_output_table_header,'tilt_spin1','tilt1') test_and_switch_param(common_output_table_header,'tilt_spin2','tilt2') if 'LI_MCMC' in name or 'FU_MCMC' in name: try: idx=common_output_table_header.index('iota') print "Inverting iota!" common_output_table_raw[:,idx]= np.pi*np.ones(len(common_output_table_raw[:,0])) - common_output_table_raw[:,idx] except: pass # try: # print "Converting phi_orb-> 2phi_orb" # idx=common_output_table_header.index('phi_orb') # common_output_table_header[idx]='2phi_orb' # common_output_table_raw[:,idx]= 2*common_output_table_raw[:,idx] # except: # pass try: print "Converting iota-> cos(iota)" idx=common_output_table_header.index('iota') common_output_table_header[idx]='cos(iota)' common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx]) except: pass #try: # print "Converting tilt1 -> cos(tilt1)" # idx=common_output_table_header.index('tilt1') # common_output_table_header[idx]='cos(tilt1)' # common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx]) #except: # pass #try: # print "Converting tilt2 -> cos(tilt2)" # idx=common_output_table_header.index('tilt2') # common_output_table_header[idx]='cos(tilt2)' # common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx]) #except: # pass try: print "Converting thetas -> cos(thetas)" idx=common_output_table_header.index('thetas') common_output_table_header[idx]='cos(thetas)' common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx]) except: pass try: print "Converting beta -> cos(beta)" idx=common_output_table_header.index('beta') common_output_table_header[idx]='cos(beta)' common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx]) except: pass try: idx=common_output_table_header.index('f_ref') injFrefs=np.unique(common_output_table_raw[:,idx]) if len(injFrefs) == 1: injFref = injFrefs[0] print "Using f_ref in results as injected value" except: injFref = None pass pos_temp=bppu.Posterior((common_output_table_header,common_output_table_raw),SimInspiralTableEntry=injection, injFref=injFref) if 'a1' in pos_temp.names and min(pos_temp['a1'].samples)[0] < 0: pos_temp.append_mapping('spin1', lambda a:a, 'a1') pos_temp.pop('a1') pos_temp.append_mapping('a1', lambda a:np.abs(a), 'spin1') if 'a2' in pos_temp.names and min(pos_temp['a2'].samples)[0] < 0: pos_temp.append_mapping('spin2', lambda a:a, 'a2') pos_temp.pop('a2') pos_temp.append_mapping('a2', lambda a:np.abs(a), 'spin2') if 'm1' in pos_temp.names and 'm2' in pos_temp.names: print "Calculating total mass" pos_temp.append_mapping('mtotal', lambda m1,m2: m1+m2, ['m1','m2']) if 'mass1' in pos_temp.names and 'mass2' in pos_temp.names: print "Calculating total mass" pos_temp.append_mapping('mtotal', lambda m1,m2: m1+m2, ['mass1','mass2']) try: idx=common_output_table_header.index('m1') idx2=common_output_table_header.index('m2') if pos_temp['m1'].mean<pos_temp['m2'].mean: print "SWAPPING MASS PARAMS!" common_output_table_header[idx]='x' common_output_table_header[idx2]='m1' common_output_table_header[idx]='m2' pos_temp=bppu.Posterior((common_output_table_header,common_output_table_raw),SimInspiralTableEntry=injection) except: pass pos_list[name]=pos_temp if common_params is None: common_params=pos_temp.names else: set_of_pars = set(pos_temp.names) common_params=list(set_of_pars.intersection(common_params)) print "Common parameters are %s"%str(common_params) if injection is None and injection_path is not None: import itertools injections = SimInspiralUtils.ReadSimInspiralFromFiles([injection_path]) injection=bppu.get_inj_by_time(injections,pos_temp.means['time']) if injection is not None: for pos in pos_list.values(): pos.set_injection(injection) set_of_pars = set(allowed_params) common_params=list(set_of_pars.intersection(common_params)) print "Using parameters %s"%str(common_params) if not os.path.exists(os.path.join(os.getcwd(),'results')): os.makedirs('results') if not os.path.exists(outdir): os.makedirs(outdir) pdfdir=os.path.join(outdir,'pdfs') if not os.path.exists(pdfdir): os.makedirs(pdfdir) greedy2savepaths=[] if common_params is not [] and common_params is not None: #If there are common parameters.... colorlst=bppu.__default_color_lst if len(common_params)>1: #If there is more than one parameter... temp=copy.copy(common_params) #Plot two param contour plots #Assign some colours to each different analysis result color_by_name={} hatches_by_name={} my_cm=mpl_cm.Dark2 cmap_size=my_cm.N color_idx=0 color_idx_max=len(names_and_pos_folders) cmap_array=my_cm(np.array(range(cmap_size))) #cmap_array=['r','g','b','c','m','k','0.5','#ffff00'] hatches=['/','\\','|','-','+','x','o','O','.','*'] ldg='auto' if not ldg_flag: ldg=None for name,infolder in names_and_pos_folders: #color_by_name=cmap_array[color_idx] color_by_name[name]=cmap_array[int(floor(color_idx*cmap_size/color_idx_max)),:] color_idx+=1 hatches_by_name[name]=hatches[color_idx] for i,j in all_pairs(temp):#Iterate over all unique pairs in the set of common parameters pplst=[i,j] rpplst=pplst[:] rpplst.reverse() pplst_cond=(pplst in twoDplots) rpplst_cond=(rpplst in twoDplots) if pplst_cond or rpplst_cond:#If this pair of parameters is in the plotting list... try: print '2d plots: building ',i,j greedy2Params={i:greedyBinSizes[i],j:greedyBinSizes[j]} except KeyError: continue name_list=[] cs_list=[] slinestyles=['solid', 'dashed', 'dashdot', 'dotted'] fig=bppu.plot_two_param_kde_greedy_levels(pos_list,greedy2Params,TwoDconfidenceLevels,color_by_name,figsize=contour_figsize,dpi=contour_dpi,figposition=contour_figposition,legend=ldg,line_styles=slinestyles,hatches_by_name=hatches_by_name,Npixels=Npixels2D) if fig is None: continue #fig=bppu.plot_two_param_greedy_bins_contour(pos_list,greedy2Params,TwoDconfidenceLevels,color_by_name,figsize=contour_figsize,dpi=contour_dpi,figposition=contour_figposition) greedy2savepaths.append('%s-%s.png'%(pplst[0],pplst[1])) fig.savefig(os.path.join(outdir,'%s-%s.png'%(pplst[0],pplst[1])),bbox_inches='tight') fig.savefig(os.path.join(pdfdir,'%s-%s.pdf'%(pplst[0],pplst[1])),bbox_inches='tight') plt.clf() oned_data={} #confidence_levels={} confidence_levels=[{},{},{},{}] confidence_uncertainty={} for param in common_params: print "Plotting comparison for '%s'"%param cl_table_header='<table><th>Run</th>' cl_table={} save_paths=[] cl_table_min_max_str='<tr><td> Min | Max </td>' level_index=0 for confidence_level in OneDconfidenceLevels: if analyticLikelihood: pdf=analyticLikelihood.pdf(param) cdf=analyticLikelihood.cdf(param) else: pdf=None cdf=None cl_table_header+='<th colspan="2">%i%% (Lower|Upper)</th>'%(int(100*confidence_level)) hist_fig,cl_intervals=compare_plots_one_param_line_hist(pos_list,param,confidence_level,color_by_name,cl_lines_flag=clf,legend=ldg,analyticPDF=pdf) hist_fig2,cl_intervals=compare_plots_one_param_line_hist_cum(pos_list,param,confidence_level,color_by_name,cl_lines_flag=clf,analyticCDF=cdf,legend=ldg) # Save confidence levels and uncertainty #confidence_levels[param]=[] confidence_levels[level_index][param]=[] for name,pos in pos_list.items(): median=pos[param].median low,high=cl_intervals[name] #confidence_levels[param].append((name,low,median,high)) confidence_levels[level_index][param].append((name,low,median,high)) level_index=level_index+1 cl_bounds=[] poses=[] for name,pos in pos_list.items(): cl_bounds.append(cl_intervals[name]) poses.append(pos[param]) confidence_uncertainty[param]=bppu.confidence_interval_uncertainty(confidence_level, cl_bounds, poses) save_path='' if hist_fig is not None: save_path=os.path.join(outdir,'%s_%i.png'%(param,int(100*confidence_level))) save_path_pdf=os.path.join(pdfdir,'%s_%i.pdf'%(param,int(100*confidence_level))) try: plt.tight_layout(hist_fig) plt.tight_layout(hist_fig2) except: pass hist_fig.savefig(save_path,bbox_inches='tight') hist_fig.savefig(save_path_pdf,bbox_inches='tight') save_paths.append(save_path) save_path=os.path.join(outdir,'%s_%i_cum.png'%(param,int(100*confidence_level))) save_path_pdf=os.path.join(pdfdir,'%s_%i_cum.pdf'%(param,int(100*confidence_level))) hist_fig2.savefig(save_path,bbox_inches='tight') hist_fig2.savefig(save_path_pdf,bbox_inches='tight') save_paths.append(save_path) min_low,max_high=cl_intervals.values()[0] for name,interval in cl_intervals.items(): low,high=interval if low<min_low: min_low=low if high>max_high: max_high=high try: cl_table[name]+='<td>%s</td><td>%s</td>'%(low,high) except: cl_table[name]='<td>%s</td><td>%s</td>'%(low,high) cl_table_min_max_str+='<td>%s</td><td>%s</td>'%(min_low,max_high) cl_table_str=cl_table_header for name,row_contents in cl_table.items(): cl_table_str+='<tr><td>%s<font color="%s"></font></td>'%(name,str(mpl_colors.rgb2hex(color_by_name[name][0:3])))#,'·'.encode('utf-8')) cl_table_str+=row_contents+'</tr>' cl_table_str+=cl_table_min_max_str+'</tr>' cl_table_str+='</table>' cl_uncer_str='<table> <th>Confidence Relative Uncertainty</th> <th>Confidence Fractional Uncertainty</th> <th>Confidence Percentile Uncertainty</th>\n' cl_uncer_str+='<tr> <td> %g </td> <td> %g </td> <td> %g </td> </tr> </table>'%(confidence_uncertainty[param][0], confidence_uncertainty[param][1], confidence_uncertainty[param][2]) ks_matrix=compute_ks_pvalue_matrix(pos_list, param) N=ks_matrix.shape[0]+1 # Make up KS-test table ks_table_str='<table><th colspan="%d"> K-S test p-value matrix </th>'%N # Column headers ks_table_str+='<tr> <td> -- </td> ' for name,pos in pos_list.items(): ks_table_str+='<td> %s </td>'%name ks_table_str+='</tr>' # Now plot rows of matrix for i in range(len(pos_list)): ks_table_str+='<tr> <td> %s </td>'%(pos_list.keys()[i]) for j in range(len(pos_list)): if i == j: ks_table_str+='<td> -- </td>' elif ks_matrix[i,j] < 0.05: # Failing at suspiciously low p-value ks_table_str+='<td> <b> %g </b> </td>'%ks_matrix[i,j] else: ks_table_str+='<td> %g </td>'%ks_matrix[i,j] ks_table_str+='</tr>' ks_table_str+='</table>' oned_data[param]=(save_paths,cl_table_str,ks_table_str,cl_uncer_str) # Watch out---using private variable _logL max_logls = [[name,max(pos._logL)] for name,pos in pos_list.items()] return greedy2savepaths,oned_data,confidence_uncertainty,confidence_levels,max_logls
parser.add_option("-c","--code",dest="c") (options, args) = parser.parse_args() if (options.i == None or options.o == None or options.j == None or options.e == None): parser.error("not enough arguments") input=options.i output=options.o injFile=options.j event=options.e code=options.c file = open(input,'r') from pylal import SimInspiralUtils injections = SimInspiralUtils.ReadSimInspiralFromFiles([injFile]) if(len(injections)<event): print "Error: You asked for event %d, but %s contains only %d injections" %(event,injfile,len(injections)) sys.exit(1) else: injection = injections[event] def grabParam(posterior,injVal,param_name): counter = 0 total = 0 countSamples = 0 column = None for line in posterior:
cp.set('data',ifo.lower()+'-channel',channel) if cp.has_option('analysis','time-slide-dump'): timeslidefiledump= cp.get('analysis','time-slide-dump') else: timeslidefiledump=None time_event=None if opts.inj and cp.has_option('analysis','events'): time_event={} events=[] times=[] raw_events=cp.get('analysis','events').replace('[','').replace(']','').split(',') from pylal import SimInspiralUtils injTable=SimInspiralUtils.ReadSimInspiralFromFiles([opts.inj]) if 'all' is raw_events or 'all' in raw_events: events=range(len(injTable)) else: for raw_event in raw_events: if ':' in raw_event: limits=raw_event.split(':') if len(limits) != 2: print "Error: in event config option; ':' must separate two numbers." exit(0) low=int(limits[0]) high=int(limits[1]) if low>high: events.extend(range(int(high),int(low)))