def makeInjObjs(injfile, posfiles):
"""
Make a list of results objects from the posfiles and injections
"""
getByTime = False
resObjs = []
if injfile:
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injfile])
if (len(injections) != len(posfiles)):
print 'Different numbers of injections and posterior files, attempting to recognise by time'
getByTime = True
i = 0
for posfile in posfiles:
peparser = bppu.PEOutputParser('common')
resObj = peparser.parse(open(posfile, 'r'))
pos = bppu.Posterior(resObj)
if not getByTime:
injection = injections[i]
else:
time = pos['time'].mean
injection = getInjByTime(time, injections)
if injection is None:
continue
pos.set_injection(injection)
resObjs.append(pos)
i = i + 1
return resObjs
def makeInjObjs(injfile,event,posfiles):
"""
Make a list of results objects from the posfiles and injections
"""
getByTime=False
resObjs=[]
if injfile:
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injfile])
i=0
for posfile in posfiles:
peparser=bppu.PEOutputParser('common')
resObj=peparser.parse(open(posfile,'r'))
pos=bppu.Posterior(resObj)
if event is not None:
injection=injections[event]
else:
time=pos['time'].mean
injection=getInjByTime(time,injections)
if injection is None:
continue
pos.set_injection(injection)
resObjs.append(pos)
i=i+1
return resObjs
def get_coincs_from_coire(self, files, stat='snr'):
"""
uses CoincInspiralUtils to get data from old-style (coire'd) coincs
"""
coincTrigs = CoincInspiralUtils.coincInspiralTable()
inspTrigs = SnglInspiralUtils.ReadSnglInspiralFromFiles(files, \
mangle_event_id = True,verbose=None)
statistic = CoincInspiralUtils.coincStatistic(stat, None, None)
coincTrigs = CoincInspiralUtils.coincInspiralTable(
inspTrigs, statistic)
try:
inspInj = SimInspiralUtils.ReadSimInspiralFromFiles(files)
coincTrigs.add_sim_inspirals(inspInj)
#FIXME: name the exception!
except:
pass
#now extract the relevant information into CoincData objects
for ctrig in coincTrigs:
coinc = CoincData()
coinc.set_ifos(ctrig.get_ifos()[1])
coinc.set_gps(
dict(
(trig.ifo, LIGOTimeGPS(trig.get_end())) for trig in ctrig))
coinc.set_snr(
dict((trig.ifo, getattr(ctrig, trig.ifo).snr)
for trig in ctrig))
coinc.set_effDs(
dict((trig.ifo, getattr(ctrig, trig.ifo).eff_distance)
for trig in ctrig))
coinc.set_masses(dict((trig.ifo,getattr(ctrig,trig.ifo).mass1) for trig in ctrig), \
dict((trig.ifo,getattr(ctrig,trig.ifo).mass2) for trig in ctrig))
try:
effDs_inj = {}
for ifo in coinc.ifo_list:
if ifo == 'H1':
effDs_inj[ifo] = getattr(ctrig, 'sim').eff_dist_h
elif ifo == 'L1':
effDs_inj[ifo] = getattr(ctrig, 'sim').eff_dist_l
elif ifo == 'V1':
effDs_inj[ifo] = getattr(ctrig, 'sim').eff_dist_v
dist_inj = getattr(ctrig, 'sim').distance
coinc.set_inj_params(getattr(ctrig,'sim').latitude,getattr(ctrig,'sim').longitude, \
getattr(ctrig,'sim').mass1,getattr(ctrig,'sim').mass2,dist_inj,effDs_inj)
coinc.is_injection = True
#FIXME: name the exception!
except:
pass
self.append(coinc)
roq_force_flow = cp.getfloat('lalinference','roq_force_flow')
print "WARNING: Forcing the f_low to ", str(roq_force_flow), "Hz"
print "WARNING: Overwriting user choice of flow, srate, seglen, and (mc_min, mc_max and q-min) or (mass1_min, mass1_max, mass2_min, mass2_max)"
if opts.gid is not None:
gid=opts.gid
elif opts.injections is not None or cp.has_option('input','injection-file'):
print "Only 0-th event in the XML table will be considered while running with ROQ\n"
# Read event 0 from Siminspiral Table
from pylal import SimInspiralUtils
if opts.injections is not None:
inxml=opts.injections
else:
inxml=cp.get('input','injection-file')
injTable=SimInspiralUtils.ReadSimInspiralFromFiles([inxml])
row=injTable[0]
else:
gid=None
row=None
roq_bounds = pipe_utils.Query_ROQ_Bounds_Type(path, roq_paths)
if roq_bounds == 'chirp_mass_q':
print 'ROQ has bounds in chirp mass and mass-ratio'
mc_priors, trigger_mchirp = pipe_utils.get_roq_mchirp_priors(path, roq_paths, roq_params, key, gid=gid, sim_inspiral=row)
elif roq_bounds == 'component_mass':
print 'ROQ has bounds in component masses'
# get component mass bounds, then compute the chirp mass that can be safely covered
# further below we pass along the component mass bounds to the sampler, not the tighter chirp-mass, q bounds
m1_priors, m2_priors, trigger_mchirp = pipe_utils.get_roq_component_mass_priors(path, roq_paths, roq_params, key, gid=gid, sim_inspiral=row)
mc_priors = {}
def compare_bayes(outdir,names_and_pos_folders,injection_path,eventnum,username,password,reload_flag,clf,ldg_flag,contour_figsize=(4.5,4.5),contour_dpi=250,contour_figposition=[0.15,0.15,0.5,0.75],fail_on_file_err=True,covarianceMatrices=None,meanVectors=None,Npixels2D=50):
injection=None
if injection_path is not None and os.path.exists(injection_path) and eventnum is not None:
eventnum=int(eventnum)
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injection_path])
if eventnum is not None:
if(len(injections)<eventnum):
print "Error: You asked for event %d, but %s contains only %d injections" %(eventnum,injection_path,len(injections))
sys.exit(1)
else:
injection=injections[eventnum]
#Create analytic likelihood functions if covariance matrices and mean vectors were given
analyticLikelihood = None
if covarianceMatrices and meanVectors:
analyticLikelihood = bppu.AnalyticLikelihood(covarianceMatrices, meanVectors)
peparser=bppu.PEOutputParser('common')
pos_list={}
tp_list={}
common_params=None
working_folder=os.getcwd()
for name,pos_folder in names_and_pos_folders:
import urlparse
pos_folder_url=urlparse.urlparse(pos_folder)
pfu_scheme,pfu_netloc,pfu_path,pfu_params,pfu_query,pfu_fragment=pos_folder_url
if 'http' in pfu_scheme:
"""
Retrieve a file over http(s).
"""
downloads_folder=os.path.join(os.getcwd(),"downloads")
pos_folder_parse=urlparse.urlparse(pos_folder)
pfp_scheme,pfp_netloc,pfp_path,pfp_params,pfp_query,pfp_fragment=pos_folder_parse
head,tail=os.path.split(pfp_path)
if tail is 'posplots.html' or tail:
pos_file_part=head
else:
pos_file_part=pfp_path
pos_file_url=urlparse.urlunsplit((pfp_scheme,pfp_netloc,os.path.join(pos_file_part,'posterior_samples.dat'),'',''))
print pos_file_url
pos_file=os.path.join(os.getcwd(),downloads_folder,"%s.dat"%name)
if not os.path.exists(pos_file):
reload_flag=True
if reload_flag:
if os.path.exists(pos_file):
os.remove(pos_file)
if not os.path.exists(downloads_folder):
os.makedirs(downloads_folder)
open_url_wget(pos_file_url,un=username,pw=password,args=["-O","%s"%pos_file])
elif pfu_scheme is '' or pfu_scheme is 'file':
pos_file=os.path.join(pos_folder,'%s.dat'%name)
# Try looking for posterior_samples.dat if name.dat doesn't exist
if not os.path.exists(pos_file):
print '%s does not exist, trying posterior_samples.dat'%(pos_file)
pos_file=os.path.join(pos_folder,'posterior_samples.dat')
else:
print "Unknown scheme for input data url: %s\nFull URL: %s"%(pfu_scheme,str(pos_folder_url))
exit(0)
print "Reading posterior samples from %s ..."%pos_file
try:
common_output_table_header,common_output_table_raw=peparser.parse(open(pos_file,'r'))
except:
print 'Unable to read file '+pos_file
continue
test_and_switch_param(common_output_table_header,'distance','dist')
test_and_switch_param(common_output_table_header,'chirpmass','mchirp')
test_and_switch_param(common_output_table_header,'mc','mchirp')
test_and_switch_param(common_output_table_header,'asym_massratio','q')
test_and_switch_param(common_output_table_header,'massratio', 'eta')
test_and_switch_param(common_output_table_header,'RA','ra')
test_and_switch_param(common_output_table_header,'rightascension','ra')
test_and_switch_param(common_output_table_header,'declination','dec')
test_and_switch_param(common_output_table_header,'tilt_spin1','tilt1')
test_and_switch_param(common_output_table_header,'tilt_spin2','tilt2')
if 'LI_MCMC' in name or 'FU_MCMC' in name:
try:
idx=common_output_table_header.index('iota')
print "Inverting iota!"
common_output_table_raw[:,idx]= np.pi*np.ones(len(common_output_table_raw[:,0])) - common_output_table_raw[:,idx]
except:
pass
# try:
# print "Converting phi_orb-> 2phi_orb"
# idx=common_output_table_header.index('phi_orb')
# common_output_table_header[idx]='2phi_orb'
# common_output_table_raw[:,idx]= 2*common_output_table_raw[:,idx]
# except:
# pass
try:
print "Converting iota-> cos(iota)"
idx=common_output_table_header.index('iota')
common_output_table_header[idx]='cos(iota)'
common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
except:
pass
#try:
# print "Converting tilt1 -> cos(tilt1)"
# idx=common_output_table_header.index('tilt1')
# common_output_table_header[idx]='cos(tilt1)'
# common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
#except:
# pass
#try:
# print "Converting tilt2 -> cos(tilt2)"
# idx=common_output_table_header.index('tilt2')
# common_output_table_header[idx]='cos(tilt2)'
# common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
#except:
# pass
try:
print "Converting thetas -> cos(thetas)"
idx=common_output_table_header.index('thetas')
common_output_table_header[idx]='cos(thetas)'
common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
except:
pass
try:
print "Converting beta -> cos(beta)"
idx=common_output_table_header.index('beta')
common_output_table_header[idx]='cos(beta)'
common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
except:
pass
try:
idx=common_output_table_header.index('f_ref')
injFrefs=np.unique(common_output_table_raw[:,idx])
if len(injFrefs) == 1:
injFref = injFrefs[0]
print "Using f_ref in results as injected value"
except:
injFref = None
pass
pos_temp=bppu.Posterior((common_output_table_header,common_output_table_raw),SimInspiralTableEntry=injection, injFref=injFref)
if 'a1' in pos_temp.names and min(pos_temp['a1'].samples)[0] < 0:
pos_temp.append_mapping('spin1', lambda a:a, 'a1')
pos_temp.pop('a1')
pos_temp.append_mapping('a1', lambda a:np.abs(a), 'spin1')
if 'a2' in pos_temp.names and min(pos_temp['a2'].samples)[0] < 0:
pos_temp.append_mapping('spin2', lambda a:a, 'a2')
pos_temp.pop('a2')
pos_temp.append_mapping('a2', lambda a:np.abs(a), 'spin2')
if 'm1' in pos_temp.names and 'm2' in pos_temp.names:
print "Calculating total mass"
pos_temp.append_mapping('mtotal', lambda m1,m2: m1+m2, ['m1','m2'])
if 'mass1' in pos_temp.names and 'mass2' in pos_temp.names:
print "Calculating total mass"
pos_temp.append_mapping('mtotal', lambda m1,m2: m1+m2, ['mass1','mass2'])
try:
idx=common_output_table_header.index('m1')
idx2=common_output_table_header.index('m2')
if pos_temp['m1'].mean<pos_temp['m2'].mean:
print "SWAPPING MASS PARAMS!"
common_output_table_header[idx]='x'
common_output_table_header[idx2]='m1'
common_output_table_header[idx]='m2'
pos_temp=bppu.Posterior((common_output_table_header,common_output_table_raw),SimInspiralTableEntry=injection)
except:
pass
pos_list[name]=pos_temp
if common_params is None:
common_params=pos_temp.names
else:
set_of_pars = set(pos_temp.names)
common_params=list(set_of_pars.intersection(common_params))
print "Common parameters are %s"%str(common_params)
if injection is None and injection_path is not None:
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injection_path])
injection=bppu.get_inj_by_time(injections,pos_temp.means['time'])
if injection is not None:
for pos in pos_list.values():
pos.set_injection(injection)
set_of_pars = set(allowed_params)
common_params=list(set_of_pars.intersection(common_params))
print "Using parameters %s"%str(common_params)
if not os.path.exists(os.path.join(os.getcwd(),'results')):
os.makedirs('results')
if not os.path.exists(outdir):
os.makedirs(outdir)
pdfdir=os.path.join(outdir,'pdfs')
if not os.path.exists(pdfdir):
os.makedirs(pdfdir)
greedy2savepaths=[]
if common_params is not [] and common_params is not None: #If there are common parameters....
colorlst=bppu.__default_color_lst
if len(common_params)>1: #If there is more than one parameter...
temp=copy.copy(common_params)
#Plot two param contour plots
#Assign some colours to each different analysis result
color_by_name={}
hatches_by_name={}
my_cm=mpl_cm.Dark2
cmap_size=my_cm.N
color_idx=0
color_idx_max=len(names_and_pos_folders)
cmap_array=my_cm(np.array(range(cmap_size)))
#cmap_array=['r','g','b','c','m','k','0.5','#ffff00']
hatches=['/','\\','|','-','+','x','o','O','.','*']
ldg='auto'
if not ldg_flag:
ldg=None
for name,infolder in names_and_pos_folders:
#color_by_name=cmap_array[color_idx]
color_by_name[name]=cmap_array[int(floor(color_idx*cmap_size/color_idx_max)),:]
color_idx+=1
hatches_by_name[name]=hatches[color_idx]
for i,j in all_pairs(temp):#Iterate over all unique pairs in the set of common parameters
pplst=[i,j]
rpplst=pplst[:]
rpplst.reverse()
pplst_cond=(pplst in twoDplots)
rpplst_cond=(rpplst in twoDplots)
if pplst_cond or rpplst_cond:#If this pair of parameters is in the plotting list...
try:
print '2d plots: building ',i,j
greedy2Params={i:greedyBinSizes[i],j:greedyBinSizes[j]}
except KeyError:
continue
name_list=[]
cs_list=[]
slinestyles=['solid', 'dashed', 'dashdot', 'dotted']
fig=bppu.plot_two_param_kde_greedy_levels(pos_list,greedy2Params,TwoDconfidenceLevels,color_by_name,figsize=contour_figsize,dpi=contour_dpi,figposition=contour_figposition,legend=ldg,line_styles=slinestyles,hatches_by_name=hatches_by_name,Npixels=Npixels2D)
if fig is None: continue
#fig=bppu.plot_two_param_greedy_bins_contour(pos_list,greedy2Params,TwoDconfidenceLevels,color_by_name,figsize=contour_figsize,dpi=contour_dpi,figposition=contour_figposition)
greedy2savepaths.append('%s-%s.png'%(pplst[0],pplst[1]))
fig.savefig(os.path.join(outdir,'%s-%s.png'%(pplst[0],pplst[1])),bbox_inches='tight')
fig.savefig(os.path.join(pdfdir,'%s-%s.pdf'%(pplst[0],pplst[1])),bbox_inches='tight')
plt.clf()
oned_data={}
#confidence_levels={}
confidence_levels=[{},{},{},{}]
confidence_uncertainty={}
for param in common_params:
print "Plotting comparison for '%s'"%param
cl_table_header='<table><th>Run</th>'
cl_table={}
save_paths=[]
cl_table_min_max_str='<tr><td> Min | Max </td>'
level_index=0
for confidence_level in OneDconfidenceLevels:
if analyticLikelihood:
pdf=analyticLikelihood.pdf(param)
cdf=analyticLikelihood.cdf(param)
else:
pdf=None
cdf=None
cl_table_header+='<th colspan="2">%i%% (Lower|Upper)</th>'%(int(100*confidence_level))
hist_fig,cl_intervals=compare_plots_one_param_line_hist(pos_list,param,confidence_level,color_by_name,cl_lines_flag=clf,legend=ldg,analyticPDF=pdf)
hist_fig2,cl_intervals=compare_plots_one_param_line_hist_cum(pos_list,param,confidence_level,color_by_name,cl_lines_flag=clf,analyticCDF=cdf,legend=ldg)
# Save confidence levels and uncertainty
#confidence_levels[param]=[]
confidence_levels[level_index][param]=[]
for name,pos in pos_list.items():
median=pos[param].median
low,high=cl_intervals[name]
#confidence_levels[param].append((name,low,median,high))
confidence_levels[level_index][param].append((name,low,median,high))
level_index=level_index+1
cl_bounds=[]
poses=[]
for name,pos in pos_list.items():
cl_bounds.append(cl_intervals[name])
poses.append(pos[param])
confidence_uncertainty[param]=bppu.confidence_interval_uncertainty(confidence_level, cl_bounds, poses)
save_path=''
if hist_fig is not None:
save_path=os.path.join(outdir,'%s_%i.png'%(param,int(100*confidence_level)))
save_path_pdf=os.path.join(pdfdir,'%s_%i.pdf'%(param,int(100*confidence_level)))
try:
plt.tight_layout(hist_fig)
plt.tight_layout(hist_fig2)
except:
pass
hist_fig.savefig(save_path,bbox_inches='tight')
hist_fig.savefig(save_path_pdf,bbox_inches='tight')
save_paths.append(save_path)
save_path=os.path.join(outdir,'%s_%i_cum.png'%(param,int(100*confidence_level)))
save_path_pdf=os.path.join(pdfdir,'%s_%i_cum.pdf'%(param,int(100*confidence_level)))
hist_fig2.savefig(save_path,bbox_inches='tight')
hist_fig2.savefig(save_path_pdf,bbox_inches='tight')
save_paths.append(save_path)
min_low,max_high=cl_intervals.values()[0]
for name,interval in cl_intervals.items():
low,high=interval
if low<min_low:
min_low=low
if high>max_high:
max_high=high
try:
cl_table[name]+='<td>%s</td><td>%s</td>'%(low,high)
except:
cl_table[name]='<td>%s</td><td>%s</td>'%(low,high)
cl_table_min_max_str+='<td>%s</td><td>%s</td>'%(min_low,max_high)
cl_table_str=cl_table_header
for name,row_contents in cl_table.items():
cl_table_str+='<tr><td>%s<font color="%s"></font></td>'%(name,str(mpl_colors.rgb2hex(color_by_name[name][0:3])))#,'·'.encode('utf-8'))
cl_table_str+=row_contents+'</tr>'
cl_table_str+=cl_table_min_max_str+'</tr>'
cl_table_str+='</table>'
cl_uncer_str='<table> <th>Confidence Relative Uncertainty</th> <th>Confidence Fractional Uncertainty</th> <th>Confidence Percentile Uncertainty</th>\n'
cl_uncer_str+='<tr> <td> %g </td> <td> %g </td> <td> %g </td> </tr> </table>'%(confidence_uncertainty[param][0], confidence_uncertainty[param][1], confidence_uncertainty[param][2])
ks_matrix=compute_ks_pvalue_matrix(pos_list, param)
N=ks_matrix.shape[0]+1
# Make up KS-test table
ks_table_str='<table><th colspan="%d"> K-S test p-value matrix </th>'%N
# Column headers
ks_table_str+='<tr> <td> -- </td> '
for name,pos in pos_list.items():
ks_table_str+='<td> %s </td>'%name
ks_table_str+='</tr>'
# Now plot rows of matrix
for i in range(len(pos_list)):
ks_table_str+='<tr> <td> %s </td>'%(pos_list.keys()[i])
for j in range(len(pos_list)):
if i == j:
ks_table_str+='<td> -- </td>'
elif ks_matrix[i,j] < 0.05:
# Failing at suspiciously low p-value
ks_table_str+='<td> <b> %g </b> </td>'%ks_matrix[i,j]
else:
ks_table_str+='<td> %g </td>'%ks_matrix[i,j]
ks_table_str+='</tr>'
ks_table_str+='</table>'
oned_data[param]=(save_paths,cl_table_str,ks_table_str,cl_uncer_str)
# Watch out---using private variable _logL
max_logls = [[name,max(pos._logL)] for name,pos in pos_list.items()]
return greedy2savepaths,oned_data,confidence_uncertainty,confidence_levels,max_logls
parser.add_option("-c","--code",dest="c")
(options, args) = parser.parse_args()
if (options.i == None or options.o == None or options.j == None or options.e == None):
parser.error("not enough arguments")
input=options.i
output=options.o
injFile=options.j
event=options.e
code=options.c
file = open(input,'r')
from pylal import SimInspiralUtils
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injFile])
if(len(injections)<event):
print "Error: You asked for event %d, but %s contains only %d injections" %(event,injfile,len(injections))
sys.exit(1)
else:
injection = injections[event]
def grabParam(posterior,injVal,param_name):
counter = 0
total = 0
countSamples = 0
column = None
for line in posterior:
cp.set('data',ifo.lower()+'-channel',channel)
if cp.has_option('analysis','time-slide-dump'):
timeslidefiledump= cp.get('analysis','time-slide-dump')
else:
timeslidefiledump=None
time_event=None
if opts.inj and cp.has_option('analysis','events'):
time_event={}
events=[]
times=[]
raw_events=cp.get('analysis','events').replace('[','').replace(']','').split(',')
from pylal import SimInspiralUtils
injTable=SimInspiralUtils.ReadSimInspiralFromFiles([opts.inj])
if 'all' is raw_events or 'all' in raw_events:
events=range(len(injTable))
else:
for raw_event in raw_events:
if ':' in raw_event:
limits=raw_event.split(':')
if len(limits) != 2:
print "Error: in event config option; ':' must separate two numbers."
exit(0)
low=int(limits[0])
high=int(limits[1])
if low>high:
events.extend(range(int(high),int(low)))