def makeInjObjs(injfile,event,posfiles):
"""
Make a list of results objects from the posfiles and injections
"""
getByTime=False
resObjs=[]
if injfile:
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injfile])
i=0
for posfile in posfiles:
peparser=bppu.PEOutputParser('common')
resObj=peparser.parse(open(posfile,'r'))
pos=bppu.Posterior(resObj)
if event is not None:
injection=injections[event]
else:
time=pos['time'].mean
injection=getInjByTime(time,injections)
if injection is None:
continue
pos.set_injection(injection)
resObjs.append(pos)
i=i+1
return resObjs
def makeInjObjs(injfile, posfiles):
"""
Make a list of results objects from the posfiles and injections
"""
getByTime = False
resObjs = []
if injfile:
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injfile])
if (len(injections) != len(posfiles)):
print 'Different numbers of injections and posterior files, attempting to recognise by time'
getByTime = True
i = 0
for posfile in posfiles:
peparser = bppu.PEOutputParser('common')
resObj = peparser.parse(open(posfile, 'r'))
pos = bppu.Posterior(resObj)
if not getByTime:
injection = injections[i]
else:
time = pos['time'].mean
injection = getInjByTime(time, injections)
if injection is None:
continue
pos.set_injection(injection)
resObjs.append(pos)
i = i + 1
return resObjs
def read_posterior_samples(f, injrow):
"""Returns a bppu posterior sample object
"""
peparser = bppu.PEOutputParser('common')
commonResultsObj = peparser.parse(open(f, 'r'))
data = bppu.Posterior(commonResultsObj,
SimInspiralTableEntry=injrow,
injFref=100.0)
# add tilts, comp masses, tidal...
data.extend_posterior()
return data
def gelman_rubin(pos_samples, param_arr, outdir):
"""
Compute Gelman-Rubin R-statistic for each parameter in param_arr.
"""
writefile = os.path.join(outdir, 'merged_files.dat')
runs = len(pos_samples)
R_arr = []
merge_files(pos_samples, param_arr, writefile)
for param in param_arr:
data = bppu.PEOutputParser('common')
inputFileObj = open(writefile)
dataObj0 = data.parse(inputFileObj)
posterior = bppu.Posterior(dataObj0)
R = posterior.gelman_rubin(param)
R_arr.append(R)
return R_arr
def get_modes(posfiles, plot=False):
"""
Return a tuple with the means and covariances of Gaussian mixture models fit
to the posterior modes
"""
fmeans = np.empty(shape=(len(posfiles), 3))
fmeans.fill(None)
fcovars = np.empty(shape=(len(posfiles), 3))
fcovars.fill(None)
# Create PE parser object and construct posterior
peparser = bppu.PEOutputParser('common')
gmms = []
for p, posfile in enumerate(posfiles):
resultsObj = peparser.parse(open(posfile, 'r'), info=[None, None])
posterior = bppu.Posterior(resultsObj)
# Get GMM for frequency posterior
gmmresult = gmm_peaks(posterior['frequency'].samples)
means = np.concatenate(gmmresult.means_)
covars = np.concatenate(gmmresult.covars_)
# Populate the fpeaks array such that the highest frequency is the first
# column
idx = np.argsort(means)[::-1]
for i in xrange(len(idx)):
fmeans[p, i] = means[idx][i]
fcovars[p, i] = means[idx][i]
if plot:
outdir = os.path.dirname(posfile)
f, ax = plot_oneDposterior(posterior,
'frequency',
histbinswidth=2.5)
freqs = np.arange(1500, 4000, 0.1)
ax.plot(freqs,
np.exp(gmmresult.score_samples(freqs)[0]),
color='r',
label='GMM')
ax.legend()
f.savefig(os.path.join(outdir, 'frequency_posterior_with_GMM.png'))
return fmeans, fcovars
def make_posteriors(self):
peparser = bppu.PEOutputParser('common')
resultsObj = peparser.parse(open(self.samples_file, 'r'))
self.posterior = bppu.Posterior(resultsObj)
# Create bppu posterior instance for easy conf intervals and
# characterisation
self.intervals = {}
for p, param in enumerate(self.params):
# --- Statistics from this posterior
toppoints, injection_cl, reses, injection_area, cl_intervals = \
bppu.greedy_bin_one_param(self.posterior,
{param:self.binSizes[param]}, self.levels)
self.intervals[param]=cl_intervals
def single_injection_results(outdir, posterior_file, bsn_file, snr_file,
waveform):
"""
Characterise the results, including producing plots, for a single injection
"""
# Mostly taken from cbcBayesBurstPostProc.py
headerfile = snr_file.replace('_snr', '_params')
# Output dir for this injection (top-level directory + event number and gps time
# taken from posterior samples filename)
currentdir = os.path.join(
outdir,
posterior_file.split('/')[-1].split('_')[-1].replace('.dat', ''))
if not os.path.isdir(currentdir):
os.makedirs(currentdir)
# Create PE parser object
peparser = bppu.PEOutputParser('common')
resultsObj = peparser.parse(open(posterior_file, 'r'),
info=[headerfile, None])
# Read Bayes and SNR files and copy them into this directory in case it's
# useful later
bfile = open(bsn_file, 'r')
BSN = bfile.read()
bfile.close()
BSN = float(BSN.split()[0])
snrstring = ""
snrf = open(snr_file, 'r')
snrs = snrf.readlines()
snrf.close()
for snr in snrs:
if snr == "\n":
continue
snrstring = snrstring + " " + str(snr[0:-1]) + " ,"
snrstring = snrstring[0:-1]
# Just return the final (usually network) snr
SNR = float(snrstring.split()[-1])
# Archive files to this directory
shutil.copyfile(posterior_file,
os.path.join(currentdir,
posterior_file.split('/')[-1]))
shutil.copyfile(bsn_file, os.path.join(currentdir,
bsn_file.split('/')[-1]))
shutil.copyfile(snr_file, os.path.join(currentdir,
snr_file.split('/')[-1]))
# Create Posterior Sample object (shouldn't matter whether we go with Posterior or BurstPosterior)
pos = bppu.Posterior(resultsObj)
# Add in derived parameters
pos = add_derived_params(pos)
oneDMenu, twoDMenu, binSizes = oneD_bin_params()
# Get true values:
truevals = {}
for item in oneDMenu:
if item == 'frequency':
truevals[item] = waveform.fpeak
else:
truevals[item] = None
# Make sure we have quality AND bandwidth samples
# TODO:
# 3) Loop over injections
# 4) webpage for injection population
# 5) plot injected / recovered waveforms
# ----------------------------------------------------------
# --- Corner plot
#corner_fig = plot_corner(pos, [0.05, 0.5, 0.95], parvals=truevals)
#corner_fig.savefig(os.path.join(currentdir,'corner.png'))
# --- 1D Posterior results (see cbcBayesBurstPostProc.py:733)
# Dictionaries to contain confidence intervals and standard accuracies
cl_intervals_allparams = {}
staccs_allparams = {}
accs_allparams = {}
for par_name in oneDMenu:
print >> sys.stdout, "Producing 1D posteriors for %s" % par_name
# Get bin params
try:
pos[par_name.lower()]
except KeyError:
print "No posterior samples for %s, skipping binning." % par_name
continue
try:
par_bin = binSizes[par_name]
except KeyError:
print "Bin size is not set for %s, skipping binning." % par_name
continue
binParams = {par_name: par_bin}
# --- Statistics from this posterior
toppoints, injection_cl, reses, injection_area, cl_intervals = \
bppu.greedy_bin_one_param(pos, binParams, [0.67, 0.9, 0.95])
# add conf intervals dictionary
cl_intervals_allparams[par_name] = cl_intervals[1]
# add standard accuracy to dictionary
staccs_allparams[par_name] = stacc(pos, par_name, truevals[par_name])
accs_allparams[par_name] = acc(pos, par_name, truevals[par_name])
# --- Plotting
#fig, ax = plot_oneDposterior(pos, par_name, cl_intervals[1],
# truevals, plotkde=False)
#
#figname=par_name+'.png'
#oneDplotPath=os.path.join(currentdir,figname)
#fig.savefig(oneDplotPath)
#pl.close(fig)
#pl.close(corner_fig)
return currentdir, BSN, SNR, pos, cl_intervals_allparams, \
staccs_allparams, accs_allparams, pos
def pulsarBayesPostProc(
outdir,
data,
upperlimit,
histbins,
#nested sampling options
priorfile,
parfile,
ns_Nlive,
# analysis data
Bkdata=None,
ifos=None,
# pulsar information
corfile=None #,
#Turn on 2D kdes
#twodkdeplots=False,
#Turn on R convergence tests
#RconvergenceTests=False
):
# check data is input
if data is None:
raise RuntimeError('You must specify an input data file')
# check output directory is input
if outdir is None:
raise RuntimeError("You must specify an output directory.")
# create output dir if it doesn't exist
if not os.path.isdir(outdir):
os.makedirs(outdir)
# list of posteriors for each ifo
poslist = []
# loop over ifos and generate posteriors
for idx, ifo in enumerate(ifos):
# nested sampling data files for IFO idx
idat = data[idx]
# if input format is xml
votfile = None
if '.xml' in data[0]:
peparser = bppu.PEOutputParser('xml')
commonResultsObj = peparser.parse(idat[0])
thefile = open(idat[0], 'r')
votfile = thefile.read()
else: # input as standard nested sampling input
peparser = bppu.PEOutputParser('ns')
commonResultsObj = peparser.parse(idat,
Nlive=ns_Nlive,
xflag=False)
# Create an instance of the posterior class using the posterior values
# loaded from the file.
pos = bppu.Posterior( commonResultsObj, SimInspiralTableEntry=None, \
votfile=votfile )
# append to list
poslist.append(pos)
# read in par file if given
if parfile is not None:
par = pppu.psr_par(parfile)
# convert phi0' and psi' to phi0 and psi
if 'phi0prime' in pos.names and 'psiprime' in pos.names:
phi0samps, psisamps = pppu.phipsiconvert( pos['phi0prime'].samples, \
pos['psiprime'].samples )
# append phi0 and psi to posterior
phi0_pos = bppu.OneDPosterior('phi0', phi0samps)
pos.append(phi0_pos)
psi_pos = bppu.OneDPosterior('psi', phi0samps)
pos.append(psi_pos)
# get parameters to plot from prior file and those with "Errors" in the par
# file
plotpars = []
pri = pppu.psr_prior(priorfile)
for pars in pppu.float_keys:
# prior file values have preference over par file values with errors in the
# nested sampling code
pars_err = pars + '_ERR'
if pri[pars] is not None:
plotpars.append(pars)
elif par[pars_err] is not None:
plotpars.append(pars)
# if h0 exists plot the pdf
if 'H0' in plotpars:
# set bounds of h0
bounds = [0, float("inf")]
plotFig, ulvals = pppu.plot_posterior_hist(poslist, 'H0'.lower(), ifos,
bounds, histbins,
upperlimit)
# output plots for each IFO
for i, ifo in enumerate(ifos):
figname = 'H0'.lower() + '_' + ifo + '.png'
oneDplotPath = os.path.join(outdir, figname)
plotFig[i].savefig(oneDplotPath)
# if the heterodyned data files have been included create time series plots
# of |B_k| and a amplitude spectral density spectrogram of the data
if Bkdata is not None:
Bkfigs, ASDfigs = pppu.plot_Bks_ASDs(Bkdata, ifos)
# output plots
for i, ifo in enumerate(ifos):
figname = 'Bk' + '_' + ifo + '.png'
Bkplotpath = os.path.join(outdir, figname)
Bkfigs[i].savefig(Bkplotpath)
figname = 'ASD' + '_' + ifo + '.png'
ASDplotpath = os.path.join(outdir, figname)
ASDfigs[i].savefig(ASDplotpath)
def cbcBayesConvergence(
outdir,
data,
#Number of live points used in data
ns_Nlive,
#Threshold for failure for gelman-rubin test
gelmanthresh=1.01
#
):
"""
This is a script which calculates convergence statistics for nested
sampling runs.
"""
if data is None:
raise RuntimeError('You must specify an input data file')
#
if outdir is None:
raise RuntimeError("You must specify an output directory.")
if not os.path.isdir(outdir):
os.makedirs(outdir)
#
import string
from numpy import loadtxt
pos_samples = []
new_data = []
for d in reversed(data):
temp = [d]
new_data.append(temp)
peparser = bppu.PEOutputParser('ns')
posfilename = os.path.join(outdir, 'posterior_samples.dat')
for i in range(len(data)):
# Create a posterior object (Npost=None avoids repeated samples which ruin the KS test)
commonResultsObj = peparser.parse(new_data[i],
Nlive=ns_Nlive,
Npost=None)
pos = bppu.Posterior(commonResultsObj)
pos.write_to_file(posfilename)
with open(posfilename) as f:
param_arr = string.split(f.readline())
loadfile = loadtxt(f)
pos_samples.append(loadfile)
#==================================================================#
#Create webpage with nested sampling convergence information
#==================================================================#
import pylal.nsconvergence as nsc
runs = len(pos_samples)
html_nsconvergence = bppu.htmlPage('Convergence Information',
css=bppu.__default_css_string)
convergencedir = os.path.join(outdir, 'convergence')
if not os.path.isdir(convergencedir):
os.makedirs(convergencedir)
#Summary Section
html_nsconvergence_stats = html_nsconvergence.add_section('Summary')
max_l, l_diff = nsc.compare_maxl(pos_samples, param_arr)
html_nsconvergence_stats.p('Max difference in loglikelihood: %f' % l_diff)
summary_table_string = ''
summary_table_header = '<table border="1"><tr><th>Run</th><th>maxloglikelihood</th>'
#maxposterior column
if param_arr.count('prior') > 0:
max_pos, pos_diff = nsc.compare_maxposterior(pos_samples, param_arr)
html_nsconvergence_stats.p('Max difference in posterior: %f' %
pos_diff)
summary_table_header += '<th>maxposterior</th>'
summary_table_header += '</tr>'
summary_table_string += summary_table_header
for i in range(runs):
max_l_val = max_l[i]
summary_table_string += '<tr><td>%i</td><td>%f</td>' % (i, max_l_val)
if param_arr.count('prior') > 0:
max_pos_val = max_pos[i]
summary_table_string += '<td>%f</td>' % max_pos_val
summary_table_string += '</tr>'
summary_table_string += '</table>'
html_nsconvergence_stats.write(summary_table_string)
#KS Test Section
html_nsconvergence_ks = html_nsconvergence.add_section('KS Test')
ks_arr = nsc.kstest(pos_samples, param_arr, convergencedir)
for index, p in enumerate(param_arr):
ks_table_string = '<table><caption>%s</caption><tr><th></th>' % p
for i in range(runs):
ks_table_string += '<th>Run%i</th>' % i
ks_table_string += '</tr>'
for i in range(runs):
ks_table_string += '<tr><th>Run%i</th>' % i
for j in range(i * runs, (i * runs) + runs):
pval = ks_arr[index][j]
ks_table_string += '<td>%f</td>' % pval
ks_table_string += '</tr>'
ks_table_string += '</table>'
html_nsconvergence_ks.write(ks_table_string)
for p in param_arr:
html_nsconvergence_ks.write(
'<a href="./convergence/ks/' + p +
'_ks.png" target="_blank"><img width="35%" src="./convergence/ks/'
+ p + '_ks.png"/></a>')
#Gelman Rubin Section
html_nsconvergence_gelman = html_nsconvergence.add_section('Gelman Rubin')
gelmandir = os.path.join(convergencedir, 'gelmanrubin')
if not os.path.isdir(gelmandir):
os.makedirs(gelmandir)
gelmanrubin = nsc.gelman_rubin(pos_samples, param_arr, gelmandir)
warn = False
warnparams = []
for index, g in enumerate(gelmanrubin):
if g > gelmanthresh:
warn = True
warnparams.append(index)
if warn:
with open(outdir + '/warning.txt', 'w') as warnfile:
warnfile.write('Gelman-Rubin threshold set to %f\n' % gelmanthresh)
for i in warnparams:
warnfile.write('%s has an R-value of %f\n' %
(param_arr[i], gelmanrubin[i]))
colors = ['b', 'r', 'g', 'c', 'k', 'm', 'y', .25, .5, .75]
for param_index, param in enumerate(param_arr):
for i in range(runs):
data_range = []
for j in range(len(pos_samples[i])):
data_range.append(j)
col = nsc.get_data_col(pos_samples[i], param_arr, param)
plt.figure(param_index)
plt.scatter(data_range, col, c=colors[i], s=5, edgecolors='none')
plt.title('R = ' + str(gelmanrubin[param_index]))
plt.xlabel('Sample')
plt.ylabel(param)
plt.xlim(0, len(pos_samples[i]))
plt.ylim(min(col), max(col))
plt.savefig(gelmandir + '/' + param)
for p in param_arr:
html_nsconvergence_gelman.write(
'<a href="./convergence/gelmanrubin/' + p +
'.png" target="_blank"><img width="35%" src="./convergence/gelmanrubin/'
+ p + '.png"/></a>')
#Write convergence page
nsconvergencepage = open(os.path.join(outdir, 'convergence.html'), 'w')
nsconvergencepage.write(str(html_nsconvergence))
nsconvergencepage.close()
def load_data(filename, header=None):
peparser = bppu.PEOutputParser('common')
commonObj = peparser.parse(open(filename, 'r'), info=[header, None])
pos = bppu.Posterior(commonObj)
return pos
def compare_bayes(outdir,names_and_pos_folders,injection_path,eventnum,username,password,reload_flag,clf,ldg_flag,contour_figsize=(4.5,4.5),contour_dpi=250,contour_figposition=[0.15,0.15,0.5,0.75],fail_on_file_err=True,covarianceMatrices=None,meanVectors=None,Npixels2D=50):
injection=None
if injection_path is not None and os.path.exists(injection_path) and eventnum is not None:
eventnum=int(eventnum)
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injection_path])
if eventnum is not None:
if(len(injections)<eventnum):
print "Error: You asked for event %d, but %s contains only %d injections" %(eventnum,injection_path,len(injections))
sys.exit(1)
else:
injection=injections[eventnum]
#Create analytic likelihood functions if covariance matrices and mean vectors were given
analyticLikelihood = None
if covarianceMatrices and meanVectors:
analyticLikelihood = bppu.AnalyticLikelihood(covarianceMatrices, meanVectors)
peparser=bppu.PEOutputParser('common')
pos_list={}
tp_list={}
common_params=None
working_folder=os.getcwd()
for name,pos_folder in names_and_pos_folders:
import urlparse
pos_folder_url=urlparse.urlparse(pos_folder)
pfu_scheme,pfu_netloc,pfu_path,pfu_params,pfu_query,pfu_fragment=pos_folder_url
if 'http' in pfu_scheme:
"""
Retrieve a file over http(s).
"""
downloads_folder=os.path.join(os.getcwd(),"downloads")
pos_folder_parse=urlparse.urlparse(pos_folder)
pfp_scheme,pfp_netloc,pfp_path,pfp_params,pfp_query,pfp_fragment=pos_folder_parse
head,tail=os.path.split(pfp_path)
if tail is 'posplots.html' or tail:
pos_file_part=head
else:
pos_file_part=pfp_path
pos_file_url=urlparse.urlunsplit((pfp_scheme,pfp_netloc,os.path.join(pos_file_part,'posterior_samples.dat'),'',''))
print pos_file_url
pos_file=os.path.join(os.getcwd(),downloads_folder,"%s.dat"%name)
if not os.path.exists(pos_file):
reload_flag=True
if reload_flag:
if os.path.exists(pos_file):
os.remove(pos_file)
if not os.path.exists(downloads_folder):
os.makedirs(downloads_folder)
open_url_wget(pos_file_url,un=username,pw=password,args=["-O","%s"%pos_file])
elif pfu_scheme is '' or pfu_scheme is 'file':
pos_file=os.path.join(pos_folder,'%s.dat'%name)
# Try looking for posterior_samples.dat if name.dat doesn't exist
if not os.path.exists(pos_file):
print '%s does not exist, trying posterior_samples.dat'%(pos_file)
pos_file=os.path.join(pos_folder,'posterior_samples.dat')
else:
print "Unknown scheme for input data url: %s\nFull URL: %s"%(pfu_scheme,str(pos_folder_url))
exit(0)
print "Reading posterior samples from %s ..."%pos_file
try:
common_output_table_header,common_output_table_raw=peparser.parse(open(pos_file,'r'))
except:
print 'Unable to read file '+pos_file
continue
test_and_switch_param(common_output_table_header,'distance','dist')
test_and_switch_param(common_output_table_header,'chirpmass','mchirp')
test_and_switch_param(common_output_table_header,'mc','mchirp')
test_and_switch_param(common_output_table_header,'asym_massratio','q')
test_and_switch_param(common_output_table_header,'massratio', 'eta')
test_and_switch_param(common_output_table_header,'RA','ra')
test_and_switch_param(common_output_table_header,'rightascension','ra')
test_and_switch_param(common_output_table_header,'declination','dec')
test_and_switch_param(common_output_table_header,'tilt_spin1','tilt1')
test_and_switch_param(common_output_table_header,'tilt_spin2','tilt2')
if 'LI_MCMC' in name or 'FU_MCMC' in name:
try:
idx=common_output_table_header.index('iota')
print "Inverting iota!"
common_output_table_raw[:,idx]= np.pi*np.ones(len(common_output_table_raw[:,0])) - common_output_table_raw[:,idx]
except:
pass
# try:
# print "Converting phi_orb-> 2phi_orb"
# idx=common_output_table_header.index('phi_orb')
# common_output_table_header[idx]='2phi_orb'
# common_output_table_raw[:,idx]= 2*common_output_table_raw[:,idx]
# except:
# pass
try:
print "Converting iota-> cos(iota)"
idx=common_output_table_header.index('iota')
common_output_table_header[idx]='cos(iota)'
common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
except:
pass
#try:
# print "Converting tilt1 -> cos(tilt1)"
# idx=common_output_table_header.index('tilt1')
# common_output_table_header[idx]='cos(tilt1)'
# common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
#except:
# pass
#try:
# print "Converting tilt2 -> cos(tilt2)"
# idx=common_output_table_header.index('tilt2')
# common_output_table_header[idx]='cos(tilt2)'
# common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
#except:
# pass
try:
print "Converting thetas -> cos(thetas)"
idx=common_output_table_header.index('thetas')
common_output_table_header[idx]='cos(thetas)'
common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
except:
pass
try:
print "Converting beta -> cos(beta)"
idx=common_output_table_header.index('beta')
common_output_table_header[idx]='cos(beta)'
common_output_table_raw[:,idx]=np.cos(common_output_table_raw[:,idx])
except:
pass
try:
idx=common_output_table_header.index('f_ref')
injFrefs=np.unique(common_output_table_raw[:,idx])
if len(injFrefs) == 1:
injFref = injFrefs[0]
print "Using f_ref in results as injected value"
except:
injFref = None
pass
pos_temp=bppu.Posterior((common_output_table_header,common_output_table_raw),SimInspiralTableEntry=injection, injFref=injFref)
if 'a1' in pos_temp.names and min(pos_temp['a1'].samples)[0] < 0:
pos_temp.append_mapping('spin1', lambda a:a, 'a1')
pos_temp.pop('a1')
pos_temp.append_mapping('a1', lambda a:np.abs(a), 'spin1')
if 'a2' in pos_temp.names and min(pos_temp['a2'].samples)[0] < 0:
pos_temp.append_mapping('spin2', lambda a:a, 'a2')
pos_temp.pop('a2')
pos_temp.append_mapping('a2', lambda a:np.abs(a), 'spin2')
if 'm1' in pos_temp.names and 'm2' in pos_temp.names:
print "Calculating total mass"
pos_temp.append_mapping('mtotal', lambda m1,m2: m1+m2, ['m1','m2'])
if 'mass1' in pos_temp.names and 'mass2' in pos_temp.names:
print "Calculating total mass"
pos_temp.append_mapping('mtotal', lambda m1,m2: m1+m2, ['mass1','mass2'])
try:
idx=common_output_table_header.index('m1')
idx2=common_output_table_header.index('m2')
if pos_temp['m1'].mean<pos_temp['m2'].mean:
print "SWAPPING MASS PARAMS!"
common_output_table_header[idx]='x'
common_output_table_header[idx2]='m1'
common_output_table_header[idx]='m2'
pos_temp=bppu.Posterior((common_output_table_header,common_output_table_raw),SimInspiralTableEntry=injection)
except:
pass
pos_list[name]=pos_temp
if common_params is None:
common_params=pos_temp.names
else:
set_of_pars = set(pos_temp.names)
common_params=list(set_of_pars.intersection(common_params))
print "Common parameters are %s"%str(common_params)
if injection is None and injection_path is not None:
import itertools
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injection_path])
injection=bppu.get_inj_by_time(injections,pos_temp.means['time'])
if injection is not None:
for pos in pos_list.values():
pos.set_injection(injection)
set_of_pars = set(allowed_params)
common_params=list(set_of_pars.intersection(common_params))
print "Using parameters %s"%str(common_params)
if not os.path.exists(os.path.join(os.getcwd(),'results')):
os.makedirs('results')
if not os.path.exists(outdir):
os.makedirs(outdir)
pdfdir=os.path.join(outdir,'pdfs')
if not os.path.exists(pdfdir):
os.makedirs(pdfdir)
greedy2savepaths=[]
if common_params is not [] and common_params is not None: #If there are common parameters....
colorlst=bppu.__default_color_lst
if len(common_params)>1: #If there is more than one parameter...
temp=copy.copy(common_params)
#Plot two param contour plots
#Assign some colours to each different analysis result
color_by_name={}
hatches_by_name={}
my_cm=mpl_cm.Dark2
cmap_size=my_cm.N
color_idx=0
color_idx_max=len(names_and_pos_folders)
cmap_array=my_cm(np.array(range(cmap_size)))
#cmap_array=['r','g','b','c','m','k','0.5','#ffff00']
hatches=['/','\\','|','-','+','x','o','O','.','*']
ldg='auto'
if not ldg_flag:
ldg=None
for name,infolder in names_and_pos_folders:
#color_by_name=cmap_array[color_idx]
color_by_name[name]=cmap_array[int(floor(color_idx*cmap_size/color_idx_max)),:]
color_idx+=1
hatches_by_name[name]=hatches[color_idx]
for i,j in all_pairs(temp):#Iterate over all unique pairs in the set of common parameters
pplst=[i,j]
rpplst=pplst[:]
rpplst.reverse()
pplst_cond=(pplst in twoDplots)
rpplst_cond=(rpplst in twoDplots)
if pplst_cond or rpplst_cond:#If this pair of parameters is in the plotting list...
try:
print '2d plots: building ',i,j
greedy2Params={i:greedyBinSizes[i],j:greedyBinSizes[j]}
except KeyError:
continue
name_list=[]
cs_list=[]
slinestyles=['solid', 'dashed', 'dashdot', 'dotted']
fig=bppu.plot_two_param_kde_greedy_levels(pos_list,greedy2Params,TwoDconfidenceLevels,color_by_name,figsize=contour_figsize,dpi=contour_dpi,figposition=contour_figposition,legend=ldg,line_styles=slinestyles,hatches_by_name=hatches_by_name,Npixels=Npixels2D)
if fig is None: continue
#fig=bppu.plot_two_param_greedy_bins_contour(pos_list,greedy2Params,TwoDconfidenceLevels,color_by_name,figsize=contour_figsize,dpi=contour_dpi,figposition=contour_figposition)
greedy2savepaths.append('%s-%s.png'%(pplst[0],pplst[1]))
fig.savefig(os.path.join(outdir,'%s-%s.png'%(pplst[0],pplst[1])),bbox_inches='tight')
fig.savefig(os.path.join(pdfdir,'%s-%s.pdf'%(pplst[0],pplst[1])),bbox_inches='tight')
plt.clf()
oned_data={}
#confidence_levels={}
confidence_levels=[{},{},{},{}]
confidence_uncertainty={}
for param in common_params:
print "Plotting comparison for '%s'"%param
cl_table_header='<table><th>Run</th>'
cl_table={}
save_paths=[]
cl_table_min_max_str='<tr><td> Min | Max </td>'
level_index=0
for confidence_level in OneDconfidenceLevels:
if analyticLikelihood:
pdf=analyticLikelihood.pdf(param)
cdf=analyticLikelihood.cdf(param)
else:
pdf=None
cdf=None
cl_table_header+='<th colspan="2">%i%% (Lower|Upper)</th>'%(int(100*confidence_level))
hist_fig,cl_intervals=compare_plots_one_param_line_hist(pos_list,param,confidence_level,color_by_name,cl_lines_flag=clf,legend=ldg,analyticPDF=pdf)
hist_fig2,cl_intervals=compare_plots_one_param_line_hist_cum(pos_list,param,confidence_level,color_by_name,cl_lines_flag=clf,analyticCDF=cdf,legend=ldg)
# Save confidence levels and uncertainty
#confidence_levels[param]=[]
confidence_levels[level_index][param]=[]
for name,pos in pos_list.items():
median=pos[param].median
low,high=cl_intervals[name]
#confidence_levels[param].append((name,low,median,high))
confidence_levels[level_index][param].append((name,low,median,high))
level_index=level_index+1
cl_bounds=[]
poses=[]
for name,pos in pos_list.items():
cl_bounds.append(cl_intervals[name])
poses.append(pos[param])
confidence_uncertainty[param]=bppu.confidence_interval_uncertainty(confidence_level, cl_bounds, poses)
save_path=''
if hist_fig is not None:
save_path=os.path.join(outdir,'%s_%i.png'%(param,int(100*confidence_level)))
save_path_pdf=os.path.join(pdfdir,'%s_%i.pdf'%(param,int(100*confidence_level)))
try:
plt.tight_layout(hist_fig)
plt.tight_layout(hist_fig2)
except:
pass
hist_fig.savefig(save_path,bbox_inches='tight')
hist_fig.savefig(save_path_pdf,bbox_inches='tight')
save_paths.append(save_path)
save_path=os.path.join(outdir,'%s_%i_cum.png'%(param,int(100*confidence_level)))
save_path_pdf=os.path.join(pdfdir,'%s_%i_cum.pdf'%(param,int(100*confidence_level)))
hist_fig2.savefig(save_path,bbox_inches='tight')
hist_fig2.savefig(save_path_pdf,bbox_inches='tight')
save_paths.append(save_path)
min_low,max_high=cl_intervals.values()[0]
for name,interval in cl_intervals.items():
low,high=interval
if low<min_low:
min_low=low
if high>max_high:
max_high=high
try:
cl_table[name]+='<td>%s</td><td>%s</td>'%(low,high)
except:
cl_table[name]='<td>%s</td><td>%s</td>'%(low,high)
cl_table_min_max_str+='<td>%s</td><td>%s</td>'%(min_low,max_high)
cl_table_str=cl_table_header
for name,row_contents in cl_table.items():
cl_table_str+='<tr><td>%s<font color="%s"></font></td>'%(name,str(mpl_colors.rgb2hex(color_by_name[name][0:3])))#,'·'.encode('utf-8'))
cl_table_str+=row_contents+'</tr>'
cl_table_str+=cl_table_min_max_str+'</tr>'
cl_table_str+='</table>'
cl_uncer_str='<table> <th>Confidence Relative Uncertainty</th> <th>Confidence Fractional Uncertainty</th> <th>Confidence Percentile Uncertainty</th>\n'
cl_uncer_str+='<tr> <td> %g </td> <td> %g </td> <td> %g </td> </tr> </table>'%(confidence_uncertainty[param][0], confidence_uncertainty[param][1], confidence_uncertainty[param][2])
ks_matrix=compute_ks_pvalue_matrix(pos_list, param)
N=ks_matrix.shape[0]+1
# Make up KS-test table
ks_table_str='<table><th colspan="%d"> K-S test p-value matrix </th>'%N
# Column headers
ks_table_str+='<tr> <td> -- </td> '
for name,pos in pos_list.items():
ks_table_str+='<td> %s </td>'%name
ks_table_str+='</tr>'
# Now plot rows of matrix
for i in range(len(pos_list)):
ks_table_str+='<tr> <td> %s </td>'%(pos_list.keys()[i])
for j in range(len(pos_list)):
if i == j:
ks_table_str+='<td> -- </td>'
elif ks_matrix[i,j] < 0.05:
# Failing at suspiciously low p-value
ks_table_str+='<td> <b> %g </b> </td>'%ks_matrix[i,j]
else:
ks_table_str+='<td> %g </td>'%ks_matrix[i,j]
ks_table_str+='</tr>'
ks_table_str+='</table>'
oned_data[param]=(save_paths,cl_table_str,ks_table_str,cl_uncer_str)
# Watch out---using private variable _logL
max_logls = [[name,max(pos._logL)] for name,pos in pos_list.items()]
return greedy2savepaths,oned_data,confidence_uncertainty,confidence_levels,max_logls
def downsample_and_evidence(data_hdf5,
deltaLogP=None,
fixedBurnin=None,
nDownsample=None,
verbose=False):
# Remove burnin from beginning of MCMC-chains, downsample by the chain's respective autocorrelation length.
# Compute the evidence for the set of parallel tempered chains through a thermodynamic integral
if not data_hdf5.lower().endswith(('.hdf5', '.hdf', '.h5')):
print 'cbcBayesMCMC2pos only suports hdf5 input, for older file formats plese revert to cbcBayesThermoInt and cbcBayesPosProc'
sys.exit(1)
peparser = bppu.PEOutputParser('hdf5')
ps, samps = peparser.parse(data_hdf5,
deltaLogP=deltaLogP,
fixedBurnins=fixedBurnin,
nDownsample=nDownsample,
tablename=None)
posterior_samples = apt.Table(samps, names=ps)
with h5py.File(data_hdf5, 'r') as inp:
assert posterior_samples['nTemps'][0] == len(inp[mcmc_group_id]), \
'Temperature ladder spread over multiple hdf5 files. Use cbcBayesCombinePTMCMCh5s.py to combine.'
highTchains = []
for i in xrange(1, int(posterior_samples['nTemps'][0])):
ps, samps = peparser.parse(data_hdf5,
deltaLogP=deltaLogP,
fixedBurnins=fixedBurnin,
nDownsample=nDownsample,
tablename='chain_' + str('%02.f' % i))
highTchains.append(apt.Table(samps, names=ps))
if verbose:
print 'chain_' + str('%02.f' % i) + ' at a temperature ' + str(
highTchains[i - 1]['temperature'].mean())
betas = np.zeros(len(highTchains) + 1)
logls = np.zeros_like(betas)
betas[0] = 1. / np.median(posterior_samples['temperature'])
logls[0] = np.median(posterior_samples['logl'])
for i in xrange(len(highTchains)):
betas[i + 1] = 1. / np.median(highTchains[i]['temperature'])
logls[i + 1] = np.median(highTchains[i]['logl'])
inds = np.argsort(betas)[::-1]
betas = betas[inds]
logls = logls[inds]
# Now extend to infinite temperature by copying the last <log(L)>.
# This works as long as the chains have extended to high enough
# temperature to sample the prior.
# If infinite temperature is already included, this 'duplicate'
# will not change the final evidence.
ebetas = np.concatenate((betas, [0.0]))
elogls = np.concatenate((logls, [logls[-1]]))
ebetas2 = np.concatenate((betas[::2], [0.0]))
elogls2 = np.concatenate((logls[::2], [logls[::2][-1]]))
evidence = -si.trapz(elogls, ebetas)
evidence2 = -si.trapz(elogls2, ebetas2)
posterior_samples['chain_log_evidence'] = evidence
posterior_samples['chain_delta_log_evidence'] = np.absolute(evidence -
evidence2)
posterior_samples['chain_log_noise_evidence'] = posterior_samples[
'nullLogL']
posterior_samples['chain_log_bayes_factor'] = posterior_samples[
'chain_log_evidence'] - posterior_samples['chain_log_noise_evidence']
if verbose:
print 'logZ = ' + str(
posterior_samples['chain_log_evidence'][0]) + '+-' + str(
posterior_samples['chain_delta_log_evidence'][0])
print 'logB_SN = ' + str(
posterior_samples['chain_log_bayes_factor'][0])
posterior_samples = reassign_metadata(posterior_samples, data_hdf5)
return posterior_samples
from optparse import OptionParser
__author__="Will M. Farr <*****@*****.**>"
__version__="git id %s"%git_version.id
__date__=git_version.date
if __name__=='__main__':
parser=OptionParser()
parser.add_option("--data",dest="data",action="store",help="file of posterior samples",metavar="FILE")
parser.add_option("--Nboxing",dest="Nboxing",action="store",default=64,type="int",metavar="N")
parser.add_option("--bootstrap", dest='bootstrap',action='store',default=1,type='int',metavar='N')
parser.add_option('--output',dest='output',action='store',default=None,metavar='FILE')
(opts,args)=parser.parse_args()
pos_parser=bp.PEOutputParser('common')
f=open(opts.data, "r")
try:
pos=bp.Posterior(pos_parser.parse(f))
finally:
f.close()
outfile=None
if opts.output:
outfile=open(opts.output,'w')
try:
for i in range(opts.bootstrap):
if i == 0:
log_ev=pos.di_evidence(boxing=opts.Nboxing)
else:
art = mpatches.Rectangle((tile[0], tile[2]),
tile[1] - tile[0],
tile[3] - tile[2],
fill=True,
facecolor=matplotlib.cm.jet(1. - tile[4]),
linewidth=0) #,edgecolor = 'k')
ax.add_patch(art)
if injCoords is not None:
plt.scatter(injCoords[0], injCoords[1], marker=(5, 1), s=250, c='k')
myfig.savefig(output + '/plot' + str(event))
###############load posterior#########
data = bppu.PEOutputParser('common')
if (urlUsed):
open_url_wget(input, output, eventNum=event)
inputFileObj = open(output + '/posteriors/posterior_' + str(event) +
'.dat')
else:
inputFileObj = open(input)
dataObj0 = data.parse(inputFileObj)
from pylal import SimInspiralUtils
injections = SimInspiralUtils.ReadSimInspiralFromFiles([injFile])
if (len(injections) < event):
print "Error: You asked for event %d, but %s contains only %d injections" % (
event, injfile, len(injections))
sys.exit(1)
else:
cbar.formatter.set_powerlimits((-1, 1))
cbar.update_ticks()
cbar.set_label('posterior probability')
cbar.solids.set_edgecolor("face")
plt.savefig(os.path.join(outpath,"comp_spin_pos.png"), bbox_inches='tight')
plt.clf()
USAGE='What this does is : '
if __name__=='__main__':
import sys
from optparse import OptionParser
from pylal import bayespputils as bppu
parser=OptionParser(USAGE)
parser.add_option("-o","--outpath", dest="outpath",default=None, help="make page and plots in DIR", metavar="DIR")
parser.add_option("-d","--data",dest="data",help="Posteriors samples file (must be in common format)")
(opts,args)=parser.parse_args()
if opts.outpath is None:
opts.outpath=os.getcwd()
if not os.path.isfile(opts.data):
print "Cannot find posterior file %s\n"%opts.data
sys.exit(1)
else:
peparser=bppu.PEOutputParser('common')
commonResultsObj=peparser.parse(open(opts.data,'r'),info=[None,None])
ps,samps = commonResultsObj
pos = bppu.Posterior(commonResultsObj)
make_disk_plot(pos,opts.outpath)