def finish(self):
livetime = rate.BinnedArray(self.counts.bins)
for offsets in self.tisi_rows:
self.seglists.offsets.update(offsets)
livetime[offsets[self.x_instrument],
offsets[self.y_instrument]] += float(
abs(self.seglists.intersection(self.seglists.keys())))
zvals = self.counts.at_centres() / livetime.at_centres()
rate.filter_array(zvals, rate.gaussian_window(3, 3))
xcoords, ycoords = self.counts.centres()
self.axes.contour(xcoords, ycoords, numpy.transpose(numpy.log(zvals)))
for offsets in self.tisi_rows:
if any(offsets):
# time slide vector is non-zero-lag
self.axes.plot((offsets[self.x_instrument], ),
(offsets[self.y_instrument], ), "k+")
else:
# time slide vector is zero-lag
self.axes.plot((offsets[self.x_instrument], ),
(offsets[self.y_instrument], ), "r+")
self.axes.set_xlim(
[self.counts.bins().min[0],
self.counts.bins().max[0]])
self.axes.set_ylim(
[self.counts.bins().min[1],
self.counts.bins().max[1]])
self.axes.set_title(
r"Coincident Event Rate vs. Instrument Time Offset (Logarithmic Rate Contours)"
)
def density_estimate(lnpdf, name, kernel=rate.gaussian_window(4.)):
"""
For internal use only.
"""
assert not numpy.isnan(lnpdf.array).any(
), "%s log likelihood ratio PDF contain NaNs" % name
rate.filter_array(lnpdf.array, kernel)
def finish(self):
self.axes.set_title("Trigger Rate vs. Peak Frequency\n(%d Triggers)" % self.nevents)
# 21 bins per filter width
rate.filter_array(self.rate.array, rate.gaussian_window(21))
xvals = self.rate.centres()[0]
self.axes.plot(xvals, self.rate.at_centres(), "k")
self.axes.semilogy()
self.axes.set_xlim((min(xvals), max(xvals)))
def finish(self):
self.axes.set_title("Trigger Rate vs. Peak Frequency\n(%d Triggers)" % self.nevents)
# 21 bins per filter width
rate.filter_array(self.rate.array, rate.gaussian_window(21))
xvals = self.rate.centres()[0]
self.axes.plot(xvals, self.rate.at_centres(), "k")
self.axes.semilogy()
self.axes.set_xlim((min(xvals), max(xvals)))
def finish(self):
self.axes.set_title("Trigger Peak Time - Injection Peak Time\n(%d Found Injections)" % self.found)
# 21 bins per filter width
filter = rate.gaussian_window(21)
rate.filter_array(self.offsets.array, filter)
rate.filter_array(self.coinc_offsets.array, filter)
self.axes.plot(self.offsets.centres()[0], self.offsets.at_centres(), "k")
self.axes.plot(self.coinc_offsets.centres()[0], self.coinc_offsets.at_centres(), "r")
self.axes.legend(["%s residuals" % self.instrument, "SNR-weighted mean of residuals in all instruments"], loc = "lower right")
def finish(self):
for instrument, data in sorted(self.data.items()):
fig, axes = SnglBurstUtils.make_burst_plot(r"$t_{\mathrm{recovered}} - t_{\mathrm{injected}}$ (s)", "Triggers per Unit Offset")
axes.semilogy()
axes.set_title("Trigger Peak Time - Injection Peak Time in %s\n(%d Found Injections)" % (instrument, data.found))
# 21 bins per filter width
rate.filter_array(data.offsets.array, rate.gaussian_window(21))
axes.plot(data.offsets.centres()[0], data.offsets.at_centres(), "k")
#axes.legend(["%s residuals" % instrument, "SNR-weighted mean of residuals in all instruments"], loc = "lower right")
yield fig
def finish(self):
self.axes.set_title("Time Between Triggers\n(%d Triggers)" % self.nevents)
rate.filter_array(self.bins.array, rate.gaussian_window(21))
xvals = self.bins.centres()[0]
yvals = self.bins.at_centres()
self.axes.plot(xvals, yvals, "k")
self.axes.set_xlim((0, xvals[-1]))
self.axes.set_ylim((1, 10.0**(int(math.log10(yvals.max())) + 1)))
def finish(self):
self.axes.set_title("Time Between Triggers\n(%d Triggers)" % self.nevents)
rate.filter_array(self.bins.array, rate.gaussian_window(21))
xvals = self.bins.centres()[0]
yvals = self.bins.at_centres()
self.axes.plot(xvals, yvals, "k")
self.axes.set_xlim((0, xvals[-1]))
self.axes.set_ylim((1, 10.0**(int(math.log10(yvals.max())) + 1)))
def finish(self):
self.axes.set_title("Trigger Peak Frequency / Injection Centre Frequency\n(%d Found Injections)" % self.found)
# 21 bins per filter width
filter = rate.gaussian_window(21)
rate.filter_array(self.offsets.array, filter)
rate.filter_array(self.coinc_offsets.array, filter)
self.axes.plot(10**self.offsets.centres()[0], self.offsets.at_centres(), "k")
self.axes.plot(10**self.coinc_offsets.centres()[0], self.coinc_offsets.at_centres(), "r")
self.axes.legend(["%s triggers" % self.instrument, "SNR-weighted mean of all matching triggers"], loc = "lower right")
ymin, ymax = self.axes.get_ylim()
if ymax / ymin > 1e6:
ymin = ymax / 1e6
self.axes.set_ylim((ymin, ymax))
def finish(self):
for key, pdf in self.densities.items():
if key.endswith("_snr2_chi2"):
rate.filter_array(pdf.array, rate.gaussian_window(11, 11, sigma = 20))
elif key.endswith("_dt") or key.endswith("_dA") or key.endswith("_df"):
rate.filter_array(pdf.array, rate.gaussian_window(11, sigma = 20))
elif key.startswith("instrumentgroup"):
# instrument group filter is a no-op
pass
else:
# shouldn't get here
raise Exception
pdf.normalize()
self.mkinterps()
def finish(self):
self.axes.set_title(r"\begin{center}Distribution of Coincident Events (%d Foreground, %d Background Events, %d Injections Found in Coincidence, Logarithmic Density Contours)\end{center}" % (self.n_foreground, self.n_background, self.n_injections))
xcoords, ycoords = self.background_bins.centres()
# prepare the data
rate.filter_array(self.foreground_bins.array, rate.gaussian_window(8, 8))
rate.filter_array(self.background_bins.array, rate.gaussian_window(8, 8))
rate.filter_array(self.coinc_injection_bins.array, rate.gaussian_window(8, 8))
rate.filter_array(self.incomplete_coinc_injection_bins.array, rate.gaussian_window(8, 8))
self.foreground_bins.logregularize()
self.background_bins.logregularize()
self.coinc_injection_bins.logregularize()
self.incomplete_coinc_injection_bins.logregularize()
# plot background contours
max_density = math.log(self.background_bins.array.max())
self.axes.contour(xcoords, ycoords, numpy.transpose(numpy.log(self.background_bins.array)), [max_density - n for n in xrange(0, 10, 1)], cmap = matplotlib.cm.Greys)
# plot foreground (zero-lag) contours
max_density = math.log(self.foreground_bins.array.max())
self.axes.contour(xcoords, ycoords, numpy.transpose(numpy.log(self.foreground_bins.array)), [max_density - n for n in xrange(0, 10, 1)], cmap = matplotlib.cm.Reds)
#self.axes.plot(self.foreground_x, self.foreground_y, "r+")
# plot coincident injection contours
max_density = math.log(self.coinc_injection_bins.array.max())
self.axes.contour(xcoords, ycoords, numpy.transpose(numpy.log(self.coinc_injection_bins.array)), [max_density - n for n in xrange(0, 10, 1)], cmap = matplotlib.cm.Blues)
# plot incomplete coincident injection contours
max_density = math.log(self.incomplete_coinc_injection_bins.array.max())
self.axes.contour(xcoords, ycoords, numpy.transpose(numpy.log(self.incomplete_coinc_injection_bins.array)), [max_density - n for n in xrange(0, 10, 1)], cmap = matplotlib.cm.Greens)
# fix axes limits
self.axes.set_xlim([self.background_bins.bins.min[0], self.background_bins.bins.max[0]])
self.axes.set_ylim([self.background_bins.bins.min[1], self.background_bins.bins.max[1]])
def finish(self):
for key, pdf in self.densities.items():
if key.endswith("_snr2_chi2"):
rate.filter_array(pdf.array, rate.gaussian_window(11, 11, sigma = 20))
elif key.endswith("_dt") or key.endswith("_dA") or key.endswith("_df"):
rate.filter_array(pdf.array, rate.gaussian_window(11, sigma = 20))
elif key.startswith("instrumentgroup"):
# instrument group filter is a no-op
pass
else:
# shouldn't get here
raise Exception
pdf.normalize()
self.mkinterps()
def create_efficiency_plot(axes, all_injections, found_injections, detection_threshold, cal_uncertainty):
filter_width = 16.7
# formats
axes.semilogx()
axes.set_position([0.10, 0.150, 0.86, 0.77])
# set desired yticks
axes.set_yticks((0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0))
axes.set_yticklabels((r"\(0\)", r"\(0.1\)", r"\(0.2\)", r"\(0.3\)", r"\(0.4\)", r"\(0.5\)", r"\(0.6\)", r"\(0.7\)", r"\(0.8\)", r"\(0.9\)", r"\(1.0\)"))
axes.xaxis.grid(True, which = "major,minor")
axes.yaxis.grid(True, which = "major,minor")
# put made and found injections in the denominators and
# numerators of the efficiency bins
bins = rate.NDBins((rate.LogarithmicBins(min(sim.amplitude for sim in all_injections), max(sim.amplitude for sim in all_injections), 400),))
efficiency_num = rate.BinnedArray(bins)
efficiency_den = rate.BinnedArray(bins)
for sim in found_injections:
efficiency_num[sim.amplitude,] += 1
for sim in all_injections:
efficiency_den[sim.amplitude,] += 1
# generate and plot trend curves. adjust window function
# normalization so that denominator array correctly
# represents the number of injections contributing to each
# bin: make w(0) = 1.0. note that this factor has no
# effect on the efficiency because it is common to the
# numerator and denominator arrays. we do this for the
# purpose of computing the Poisson error bars, which
# requires us to know the counts for the bins
windowfunc = rate.gaussian_window(filter_width)
windowfunc /= windowfunc[len(windowfunc) / 2 + 1]
rate.filter_array(efficiency_num.array, windowfunc)
rate.filter_array(efficiency_den.array, windowfunc)
# regularize: adjust unused bins so that the efficiency is
# 0, not NaN
assert (efficiency_num.array <= efficiency_den.array).all()
efficiency_den.array[(efficiency_num.array == 0) & (efficiency_den.array == 0)] = 1
line1, A50, A50_err = render_data_from_bins(file("string_efficiency.dat", "w"), axes, efficiency_num, efficiency_den, cal_uncertainty, filter_width, colour = "k", linestyle = "-", erroralpha = 0.2)
# add a legend to the axes
axes.legend((line1,), (r"\noindent Injections recovered with $\log \Lambda > %.2f$" % detection_threshold,), loc = "lower right")
# adjust limits
axes.set_xlim([3e-22, 3e-19])
axes.set_ylim([0.0, 1.0])
def finish(self):
self.axes.set_title(
"Trigger Peak Time - Injection Peak Time\n(%d Found Injections)" %
self.found)
# 21 bins per filter width
filter = rate.gaussian_window(21)
rate.filter_array(self.offsets.array, filter)
rate.filter_array(self.coinc_offsets.array, filter)
self.axes.plot(self.offsets.centres()[0], self.offsets.at_centres(),
"k")
self.axes.plot(self.coinc_offsets.centres()[0],
self.coinc_offsets.at_centres(), "r")
self.axes.legend([
"%s residuals" % self.instrument,
"SNR-weighted mean of residuals in all instruments"
],
loc="lower right")
def finish(self): livetime = rate.BinnedArray(self.counts.bins) for offsets in self.tisi_rows: self.seglists.offsets.update(offsets) livetime[offsets[self.x_instrument], offsets[self.y_instrument]] += float(abs(self.seglists.intersection(self.seglists.keys()))) zvals = self.counts.at_centres() / livetime.at_centres() rate.filter_array(zvals, rate.gaussian_window(3, 3)) xcoords, ycoords = self.counts.centres() self.axes.contour(xcoords, ycoords, numpy.transpose(numpy.log(zvals))) for offsets in self.tisi_rows: if any(offsets): # time slide vector is non-zero-lag self.axes.plot((offsets[self.x_instrument],), (offsets[self.y_instrument],), "k+") else: # time slide vector is zero-lag self.axes.plot((offsets[self.x_instrument],), (offsets[self.y_instrument],), "r+") self.axes.set_xlim([self.counts.bins().min[0], self.counts.bins().max[0]]) self.axes.set_ylim([self.counts.bins().min[1], self.counts.bins().max[1]]) self.axes.set_title(r"Coincident Event Rate vs. Instrument Time Offset (Logarithmic Rate Contours)")
def finish(self):
self.axes.set_title(
"Trigger Peak Frequency / Injection Centre Frequency\n(%d Found Injections)"
% self.found)
# 21 bins per filter width
filter = rate.gaussian_window(21)
rate.filter_array(self.offsets.array, filter)
rate.filter_array(self.coinc_offsets.array, filter)
self.axes.plot(10**self.offsets.centres()[0],
self.offsets.at_centres(), "k")
self.axes.plot(10**self.coinc_offsets.centres()[0],
self.coinc_offsets.at_centres(), "r")
self.axes.legend([
"%s triggers" % self.instrument,
"SNR-weighted mean of all matching triggers"
],
loc="lower right")
ymin, ymax = self.axes.get_ylim()
if ymax / ymin > 1e6:
ymin = ymax / 1e6
self.axes.set_ylim((ymin, ymax))
def rate_posterior_from_samples(samples):
"""
Construct and return a BinnedArray containing a histogram of a
sequence of samples. If limits is None (default) then the limits
of the binning will be determined automatically from the sequence,
otherwise limits is expected to be a tuple or other two-element
sequence providing the (low, high) limits, and in that case the
sequence can be a generator.
"""
nbins = int(math.sqrt(len(samples)) / 40.)
assert nbins >= 1, "too few samples to construct histogram"
lo = samples.min() * (1. - nbins / len(samples))
hi = samples.max() * (1. + nbins / len(samples))
ln_pdf = rate.BinnedLnPDF(
rate.NDBins((rate.LogarithmicBins(lo, hi, nbins), )))
count = ln_pdf.count # avoid attribute look-up in loop
for sample in samples:
count[sample, ] += 1.
rate.filter_array(ln_pdf.array, rate.gaussian_window(5), use_fft=False)
ln_pdf.normalize()
return ln_pdf
def finish(self):
self.axes.set_title(
r"\begin{center}Distribution of Coincident Events (%d Foreground, %d Background Events, %d Injections Found in Coincidence, Logarithmic Density Contours)\end{center}"
% (self.n_foreground, self.n_background, self.n_injections))
xcoords, ycoords = self.background_bins.centres()
# prepare the data
rate.filter_array(self.foreground_bins.array,
rate.gaussian_window(8, 8))
rate.filter_array(self.background_bins.array,
rate.gaussian_window(8, 8))
rate.filter_array(self.coinc_injection_bins.array,
rate.gaussian_window(8, 8))
rate.filter_array(self.incomplete_coinc_injection_bins.array,
rate.gaussian_window(8, 8))
self.foreground_bins.logregularize()
self.background_bins.logregularize()
self.coinc_injection_bins.logregularize()
self.incomplete_coinc_injection_bins.logregularize()
# plot background contours
max_density = math.log(self.background_bins.array.max())
self.axes.contour(xcoords,
ycoords,
numpy.transpose(numpy.log(
self.background_bins.array)),
[max_density - n for n in xrange(0, 10, 1)],
cmap=matplotlib.cm.Greys)
# plot foreground (zero-lag) contours
max_density = math.log(self.foreground_bins.array.max())
self.axes.contour(xcoords,
ycoords,
numpy.transpose(numpy.log(
self.foreground_bins.array)),
[max_density - n for n in xrange(0, 10, 1)],
cmap=matplotlib.cm.Reds)
#self.axes.plot(self.foreground_x, self.foreground_y, "r+")
# plot coincident injection contours
max_density = math.log(self.coinc_injection_bins.array.max())
self.axes.contour(xcoords,
ycoords,
numpy.transpose(
numpy.log(self.coinc_injection_bins.array)),
[max_density - n for n in xrange(0, 10, 1)],
cmap=matplotlib.cm.Blues)
# plot incomplete coincident injection contours
max_density = math.log(
self.incomplete_coinc_injection_bins.array.max())
self.axes.contour(xcoords,
ycoords,
numpy.transpose(
numpy.log(
self.incomplete_coinc_injection_bins.array)),
[max_density - n for n in xrange(0, 10, 1)],
cmap=matplotlib.cm.Greens)
# fix axes limits
self.axes.set_xlim([
self.background_bins.bins.min[0], self.background_bins.bins.max[0]
])
self.axes.set_ylim([
self.background_bins.bins.min[1], self.background_bins.bins.max[1]
])
def finish(self):
for key, pdf in self.densities.items():
rate.filter_array(pdf.array, rate.gaussian_window(11, 5))
pdf.normalize()
self.mkinterps()
def integrate(self, func, *args, **kwargs):
"""
Integrate func, by using n sample points. Right now, all params defined must be passed to args must be provided, but this will change soon.
Limitations:
func's signature must contain all parameters currently defined by the sampler, and with the same names. This is required so that the sample values can be passed consistently.
kwargs:
nmax -- total allowed number of sample points, will throw a warning if this number is reached before neff.
nmin -- minimum number of samples to allow, by default will be set to the end of the 'burnin' at n_adapt * n
neff -- Effective samples to collect before terminating. If not given, assume infinity
n -- Number of samples to integrate in a 'chunk' -- default is 1000
save_integrand -- Save the evaluated value of the integrand at the sample points with the sample point
history_mult -- Number of chunks (of size n) to use in the adaptive histogramming: only useful if there are parameters with adaptation enabled
tempering_exp -- Exponent to raise the weights of the 1-D marginalized histograms for adaptive sampling prior generation, by default it is 0 which will turn off adaptive sampling regardless of other settings
n_adapt -- number of chunks over which to allow the pdf to adapt. Default is zero, which will turn off adaptive sampling regardless of other settings
convergence_tests - dictionary of function pointers, each accepting self._rvs and self.params as arguments. CURRENTLY ONLY USED FOR REPORTING
maxval - Guess at the maximum value of the integrand -- used as a seed for the maxval counter
Pinning a value: By specifying a kwarg with the same of an existing parameter, it is possible to "pin" it. The sample draws will always be that value, and the sampling prior will use a delta function at that value.
"""
#
# Pin values
#
tempcdfdict, temppdfdict, temppriordict, temppdfnormdict = {}, {}, {}, {}
temppdfnormdict = defaultdict(lambda: 1.0)
for p, val in kwargs.iteritems():
if p in self.params:
# Store the previous pdf/cdf in case it's already defined
tempcdfdict[p] = self.cdf_inv[p]
temppdfdict[p] = self.pdf[p]
temppdfnormdict[p] = self._pdf_norm[p]
temppriordict[p] = self.prior_pdf[p]
# Set a new one to always return the same value
self.pdf[p] = functools.partial(delta_func_pdf_vector, val)
self._pdf_norm[p] = 1.0
self.prior_pdf[p] = functools.partial(delta_func_pdf_vector, val)
self.cdf_inv[p] = functools.partial(delta_func_samp_vector, val)
# This is a semi-hack to ensure that the integrand is called with
# the arguments in the right order
# FIXME: How dangerous is this?
args = func.func_code.co_varnames[:func.func_code.co_argcount]
if not MCSampler.match_params_from_args(args, self.params):
raise ValueError("All integrand variables must be represented by integral parameters.")
#
# Determine stopping conditions
#
nmax = kwargs["nmax"] if kwargs.has_key("nmax") else float("inf")
neff = kwargs["neff"] if kwargs.has_key("neff") else numpy.float128("inf")
n = kwargs["n"] if kwargs.has_key("n") else min(1000, nmax)
convergence_tests = kwargs["convergence_tests"] if kwargs.has_key("convergence_tests") else None
#
# Adaptive sampling parameters
#
n_history = int(kwargs["history_mult"]*n) if kwargs.has_key("history_mult") else None
tempering_exp = kwargs["tempering_exp"] if kwargs.has_key("tempering_exp") else 1.0
n_adapt = int(kwargs["n_adapt"]*n) if kwargs.has_key("n_adapt") else 0
nmax = kwargs["nmax"] if kwargs.has_key("nmax") else n_adapt
nmin = kwargs["nmin"] if kwargs.has_key("nmin") else n_adapt
save_intg = kwargs["save_intg"] if kwargs.has_key("save_intg") else False
nkeep = kwargs["save_intg"] if kwargs.has_key("save_intg") else None
# Corner case: we want all the samples, and we don't want them messed
# with, so everything is saved, but no sort is done
if nkeep is True:
nkeep = None
# FIXME: The adaptive step relies on the _rvs cache, so this has to be
# on in order to work
if n_adapt > 0 and tempering_exp > 0.0:
save_intg = True
deltalnL = kwargs['igrand_threshold_deltalnL'] if kwargs.has_key('igrand_threshold_deltalnL') else None # default is to return all
deltaP = kwargs["igrand_threshold_p"] if kwargs.has_key('igrand_threshold_p') else 0 # default is to omit 1e-7 of probability
show_evaluation_log = kwargs['verbose'] if kwargs.has_key('verbose') else False
if show_evaluation_log:
print " .... mcsampler : providing verbose output ..... "
int_val1 = numpy.float128(0)
self.ntotal = 0
maxval = kwargs["maxval"] if "maxval" in kwargs else -float("Inf")
old_maxval = maxval
maxlnL = -float("Inf")
eff_samp = 0
mean, var = None, None
last_convergence_test = defaultdict(lambda: False) # initialize record of tests
if show_evaluation_log:
print "walltime : iteration Neff ln(maxweight) lnLmarg ln(Z/Lmax) int_var"
socket = None
while self.ntotal < nmin or (eff_samp < neff and self.ntotal < nmax):
# Draw our sample points
p_s, p_prior, rv = self.draw(n, *self.params)
# Calculate the overall p_s assuming each pdf is independent
joint_p_s = numpy.prod(p_s, axis=0)
joint_p_prior = numpy.prod(p_prior, axis=0)
#
# Prevent zeroes in the sampling prior
#
# FIXME: If we get too many of these, we should bail
if (isinstance(joint_p_s, numpy.ndarray) and any(joint_p_s <= 0)) \
or (not isinstance(joint_p_s, numpy.ndarray) and joint_p_s <= 0):
for p in self.params:
self._rvs[p] = numpy.resize(self._rvs[p], len(self._rvs[p])-n)
print >>sys.stderr, "Zero prior value detected, skipping."
continue
#
# Unpack rvs and evaluate integrand
#
if len(rv[0].shape) != 1:
rv = rv[0]
params = []
for item in self.params:
if isinstance(item, tuple):
params.extend(item)
else:
params.append(item)
unpacked = numpy.hstack([r.flatten() for r in rv]).reshape(len(args), -1)
unpacked = dict(zip(params, unpacked))
fval = func(**unpacked)
#
# Check if there is any practical contribution to the integral
#
# FIXME: While not technically a fatal error, this will kill the
# adaptive sampling
if fval.sum() == 0:
for p in self.params:
self._rvs[p] = numpy.resize(self._rvs[p], len(self._rvs[p])-n)
print >>sys.stderr, "No contribution to integral, skipping."
continue
sample_n = numpy.arange(self.ntotal, self.ntotal + len(fval))
if save_intg:
# FIXME: The joint_prior, if not specified is set to one and
# will come out as a scalar here, hence the hack
if not isinstance(joint_p_prior, numpy.ndarray):
joint_p_prior = numpy.ones(fval.shape)*joint_p_prior
# FIXME: See warning at beginning of function. The prior values
# need to be moved out of this, as they are not part of MC
# integration
if self._rvs.has_key("integrand"):
self._rvs["integrand"] = numpy.hstack( (self._rvs["integrand"], fval) )
self._rvs["joint_prior"] = numpy.hstack( (self._rvs["joint_prior"], joint_p_prior) )
self._rvs["joint_s_prior"] = numpy.hstack( (self._rvs["joint_s_prior"], joint_p_s) )
self._rvs["weights"] = numpy.hstack( (self._rvs["weights"], fval*joint_p_prior/joint_p_s) )
self._rvs["sample_n"] = numpy.hstack( (self._rvs["sample_n"], sample_n) )
else:
self._rvs["integrand"] = fval
self._rvs["joint_prior"] = joint_p_prior
self._rvs["joint_s_prior"] = joint_p_s
self._rvs["weights"] = fval*joint_p_prior/joint_p_s
self._rvs["sample_n"] = sample_n
# Calculate the integral over this chunk
int_val = fval * joint_p_prior / joint_p_s
# Calculate max L (a useful convergence feature) for debug
# reporting. Not used for integration
# Try to avoid nan's
maxlnL = numpy.log(numpy.max([numpy.exp(maxlnL), numpy.max(fval),numpy.exp(-100)])) # note if f<0, this will return nearly 0
# Calculate the effective samples via max over the current
# evaluations
maxval = [max(maxval, int_val[0]) if int_val[0] != 0 else maxval]
for v in int_val[1:]:
maxval.append( v if v > maxval[-1] and v != 0 else maxval[-1] )
# running variance
var = statutils.cumvar(int_val, mean, var, self.ntotal)[-1]
# running integral
int_val1 += int_val.sum()
# running number of evaluations
self.ntotal += n
# FIXME: Likely redundant with int_val1
mean = int_val1/self.ntotal
maxval = maxval[-1]
eff_samp = int_val1/maxval
# Throw exception if we get infinity or nan
if math.isnan(eff_samp):
raise NanOrInf("Effective samples = nan")
if maxlnL is float("Inf"):
raise NanOrInf("maxlnL = inf")
if show_evaluation_log:
print int(time.time()), ": ", self.ntotal, eff_samp, math.log(maxval), numpy.log(int_val1/self.ntotal), numpy.log(int_val1/self.ntotal)-maxlnL, numpy.sqrt(var*self.ntotal)/int_val1
if (not convergence_tests) and self.ntotal >= nmin and self.ntotal >= nmax and neff != float("inf"):
print >>sys.stderr, "WARNING: User requested maximum number of samples reached... bailing."
#
# Convergence tests
#
if convergence_tests:
converged = True
for key in convergence_tests.keys():
last_convergence_test[key] = convergence_tests[key](self._rvs, self.params)
converged &= las_convergence_test[key]
if convergence_tests and show_evaluation_log: # Print status of each test
for key in convergence_tests:
print " -- Convergence test status : ", key, last_convergence_test[key]
self._address, self._port = "pcdev2.nemo.phys.uwm.edu", 1890
#if self._address is not None:
if False:
dims = ("distance", "inclination", "right_ascension",
"declination", "integrand", "joint_prior", "joint_s_prior")
send_data = synchlib.prepare_data(self._rvs, dims, self.ntotal - n)
self.socket = synchlib.send_samples(send_data, self._address, self._port, verbose=True, socket=self.socket)
#
# The total number of adaptive steps is reached
#
# FIXME: We need a better stopping condition here
if self.ntotal >= n_adapt and maxval == old_maxval:
# Downsample points
if save_intg and nkeep is not None:
pt_sort = self._rvs["weights"].argsort()[-nkeep:]
for key in self._rvs:
if len(self._rvs[key].shape) > 1:
self._rvs[key] = self._rvs[key][:,pt_sort]
else:
self._rvs[key] = self._rvs[key][pt_sort]
continue
old_maxval = maxval
#
# Iterate through each of the parameters, updating the sampling
# prior PDF according to the 1-D marginalization
#
for itr, p in enumerate(self.params):
# FIXME: The second part of this condition should be made more
# specific to pinned parameters
if p not in self.adaptive or p in kwargs.keys():
continue
points = self._rvs[p][-n_history:]
weights = (self._rvs["integrand"][-n_history:]*self._rvs["joint_prior"][-n_history:])**tempering_exp
self._hist[p] = statutils.get_adaptive_binning(points, (self.llim[p], self.rlim[p]))
for pt, w in zip(points, weights):
self._hist[p][pt,] += w
self._hist[p].array += self._hist[p].array.mean()
rate.filter_array(self._hist[p].array, rate.tophat_window(3))
norm = numpy.sum(self._hist[p].array * self._hist[p].bins.volumes())
self._hist[p].array /= norm
# FIXME: Stupid pet trick while numpy version is lacking
self.pdf[p] = numpy.frompyfunc(rate.InterpBinnedArray(self._hist[p]), 1, 1)
#with open("%s_%d_hist.txt" % (p, self.ntotal), "w") as fout:
#for c, pdf in zip(self._hist[p].centres()[0], self._hist[p].array):
#print >>fout, "%f %g" % (c, pdf)
self.cdf[p] = self.cdf_function(p)
self.cdf_inv[p] = self.cdf_inverse(p)
# If we were pinning any values, undo the changes we did before
self.cdf_inv.update(tempcdfdict)
self.pdf.update(temppdfdict)
self._pdf_norm.update(temppdfnormdict)
self.prior_pdf.update(temppriordict)
# Clean out the _rvs arrays for 'irrelevant' points
# - create the cumulative weights
if "weights" in self._rvs and deltaP > 0:
# Sort the weights with the deltaL masked applied
sorted_weights = self._rvs["weights"].argsort()
# Make the (unnormalized) CDF
total_weight = self._rvs["weights"][sorted_weights].cumsum()
# Find the deltaP cutoff index
idx = numpy.searchsorted(total_weight, deltaP*total_weight[-1], 'left')
sorted_weights = sorted_weights[idx:]
# Remove all samples which contribute to smallest 1e-3 of cumulative
# probability
for key in self._rvs.keys():
if isinstance(key, tuple):
self._rvs[key] = self._rvs[key][:,sorted_weights]
else:
self._rvs[key] = self._rvs[key][sorted_weights]
if "integrand" in self._rvs and deltalnL is not None:
deltal_mask = numpy.log(self._rvs["integrand"]) > (maxlnL - deltalnL)
# Remove all samples which do not have L > maxlnL - deltalnL
for key in self._rvs.keys():
if isinstance(key, tuple):
self._rvs[key] = self._rvs[key][:,deltal_mask]
else:
self._rvs[key] = self._rvs[key][deltal_mask]
# Create extra dictionary to return things
dict_return ={}
if convergence_tests is not None:
dict_return["convergence_test_results"] = last_convergence_test
return int_val1/self.ntotal, var/self.ntotal, eff_samp, dict_return
def finish(self, threshold):
# bin the injections
self._bin_events()
# use the same binning for the found injection density as
# was constructed for the efficiency
self.found_density = rate.BinnedArray(self.efficiency.denominator.bins)
# construct the amplitude weighting function
amplitude_weight = rate.BinnedArray(rate.NDBins((rate.LinearBins(threshold - 100, threshold + 100, 10001),)))
# gaussian window's width is the number of bins
# corresponding to 10 units of amplitude, which is computed
# by dividing 10 by the "volume" of the bin corresponding
# to threshold. index is the index of the element in
# amplitude_weight corresponding to the threshold.
index, = amplitude_weight.bins[threshold,]
window = rate.gaussian_window(10.0 / amplitude_weight.bins.volumes()[index])
window /= 10 * window[(len(window) - 1) / 2]
# set the window data into the BinnedArray object. the
# Gaussian peaks on the middle element of the window, which
# we want to place at index in the amplitude_weight array.
lo = index - (len(window) - 1) / 2
hi = lo + len(window)
if lo < 0 or hi > len(amplitude_weight.array):
raise ValueError("amplitude weighting window too large")
amplitude_weight.array[lo:hi] = window
# store the recovered injections in the found density bins
# weighted by amplitude
for x, y, z in self.recovered_xyz:
try:
weight = amplitude_weight[z,]
except IndexError:
# beyond the edge of the window
weight = 0.0
self.found_density[x, y] += weight
# the efficiency is only valid up to the highest energy
# that has been injected. this creates problems later on
# so, instead, for each frequency, identify the highest
# energy that has been measured and copy the values for
# that bin's numerator and denominator into the bins for
# all higher energies. do the same for the counts in the
# found injection density bins.
#
# numpy.indices() returns two arrays array, the first of
# which has each element set equal to its x index, the
# second has each element set equal to its y index, we keep
# the latter. meanwhile numpy.roll() cyclically permutes
# the efficiency bins down one along the y (energy) axis.
# from this, the conditional finds bins where the
# efficiency is greater than 0.9 but <= 0.9 in the bin
# immediately above it in energy. we select the elements
# from the y index array where the conditional is true, and
# then use numpy.max() along the y direction to return the
# highest such y index for each x index, which is a 1-D
# array. finally, enumerate() is used to iterate over x
# index and corresponding y index, and if the y index is
# not negative (was found) the value from that x-y bin is
# copied to all higher bins.
n = self.efficiency.numerator.array
d = self.efficiency.denominator.array
f = self.found_density.array
bady = -1
for x, y in enumerate(numpy.max(numpy.where((d > 0) & (numpy.roll(d, -1, axis = 1) <= 0), numpy.indices(d.shape)[1], bady), axis = 1)):
if y != bady:
n[x, y + 1:] = n[x, y]
d[x, y + 1:] = d[x, y]
f[x, y + 1:] = f[x, y]
# now do the same for the bins at energies below those that
# have been measured.
bady = d.shape[1]
for x, y in enumerate(numpy.min(numpy.where((d > 0) & (numpy.roll(d, 1, axis = 1) <= 0), numpy.indices(d.shape)[1], bady), axis = 1)):
if y != bady:
n[x, 0:y] = n[x, y]
d[x, 0:y] = d[x, y]
f[x, 0:y] = f[x, y]
diagnostic_plot(self.efficiency.numerator.array, self.efficiency.denominator.bins, r"Efficiency Numerator (Before Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_numerator_before.png")
diagnostic_plot(self.efficiency.denominator.array, self.efficiency.denominator.bins, r"Efficiency Denominator (Before Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_denominator_before.png")
diagnostic_plot(self.found_density.array, self.efficiency.denominator.bins, r"Injections Lost / Unit of Threshold (Before Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_found_density_before.png")
# smooth the efficiency bins and the found injection
# density bins using the same 2D window.
window = rate.gaussian_window(self.window_size_x, self.window_size_y)
window /= window[tuple((numpy.array(window.shape, dtype = "double") - 1) / 2)]
rate.filter_binned_ratios(self.efficiency, window)
rate.filter_array(self.found_density.array, window)
diagnostic_plot(self.efficiency.numerator.array, self.efficiency.denominator.bins, r"Efficiency Numerator (After Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_numerator_after.png")
diagnostic_plot(self.efficiency.denominator.array, self.efficiency.denominator.bins, r"Efficiency Denominator (After Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_denominator_after.png")
diagnostic_plot(self.found_density.array, self.efficiency.denominator.bins, r"Injections Lost / Unit of Threshold (After Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_found_density_after.png")
# compute the uncertainties in the efficiency and its
# derivative by assuming these to be the binomial counting
# fluctuations in the numerators.
p = self.efficiency.ratio()
self.defficiency = numpy.sqrt(p * (1 - p) / self.efficiency.denominator.array)
p = self.found_density.array / self.efficiency.denominator.array
self.dfound_density = numpy.sqrt(p * (1 - p) / self.efficiency.denominator.array)
def twoD_SearchVolume(self, instruments, dbin=None, FAR=None, bootnum=None, derr=0.197, dsys=0.074):
"""
Compute the search volume in the mass/mass plane, bootstrap
and measure the first and second moment (assumes the underlying
distribution can be characterized by those two parameters)
This is gonna be brutally slow
derr = (0.134**2+.103**2+.102**2)**.5 = 0.197 which is the 3 detector
calibration uncertainty in quadrature. This is conservative since some injections
will be H1L1 and have a lower error of .17
the dsys is the DC offset which is the max offset of .074.
"""
if not FAR: FAR = self.far[instruments]
found, missed = self.get_injections(instruments, FAR)
twodbin = self.twoDMassBins
wnfunc = self.gw
livetime = self.livetime[instruments]
if not bootnum: bootnum = self.bootnum
if wnfunc: wnfunc /= wnfunc[(wnfunc.shape[0]-1) / 2, (wnfunc.shape[1]-1) / 2]
x = twodbin.shape[0]
y = twodbin.shape[1]
z = int(self.opts.dist_bins)
rArrays = []
volArray=rate.BinnedArray(twodbin)
volArray2=rate.BinnedArray(twodbin)
#set up ratio arrays for each distance bin
for k in range(z):
rArrays.append(rate.BinnedRatios(twodbin))
# Bootstrap to account for errors
for n in range(bootnum):
#initialize by setting these to zero
for k in range(z):
rArrays[k].numerator.array = numpy.zeros(rArrays[k].numerator.bins.shape)
rArrays[k].denominator.array = numpy.zeros(rArrays[k].numerator.bins.shape)
#Scramble the inj population and distances
if bootnum > 1:
sm, sf = self._scramble_pop(missed, found)
# I make a separate array of distances to speed up this calculation
f_dist = self._scramble_dist(sf, derr, dsys)
else:
sm, sf = missed, found
f_dist = numpy.array([l.distance for l in found])
# compute the distance bins
if not dbin:
dbin = rate.LogarithmicBins(min(f_dist),max(f_dist), z)
#else: print dbin.centres()
# get rid of all missed injections outside the distance bins
# to prevent binning errors
sm, m_dist = self.cut_distance(sm, dbin)
sf, f_dist = self.cut_distance(sf, dbin)
for i, l in enumerate(sf):#found:
tbin = rArrays[dbin[f_dist[i]]]
tbin.incnumerator( (l.mass1, l.mass2) )
for i, l in enumerate(sm):#missed:
tbin = rArrays[dbin[m_dist[i]]]
tbin.incdenominator( (l.mass1, l.mass2) )
tmpArray2=rate.BinnedArray(twodbin) #start with a zero array to compute the mean square
for k in range(z):
tbins = rArrays[k]
tbins.denominator.array += tbins.numerator.array
if wnfunc: rate.filter_array(tbins.denominator.array,wnfunc)
if wnfunc: rate.filter_array(tbins.numerator.array,wnfunc)
tbins.regularize()
# logarithmic(d)
integrand = 4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
volArray.array += integrand
tmpArray2.array += integrand #4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
print("bootstrapping:\t%.1f%% and Calculating smoothed volume:\t%.1f%%\r" % ((100.0 * n / bootnum), (100.0 * k / z)), end=' ', file=sys.stderr)
tmpArray2.array *= tmpArray2.array
volArray2.array += tmpArray2.array
print("", file=sys.stderr)
#Mean and variance
volArray.array /= bootnum
volArray2.array /= bootnum
volArray2.array -= volArray.array**2 # Variance
volArray.array *= livetime
volArray2.array *= livetime*livetime # this gets two powers of live time
return volArray, volArray2
def finish(self, threshold):
# bin the injections
self._bin_events()
# use the same binning for the found injection density as
# was constructed for the efficiency
self.found_density = rate.BinnedArray(self.efficiency.denominator.bins)
# construct the amplitude weighting function
amplitude_weight = rate.BinnedArray(rate.NDBins((rate.LinearBins(threshold - 100, threshold + 100, 10001),)))
# gaussian window's width is the number of bins
# corresponding to 10 units of amplitude, which is computed
# by dividing 10 by the "volume" of the bin corresponding
# to threshold. index is the index of the element in
# amplitude_weight corresponding to the threshold.
index, = amplitude_weight.bins[threshold,]
window = rate.gaussian_window(10.0 / amplitude_weight.bins.volumes()[index])
window /= 10 * window[(len(window) - 1) / 2]
# set the window data into the BinnedArray object. the
# Gaussian peaks on the middle element of the window, which
# we want to place at index in the amplitude_weight array.
lo = index - (len(window) - 1) / 2
hi = lo + len(window)
if lo < 0 or hi > len(amplitude_weight.array):
raise ValueError("amplitude weighting window too large")
amplitude_weight.array[lo:hi] = window
# store the recovered injections in the found density bins
# weighted by amplitude
for x, y, z in self.recovered_xyz:
try:
weight = amplitude_weight[z,]
except IndexError:
# beyond the edge of the window
weight = 0.0
self.found_density[x, y] += weight
# the efficiency is only valid up to the highest energy
# that has been injected. this creates problems later on
# so, instead, for each frequency, identify the highest
# energy that has been measured and copy the values for
# that bin's numerator and denominator into the bins for
# all higher energies. do the same for the counts in the
# found injection density bins.
#
# numpy.indices() returns two arrays array, the first of
# which has each element set equal to its x index, the
# second has each element set equal to its y index, we keep
# the latter. meanwhile numpy.roll() cyclically permutes
# the efficiency bins down one along the y (energy) axis.
# from this, the conditional finds bins where the
# efficiency is greater than 0.9 but <= 0.9 in the bin
# immediately above it in energy. we select the elements
# from the y index array where the conditional is true, and
# then use numpy.max() along the y direction to return the
# highest such y index for each x index, which is a 1-D
# array. finally, enumerate() is used to iterate over x
# index and corresponding y index, and if the y index is
# not negative (was found) the value from that x-y bin is
# copied to all higher bins.
n = self.efficiency.numerator.array
d = self.efficiency.denominator.array
f = self.found_density.array
bady = -1
for x, y in enumerate(numpy.max(numpy.where((d > 0) & (numpy.roll(d, -1, axis = 1) <= 0), numpy.indices(d.shape)[1], bady), axis = 1)):
if y != bady:
n[x, y + 1:] = n[x, y]
d[x, y + 1:] = d[x, y]
f[x, y + 1:] = f[x, y]
# now do the same for the bins at energies below those that
# have been measured.
bady = d.shape[1]
for x, y in enumerate(numpy.min(numpy.where((d > 0) & (numpy.roll(d, 1, axis = 1) <= 0), numpy.indices(d.shape)[1], bady), axis = 1)):
if y != bady:
n[x, 0:y] = n[x, y]
d[x, 0:y] = d[x, y]
f[x, 0:y] = f[x, y]
diagnostic_plot(self.efficiency.numerator.array, self.efficiency.denominator.bins, r"Efficiency Numerator (Before Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_numerator_before.png")
diagnostic_plot(self.efficiency.denominator.array, self.efficiency.denominator.bins, r"Efficiency Denominator (Before Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_denominator_before.png")
diagnostic_plot(self.found_density.array, self.efficiency.denominator.bins, r"Injections Lost / Unit of Threshold (Before Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_found_density_before.png")
# smooth the efficiency bins and the found injection
# density bins using the same 2D window.
window = rate.gaussian_window(self.window_size_x, self.window_size_y)
window /= window[tuple((numpy.array(window.shape, dtype = "double") - 1) / 2)]
rate.filter_binned_ratios(self.efficiency, window)
rate.filter_array(self.found_density.array, window)
diagnostic_plot(self.efficiency.numerator.array, self.efficiency.denominator.bins, r"Efficiency Numerator (After Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_numerator_after.png")
diagnostic_plot(self.efficiency.denominator.array, self.efficiency.denominator.bins, r"Efficiency Denominator (After Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_efficiency_denominator_after.png")
diagnostic_plot(self.found_density.array, self.efficiency.denominator.bins, r"Injections Lost / Unit of Threshold (After Averaging)", self.amplitude_lbl, "lalapps_excesspowerfinal_found_density_after.png")
# compute the uncertainties in the efficiency and its
# derivative by assuming these to be the binomial counting
# fluctuations in the numerators.
p = self.efficiency.ratio()
self.defficiency = numpy.sqrt(p * (1 - p) / self.efficiency.denominator.array)
p = self.found_density.array / self.efficiency.denominator.array
self.dfound_density = numpy.sqrt(p * (1 - p) / self.efficiency.denominator.array)
weeks = float(abs(segment)) / 86400.0 / 7.0 # FIXME: leep seconds?
border = (0.5, 0.75, 0.125, 0.625) # inches
width = max(10.0, weeks * 3.0 + border[0] + border[2]) # inches
height = 8.0 # inches
fig = figure.Figure()
canvas = FigureCanvas(fig)
fig.set_size_inches(width, height)
fig.gca().set_position([border[0] / width, border[1] / height, (width - border[0] - border[2]) / width, (height - border[1] - border[3]) / height])
return fig
fig = newfig(options.segment)
axes = fig.gca()
rate.filter_array(trigger_rate.array, rate.gaussian_window(bins_per_filterwidth))
axes.plot(trigger_rate.centres()[0], trigger_rate.at_centres())
axes.set_xlim(list(options.segment))
axes.grid(True)
for seg in ~seglist & segments.segmentlist([options.segment]):
axes.axvspan(seg[0], seg[1], facecolor = "k", alpha = 0.2)
axes.set_title("%s Excess Power Trigger Rate vs. Time\n(%d Triggers, %g s Moving Average)" % (options.instrument, num_triggers, options.window))
def finish(self):
fig, axes = SnglBurstUtils.make_burst_plot(r"Injection Amplitude (\(\mathrm{s}^{-\frac{1}{3}}\))", "Detection Efficiency", width = 108.0)
axes.set_title(r"Detection Efficiency vs.\ Amplitude")
axes.semilogx()
axes.set_position([0.10, 0.150, 0.86, 0.77])
# set desired yticks
axes.set_yticks((0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0))
axes.set_yticklabels((r"\(0\)", r"\(0.1\)", r"\(0.2\)", r"\(0.3\)", r"\(0.4\)", r"\(0.5\)", r"\(0.6\)", r"\(0.7\)", r"\(0.8\)", r"\(0.9\)", r"\(1.0\)"))
axes.xaxis.grid(True, which = "major,minor")
axes.yaxis.grid(True, which = "major,minor")
# put made and found injections in the denominators and
# numerators of the efficiency bins
bins = rate.NDBins((rate.LogarithmicBins(min(sim.amplitude for sim in self.all), max(sim.amplitude for sim in self.all), 400),))
efficiency_num = rate.BinnedArray(bins)
efficiency_den = rate.BinnedArray(bins)
for sim in self.found:
efficiency_num[sim.amplitude,] += 1
for sim in self.all:
efficiency_den[sim.amplitude,] += 1
# generate and plot trend curves. adjust window function
# normalization so that denominator array correctly
# represents the number of injections contributing to each
# bin: make w(0) = 1.0. note that this factor has no
# effect on the efficiency because it is common to the
# numerator and denominator arrays. we do this for the
# purpose of computing the Poisson error bars, which
# requires us to know the counts for the bins
windowfunc = rate.gaussian_window(self.filter_width)
windowfunc /= windowfunc[len(windowfunc) / 2 + 1]
rate.filter_array(efficiency_num.array, windowfunc)
rate.filter_array(efficiency_den.array, windowfunc)
# regularize: adjust unused bins so that the efficiency is
# 0, not NaN
assert (efficiency_num.array <= efficiency_den.array).all()
efficiency_den.array[(efficiency_num.array == 0) & (efficiency_den.array == 0)] = 1
line1, A50, A50_err = render_data_from_bins(file("string_efficiency.dat", "w"), axes, efficiency_num, efficiency_den, self.cal_uncertainty, self.filter_width, colour = "k", linestyle = "-", erroralpha = 0.2)
print("Pipeline's 50%% efficiency point for all detections = %g +/- %g%%\n" % (A50, A50_err * 100), file=sys.stderr)
# add a legend to the axes
axes.legend((line1,), (r"\noindent Injections recovered with $\log \Lambda > %.2f$" % self.detection_threshold,), loc = "lower right")
# adjust limits
axes.set_xlim([3e-22, 3e-19])
axes.set_ylim([0.0, 1.0])
#
# dump some information about the highest-amplitude missed
# and quietest-amplitude found injections
#
self.loudest_missed.sort(reverse = True)
self.quietest_found.sort(reverse = True)
f = file("string_loud_missed_injections.txt", "w")
print("Highest Amplitude Missed Injections", file=f)
print("===================================", file=f)
for amplitude, sim, offsetvector, filename, ln_likelihood_ratio in self.loudest_missed:
print(file=f)
print("%s in %s:" % (str(sim.simulation_id), filename), file=f)
if ln_likelihood_ratio is None:
print("Not recovered", file=f)
else:
print("Recovered with \\log \\Lambda = %.16g, detection threshold was %.16g" % (ln_likelihood_ratio, self.detection_threshold), file=f)
for instrument in self.seglists:
print("In %s:" % instrument, file=f)
print("\tInjected amplitude:\t%.16g" % SimBurstUtils.string_amplitude_in_instrument(sim, instrument, offsetvector), file=f)
print("\tTime of injection:\t%s s" % sim.time_at_instrument(instrument, offsetvector), file=f)
print("Amplitude in waveframe:\t%.16g" % sim.amplitude, file=f)
t = sim.get_time_geocent()
print("Time at geocentre:\t%s s" % t, file=f)
print("Segments within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.seglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.seglists)), file=f)
print("Vetoes within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.vetoseglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.vetoseglists)), file=f)
f = file("string_quiet_found_injections.txt", "w")
print("Lowest Amplitude Found Injections", file=f)
print("=================================", file=f)
for inv_amplitude, sim, offsetvector, filename, ln_likelihood_ratio in self.quietest_found:
print(file=f)
print("%s in %s:" % (str(sim.simulation_id), filename), file=f)
if ln_likelihood_ratio is None:
print("Not recovered", file=f)
else:
print("Recovered with \\log \\Lambda = %.16g, detection threshold was %.16g" % (ln_likelihood_ratio, self.detection_threshold), file=f)
for instrument in self.seglists:
print("In %s:" % instrument, file=f)
print("\tInjected amplitude:\t%.16g" % SimBurstUtils.string_amplitude_in_instrument(sim, instrument, offsetvector), file=f)
print("\tTime of injection:\t%s s" % sim.time_at_instrument(instrument, offsetvector), file=f)
print("Amplitude in waveframe:\t%.16g" % sim.amplitude, file=f)
t = sim.get_time_geocent()
print("Time at geocentre:\t%s s" % t, file=f)
print("Segments within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.seglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.seglists)), file=f)
print("Vetoes within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.vetoseglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.vetoseglists)), file=f)
#
# done
#
return fig,
def twoD_SearchVolume(found,
missed,
twodbin,
dbin,
wnfunc,
livetime,
bootnum=1,
derr=0.197,
dsys=0.074):
"""
Compute the search volume in the mass/mass plane, bootstrap
and measure the first and second moment (assumes the underlying
distribution can be characterized by those two parameters)
This is gonna be brutally slow
derr = (0.134**2+.103**2+.102**2)**.5 = 0.197 which is the 3 detector
calibration uncertainty in quadrature. This is conservative since some injections
will be H1L1 and have a lower error of .17
the dsys is the DC offset which is the max offset of .074.
"""
if wnfunc:
wnfunc /= wnfunc[(wnfunc.shape[0] - 1) / 2, (wnfunc.shape[1] - 1) / 2]
x = twodbin.shape[0]
y = twodbin.shape[1]
z = dbin.n
rArrays = []
volArray = rate.BinnedArray(twodbin)
volArray2 = rate.BinnedArray(twodbin)
#set up ratio arrays for each distance bin
for k in range(z):
rArrays.append(rate.BinnedRatios(twodbin))
# Bootstrap to account for errors
for n in range(bootnum):
#initialize by setting these to zero
for k in range(z):
rArrays[k].numerator.array = numpy.zeros(
rArrays[k].numerator.bins.shape)
rArrays[k].denominator.array = numpy.zeros(
rArrays[k].numerator.bins.shape)
#Scramble the inj population
if bootnum > 1: sm, sf = scramble_pop(missed, found)
else: sm, sf = missed, found
for l in sf: #found:
tbin = rArrays[dbin[scramble_dist(l.distance, derr, dsys)]]
tbin.incnumerator((l.mass1, l.mass2))
for l in sm: #missed:
tbin = rArrays[dbin[scramble_dist(l.distance, derr, dsys)]]
tbin.incdenominator((l.mass1, l.mass2))
tmpArray2 = rate.BinnedArray(
twodbin) #start with a zero array to compute the mean square
for k in range(z):
tbins = rArrays[k]
tbins.denominator.array += tbins.numerator.array
if wnfunc: rate.filter_array(tbins.denominator.array, wnfunc)
if wnfunc: rate.filter_array(tbins.numerator.array, wnfunc)
tbins.regularize()
# logarithmic(d)
integrand = 4.0 * pi * tbins.ratio() * dbin.centres(
)[k]**3 * dbin.delta
volArray.array += integrand
tmpArray2.array += integrand #4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
print(
"bootstrapping:\t%.1f%% and Calculating smoothed volume:\t%.1f%%\r"
% ((100.0 * n / bootnum), (100.0 * k / z)),
end=' ',
file=sys.stderr)
tmpArray2.array *= tmpArray2.array
volArray2.array += tmpArray2.array
print("", file=sys.stderr)
#Mean and variance
volArray.array /= bootnum
volArray2.array /= bootnum
volArray2.array -= volArray.array**2 # Variance
volArray.array *= livetime
volArray2.array *= livetime * livetime # this gets two powers of live time
return volArray, volArray2
def finish(self): for key, pdf in self.densities.items(): rate.filter_array(pdf.array, rate.gaussian_window(11, 5)) pdf.normalize() self.mkinterps()
def SNRPDF(instruments,
horizon_distances,
snr_cutoff,
n_samples=200000,
bins=rate.ATanLogarithmicBins(3.6, 1e3, 150)):
"""
Precomputed SNR PDF for each detector
Returns a BinnedArray containing
P(snr_{inst1}, snr_{inst2}, ... | signal seen in exactly
{inst1, inst2, ...} in a network of instruments
with a given set of horizon distances)
i.e., the joint probability density of observing a set of
SNRs conditional on them being the result of signal that
has been recovered in a given subset of the instruments in
a network of instruments with a given set of horizon
distances.
The axes of the PDF correspond to the instruments in
alphabetical order. The binning used for all axes is set
with the bins parameter.
The n_samples parameter sets the number of iterations for
the internal Monte Carlo sampling loop.
"""
if n_samples < 1:
raise ValueError("n_samples=%d must be >= 1" % n_samples)
# get instrument names
instruments = sorted(instruments)
if len(instruments) < 1:
raise ValueError(instruments)
# get the horizon distances in the same order
DH_times_8 = 8. * numpy.array(
[horizon_distances[inst] for inst in instruments])
# get detector responses in the same order
resps = [
lalsimulation.DetectorPrefixToLALDetector(str(inst)).response
for inst in instruments
]
# get horizon distances and responses of remaining
# instruments (order doesn't matter as long as they're in
# the same order)
DH_times_8_other = 8. * numpy.array([
dist
for inst, dist in horizon_distances.items() if inst not in instruments
])
resps_other = tuple(
lalsimulation.DetectorPrefixToLALDetector(str(inst)).response
for inst in horizon_distances if inst not in instruments)
# initialize the PDF array, and pre-construct the sequence of
# snr, d(snr) tuples. since the last SNR bin probably has
# infinite size, we remove it from the sequence
# (meaning the PDF will be left 0 in that bin)
pdf = rate.BinnedArray(rate.NDBins([bins] * len(instruments)))
snr_sequence = rate.ATanLogarithmicBins(3.6, 1e3, 500)
snr_snrlo_snrhi_sequence = numpy.array(
zip(snr_sequence.centres(), snr_sequence.lower(),
snr_sequence.upper())[:-1])
# compute the SNR at which to begin iterations over bins
assert type(snr_cutoff) is float
snr_min = snr_cutoff - 3.0
assert snr_min > 0.0
# we select random uniformly-distributed right assensions
# so there's no point in also choosing random GMSTs and any
# vlaue is as good as any other
gmst = 0.0
# run the sample the requested # of iterations. save some
# symbols to avoid doing module attribute look-ups in the
# loop
acos = math.acos
random_uniform = random.uniform
twopi = 2. * math.pi
pi_2 = math.pi / 2.
xlal_am_resp = lal.ComputeDetAMResponse
# FIXME: scipy.stats.rice.rvs broken on reference OS.
# switch to it when we can rely on a new-enough scipy
#rice_rvs = stats.rice.rvs # broken on reference OS
# the .reshape is needed in the event that x is a 1x1
# array: numpy returns a scalar from sqrt(), but we must
# have something that we can iterate over
rice_rvs = lambda x: numpy.sqrt(stats.ncx2.rvs(2., x**2.)).reshape(x.shape)
for i in xrange(n_samples):
# select random sky location and source orbital
# plane inclination
# the signal is linearly polaraized, and h_cross = 0
# is assumed, so we need only F+ (its absolute value).
ra = random_uniform(0., twopi)
dec = pi_2 - acos(random_uniform(-1., 1.))
psi = random_uniform(0., twopi)
fplus = tuple(
abs(xlal_am_resp(resp, ra, dec, psi, gmst)[0]) for resp in resps)
# 1/8 ratio of inverse SNR to distance for each instrument
# (1/8 because horizon distance is defined for an SNR of 8,
# and we've omitted that factor for performance)
snr_times_D = DH_times_8 * fplus
# snr * D in instrument whose SNR grows fastest
# with decreasing D
max_snr_times_D = snr_times_D.max()
# snr_times_D.min() / snr_min = the furthest a
# source can be and still be above snr_min in all
# instruments involved. max_snr_times_D / that
# distance = the SNR that distance corresponds to
# in the instrument whose SNR grows fastest with
# decreasing distance --- the SNR the source has in
# the most sensitive instrument when visible to all
# instruments in the combo
try:
start_index = snr_sequence[max_snr_times_D /
(snr_times_D.min() / snr_min)]
except ZeroDivisionError:
# one of the instruments that must be able
# to see the event is blind to it
continue
# min_D_other is minimum distance at which source
# becomes visible in an instrument that isn't
# involved. max_snr_times_D / min_D_other gives
# the SNR in the most sensitive instrument at which
# the source becomes visible to one of the
# instruments not allowed to participate
if len(DH_times_8_other):
min_D_other = (DH_times_8_other * fplus).min() / snr_cutoff
try:
end_index = snr_sequence[max_snr_times_D / min_D_other] + 1
except ZeroDivisionError:
# all instruments that must not see
# it are blind to it
end_index = None
else:
# there are no other instruments
end_index = None
# if start_index >= end_index then in order for the
# source to be close enough to be visible in all
# the instruments that must see it it is already
# visible to one or more instruments that must not.
# don't need to check for this, the for loop that
# comes next will simply not have any iterations.
# iterate over the nominal SNRs (= noise-free SNR
# in the most sensitive instrument) at which we
# will add weight to the PDF. from the SNR in
# most sensitive instrument, the distance to the
# source is:
#
# D = max_snr_times_D / snr
#
# and the (noise-free) SNRs in all instruments are:
#
# snr_times_D / D
#
# scipy's Rice-distributed RV code is used to
# add the effect of background noise, converting
# the noise-free SNRs into simulated observed SNRs
#
# number of sources b/w Dlo and Dhi:
#
# d count \propto D^2 |dD|
# count \propto Dhi^3 - Dlo**3
D_Dhi_Dlo_sequence = max_snr_times_D / snr_snrlo_snrhi_sequence[
start_index:end_index]
for snr, weight in zip(
rice_rvs(snr_times_D /
numpy.reshape(D_Dhi_Dlo_sequence[:, 0],
(len(D_Dhi_Dlo_sequence), 1))),
D_Dhi_Dlo_sequence[:, 1]**3. - D_Dhi_Dlo_sequence[:, 2]**3.):
pdf[tuple(snr)] += weight
# check for divide-by-zeros that weren't caught. also
# finds nans if they are there
assert numpy.isfinite(pdf.array).all()
# convolve samples with gaussian kernel
rate.filter_array(pdf.array,
rate.gaussian_window(*(1.875, ) * len(pdf.array.shape)))
# protect against round-off in FFT convolution leading to
# negative valuesin the PDF
numpy.clip(pdf.array, 0., PosInf, pdf.array)
# zero counts in bins that are below the trigger threshold.
# have to convert SNRs to indexes ourselves and adjust so
# that we don't zero the bin in which the SNR threshold
# falls
range_all = slice(None, None)
range_low = slice(None, pdf.bins[0][snr_cutoff])
for i in xrange(len(instruments)):
slices = [range_all] * len(instruments)
slices[i] = range_low
# convert bin counts to normalized PDF
pdf.to_pdf()
# one last sanity check
assert numpy.isfinite(pdf.array).all()
# done
return pdf
def twoD_SearchVolume(found, missed, twodbin, dbin, wnfunc, livetime, bootnum=1, derr=0.197, dsys=0.074):
"""
Compute the search volume in the mass/mass plane, bootstrap
and measure the first and second moment (assumes the underlying
distribution can be characterized by those two parameters)
This is gonna be brutally slow
derr = (0.134**2+.103**2+.102**2)**.5 = 0.197 which is the 3 detector
calibration uncertainty in quadrature. This is conservative since some injections
will be H1L1 and have a lower error of .17
the dsys is the DC offset which is the max offset of .074.
"""
if wnfunc: wnfunc /= wnfunc[(wnfunc.shape[0]-1) / 2, (wnfunc.shape[1]-1) / 2]
x = twodbin.shape[0]
y = twodbin.shape[1]
z = dbin.n
rArrays = []
volArray=rate.BinnedArray(twodbin)
volArray2=rate.BinnedArray(twodbin)
#set up ratio arrays for each distance bin
for k in range(z):
rArrays.append(rate.BinnedRatios(twodbin))
# Bootstrap to account for errors
for n in range(bootnum):
#initialize by setting these to zero
for k in range(z):
rArrays[k].numerator.array = numpy.zeros(rArrays[k].numerator.bins.shape)
rArrays[k].denominator.array = numpy.zeros(rArrays[k].numerator.bins.shape)
#Scramble the inj population
if bootnum > 1: sm, sf = scramble_pop(missed, found)
else: sm, sf = missed, found
for l in sf:#found:
tbin = rArrays[dbin[scramble_dist(l.distance,derr,dsys)]]
tbin.incnumerator( (l.mass1, l.mass2) )
for l in sm:#missed:
tbin = rArrays[dbin[scramble_dist(l.distance,derr,dsys)]]
tbin.incdenominator( (l.mass1, l.mass2) )
tmpArray2=rate.BinnedArray(twodbin) #start with a zero array to compute the mean square
for k in range(z):
tbins = rArrays[k]
tbins.denominator.array += tbins.numerator.array
if wnfunc: rate.filter_array(tbins.denominator.array,wnfunc)
if wnfunc: rate.filter_array(tbins.numerator.array,wnfunc)
tbins.regularize()
# logarithmic(d)
integrand = 4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
volArray.array += integrand
tmpArray2.array += integrand #4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
print("bootstrapping:\t%.1f%% and Calculating smoothed volume:\t%.1f%%\r" % ((100.0 * n / bootnum), (100.0 * k / z)), end=' ', file=sys.stderr)
tmpArray2.array *= tmpArray2.array
volArray2.array += tmpArray2.array
print("", file=sys.stderr)
#Mean and variance
volArray.array /= bootnum
volArray2.array /= bootnum
volArray2.array -= volArray.array**2 # Variance
volArray.array *= livetime
volArray2.array *= livetime*livetime # this gets two powers of live time
return volArray, volArray2
def integrate(self, func, *args, **kwargs):
"""
Integrate func, by using n sample points. Right now, all params defined must be passed to args must be provided, but this will change soon.
Limitations:
func's signature must contain all parameters currently defined by the sampler, and with the same names. This is required so that the sample values can be passed consistently.
kwargs:
nmax -- total allowed number of sample points, will throw a warning if this number is reached before neff.
nmin -- minimum number of samples to allow, by default will be set to the end of the 'burnin' at n_adapt * n
neff -- Effective samples to collect before terminating. If not given, assume infinity
n -- Number of samples to integrate in a 'chunk' -- default is 1000
save_integrand -- Save the evaluated value of the integrand at the sample points with the sample point
history_mult -- Number of chunks (of size n) to use in the adaptive histogramming: only useful if there are parameters with adaptation enabled
tempering_exp -- Exponent to raise the weights of the 1-D marginalized histograms for adaptive sampling prior generation, by default it is 0 which will turn off adaptive sampling regardless of other settings
n_adapt -- number of chunks over which to allow the pdf to adapt. Default is zero, which will turn off adaptive sampling regardless of other settings
convergence_tests - dictionary of function pointers, each accepting self._rvs and self.params as arguments. CURRENTLY ONLY USED FOR REPORTING
maxval - Guess at the maximum value of the integrand -- used as a seed for the maxval counter
Pinning a value: By specifying a kwarg with the same of an existing parameter, it is possible to "pin" it. The sample draws will always be that value, and the sampling prior will use a delta function at that value.
"""
#
# Pin values
#
tempcdfdict, temppdfdict, temppriordict, temppdfnormdict = {}, {}, {}, {}
temppdfnormdict = defaultdict(lambda: 1.0)
for p, val in kwargs.iteritems():
if p in self.params:
# Store the previous pdf/cdf in case it's already defined
tempcdfdict[p] = self.cdf_inv[p]
temppdfdict[p] = self.pdf[p]
temppdfnormdict[p] = self._pdf_norm[p]
temppriordict[p] = self.prior_pdf[p]
# Set a new one to always return the same value
self.pdf[p] = functools.partial(delta_func_pdf_vector, val)
self._pdf_norm[p] = 1.0
self.prior_pdf[p] = functools.partial(delta_func_pdf_vector, val)
self.cdf_inv[p] = functools.partial(delta_func_samp_vector, val)
# This is a semi-hack to ensure that the integrand is called with
# the arguments in the right order
# FIXME: How dangerous is this?
args = func.func_code.co_varnames[:func.func_code.co_argcount]
if not MCSampler.match_params_from_args(args, self.params):
raise ValueError("All integrand variables must be represented by integral parameters.")
#
# Determine stopping conditions
#
nmax = kwargs["nmax"] if kwargs.has_key("nmax") else float("inf")
neff = kwargs["neff"] if kwargs.has_key("neff") else numpy.float128("inf")
n = kwargs["n"] if kwargs.has_key("n") else min(1000, nmax)
convergence_tests = kwargs["convergence_tests"] if kwargs.has_key("convergence_tests") else None
#
# Adaptive sampling parameters
#
n_history = int(kwargs["history_mult"]*n) if kwargs.has_key("history_mult") else None
tempering_exp = kwargs["tempering_exp"] if kwargs.has_key("tempering_exp") else 1.0
n_adapt = int(kwargs["n_adapt"]*n) if kwargs.has_key("n_adapt") else 0
nmax = kwargs["nmax"] if kwargs.has_key("nmax") else n_adapt
nmin = kwargs["nmin"] if kwargs.has_key("nmin") else n_adapt
save_intg = kwargs["save_intg"] if kwargs.has_key("save_intg") else False
nkeep = kwargs["save_intg"] if kwargs.has_key("save_intg") else None
# Corner case: we want all the samples, and we don't want them messed
# with, so everything is saved, but no sort is done
if nkeep is True:
nkeep = None
# FIXME: The adaptive step relies on the _rvs cache, so this has to be
# on in order to work
if n_adapt > 0 and tempering_exp > 0.0:
save_intg = True
deltalnL = kwargs['igrand_threshold_deltalnL'] if kwargs.has_key('igrand_threshold_deltalnL') else None # default is to return all
deltaP = kwargs["igrand_threshold_p"] if kwargs.has_key('igrand_threshold_p') else 0 # default is to omit 1e-7 of probability
show_evaluation_log = kwargs['verbose'] if kwargs.has_key('verbose') else False
if show_evaluation_log:
print(" .... mcsampler : providing verbose output ..... ")
int_val1 = numpy.float128(0)
self.ntotal = 0
maxval = kwargs["maxval"] if "maxval" in kwargs else -float("Inf")
old_maxval = maxval
maxlnL = -float("Inf")
eff_samp = 0
mean, var = None, None
last_convergence_test = defaultdict(lambda: False) # initialize record of tests
if show_evaluation_log:
print("walltime : iteration Neff ln(maxweight) lnLmarg ln(Z/Lmax) int_var")
socket = None
while self.ntotal < nmin or (eff_samp < neff and self.ntotal < nmax):
# Draw our sample points
p_s, p_prior, rv = self.draw(n, *self.params)
# Calculate the overall p_s assuming each pdf is independent
joint_p_s = numpy.prod(p_s, axis=0)
joint_p_prior = numpy.prod(p_prior, axis=0)
#
# Prevent zeroes in the sampling prior
#
# FIXME: If we get too many of these, we should bail
if (isinstance(joint_p_s, numpy.ndarray) and any(joint_p_s <= 0)) \
or (not isinstance(joint_p_s, numpy.ndarray) and joint_p_s <= 0):
for p in self.params:
self._rvs[p] = numpy.resize(self._rvs[p], len(self._rvs[p])-n)
print("Zero prior value detected, skipping.", file=sys.stderr)
continue
#
# Unpack rvs and evaluate integrand
#
if len(rv[0].shape) != 1:
rv = rv[0]
params = []
for item in self.params:
if isinstance(item, tuple):
params.extend(item)
else:
params.append(item)
unpacked = numpy.hstack([r.flatten() for r in rv]).reshape(len(args), -1)
unpacked = dict(zip(params, unpacked))
fval = func(**unpacked)
#
# Check if there is any practical contribution to the integral
#
# FIXME: While not technically a fatal error, this will kill the
# adaptive sampling
if fval.sum() == 0:
for p in self.params:
self._rvs[p] = numpy.resize(self._rvs[p], len(self._rvs[p])-n)
print("No contribution to integral, skipping.", file=sys.stderr)
continue
sample_n = numpy.arange(self.ntotal, self.ntotal + len(fval))
if save_intg:
# FIXME: The joint_prior, if not specified is set to one and
# will come out as a scalar here, hence the hack
if not isinstance(joint_p_prior, numpy.ndarray):
joint_p_prior = numpy.ones(fval.shape)*joint_p_prior
# FIXME: See warning at beginning of function. The prior values
# need to be moved out of this, as they are not part of MC
# integration
if self._rvs.has_key("integrand"):
self._rvs["integrand"] = numpy.hstack( (self._rvs["integrand"], fval) )
self._rvs["joint_prior"] = numpy.hstack( (self._rvs["joint_prior"], joint_p_prior) )
self._rvs["joint_s_prior"] = numpy.hstack( (self._rvs["joint_s_prior"], joint_p_s) )
self._rvs["weights"] = numpy.hstack( (self._rvs["weights"], fval*joint_p_prior/joint_p_s) )
self._rvs["sample_n"] = numpy.hstack( (self._rvs["sample_n"], sample_n) )
else:
self._rvs["integrand"] = fval
self._rvs["joint_prior"] = joint_p_prior
self._rvs["joint_s_prior"] = joint_p_s
self._rvs["weights"] = fval*joint_p_prior/joint_p_s
self._rvs["sample_n"] = sample_n
# Calculate the integral over this chunk
int_val = fval * joint_p_prior / joint_p_s
# Calculate max L (a useful convergence feature) for debug
# reporting. Not used for integration
# Try to avoid nan's
maxlnL = numpy.log(numpy.max([numpy.exp(maxlnL), numpy.max(fval),numpy.exp(-100)])) # note if f<0, this will return nearly 0
# Calculate the effective samples via max over the current
# evaluations
maxval = [max(maxval, int_val[0]) if int_val[0] != 0 else maxval]
for v in int_val[1:]:
maxval.append( v if v > maxval[-1] and v != 0 else maxval[-1] )
# running variance
var = statutils.cumvar(int_val, mean, var, self.ntotal)[-1]
# running integral
int_val1 += int_val.sum()
# running number of evaluations
self.ntotal += n
# FIXME: Likely redundant with int_val1
mean = int_val1/self.ntotal
maxval = maxval[-1]
eff_samp = int_val1/maxval
# Throw exception if we get infinity or nan
if math.isnan(eff_samp):
raise NanOrInf("Effective samples = nan")
if maxlnL is float("Inf"):
raise NanOrInf("maxlnL = inf")
if show_evaluation_log:
print("{0:.3f} : {1:d} {2:.5f} {3:.2f} {4:.2f} {5:.2f} {6:.3f}".format(time.time(), self.ntotal, eff_samp, math.log(maxval), numpy.log(int_val1 / self.ntotal), numpy.log(int_val1 / self.ntotal) - maxlnL, numpy.sqrt(var * self.ntotal) / int_val1))
if (not convergence_tests) and self.ntotal >= nmin and self.ntotal >= nmax and neff != float("inf"):
print("WARNING: User requested maximum number of samples reached... bailing.", file=sys.stderr)
#
# Convergence tests
#
if convergence_tests:
converged = True
for key in convergence_tests.keys():
last_convergence_test[key] = convergence_tests[key](self._rvs, self.params)
converged &= las_convergence_test[key]
if convergence_tests and show_evaluation_log: # Print status of each test
for key in convergence_tests:
print(" -- Convergence test status : ", key, last_convergence_test[key])
self._address, self._port = "pcdev2.nemo.phys.uwm.edu", 1890
#if self._address is not None:
if False:
dims = ("distance", "inclination", "right_ascension",
"declination", "integrand", "joint_prior", "joint_s_prior")
send_data = synchlib.prepare_data(self._rvs, dims, self.ntotal - n)
self.socket = synchlib.send_samples(send_data, self._address, self._port, verbose=True, socket=self.socket)
#
# The total number of adaptive steps is reached
#
# FIXME: We need a better stopping condition here
if self.ntotal >= n_adapt and maxval == old_maxval:
# Downsample points
if save_intg and nkeep is not None:
pt_sort = self._rvs["weights"].argsort()[-nkeep:]
for key in self._rvs:
if len(self._rvs[key].shape) > 1:
self._rvs[key] = self._rvs[key][:,pt_sort]
else:
self._rvs[key] = self._rvs[key][pt_sort]
continue
old_maxval = maxval
#
# Iterate through each of the parameters, updating the sampling
# prior PDF according to the 1-D marginalization
#
for itr, p in enumerate(self.params):
# FIXME: The second part of this condition should be made more
# specific to pinned parameters
if p not in self.adaptive or p in kwargs.keys():
continue
points = self._rvs[p][-n_history:]
weights = (self._rvs["integrand"][-n_history:]*self._rvs["joint_prior"][-n_history:])**tempering_exp
self._hist[p] = statutils.get_adaptive_binning(points, (self.llim[p], self.rlim[p]))
for pt, w in zip(points, weights):
self._hist[p][pt,] += w
self._hist[p].array += self._hist[p].array.mean()
rate.filter_array(self._hist[p].array, rate.tophat_window(3))
norm = numpy.sum(self._hist[p].array * self._hist[p].bins.volumes())
self._hist[p].array /= norm
# FIXME: Stupid pet trick while numpy version is lacking
self.pdf[p] = numpy.frompyfunc(rate.InterpBinnedArray(self._hist[p]), 1, 1)
#with open("%s_%d_hist.txt" % (p, self.ntotal), "w") as fout:
#for c, pdf in zip(self._hist[p].centres()[0], self._hist[p].array):
#print >>fout, "%f %g" % (c, pdf)
self.cdf[p] = self.cdf_function(p)
self.cdf_inv[p] = self.cdf_inverse(p)
# If we were pinning any values, undo the changes we did before
self.cdf_inv.update(tempcdfdict)
self.pdf.update(temppdfdict)
self._pdf_norm.update(temppdfnormdict)
self.prior_pdf.update(temppriordict)
# Clean out the _rvs arrays for 'irrelevant' points
# - create the cumulative weights
if "weights" in self._rvs and deltaP > 0:
# Sort the weights with the deltaL masked applied
sorted_weights = self._rvs["weights"].argsort()
# Make the (unnormalized) CDF
total_weight = self._rvs["weights"][sorted_weights].cumsum()
# Find the deltaP cutoff index
idx = numpy.searchsorted(total_weight, deltaP*total_weight[-1], 'left')
sorted_weights = sorted_weights[idx:]
# Remove all samples which contribute to smallest 1e-3 of cumulative
# probability
for key in self._rvs.keys():
if isinstance(key, tuple):
self._rvs[key] = self._rvs[key][:,sorted_weights]
else:
self._rvs[key] = self._rvs[key][sorted_weights]
if "integrand" in self._rvs and deltalnL is not None:
deltal_mask = numpy.log(self._rvs["integrand"]) > (maxlnL - deltalnL)
# Remove all samples which do not have L > maxlnL - deltalnL
for key in self._rvs.keys():
if isinstance(key, tuple):
self._rvs[key] = self._rvs[key][:,deltal_mask]
else:
self._rvs[key] = self._rvs[key][deltal_mask]
# Create extra dictionary to return things
dict_return ={}
if convergence_tests is not None:
dict_return["convergence_test_results"] = last_convergence_test
return int_val1/self.ntotal, var/self.ntotal, eff_samp, dict_return
def finish(self):
snrsq_kernel_width_at_64 = 16.,
chisq_kernel_width = 0.02,
sigma = 10.
for key, lnpdf in self.densities.items():
if key.endswith("_snr2_chi2"):
numsamples = max(lnpdf.array.sum() / 10. + 1., 1e3)
snrsq_bins, chisq_bins = lnpdf.bins
snrsq_per_bin_at_64 = (snrsq_bins.upper() -
snrsq_bins.lower())[snrsq_bins[64.]]
chisq_per_bin_at_0_02 = (chisq_bins.upper() -
chisq_bins.lower())[chisq_bins[0.02]]
# apply Silverman's rule so that the width scales
# with numsamples**(-1./6.) for a 2D PDF
snrsq_kernel_bins = snrsq_kernel_width_at_64 / snrsq_per_bin_at_64 / numsamples**(
1. / 6.)
chisq_kernel_bins = chisq_kernel_width / chisq_per_bin_at_0_02 / numsamples**(
1. / 6.)
# check the size of the kernel. We don't ever let
# it get smaller than the 2.5 times the bin size
if snrsq_kernel_bins < 2.5:
snrsq_kernel_bins = 2.5
warnings.warn("Replacing snrsq kernel bins with 2.5")
if chisq_kernel_bins < 2.5:
chisq_kernel_bins = 2.5
warnings.warn("Replacing chisq kernel bins with 2.5")
# convolve bin count with density estimation kernel
rate.filter_array(
lnpdf.array,
rate.gaussian_window(snrsq_kernel_bins,
chisq_kernel_bins,
sigma=sigma))
# zero everything below the SNR cutoff. need to do the slicing
# ourselves to avoid zero-ing the at-threshold bin
lnpdf.array[:lnpdf.bins[0][self.snr_threshold], :] = 0.
elif key.endswith("_snr2_duration"):
# FIXME the duration filter kernel is left as a guess
rate.filter_array(
lnpdf.array,
rate.gaussian_window(snrsq_kernel_bins, 11, sigma=sigma))
# zero everything below the SNR cutoff. need to do the slicing
# ourselves to avoid zero-ing the at-threshold bin
lnpdf.array[:lnpdf.bins[0][self.snr_threshold], :] = 0.
else:
# shouldn't get here
raise Exception
lnpdf.normalize()
self.mkinterps()
#
# never allow PDFs that have had the density estimation
# transform applied to be written to disk: on-disk files
# must only ever provide raw counts. also don't allow
# density estimation to be applied twice
#
def to_xml(*args, **kwargs):
raise NotImplementedError(
"writing .finish()'ed LnLRDensity object to disk is forbidden")
self.to_xml = to_xml
def finish(*args, **kwargs):
raise NotImplementedError(
".finish()ing a .finish()ed LnLRDensity object is forbidden")
self.finish = finish
def twoD_SearchVolume(self,
instruments,
dbin=None,
FAR=None,
bootnum=None,
derr=0.197,
dsys=0.074):
"""
Compute the search volume in the mass/mass plane, bootstrap
and measure the first and second moment (assumes the underlying
distribution can be characterized by those two parameters)
This is gonna be brutally slow
derr = (0.134**2+.103**2+.102**2)**.5 = 0.197 which is the 3 detector
calibration uncertainty in quadrature. This is conservative since some injections
will be H1L1 and have a lower error of .17
the dsys is the DC offset which is the max offset of .074.
"""
if not FAR: FAR = self.far[instruments]
found, missed = self.get_injections(instruments, FAR)
twodbin = self.twoDMassBins
wnfunc = self.gw
livetime = self.livetime[instruments]
if not bootnum: bootnum = self.bootnum
if wnfunc:
wnfunc /= wnfunc[(wnfunc.shape[0] - 1) / 2,
(wnfunc.shape[1] - 1) / 2]
x = twodbin.shape[0]
y = twodbin.shape[1]
z = int(self.opts.dist_bins)
rArrays = []
volArray = rate.BinnedArray(twodbin)
volArray2 = rate.BinnedArray(twodbin)
#set up ratio arrays for each distance bin
for k in range(z):
rArrays.append(rate.BinnedRatios(twodbin))
# Bootstrap to account for errors
for n in range(bootnum):
#initialize by setting these to zero
for k in range(z):
rArrays[k].numerator.array = numpy.zeros(
rArrays[k].numerator.bins.shape)
rArrays[k].denominator.array = numpy.zeros(
rArrays[k].numerator.bins.shape)
#Scramble the inj population and distances
if bootnum > 1:
sm, sf = self._scramble_pop(missed, found)
# I make a separate array of distances to speed up this calculation
f_dist = self._scramble_dist(sf, derr, dsys)
else:
sm, sf = missed, found
f_dist = numpy.array([l.distance for l in found])
# compute the distance bins
if not dbin:
dbin = rate.LogarithmicBins(min(f_dist), max(f_dist), z)
#else: print dbin.centres()
# get rid of all missed injections outside the distance bins
# to prevent binning errors
sm, m_dist = self.cut_distance(sm, dbin)
sf, f_dist = self.cut_distance(sf, dbin)
for i, l in enumerate(sf): #found:
tbin = rArrays[dbin[f_dist[i]]]
tbin.incnumerator((l.mass1, l.mass2))
for i, l in enumerate(sm): #missed:
tbin = rArrays[dbin[m_dist[i]]]
tbin.incdenominator((l.mass1, l.mass2))
tmpArray2 = rate.BinnedArray(
twodbin) #start with a zero array to compute the mean square
for k in range(z):
tbins = rArrays[k]
tbins.denominator.array += tbins.numerator.array
if wnfunc: rate.filter_array(tbins.denominator.array, wnfunc)
if wnfunc: rate.filter_array(tbins.numerator.array, wnfunc)
tbins.regularize()
# logarithmic(d)
integrand = 4.0 * pi * tbins.ratio() * dbin.centres(
)[k]**3 * dbin.delta
volArray.array += integrand
tmpArray2.array += integrand #4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
print(
"bootstrapping:\t%.1f%% and Calculating smoothed volume:\t%.1f%%\r"
% ((100.0 * n / bootnum), (100.0 * k / z)),
end=' ',
file=sys.stderr)
tmpArray2.array *= tmpArray2.array
volArray2.array += tmpArray2.array
print("", file=sys.stderr)
#Mean and variance
volArray.array /= bootnum
volArray2.array /= bootnum
volArray2.array -= volArray.array**2 # Variance
volArray.array *= livetime
volArray2.array *= livetime * livetime # this gets two powers of live time
return volArray, volArray2
height = 8.0 # inches
fig = figure.Figure()
canvas = FigureCanvas(fig)
fig.set_size_inches(width, height)
fig.gca().set_position([
border[0] / width, border[1] / height,
(width - border[0] - border[2]) / width,
(height - border[1] - border[3]) / height
])
return fig
fig = newfig(options.segment)
axes = fig.gca()
rate.filter_array(trigger_rate.array,
rate.gaussian_window(bins_per_filterwidth))
axes.plot(trigger_rate.centres()[0], trigger_rate.at_centres())
axes.set_xlim(list(options.segment))
axes.grid(True)
for seg in ~seglist & segments.segmentlist([options.segment]):
axes.axvspan(seg[0], seg[1], facecolor="k", alpha=0.2)
axes.set_title(
"%s Excess Power Trigger Rate vs. Time\n(%d Triggers, %g s Moving Average)"
% (options.instrument, num_triggers, options.window))
ticks = make_xticks(options.segment)
axes.set_xticks(ticks[0])