def get_volume_derivative(self,instruments):
injfnames = self.inj_fnames
FAR = self.far[instruments]
zero_lag_segments = self.zero_lag_segments[instruments]
gw = self.gw
twoDMassBins = self.twoDMassBins
#determine binning up front for infinite FAR
found, missed = self.get_injections(instruments, FAR=float("inf"))
dbin = rate.LogarithmicBins(min([l.distance for l in found]),max([l.distance for l in found]), int(self.opts.dist_bins))
livetime = float(abs(zero_lag_segments))
FARh = FAR*100000
FARl = FAR*0.001
nbins = 5
FARS = rate.LogarithmicBins(FARl, FARh, nbins)
vA = []
vA2 = []
for far in FARS.centres():
print >>sys.stderr, "computing volume at FAR " + str(far)
vAt, vA2t = self.twoD_SearchVolume(instruments, dbin=dbin, FAR = far, bootnum=1)
# we need to compute derivitive of log according to ul paper
vAt.array = scipy.log10(vAt.array + 0.001)
vA.append(vAt)
# the derivitive is calcuated with respect to FAR * t
FARTS = rate.LogarithmicBins(FARl * livetime, FARh * livetime, nbins)
return self._derivitave_fit(FARTS, FAR * livetime, vA, twoDMassBins)
def _bin_events(self, binning = None):
# called internally by finish()
if binning is None:
minx, maxx = min(self.injected_x), max(self.injected_x)
miny, maxy = min(self.injected_y), max(self.injected_y)
binning = rate.NDBins((rate.LogarithmicBins(minx, maxx, 256), rate.LogarithmicBins(miny, maxy, 256)))
self.efficiency = rate.BinnedRatios(binning)
for xy in zip(self.injected_x, self.injected_y):
self.efficiency.incdenominator(xy)
for xy in zip(self.found_x, self.found_y):
self.efficiency.incnumerator(xy)
# 1 / error^2 is the number of injections that need to be
# within the window in order for the fractional uncertainty
# in that number to be = error. multiplying by
# bins_per_inj tells us how many bins the window needs to
# cover, and taking the square root translates that into
# the window's length on a side in bins. because the
# contours tend to run parallel to the x axis, the window
# is dilated in that direction to improve resolution.
bins_per_inj = self.efficiency.used() / float(len(self.injected_x))
self.window_size_x = self.window_size_y = math.sqrt(bins_per_inj / self.error**2)
self.window_size_x *= math.sqrt(2)
self.window_size_y /= math.sqrt(2)
if self.window_size_x > 100 or self.window_size_y > 100:
# program will take too long to run
raise ValueError("smoothing filter too large (not enough injections)")
print >>sys.stderr, "The smoothing window for %s is %g x %g bins" % ("+".join(self.instruments), self.window_size_x, self.window_size_y),
print >>sys.stderr, "which is %g%% x %g%% of the binning" % (100.0 * self.window_size_x / binning[0].n, 100.0 * self.window_size_y / binning[1].n)
def get_volume_derivative(injfnames, twoDMassBins, dBin, FAR,
zero_lag_segments, gw):
if (FAR == 0):
print "\n\nFAR = 0\n \n"
# FIXME lambda = ~inf if loudest event is above loudest timeslide?
output = rate.BinnedArray(twoDMassBins)
output.array = 10**6 * numpy.ones(output.array.shape)
return output
livetime = float(abs(zero_lag_segments))
FARh = FAR * 100000
FARl = FAR * 0.001
nbins = 5
FARS = rate.LogarithmicBins(FARl, FARh, nbins)
vA = []
vA2 = []
for far in FARS.centres():
m, f = get_injections(injfnames, far, zero_lag_segments)
print >> sys.stderr, "computing volume at FAR " + str(far)
vAt, vA2t = twoD_SearchVolume(f, m, twoDMassBins, dBin, gw, livetime,
1)
# we need to compute derivitive of log according to ul paper
vAt.array = scipy.log10(vAt.array + 0.001)
vA.append(vAt)
# the derivitive is calcuated with respect to FAR * t
FARS = rate.LogarithmicBins(FARl * livetime, FARh * livetime, nbins)
return derivitave_fit(FARS, FAR * livetime, vA, twoDMassBins)
def make_binning(plots): plots = [plot for instrument in plots.keys() for plot in plots[instrument] if isinstance(plot, SimBurstUtils.Efficiency_hrss_vs_freq)] if not plots: return None minx = min([min(plot.injected_x) for plot in plots]) maxx = max([max(plot.injected_x) for plot in plots]) miny = min([min(plot.injected_y) for plot in plots]) maxy = max([max(plot.injected_y) for plot in plots]) return rate.NDBins((rate.LogarithmicBins(minx, maxx, 512), rate.LogarithmicBins(miny, maxy, 512)))
def __init__(self, x_instrument, y_instrument, magnitude, desc,
min_magnitude, max_magnitude):
self.fig, self.axes = SnglBurstUtils.make_burst_plot(
"%s %s" % (x_instrument, desc), "%s %s" % (y_instrument, desc))
self.fig.set_size_inches(6, 6)
self.axes.loglog()
self.x_instrument = x_instrument
self.y_instrument = y_instrument
self.magnitude = magnitude
self.foreground_x = []
self.foreground_y = []
self.n_foreground = 0
self.n_background = 0
self.n_injections = 0
self.foreground_bins = rate.BinnedArray(
rate.NDBins(
(rate.LogarithmicBins(min_magnitude, max_magnitude, 1024),
rate.LogarithmicBins(min_magnitude, max_magnitude, 1024))))
self.background_bins = rate.BinnedArray(
rate.NDBins(
(rate.LogarithmicBins(min_magnitude, max_magnitude, 1024),
rate.LogarithmicBins(min_magnitude, max_magnitude, 1024))))
self.coinc_injection_bins = rate.BinnedArray(
rate.NDBins(
(rate.LogarithmicBins(min_magnitude, max_magnitude, 1024),
rate.LogarithmicBins(min_magnitude, max_magnitude, 1024))))
self.incomplete_coinc_injection_bins = rate.BinnedArray(
rate.NDBins(
(rate.LogarithmicBins(min_magnitude, max_magnitude, 1024),
rate.LogarithmicBins(min_magnitude, max_magnitude, 1024))))
def get_distance_bins(self, instruments, found=None, missed=None):
if not found and not missed: found, missed = self.get_injections(instruments)
if not found:
print >>sys.stderr,"Found no injections cannot compute distance bins ABORTING"
sys.exit(1)
#Give the bins some padding based on the errors
maxdist = max([s.distance for s in found])
mindist = min([s.distance for s in found])
if (maxdist < 0) or (mindist < 0) or (mindist > maxdist):
print >>sys.stderr, "minimum and maximum distances are screwy, maybe the distance errors given in the options don't make sense? ABORTING"
sys.exit(1)
self.dBin[instruments] = rate.LogarithmicBins(mindist,maxdist,self.opts.dist_bins)
def plotspectrogram(sequencelist, outfile, epoch=0, deltaT=1, f0=0, deltaF=1,\
t0=0, ydata=None, **kwargs):
"""
Plots a list of REAL?VectorSequences on a time-frequency-amplitude colour
map. The epochand deltaT arguments define the spacing in the x direction
for the given VectorSequence, and similarly f0 and deltaF in the
y-direction. If a list of VectorSequences is given, any of these arguments
can be in list form, one for each sequence.
ydata can be given as to explicitly define the frequencies at which the
sequences are sampled.
"""
# construct list of series
if not hasattr(sequencelist, "__contains__"):
sequencelist = [sequencelist]
numseq = len(sequencelist)
# format variables
if isinstance(epoch, numbers.Number) or isinstance(epoch, lal.LIGOTimeGPS):
epoch = [epoch] * numseq
epoch = map(float, epoch)
if not len(epoch) == numseq:
raise ValueError("Wrong number of epoch arguments given.")
if isinstance(deltaT, numbers.Number):
deltaT = [deltaT] * numseq
deltaT = map(float, deltaT)
if not len(deltaT) == numseq:
raise ValueError("Wrong number of deltaT arguments given.")
if not ydata is None:
if isinstance(f0, numbers.Number):
f0 = [f0] * numseq
f0 = map(float, f0)
if not len(f0) == numseq:
raise ValueError("Wrong number of f0 arguments given.")
if isinstance(deltaF, numbers.Number):
deltaF = [deltaF] * numseq
deltaF = map(float, deltaF)
if not len(deltaF) == numseq:
raise ValueError("Wrong number of deltaF arguments given.")
# get limits
xlim = kwargs.pop("xlim", None)
ylim = kwargs.pop("ylim", None)
colorlim = kwargs.pop("colorlim", None)
if xlim:
start, end = xlim
else:
start = min(epoch)
end = max(e + l.length * dt\
for e,l,dt in zip(epoch, sequencelist, deltaT))
if not ydata is None and not ylim:
ylim = [ydata.min(), ydata.max()]
# get axis scales
logx = kwargs.pop("logx", False)
logy = kwargs.pop("logy", False)
logcolor = kwargs.pop("logcolor", False)
# get legend loc
loc = kwargs.pop("loc", 0)
alpha = kwargs.pop("alpha", 0.8)
# get colorbar options
hidden_colorbar = kwargs.pop("hidden_colorbar", False)
# get savefig option
bbox_inches = kwargs.pop("bbox_inches", None)
#
# get labels
#
xlabel = kwargs.pop("xlabel", None)
if xlabel:
unit = 1
if not xlabel:
unit, timestr = plotutils.time_axis_unit(end - start)
if not t0:
t0 = start
t0 = lal.LIGOTimeGPS(t0)
if int(t0.gpsNanoSeconds) == 0:
xlabel = datetime.datetime(*lal.GPSToUTC(int(t0))[:6])\
.strftime("%B %d %Y, %H:%M:%S %ZUTC")
xlabel = "Time (%s) since %s (%s)" % (timestr, xlabel, int(t0))
else:
xlabel = datetime.datetime(*lal.GPSToUTC(t0.gpsSeconds)[:6])\
.strftime("%B %d %Y, %H:%M:%S %ZUTC")
xlabel = "Time (%s) since %s (%s)"\
% (timestr,
xlabel.replace(" UTC",".%.3s UTC" % t0.gpsNanoSeconds),\
t0)
t0 = float(t0)
ylabel = kwargs.pop("ylabel", "Frequency (Hz)")
colorlabel = kwargs.pop("colorlabel", "Amplitude")
title = kwargs.pop("title", "")
subtitle = kwargs.pop("subtitle", "")
#
# restrict data to the correct limits for plotting
#
interpolate = logy and ydata is None
for i, sequence in enumerate(sequencelist):
if interpolate:
# interpolate the data onto a log-scale
sequence, ydata = loginterpolate(sequence, f0[i], deltaF[i])
if logy and ylim:
plotted = (ydata > ylim[0]) & (ydata <= ylim[1])
newVectorLength = int(plotted.sum())
newsequence = lal.CreateREAL8VectorSequence(sequence.length,\
newVectorLength)
for j in range(sequence.length):
newsequence.data[j, :] = sequence.data[j, :][plotted]
del sequence
sequencelist[i] = newsequence
if len(sequencelist) and logy and ylim:
ydata = ydata[plotted]
#
# format bins
#
xbins = []
for i in range(numseq):
xmin = epoch[i]
xmax = epoch[i] + sequencelist[i].length * deltaT[i]
xbins.append(rate.LinearBins(float(xmin-t0)/unit, float(xmax-t0)/unit,\
2))
ybins = []
for i in range(numseq):
if ydata is not None:
ydata = numpy.asarray(ydata)
ymin = ydata.min()
ymax = ydata.max()
else:
ymin = f0[i]
ymax = f0[i] + sequencelist[i].vectorLength * deltaF[i]
if logy:
if ymin == 0:
ymin = deltaF[i]
ybins.append(rate.LogarithmicBins(ymin, ymax, 2))
else:
ybins.append(rate.LinearBins(ymin, ymax, 2))
#
# plot
#
kwargs.setdefault("interpolation", "kaiser")
plot = plotutils.ImagePlot(xlabel=xlabel, ylabel=ylabel, title=title,\
subtitle=subtitle, colorlabel=colorlabel)
for sequence, x, y in zip(sequencelist, xbins, ybins):
data = numpy.ma.masked_where(numpy.isnan(sequence.data), sequence.data,\
copy=False)
plot.add_content(data.T, x, y, **kwargs)
# finalize
plot.finalize(colorbar=True, logcolor=logcolor, minorticks=True,\
clim=colorlim)
if hidden_colorbar:
plotutils.add_colorbar(plot.ax, visible=False)
# set logscale
if logx:
plot.ax.xaxis.set_scale("log")
if logy:
plot.ax.yaxis.set_scale("log")
plot.ax._update_transScale()
# format axes
if xlim:
xlim = (numpy.asarray(xlim).astype(float) - t0) / unit
plot.ax.set_xlim(xlim)
if ylim:
plot.ax.set_ylim(ylim)
# set grid and ticks
plot.ax.grid(True, which="both")
plotutils.set_time_ticks(plot.ax)
plotutils.set_minor_ticks(plot.ax)
# save and close
plot.savefig(outfile, bbox_inches=bbox_inches,\
bbox_extra_artists=plot.ax.texts)
plot.close()
def twoD_SearchVolume(self, instruments, dbin=None, FAR=None, bootnum=None, derr=0.197, dsys=0.074):
"""
Compute the search volume in the mass/mass plane, bootstrap
and measure the first and second moment (assumes the underlying
distribution can be characterized by those two parameters)
This is gonna be brutally slow
derr = (0.134**2+.103**2+.102**2)**.5 = 0.197 which is the 3 detector
calibration uncertainty in quadrature. This is conservative since some injections
will be H1L1 and have a lower error of .17
the dsys is the DC offset which is the max offset of .074.
"""
if not FAR: FAR = self.far[instruments]
found, missed = self.get_injections(instruments, FAR)
twodbin = self.twoDMassBins
wnfunc = self.gw
livetime = self.livetime[instruments]
if not bootnum: bootnum = self.bootnum
if wnfunc: wnfunc /= wnfunc[(wnfunc.shape[0]-1) / 2, (wnfunc.shape[1]-1) / 2]
x = twodbin.shape[0]
y = twodbin.shape[1]
z = int(self.opts.dist_bins)
rArrays = []
volArray=rate.BinnedArray(twodbin)
volArray2=rate.BinnedArray(twodbin)
#set up ratio arrays for each distance bin
for k in range(z):
rArrays.append(rate.BinnedRatios(twodbin))
# Bootstrap to account for errors
for n in range(bootnum):
#initialize by setting these to zero
for k in range(z):
rArrays[k].numerator.array = numpy.zeros(rArrays[k].numerator.bins.shape)
rArrays[k].denominator.array = numpy.zeros(rArrays[k].numerator.bins.shape)
#Scramble the inj population and distances
if bootnum > 1:
sm, sf = self._scramble_pop(missed, found)
# I make a separate array of distances to speed up this calculation
f_dist = self._scramble_dist(sf, derr, dsys)
else:
sm, sf = missed, found
f_dist = numpy.array([l.distance for l in found])
# compute the distance bins
if not dbin:
dbin = rate.LogarithmicBins(min(f_dist),max(f_dist), z)
#else: print dbin.centres()
# get rid of all missed injections outside the distance bins
# to prevent binning errors
sm, m_dist = self.cut_distance(sm, dbin)
sf, f_dist = self.cut_distance(sf, dbin)
for i, l in enumerate(sf):#found:
tbin = rArrays[dbin[f_dist[i]]]
tbin.incnumerator( (l.mass1, l.mass2) )
for i, l in enumerate(sm):#missed:
tbin = rArrays[dbin[m_dist[i]]]
tbin.incdenominator( (l.mass1, l.mass2) )
tmpArray2=rate.BinnedArray(twodbin) #start with a zero array to compute the mean square
for k in range(z):
tbins = rArrays[k]
tbins.denominator.array += tbins.numerator.array
if wnfunc: rate.filter_array(tbins.denominator.array,wnfunc)
if wnfunc: rate.filter_array(tbins.numerator.array,wnfunc)
tbins.regularize()
# logarithmic(d)
integrand = 4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
volArray.array += integrand
tmpArray2.array += integrand #4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta
print >>sys.stderr, "bootstrapping:\t%.1f%% and Calculating smoothed volume:\t%.1f%%\r" % ((100.0 * n / bootnum), (100.0 * k / z)),
tmpArray2.array *= tmpArray2.array
volArray2.array += tmpArray2.array
print >>sys.stderr, ""
#Mean and variance
volArray.array /= bootnum
volArray2.array /= bootnum
volArray2.array -= volArray.array**2 # Variance
volArray.array *= livetime
volArray2.array *= livetime*livetime # this gets two powers of live time
return volArray, volArray2
else: Found = get_burst_injections(opts.burst_found) Missed = get_burst_injections(opts.burst_missed) # restrict the sims to a distance range Found = cut_distance(Found, 1, max_dist) Missed = cut_distance(Missed, 1, max_dist) # get a 2D mass binning twoDMassBins = get_2d_mass_bins(min_mass, max_mass, mass_bins) # get log distance bins dBin = rate.LogarithmicBins(0.1,max_dist*1.25,dist_bins) # Someday we could try a Gaussian smoothing function #gw = rate.gaussian_window2d(2,2,8) gw = None #Get derivative of volume with respect to FAR #dvA = get_volume_derivative(opts.injfnames, twoDMassBins, dBin, FAR, zero_lag_segments, gw) vA, vA2 = twoD_SearchVolume(Found, Missed, twoDMassBins, dBin, gw, 1.0, bootnum=int(opts.bootstrap_iterations)) # FIXME convert to years (use some lal or pylal thing in the future) #vA.array /= secs_in_year #vA2.array /= secs_in_year * secs_in_year #two powers for this squared quantity #Trim the array to have sane values outside the total mass area of interest
def finish(self):
fig, axes = SnglBurstUtils.make_burst_plot(r"Injection Amplitude (\(\mathrm{s}^{-\frac{1}{3}}\))", "Detection Efficiency", width = 108.0)
axes.set_title(r"Detection Efficiency vs.\ Amplitude")
axes.semilogx()
axes.set_position([0.10, 0.150, 0.86, 0.77])
# set desired yticks
axes.set_yticks((0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0))
axes.set_yticklabels((r"\(0\)", r"\(0.1\)", r"\(0.2\)", r"\(0.3\)", r"\(0.4\)", r"\(0.5\)", r"\(0.6\)", r"\(0.7\)", r"\(0.8\)", r"\(0.9\)", r"\(1.0\)"))
axes.xaxis.grid(True, which = "major,minor")
axes.yaxis.grid(True, which = "major,minor")
# put made and found injections in the denominators and
# numerators of the efficiency bins
bins = rate.NDBins((rate.LogarithmicBins(min(sim.amplitude for sim in self.all), max(sim.amplitude for sim in self.all), 400),))
efficiency_num = rate.BinnedArray(bins)
efficiency_den = rate.BinnedArray(bins)
for sim in self.found:
efficiency_num[sim.amplitude,] += 1
for sim in self.all:
efficiency_den[sim.amplitude,] += 1
# generate and plot trend curves. adjust window function
# normalization so that denominator array correctly
# represents the number of injections contributing to each
# bin: make w(0) = 1.0. note that this factor has no
# effect on the efficiency because it is common to the
# numerator and denominator arrays. we do this for the
# purpose of computing the Poisson error bars, which
# requires us to know the counts for the bins
windowfunc = rate.gaussian_window(self.filter_width)
windowfunc /= windowfunc[len(windowfunc) / 2 + 1]
rate.filter_binned_array(efficiency_num, windowfunc)
rate.filter_binned_array(efficiency_den, windowfunc)
# regularize: adjust unused bins so that the efficiency is
# 0, not NaN
assert (efficiency_num <= efficiency_den).all()
efficiency_den[efficiency_num == 0 & efficiency_den == 0] = 1
line1, A50, A50_err = render_data_from_bins(file("string_efficiency.dat", "w"), axes, efficiency_num, efficiency_den, self.cal_uncertainty, self.filter_width, colour = "k", linestyle = "-", erroralpha = 0.2)
print >>sys.stderr, "Pipeline's 50%% efficiency point for all detections = %g +/- %g%%\n" % (A50, A50_err * 100)
# add a legend to the axes
axes.legend((line1,), (r"\noindent Injections recovered with $\Lambda > %s$" % SnglBurstUtils.latexnumber("%.2e" % self.detection_threshold),), loc = "lower right")
# adjust limits
axes.set_xlim([1e-21, 2e-18])
axes.set_ylim([0.0, 1.0])
#
# dump some information about the highest-amplitude missed
# and quietest-amplitude found injections
#
self.loudest_missed.sort(reverse = True)
self.quietest_found.sort(reverse = True)
f = file("string_loud_missed_injections.txt", "w")
print >>f, "Highest Amplitude Missed Injections"
print >>f, "==================================="
for amplitude, sim, offsetvector, filename, likelihood_ratio in self.loudest_missed:
print >>f
print >>f, "%s in %s:" % (str(sim.simulation_id), filename)
if likelihood_ratio is None:
print >>f, "Not recovered"
else:
print >>f, "Recovered with \\Lambda = %.16g, detection threshold was %.16g" % (likelihood_ratio, self.detection_threshold)
for instrument in self.seglists:
print >>f, "In %s:" % instrument
print >>f, "\tInjected amplitude:\t%.16g" % SimBurstUtils.string_amplitude_in_instrument(sim, instrument, offsetvector)
print >>f, "\tTime of injection:\t%s s" % sim.time_at_instrument(instrument, offsetvector)
print >>f, "Amplitude in waveframe:\t%.16g" % sim.amplitude
t = sim.get_time_geocent()
print >>f, "Time at geocentre:\t%s s" % t
print >>f, "Segments within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.seglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.seglists))
print >>f, "Vetoes within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.vetoseglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.vetoseglists))
f = file("string_quiet_found_injections.txt", "w")
print >>f, "Lowest Amplitude Found Injections"
print >>f, "================================="
for inv_amplitude, sim, offsetvector, filename, likelihood_ratio in self.quietest_found:
print >>f
print >>f, "%s in %s:" % (str(sim.simulation_id), filename)
if likelihood_ratio is None:
print >>f, "Not recovered"
else:
print >>f, "Recovered with \\Lambda = %.16g, detection threshold was %.16g" % (likelihood_ratio, self.detection_threshold)
for instrument in self.seglists:
print >>f, "In %s:" % instrument
print >>f, "\tInjected amplitude:\t%.16g" % SimBurstUtils.string_amplitude_in_instrument(sim, instrument, offsetvector)
print >>f, "\tTime of injection:\t%s s" % sim.time_at_instrument(instrument, offsetvector)
print >>f, "Amplitude in waveframe:\t%.16g" % sim.amplitude
t = sim.get_time_geocent()
print >>f, "Time at geocentre:\t%s s" % t
print >>f, "Segments within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.seglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.seglists))
print >>f, "Vetoes within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.vetoseglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.vetoseglists))
#
# done
#
return fig,