def get_volume_derivative(self,instruments): injfnames = self.inj_fnames FAR = self.far[instruments] zero_lag_segments = self.zero_lag_segments[instruments] gw = self.gw twoDMassBins = self.twoDMassBins #determine binning up front for infinite FAR found, missed = self.get_injections(instruments, FAR=float("inf")) dbin = rate.LogarithmicBins(min([l.distance for l in found]),max([l.distance for l in found]), int(self.opts.dist_bins)) livetime = float(abs(zero_lag_segments)) FARh = FAR*100000 FARl = FAR*0.001 nbins = 5 FARS = rate.LogarithmicBins(FARl, FARh, nbins) vA = [] vA2 = [] for far in FARS.centres(): print >>sys.stderr, "computing volume at FAR " + str(far) vAt, vA2t = self.twoD_SearchVolume(instruments, dbin=dbin, FAR = far, bootnum=1) # we need to compute derivitive of log according to ul paper vAt.array = scipy.log10(vAt.array + 0.001) vA.append(vAt) # the derivitive is calcuated with respect to FAR * t FARTS = rate.LogarithmicBins(FARl * livetime, FARh * livetime, nbins) return self._derivitave_fit(FARTS, FAR * livetime, vA, twoDMassBins)
def _bin_events(self, binning = None): # called internally by finish() if binning is None: minx, maxx = min(self.injected_x), max(self.injected_x) miny, maxy = min(self.injected_y), max(self.injected_y) binning = rate.NDBins((rate.LogarithmicBins(minx, maxx, 256), rate.LogarithmicBins(miny, maxy, 256))) self.efficiency = rate.BinnedRatios(binning) for xy in zip(self.injected_x, self.injected_y): self.efficiency.incdenominator(xy) for xy in zip(self.found_x, self.found_y): self.efficiency.incnumerator(xy) # 1 / error^2 is the number of injections that need to be # within the window in order for the fractional uncertainty # in that number to be = error. multiplying by # bins_per_inj tells us how many bins the window needs to # cover, and taking the square root translates that into # the window's length on a side in bins. because the # contours tend to run parallel to the x axis, the window # is dilated in that direction to improve resolution. bins_per_inj = self.efficiency.used() / float(len(self.injected_x)) self.window_size_x = self.window_size_y = math.sqrt(bins_per_inj / self.error**2) self.window_size_x *= math.sqrt(2) self.window_size_y /= math.sqrt(2) if self.window_size_x > 100 or self.window_size_y > 100: # program will take too long to run raise ValueError("smoothing filter too large (not enough injections)") print >>sys.stderr, "The smoothing window for %s is %g x %g bins" % ("+".join(self.instruments), self.window_size_x, self.window_size_y), print >>sys.stderr, "which is %g%% x %g%% of the binning" % (100.0 * self.window_size_x / binning[0].n, 100.0 * self.window_size_y / binning[1].n)
def get_volume_derivative(injfnames, twoDMassBins, dBin, FAR, zero_lag_segments, gw): if (FAR == 0): print "\n\nFAR = 0\n \n" # FIXME lambda = ~inf if loudest event is above loudest timeslide? output = rate.BinnedArray(twoDMassBins) output.array = 10**6 * numpy.ones(output.array.shape) return output livetime = float(abs(zero_lag_segments)) FARh = FAR * 100000 FARl = FAR * 0.001 nbins = 5 FARS = rate.LogarithmicBins(FARl, FARh, nbins) vA = [] vA2 = [] for far in FARS.centres(): m, f = get_injections(injfnames, far, zero_lag_segments) print >> sys.stderr, "computing volume at FAR " + str(far) vAt, vA2t = twoD_SearchVolume(f, m, twoDMassBins, dBin, gw, livetime, 1) # we need to compute derivitive of log according to ul paper vAt.array = scipy.log10(vAt.array + 0.001) vA.append(vAt) # the derivitive is calcuated with respect to FAR * t FARS = rate.LogarithmicBins(FARl * livetime, FARh * livetime, nbins) return derivitave_fit(FARS, FAR * livetime, vA, twoDMassBins)
def make_binning(plots): plots = [plot for instrument in plots.keys() for plot in plots[instrument] if isinstance(plot, SimBurstUtils.Efficiency_hrss_vs_freq)] if not plots: return None minx = min([min(plot.injected_x) for plot in plots]) maxx = max([max(plot.injected_x) for plot in plots]) miny = min([min(plot.injected_y) for plot in plots]) maxy = max([max(plot.injected_y) for plot in plots]) return rate.NDBins((rate.LogarithmicBins(minx, maxx, 512), rate.LogarithmicBins(miny, maxy, 512)))
def __init__(self, x_instrument, y_instrument, magnitude, desc, min_magnitude, max_magnitude): self.fig, self.axes = SnglBurstUtils.make_burst_plot( "%s %s" % (x_instrument, desc), "%s %s" % (y_instrument, desc)) self.fig.set_size_inches(6, 6) self.axes.loglog() self.x_instrument = x_instrument self.y_instrument = y_instrument self.magnitude = magnitude self.foreground_x = [] self.foreground_y = [] self.n_foreground = 0 self.n_background = 0 self.n_injections = 0 self.foreground_bins = rate.BinnedArray( rate.NDBins( (rate.LogarithmicBins(min_magnitude, max_magnitude, 1024), rate.LogarithmicBins(min_magnitude, max_magnitude, 1024)))) self.background_bins = rate.BinnedArray( rate.NDBins( (rate.LogarithmicBins(min_magnitude, max_magnitude, 1024), rate.LogarithmicBins(min_magnitude, max_magnitude, 1024)))) self.coinc_injection_bins = rate.BinnedArray( rate.NDBins( (rate.LogarithmicBins(min_magnitude, max_magnitude, 1024), rate.LogarithmicBins(min_magnitude, max_magnitude, 1024)))) self.incomplete_coinc_injection_bins = rate.BinnedArray( rate.NDBins( (rate.LogarithmicBins(min_magnitude, max_magnitude, 1024), rate.LogarithmicBins(min_magnitude, max_magnitude, 1024))))
def get_distance_bins(self, instruments, found=None, missed=None): if not found and not missed: found, missed = self.get_injections(instruments) if not found: print >>sys.stderr,"Found no injections cannot compute distance bins ABORTING" sys.exit(1) #Give the bins some padding based on the errors maxdist = max([s.distance for s in found]) mindist = min([s.distance for s in found]) if (maxdist < 0) or (mindist < 0) or (mindist > maxdist): print >>sys.stderr, "minimum and maximum distances are screwy, maybe the distance errors given in the options don't make sense? ABORTING" sys.exit(1) self.dBin[instruments] = rate.LogarithmicBins(mindist,maxdist,self.opts.dist_bins)
def plotspectrogram(sequencelist, outfile, epoch=0, deltaT=1, f0=0, deltaF=1,\ t0=0, ydata=None, **kwargs): """ Plots a list of REAL?VectorSequences on a time-frequency-amplitude colour map. The epochand deltaT arguments define the spacing in the x direction for the given VectorSequence, and similarly f0 and deltaF in the y-direction. If a list of VectorSequences is given, any of these arguments can be in list form, one for each sequence. ydata can be given as to explicitly define the frequencies at which the sequences are sampled. """ # construct list of series if not hasattr(sequencelist, "__contains__"): sequencelist = [sequencelist] numseq = len(sequencelist) # format variables if isinstance(epoch, numbers.Number) or isinstance(epoch, lal.LIGOTimeGPS): epoch = [epoch] * numseq epoch = map(float, epoch) if not len(epoch) == numseq: raise ValueError("Wrong number of epoch arguments given.") if isinstance(deltaT, numbers.Number): deltaT = [deltaT] * numseq deltaT = map(float, deltaT) if not len(deltaT) == numseq: raise ValueError("Wrong number of deltaT arguments given.") if not ydata is None: if isinstance(f0, numbers.Number): f0 = [f0] * numseq f0 = map(float, f0) if not len(f0) == numseq: raise ValueError("Wrong number of f0 arguments given.") if isinstance(deltaF, numbers.Number): deltaF = [deltaF] * numseq deltaF = map(float, deltaF) if not len(deltaF) == numseq: raise ValueError("Wrong number of deltaF arguments given.") # get limits xlim = kwargs.pop("xlim", None) ylim = kwargs.pop("ylim", None) colorlim = kwargs.pop("colorlim", None) if xlim: start, end = xlim else: start = min(epoch) end = max(e + l.length * dt\ for e,l,dt in zip(epoch, sequencelist, deltaT)) if not ydata is None and not ylim: ylim = [ydata.min(), ydata.max()] # get axis scales logx = kwargs.pop("logx", False) logy = kwargs.pop("logy", False) logcolor = kwargs.pop("logcolor", False) # get legend loc loc = kwargs.pop("loc", 0) alpha = kwargs.pop("alpha", 0.8) # get colorbar options hidden_colorbar = kwargs.pop("hidden_colorbar", False) # get savefig option bbox_inches = kwargs.pop("bbox_inches", None) # # get labels # xlabel = kwargs.pop("xlabel", None) if xlabel: unit = 1 if not xlabel: unit, timestr = plotutils.time_axis_unit(end - start) if not t0: t0 = start t0 = lal.LIGOTimeGPS(t0) if int(t0.gpsNanoSeconds) == 0: xlabel = datetime.datetime(*lal.GPSToUTC(int(t0))[:6])\ .strftime("%B %d %Y, %H:%M:%S %ZUTC") xlabel = "Time (%s) since %s (%s)" % (timestr, xlabel, int(t0)) else: xlabel = datetime.datetime(*lal.GPSToUTC(t0.gpsSeconds)[:6])\ .strftime("%B %d %Y, %H:%M:%S %ZUTC") xlabel = "Time (%s) since %s (%s)"\ % (timestr, xlabel.replace(" UTC",".%.3s UTC" % t0.gpsNanoSeconds),\ t0) t0 = float(t0) ylabel = kwargs.pop("ylabel", "Frequency (Hz)") colorlabel = kwargs.pop("colorlabel", "Amplitude") title = kwargs.pop("title", "") subtitle = kwargs.pop("subtitle", "") # # restrict data to the correct limits for plotting # interpolate = logy and ydata is None for i, sequence in enumerate(sequencelist): if interpolate: # interpolate the data onto a log-scale sequence, ydata = loginterpolate(sequence, f0[i], deltaF[i]) if logy and ylim: plotted = (ydata > ylim[0]) & (ydata <= ylim[1]) newVectorLength = int(plotted.sum()) newsequence = lal.CreateREAL8VectorSequence(sequence.length,\ newVectorLength) for j in range(sequence.length): newsequence.data[j, :] = sequence.data[j, :][plotted] del sequence sequencelist[i] = newsequence if len(sequencelist) and logy and ylim: ydata = ydata[plotted] # # format bins # xbins = [] for i in range(numseq): xmin = epoch[i] xmax = epoch[i] + sequencelist[i].length * deltaT[i] xbins.append(rate.LinearBins(float(xmin-t0)/unit, float(xmax-t0)/unit,\ 2)) ybins = [] for i in range(numseq): if ydata is not None: ydata = numpy.asarray(ydata) ymin = ydata.min() ymax = ydata.max() else: ymin = f0[i] ymax = f0[i] + sequencelist[i].vectorLength * deltaF[i] if logy: if ymin == 0: ymin = deltaF[i] ybins.append(rate.LogarithmicBins(ymin, ymax, 2)) else: ybins.append(rate.LinearBins(ymin, ymax, 2)) # # plot # kwargs.setdefault("interpolation", "kaiser") plot = plotutils.ImagePlot(xlabel=xlabel, ylabel=ylabel, title=title,\ subtitle=subtitle, colorlabel=colorlabel) for sequence, x, y in zip(sequencelist, xbins, ybins): data = numpy.ma.masked_where(numpy.isnan(sequence.data), sequence.data,\ copy=False) plot.add_content(data.T, x, y, **kwargs) # finalize plot.finalize(colorbar=True, logcolor=logcolor, minorticks=True,\ clim=colorlim) if hidden_colorbar: plotutils.add_colorbar(plot.ax, visible=False) # set logscale if logx: plot.ax.xaxis.set_scale("log") if logy: plot.ax.yaxis.set_scale("log") plot.ax._update_transScale() # format axes if xlim: xlim = (numpy.asarray(xlim).astype(float) - t0) / unit plot.ax.set_xlim(xlim) if ylim: plot.ax.set_ylim(ylim) # set grid and ticks plot.ax.grid(True, which="both") plotutils.set_time_ticks(plot.ax) plotutils.set_minor_ticks(plot.ax) # save and close plot.savefig(outfile, bbox_inches=bbox_inches,\ bbox_extra_artists=plot.ax.texts) plot.close()
def twoD_SearchVolume(self, instruments, dbin=None, FAR=None, bootnum=None, derr=0.197, dsys=0.074): """ Compute the search volume in the mass/mass plane, bootstrap and measure the first and second moment (assumes the underlying distribution can be characterized by those two parameters) This is gonna be brutally slow derr = (0.134**2+.103**2+.102**2)**.5 = 0.197 which is the 3 detector calibration uncertainty in quadrature. This is conservative since some injections will be H1L1 and have a lower error of .17 the dsys is the DC offset which is the max offset of .074. """ if not FAR: FAR = self.far[instruments] found, missed = self.get_injections(instruments, FAR) twodbin = self.twoDMassBins wnfunc = self.gw livetime = self.livetime[instruments] if not bootnum: bootnum = self.bootnum if wnfunc: wnfunc /= wnfunc[(wnfunc.shape[0]-1) / 2, (wnfunc.shape[1]-1) / 2] x = twodbin.shape[0] y = twodbin.shape[1] z = int(self.opts.dist_bins) rArrays = [] volArray=rate.BinnedArray(twodbin) volArray2=rate.BinnedArray(twodbin) #set up ratio arrays for each distance bin for k in range(z): rArrays.append(rate.BinnedRatios(twodbin)) # Bootstrap to account for errors for n in range(bootnum): #initialize by setting these to zero for k in range(z): rArrays[k].numerator.array = numpy.zeros(rArrays[k].numerator.bins.shape) rArrays[k].denominator.array = numpy.zeros(rArrays[k].numerator.bins.shape) #Scramble the inj population and distances if bootnum > 1: sm, sf = self._scramble_pop(missed, found) # I make a separate array of distances to speed up this calculation f_dist = self._scramble_dist(sf, derr, dsys) else: sm, sf = missed, found f_dist = numpy.array([l.distance for l in found]) # compute the distance bins if not dbin: dbin = rate.LogarithmicBins(min(f_dist),max(f_dist), z) #else: print dbin.centres() # get rid of all missed injections outside the distance bins # to prevent binning errors sm, m_dist = self.cut_distance(sm, dbin) sf, f_dist = self.cut_distance(sf, dbin) for i, l in enumerate(sf):#found: tbin = rArrays[dbin[f_dist[i]]] tbin.incnumerator( (l.mass1, l.mass2) ) for i, l in enumerate(sm):#missed: tbin = rArrays[dbin[m_dist[i]]] tbin.incdenominator( (l.mass1, l.mass2) ) tmpArray2=rate.BinnedArray(twodbin) #start with a zero array to compute the mean square for k in range(z): tbins = rArrays[k] tbins.denominator.array += tbins.numerator.array if wnfunc: rate.filter_array(tbins.denominator.array,wnfunc) if wnfunc: rate.filter_array(tbins.numerator.array,wnfunc) tbins.regularize() # logarithmic(d) integrand = 4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta volArray.array += integrand tmpArray2.array += integrand #4.0 * pi * tbins.ratio() * dbin.centres()[k]**3 * dbin.delta print >>sys.stderr, "bootstrapping:\t%.1f%% and Calculating smoothed volume:\t%.1f%%\r" % ((100.0 * n / bootnum), (100.0 * k / z)), tmpArray2.array *= tmpArray2.array volArray2.array += tmpArray2.array print >>sys.stderr, "" #Mean and variance volArray.array /= bootnum volArray2.array /= bootnum volArray2.array -= volArray.array**2 # Variance volArray.array *= livetime volArray2.array *= livetime*livetime # this gets two powers of live time return volArray, volArray2
else: Found = get_burst_injections(opts.burst_found) Missed = get_burst_injections(opts.burst_missed) # restrict the sims to a distance range Found = cut_distance(Found, 1, max_dist) Missed = cut_distance(Missed, 1, max_dist) # get a 2D mass binning twoDMassBins = get_2d_mass_bins(min_mass, max_mass, mass_bins) # get log distance bins dBin = rate.LogarithmicBins(0.1,max_dist*1.25,dist_bins) # Someday we could try a Gaussian smoothing function #gw = rate.gaussian_window2d(2,2,8) gw = None #Get derivative of volume with respect to FAR #dvA = get_volume_derivative(opts.injfnames, twoDMassBins, dBin, FAR, zero_lag_segments, gw) vA, vA2 = twoD_SearchVolume(Found, Missed, twoDMassBins, dBin, gw, 1.0, bootnum=int(opts.bootstrap_iterations)) # FIXME convert to years (use some lal or pylal thing in the future) #vA.array /= secs_in_year #vA2.array /= secs_in_year * secs_in_year #two powers for this squared quantity #Trim the array to have sane values outside the total mass area of interest
def finish(self): fig, axes = SnglBurstUtils.make_burst_plot(r"Injection Amplitude (\(\mathrm{s}^{-\frac{1}{3}}\))", "Detection Efficiency", width = 108.0) axes.set_title(r"Detection Efficiency vs.\ Amplitude") axes.semilogx() axes.set_position([0.10, 0.150, 0.86, 0.77]) # set desired yticks axes.set_yticks((0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)) axes.set_yticklabels((r"\(0\)", r"\(0.1\)", r"\(0.2\)", r"\(0.3\)", r"\(0.4\)", r"\(0.5\)", r"\(0.6\)", r"\(0.7\)", r"\(0.8\)", r"\(0.9\)", r"\(1.0\)")) axes.xaxis.grid(True, which = "major,minor") axes.yaxis.grid(True, which = "major,minor") # put made and found injections in the denominators and # numerators of the efficiency bins bins = rate.NDBins((rate.LogarithmicBins(min(sim.amplitude for sim in self.all), max(sim.amplitude for sim in self.all), 400),)) efficiency_num = rate.BinnedArray(bins) efficiency_den = rate.BinnedArray(bins) for sim in self.found: efficiency_num[sim.amplitude,] += 1 for sim in self.all: efficiency_den[sim.amplitude,] += 1 # generate and plot trend curves. adjust window function # normalization so that denominator array correctly # represents the number of injections contributing to each # bin: make w(0) = 1.0. note that this factor has no # effect on the efficiency because it is common to the # numerator and denominator arrays. we do this for the # purpose of computing the Poisson error bars, which # requires us to know the counts for the bins windowfunc = rate.gaussian_window(self.filter_width) windowfunc /= windowfunc[len(windowfunc) / 2 + 1] rate.filter_binned_array(efficiency_num, windowfunc) rate.filter_binned_array(efficiency_den, windowfunc) # regularize: adjust unused bins so that the efficiency is # 0, not NaN assert (efficiency_num <= efficiency_den).all() efficiency_den[efficiency_num == 0 & efficiency_den == 0] = 1 line1, A50, A50_err = render_data_from_bins(file("string_efficiency.dat", "w"), axes, efficiency_num, efficiency_den, self.cal_uncertainty, self.filter_width, colour = "k", linestyle = "-", erroralpha = 0.2) print >>sys.stderr, "Pipeline's 50%% efficiency point for all detections = %g +/- %g%%\n" % (A50, A50_err * 100) # add a legend to the axes axes.legend((line1,), (r"\noindent Injections recovered with $\Lambda > %s$" % SnglBurstUtils.latexnumber("%.2e" % self.detection_threshold),), loc = "lower right") # adjust limits axes.set_xlim([1e-21, 2e-18]) axes.set_ylim([0.0, 1.0]) # # dump some information about the highest-amplitude missed # and quietest-amplitude found injections # self.loudest_missed.sort(reverse = True) self.quietest_found.sort(reverse = True) f = file("string_loud_missed_injections.txt", "w") print >>f, "Highest Amplitude Missed Injections" print >>f, "===================================" for amplitude, sim, offsetvector, filename, likelihood_ratio in self.loudest_missed: print >>f print >>f, "%s in %s:" % (str(sim.simulation_id), filename) if likelihood_ratio is None: print >>f, "Not recovered" else: print >>f, "Recovered with \\Lambda = %.16g, detection threshold was %.16g" % (likelihood_ratio, self.detection_threshold) for instrument in self.seglists: print >>f, "In %s:" % instrument print >>f, "\tInjected amplitude:\t%.16g" % SimBurstUtils.string_amplitude_in_instrument(sim, instrument, offsetvector) print >>f, "\tTime of injection:\t%s s" % sim.time_at_instrument(instrument, offsetvector) print >>f, "Amplitude in waveframe:\t%.16g" % sim.amplitude t = sim.get_time_geocent() print >>f, "Time at geocentre:\t%s s" % t print >>f, "Segments within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.seglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.seglists)) print >>f, "Vetoes within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.vetoseglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.vetoseglists)) f = file("string_quiet_found_injections.txt", "w") print >>f, "Lowest Amplitude Found Injections" print >>f, "=================================" for inv_amplitude, sim, offsetvector, filename, likelihood_ratio in self.quietest_found: print >>f print >>f, "%s in %s:" % (str(sim.simulation_id), filename) if likelihood_ratio is None: print >>f, "Not recovered" else: print >>f, "Recovered with \\Lambda = %.16g, detection threshold was %.16g" % (likelihood_ratio, self.detection_threshold) for instrument in self.seglists: print >>f, "In %s:" % instrument print >>f, "\tInjected amplitude:\t%.16g" % SimBurstUtils.string_amplitude_in_instrument(sim, instrument, offsetvector) print >>f, "\tTime of injection:\t%s s" % sim.time_at_instrument(instrument, offsetvector) print >>f, "Amplitude in waveframe:\t%.16g" % sim.amplitude t = sim.get_time_geocent() print >>f, "Time at geocentre:\t%s s" % t print >>f, "Segments within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.seglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.seglists)) print >>f, "Vetoes within 60 seconds:\t%s" % segmentsUtils.segmentlistdict_to_short_string(self.vetoseglists & segments.segmentlistdict((instrument, segments.segmentlist([segments.segment(t-offsetvector[instrument]-60, t-offsetvector[instrument]+60)])) for instrument in self.vetoseglists)) # # done # return fig,