def get_far_threshold_and_segments(zerofname, instruments, live_time_program, veto_seg_name="vetoes", verbose=True): """ return the false alarm rate of the most rare zero-lag coinc, and a dictionary of the thinca segments indexed by instrument. """ # open database working_filename = dbtables.get_connection_filename(zerofname, verbose=verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) # extract false alarm rate threshold query = 'SELECT MIN(coinc_inspiral.false_alarm_rate) FROM coinc_inspiral JOIN coinc_event ON (coinc_event.coinc_event_id == coinc_inspiral.coinc_event_id) WHERE ( coinc_event.instruments = "' + instruments + '" AND NOT EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0) );' print("\n", query) far, = connection.cursor().execute(query).fetchone() # extract segments. seglists = db_thinca_rings.get_thinca_zero_lag_segments( connection, program_name=live_time_program) # done connection.close() dbtables.discard_connection_filename(zerofname, working_filename, verbose=verbose) dbtables.DBTable_set_connection(None) return far, seglists
def get_far_threshold_and_segments(zerofname, live_time_program, instruments, verbose=False): """ return the false alarm rate of the most rare zero-lag coinc, and a dictionary of the thinca segments indexed by instrument. """ # open database working_filename = dbtables.get_connection_filename(zerofname, verbose=verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) # extract false alarm rate threshold query = 'SELECT MIN(coinc_inspiral.combined_far) FROM coinc_inspiral JOIN coinc_event ON (coinc_event.coinc_event_id == coinc_inspiral.coinc_event_id) WHERE (coinc_event.instruments == "' + str( instruments ) + '") AND NOT EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0);' print(query) far, = connection.cursor().execute(query).fetchone() # extract segments. seglists = db_thinca_rings.get_thinca_zero_lag_segments( connection, program_name=live_time_program) # done connection.close() dbtables.discard_connection_filename(zerofname, working_filename, verbose=verbose) dbtables.DBTable_set_connection(None) print("WARNING replacing far with 10^-7", file=sys.stderr) far = 1.0e-7 return far, seglists
def get_vetoes(fname, veto_segments_name="vetoes", verbose=True): working_filename = dbtables.get_connection_filename(fname, verbose=verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) veto_segments = db_thinca_rings.get_veto_segments(connection, veto_segments_name) connection.close() dbtables.discard_connection_filename(fname, working_filename, verbose=verbose) dbtables.DBTable_set_connection(None) return veto_segments
def get_injections(injfnames, FAR, zero_lag_segments, verbose = False): """ """ def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments): """ return True if injection was made in the given segmentlist """ return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments found = [] missed = [] print >>sys.stderr, "" for cnt, f in enumerate(injfnames): print >>sys.stderr, "getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),), working_filename = dbtables.get_connection_filename(f, tmp_path = None, verbose = verbose) connection = sqlite3.connect(working_filename) connection.create_function("injection_was_made", 2, injection_was_made) make_sim_inspiral = lsctables.table.get_table(dbtables.get_xml(connection), lsctables.SimInspiralTable.tableName)._row_from_cols for values in connection.cursor().execute(""" SELECT sim_inspiral.*, -- true if injection matched a coinc below the false alarm rate threshold EXISTS ( SELECT * FROM coinc_event_map AS mapa JOIN coinc_event_map AS mapb ON ( mapa.coinc_event_id == mapb.coinc_event_id ) JOIN coinc_inspiral ON ( mapb.table_name == "coinc_event" AND mapb.event_id == coinc_inspiral.coinc_event_id ) WHERE mapa.table_name == "sim_inspiral" AND mapa.event_id == sim_inspiral.simulation_id AND coinc_inspiral.combined_far < ? ) FROM sim_inspiral WHERE -- only interested in injections that were injected injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns) """, (FAR,)): sim = make_sim_inspiral(values) if values[-1]: found.append(sim) else: missed.append(sim) # done connection.close() dbtables.discard_connection_filename(f, working_filename, verbose = verbose) dbtables.DBTable_set_connection(None) print >>sys.stderr, "\nFound = %d Missed = %d" % (len(found), len(missed)) return found, missed
def update_coincs(self, fnames): """ This function iterates over the databases and updates the the likelihood column with the proper lv stat """ for f in fnames: working_filename = dbtables.get_connection_filename(zerofname, verbose=True) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) connection.create_function("lvstat", 3, self.lvstat) query = "UPDATE coinc_event SET likelihood = (SELECT lvstat(coinc_event.instruments, coinc_inspiral.ifos, coinc_inspiral.false_alarm_rate) FROM coinc_inspiral WHERE coinc_event.coinc_event_id == coinc_inspiral.coinc_event_id)" print(query) connection.cursor().execute(query) connection.commit() connection.close() dbtables.discard_connection_filename(zerofname, working_filename, verbose=True) dbtables.DBTable_set_connection(None)
def get_ifos(zerofname, verbose=True): # open database working_filename = dbtables.get_connection_filename(zerofname, verbose=verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) # extract false alarm rate threshold # FIXME This may not be robust if triggers are missing from a given category, for # example no triples in zero lag or time slides. query = 'SELECT distinct(ifos) FROM coinc_inspiral' ifo_list = [] for i in connection.cursor().execute(query): ifo_list.append(i) # done connection.close() dbtables.discard_connection_filename(zerofname, working_filename, verbose=verbose) dbtables.DBTable_set_connection(None) return ifo_list
def open_pipedown_database(database_filename,tmp_space): """ Open the connection to the pipedown database """ if not os.access(database_filename,os.R_OK): raise Exception('Unable to open input file: %s'%(database_filename)) from glue.ligolw import dbtables try: import sqlite3 except ImportError: # Pre-2.5 from pysqlite2 import dbapi2 as sqlite3 working_filename=dbtables.get_connection_filename(database_filename,tmp_path=tmp_space) connection = sqlite3.connect(working_filename) if tmp_space: dbtables.set_temp_store_directory(connection,tmp_space) dbtables.DBTable_set_connection(connection) return (connection,working_filename)
def __init__(self, opts, flist): self.segments = segments.segmentlistdict() self.non_inj_fnames = [] self.inj_fnames = [] self.found = {} self.missed = {} self.opts = opts self.veto_segments = segments.segmentlistdict() self.zero_lag_segments = {} self.instruments = [] self.livetime = {} self.multi_burst_table = None self.coinc_inspiral_table = None for f in flist: if opts.verbose: print("Gathering stats from: %s...." % (f, ), file=sys.stderr) working_filename = dbtables.get_connection_filename( f, tmp_path=opts.tmp_space, verbose=opts.verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) xmldoc = dbtables.get_xml(connection) # look for a sim table try: sim_inspiral_table = dbtables.lsctables.SimInspiralTable.get_table( xmldoc) self.inj_fnames.append(f) sim = True except ValueError: self.non_inj_fnames.append(f) sim = False # FIGURE OUT IF IT IS A BURST OR INSPIRAL RUN try: self.multi_burst_table = dbtables.lsctables.MultiBurstTable.get_table( xmldoc) except ValueError: self.multi_burst_table = None try: self.coinc_inspiral_table = dbtables.lsctables.CoincInspiralTable.get_table( xmldoc) except ValueError: self.coinc_inspiral_table = None if self.multi_burst_table and self.coinc_inspiral_table: print("both burst and inspiral tables found. Aborting", file=sys.stderr) raise ValueError if not sim: self.get_instruments(connection) self.segments += self.get_segments(connection, xmldoc) #FIXME, don't assume veto segments are the same in every file! self.veto_segments = self.get_veto_segments(connection) dbtables.discard_connection_filename(f, working_filename, verbose=opts.verbose) dbtables.DBTable_set_connection(None) # remove redundant instruments self.instruments = list(set(self.instruments)) # FIXME Do these have to be done by instruments? self.segments -= self.veto_segments # segments and livetime by instruments for i in self.instruments: self.zero_lag_segments[i] = self.segments.intersection( i) - self.segments.union(set(self.segments.keys()) - i) self.livetime[i] = float(abs(self.zero_lag_segments[i]))
def get_injections(self, instruments, FAR=float("inf")): injfnames = self.inj_fnames zero_lag_segments = self.zero_lag_segments[instruments] verbose = self.opts.verbose found = [] missed = [] print("", file=sys.stderr) for cnt, f in enumerate(injfnames): print("getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames), ), end=' ', file=sys.stderr) working_filename = dbtables.get_connection_filename( f, tmp_path=opts.tmp_space, verbose=verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) xmldoc = dbtables.get_xml(connection) # DON'T BOTHER CONTINUING IF THE INSTRUMENTS OF INTEREST ARE NOT HERE instruments_in_this_file = [] for i in connection.cursor().execute( 'SELECT DISTINCT(instruments) FROM coinc_event'): if i[0]: instruments_in_this_file.append( frozenset(lsctables.instrument_set_from_ifos(i[0]))) if instruments not in instruments_in_this_file: connection.close() dbtables.discard_connection_filename(f, working_filename, verbose=verbose) dbtables.DBTable_set_connection(None) continue # WORK OUT CORRECT SEGMENTS FOR THIS FILE WHERE WE SHOULD SEE INJECTIONS segments = self.get_segments(connection, xmldoc) segments -= self.veto_segments #print thincasegments zero_lag_segments = segments.intersection( instruments) - segments.union( set(segments.keys()) - instruments) ############### # DEFINE THE INJECTION WAS MADE FUNCTION def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments=zero_lag_segments): """ return True if injection was made in the given segmentlist """ return lsctables.LIGOTimeGPS( geocent_end_time, geocent_end_time_ns) in zero_lag_segments connection.create_function("injection_was_made", 2, injection_was_made) make_sim_inspiral = lsctables.SimInspiralTable.get_table( dbtables.get_xml(connection)).row_from_cols # INSPIRAL if self.coinc_inspiral_table: for values in connection.cursor().execute( """ SELECT sim_inspiral.*, -- true if injection matched a coinc below the false alarm rate threshold EXISTS ( SELECT * FROM coinc_event_map AS mapa JOIN coinc_event_map AS mapb ON ( mapa.coinc_event_id == mapb.coinc_event_id ) JOIN coinc_inspiral ON ( mapb.table_name == "coinc_event" AND mapb.event_id == coinc_inspiral.coinc_event_id ) WHERE mapa.table_name == "sim_inspiral" AND mapa.event_id == sim_inspiral.simulation_id AND coinc_inspiral.combined_far < ? ) FROM sim_inspiral WHERE -- only interested in injections that were injected injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns) """, (FAR, )): sim = make_sim_inspiral(values) if values[-1]: found.append(sim) else: missed.append(sim) # BURSTS if self.multi_burst_table: for values in connection.cursor().execute( """ SELECT sim_inspiral.*, -- true if injection matched a coinc below the false alarm rate threshold EXISTS ( SELECT * FROM coinc_event_map AS mapa JOIN coinc_event_map AS mapb ON ( mapa.coinc_event_id == mapb.coinc_event_id ) JOIN multi_burst ON ( mapb.table_name == "coinc_event" AND mapb.event_id == multi_burst.coinc_event_id ) WHERE mapa.table_name == "sim_inspiral" AND mapa.event_id == sim_inspiral.simulation_id AND multi_burst.false_alarm_rate < ? ) FROM sim_inspiral WHERE -- only interested in injections that were injected injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns) """, (FAR, )): sim = make_sim_inspiral(values) if values[-1]: found.append(sim) else: missed.append(sim) # done dbtables.discard_connection_filename(f, working_filename, verbose=verbose) dbtables.DBTable_set_connection(None) print("\nFound = %d Missed = %d" % (len(found), len(missed)), file=sys.stderr) return found, missed
def __init__(self, flist, opts): self.far = {} self.segments = segments.segmentlistdict() self.non_inj_fnames = [] self.inj_fnames = [] #self.der_fit = None self.twoDMassBins = None #self.dBin = {} self.gw = None self.found = {} self.missed = {} self.wnfunc = None self.opts = opts if opts.bootstrap_iterations: self.bootnum = int(opts.bootstrap_iterations) else: self.bootnum = 100 self.veto_segments = segments.segmentlistdict() self.zero_lag_segments = {} self.instruments = [] self.livetime = {} self.minmass = None self.maxmass = None self.mintotal = None self.maxtotal = None for f in flist: if opts.verbose: print >> sys.stderr, "Gathering stats from: %s...." % (f,) working_filename = dbtables.get_connection_filename(f, verbose = opts.verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) xmldoc = dbtables.get_xml(connection) # look for a sim table try: sim_inspiral_table = table.get_table(xmldoc, dbtables.lsctables.SimInspiralTable.tableName) self.inj_fnames.append(f) sim = True except ValueError: self.non_inj_fnames.append(f) sim = False if not sim: if opts.veto_segments_name is not None: self.veto_segments = db_thinca_rings.get_veto_segments(connection, opts.veto_segments_name) self.get_instruments(connection) self.segments += db_thinca_rings.get_thinca_zero_lag_segments(connection, program_name = opts.live_time_program) self.get_far_thresholds(connection) else: self.get_mass_ranges(connection) #connection.close() dbtables.discard_connection_filename(f, working_filename, verbose = opts.verbose) dbtables.DBTable_set_connection(None) # FIXME Do these have to be done by instruments? self.segments -= self.veto_segments # compute far, segments and livetime by instruments for i in self.instruments: self.far[i] = min(self.far[i]) # FIXME this bombs if any of the FARS are zero. maybe it should continue # and just remove that instrument combo from the calculation if self.far[i] == 0: print >> sys.stderr, "Encountered 0 FAR in %s, ABORTING" % (i,) sys.exit(1) self.zero_lag_segments[i] = self.segments.intersection(i) - self.segments.union(set(self.segments.keys()) - i) # Livetime must have playground removed self.livetime[i] = float(abs(self.zero_lag_segments[i] - segmentsUtils.S2playground(self.segments.extent_all()))) if opts.verbose: print >> sys.stderr, "%s FAR %e, livetime %f" % (",".join(sorted(list(i))), self.far[i], self.livetime[i]) # get a 2D mass binning self.twoDMassBins = self.get_2d_mass_bins(self.minmass, self.maxmass, opts.mass_bins)
def get_injections(injfnames, zero_lag_segments, ifos="H1,H2,L1", FAR=1.0, verbose=True): """ The LV stat uses simulations above threshold, not some IFAR of the loudest event, so the default should be "inf" """ def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments=zero_lag_segments): """ return True if injection was made in the given segmentlist """ return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments found = [] missed = [] print("", file=sys.stderr) for cnt, f in enumerate(injfnames): print("getting injections: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames), ), end=' ', file=sys.stderr) working_filename = dbtables.get_connection_filename(f, tmp_path=None, verbose=verbose) connection = sqlite3.connect(working_filename) connection.create_function("injection_was_made", 2, injection_was_made) make_sim_inspiral = lsctables.SimInspiralTable.get_table( dbtables.get_xml(connection))._row_from_cols # FIXME may not be done correctly if injections are done in timeslides # FIXME may not be done correctly if injections aren't logarithmic in d # do we really want d^3 waiting? # FIXME do we really want injections independent of their FAR for values in connection.cursor().execute( """ SELECT sim_inspiral.*, -- true if injection matched a coinc below the false alarm rate threshold EXISTS ( SELECT * FROM coinc_event_map AS mapa JOIN coinc_event_map AS mapb ON ( mapa.coinc_event_id == mapb.coinc_event_id ) JOIN coinc_inspiral ON ( mapb.table_name == "coinc_event" AND mapb.event_id == coinc_inspiral.coinc_event_id ) WHERE mapa.table_name == "sim_inspiral" AND mapa.event_id == sim_inspiral.simulation_id AND coinc_inspiral.false_alarm_rate < ? AND coinc_inspiral.ifos == ? ) FROM sim_inspiral WHERE -- only interested in injections that were injected injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns) """, ( FAR, ifos, )): sim = make_sim_inspiral(values) if values[-1]: found.append(sim) else: missed.append(sim) # done connection.close() dbtables.discard_connection_filename(f, working_filename, verbose=verbose) dbtables.DBTable_set_connection(None) print("\nFound = %d Missed = %d" % (len(found), len(missed)), file=sys.stderr) return found, missed
(opts, args) = parser.parse_args() inj_files = glob.glob(opts.injections) fulldata_files = glob.glob(opts.fulldata) timeslide_likelihood = [] timeslide_snr = [] zerolag_likelihood = [] zerolag_snr = [] for filename in fulldata_files: local_disk = None #"/tmp" working_filename = dbtables.get_connection_filename(filename, tmp_path=local_disk, verbose=True) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) xmldoc = dbtables.get_xml(connection) cursor = connection.cursor() for likelihood, snr, is_background in connection.cursor().execute(""" SELECT insp_coinc_event.likelihood, coinc_inspiral.snr, EXISTS ( SELECT * FROM time_slide WHERE time_slide.time_slide_id == insp_coinc_event.time_slide_id AND time_slide.offset != 0 )