def get_far_threshold_and_segments(zerofname,
instruments,
live_time_program,
veto_seg_name="vetoes",
verbose=True):
"""
return the false alarm rate of the most rare zero-lag coinc, and a
dictionary of the thinca segments indexed by instrument.
"""
# open database
working_filename = dbtables.get_connection_filename(zerofname,
verbose=verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
# extract false alarm rate threshold
query = 'SELECT MIN(coinc_inspiral.false_alarm_rate) FROM coinc_inspiral JOIN coinc_event ON (coinc_event.coinc_event_id == coinc_inspiral.coinc_event_id) WHERE ( coinc_event.instruments = "' + instruments + '" AND NOT EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0) );'
print("\n", query)
far, = connection.cursor().execute(query).fetchone()
# extract segments.
seglists = db_thinca_rings.get_thinca_zero_lag_segments(
connection, program_name=live_time_program)
# done
connection.close()
dbtables.discard_connection_filename(zerofname,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
return far, seglists
def get_far_threshold_and_segments(zerofname,
live_time_program,
instruments,
verbose=False):
"""
return the false alarm rate of the most rare zero-lag coinc, and a
dictionary of the thinca segments indexed by instrument.
"""
# open database
working_filename = dbtables.get_connection_filename(zerofname,
verbose=verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
# extract false alarm rate threshold
query = 'SELECT MIN(coinc_inspiral.combined_far) FROM coinc_inspiral JOIN coinc_event ON (coinc_event.coinc_event_id == coinc_inspiral.coinc_event_id) WHERE (coinc_event.instruments == "' + str(
instruments
) + '") AND NOT EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0);'
print(query)
far, = connection.cursor().execute(query).fetchone()
# extract segments.
seglists = db_thinca_rings.get_thinca_zero_lag_segments(
connection, program_name=live_time_program)
# done
connection.close()
dbtables.discard_connection_filename(zerofname,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
print("WARNING replacing far with 10^-7", file=sys.stderr)
far = 1.0e-7
return far, seglists
def get_vetoes(fname, veto_segments_name="vetoes", verbose=True):
working_filename = dbtables.get_connection_filename(fname, verbose=verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
veto_segments = db_thinca_rings.get_veto_segments(connection,
veto_segments_name)
connection.close()
dbtables.discard_connection_filename(fname,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
return veto_segments
def get_injections(injfnames, FAR, zero_lag_segments, verbose = False):
"""
"""
def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments
found = []
missed = []
print >>sys.stderr, ""
for cnt, f in enumerate(injfnames):
print >>sys.stderr, "getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),),
working_filename = dbtables.get_connection_filename(f, tmp_path = None, verbose = verbose)
connection = sqlite3.connect(working_filename)
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.table.get_table(dbtables.get_xml(connection), lsctables.SimInspiralTable.tableName)._row_from_cols
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
print >>sys.stderr, "\nFound = %d Missed = %d" % (len(found), len(missed))
return found, missed
def update_coincs(self, fnames):
"""
This function iterates over the databases and updates the the likelihood
column with the proper lv stat
"""
for f in fnames:
working_filename = dbtables.get_connection_filename(zerofname,
verbose=True)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
connection.create_function("lvstat", 3, self.lvstat)
query = "UPDATE coinc_event SET likelihood = (SELECT lvstat(coinc_event.instruments, coinc_inspiral.ifos, coinc_inspiral.false_alarm_rate) FROM coinc_inspiral WHERE coinc_event.coinc_event_id == coinc_inspiral.coinc_event_id)"
print(query)
connection.cursor().execute(query)
connection.commit()
connection.close()
dbtables.discard_connection_filename(zerofname,
working_filename,
verbose=True)
dbtables.DBTable_set_connection(None)
def get_ifos(zerofname, verbose=True):
# open database
working_filename = dbtables.get_connection_filename(zerofname,
verbose=verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
# extract false alarm rate threshold
# FIXME This may not be robust if triggers are missing from a given category, for
# example no triples in zero lag or time slides.
query = 'SELECT distinct(ifos) FROM coinc_inspiral'
ifo_list = []
for i in connection.cursor().execute(query):
ifo_list.append(i)
# done
connection.close()
dbtables.discard_connection_filename(zerofname,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
return ifo_list
def open_pipedown_database(database_filename,tmp_space):
"""
Open the connection to the pipedown database
"""
if not os.access(database_filename,os.R_OK):
raise Exception('Unable to open input file: %s'%(database_filename))
from glue.ligolw import dbtables
try:
import sqlite3
except ImportError:
# Pre-2.5
from pysqlite2 import dbapi2 as sqlite3
working_filename=dbtables.get_connection_filename(database_filename,tmp_path=tmp_space)
connection = sqlite3.connect(working_filename)
if tmp_space:
dbtables.set_temp_store_directory(connection,tmp_space)
dbtables.DBTable_set_connection(connection)
return (connection,working_filename)
def __init__(self, opts, flist):
self.segments = segments.segmentlistdict()
self.non_inj_fnames = []
self.inj_fnames = []
self.found = {}
self.missed = {}
self.opts = opts
self.veto_segments = segments.segmentlistdict()
self.zero_lag_segments = {}
self.instruments = []
self.livetime = {}
self.multi_burst_table = None
self.coinc_inspiral_table = None
for f in flist:
if opts.verbose:
print("Gathering stats from: %s...." % (f, ), file=sys.stderr)
working_filename = dbtables.get_connection_filename(
f, tmp_path=opts.tmp_space, verbose=opts.verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# look for a sim table
try:
sim_inspiral_table = dbtables.lsctables.SimInspiralTable.get_table(
xmldoc)
self.inj_fnames.append(f)
sim = True
except ValueError:
self.non_inj_fnames.append(f)
sim = False
# FIGURE OUT IF IT IS A BURST OR INSPIRAL RUN
try:
self.multi_burst_table = dbtables.lsctables.MultiBurstTable.get_table(
xmldoc)
except ValueError:
self.multi_burst_table = None
try:
self.coinc_inspiral_table = dbtables.lsctables.CoincInspiralTable.get_table(
xmldoc)
except ValueError:
self.coinc_inspiral_table = None
if self.multi_burst_table and self.coinc_inspiral_table:
print("both burst and inspiral tables found. Aborting",
file=sys.stderr)
raise ValueError
if not sim:
self.get_instruments(connection)
self.segments += self.get_segments(connection, xmldoc)
#FIXME, don't assume veto segments are the same in every file!
self.veto_segments = self.get_veto_segments(connection)
dbtables.discard_connection_filename(f,
working_filename,
verbose=opts.verbose)
dbtables.DBTable_set_connection(None)
# remove redundant instruments
self.instruments = list(set(self.instruments))
# FIXME Do these have to be done by instruments?
self.segments -= self.veto_segments
# segments and livetime by instruments
for i in self.instruments:
self.zero_lag_segments[i] = self.segments.intersection(
i) - self.segments.union(set(self.segments.keys()) - i)
self.livetime[i] = float(abs(self.zero_lag_segments[i]))
def get_injections(self, instruments, FAR=float("inf")):
injfnames = self.inj_fnames
zero_lag_segments = self.zero_lag_segments[instruments]
verbose = self.opts.verbose
found = []
missed = []
print("", file=sys.stderr)
for cnt, f in enumerate(injfnames):
print("getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" %
(100.0 * cnt / len(injfnames), ),
end=' ',
file=sys.stderr)
working_filename = dbtables.get_connection_filename(
f, tmp_path=opts.tmp_space, verbose=verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# DON'T BOTHER CONTINUING IF THE INSTRUMENTS OF INTEREST ARE NOT HERE
instruments_in_this_file = []
for i in connection.cursor().execute(
'SELECT DISTINCT(instruments) FROM coinc_event'):
if i[0]:
instruments_in_this_file.append(
frozenset(lsctables.instrument_set_from_ifos(i[0])))
if instruments not in instruments_in_this_file:
connection.close()
dbtables.discard_connection_filename(f,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
continue
# WORK OUT CORRECT SEGMENTS FOR THIS FILE WHERE WE SHOULD SEE INJECTIONS
segments = self.get_segments(connection, xmldoc)
segments -= self.veto_segments
#print thincasegments
zero_lag_segments = segments.intersection(
instruments) - segments.union(
set(segments.keys()) - instruments)
###############
# DEFINE THE INJECTION WAS MADE FUNCTION
def injection_was_made(geocent_end_time,
geocent_end_time_ns,
zero_lag_segments=zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(
geocent_end_time, geocent_end_time_ns) in zero_lag_segments
connection.create_function("injection_was_made", 2,
injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(
dbtables.get_xml(connection)).row_from_cols
# INSPIRAL
if self.coinc_inspiral_table:
for values in connection.cursor().execute(
"""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR, )):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# BURSTS
if self.multi_burst_table:
for values in connection.cursor().execute(
"""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN multi_burst ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == multi_burst.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND multi_burst.false_alarm_rate < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR, )):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
dbtables.discard_connection_filename(f,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
print("\nFound = %d Missed = %d" % (len(found), len(missed)),
file=sys.stderr)
return found, missed
def __init__(self, flist, opts):
self.far = {}
self.segments = segments.segmentlistdict()
self.non_inj_fnames = []
self.inj_fnames = []
#self.der_fit = None
self.twoDMassBins = None
#self.dBin = {}
self.gw = None
self.found = {}
self.missed = {}
self.wnfunc = None
self.opts = opts
if opts.bootstrap_iterations: self.bootnum = int(opts.bootstrap_iterations)
else: self.bootnum = 100
self.veto_segments = segments.segmentlistdict()
self.zero_lag_segments = {}
self.instruments = []
self.livetime = {}
self.minmass = None
self.maxmass = None
self.mintotal = None
self.maxtotal = None
for f in flist:
if opts.verbose: print >> sys.stderr, "Gathering stats from: %s...." % (f,)
working_filename = dbtables.get_connection_filename(f, verbose = opts.verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# look for a sim table
try:
sim_inspiral_table = table.get_table(xmldoc, dbtables.lsctables.SimInspiralTable.tableName)
self.inj_fnames.append(f)
sim = True
except ValueError:
self.non_inj_fnames.append(f)
sim = False
if not sim:
if opts.veto_segments_name is not None: self.veto_segments = db_thinca_rings.get_veto_segments(connection, opts.veto_segments_name)
self.get_instruments(connection)
self.segments += db_thinca_rings.get_thinca_zero_lag_segments(connection, program_name = opts.live_time_program)
self.get_far_thresholds(connection)
else:
self.get_mass_ranges(connection)
#connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = opts.verbose)
dbtables.DBTable_set_connection(None)
# FIXME Do these have to be done by instruments?
self.segments -= self.veto_segments
# compute far, segments and livetime by instruments
for i in self.instruments:
self.far[i] = min(self.far[i])
# FIXME this bombs if any of the FARS are zero. maybe it should continue
# and just remove that instrument combo from the calculation
if self.far[i] == 0:
print >> sys.stderr, "Encountered 0 FAR in %s, ABORTING" % (i,)
sys.exit(1)
self.zero_lag_segments[i] = self.segments.intersection(i) - self.segments.union(set(self.segments.keys()) - i)
# Livetime must have playground removed
self.livetime[i] = float(abs(self.zero_lag_segments[i] - segmentsUtils.S2playground(self.segments.extent_all())))
if opts.verbose: print >> sys.stderr, "%s FAR %e, livetime %f" % (",".join(sorted(list(i))), self.far[i], self.livetime[i])
# get a 2D mass binning
self.twoDMassBins = self.get_2d_mass_bins(self.minmass, self.maxmass, opts.mass_bins)
def get_injections(injfnames,
zero_lag_segments,
ifos="H1,H2,L1",
FAR=1.0,
verbose=True):
"""
The LV stat uses simulations above threshold, not some IFAR of the loudest event, so the default should be "inf"
"""
def injection_was_made(geocent_end_time,
geocent_end_time_ns,
zero_lag_segments=zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time,
geocent_end_time_ns) in zero_lag_segments
found = []
missed = []
print("", file=sys.stderr)
for cnt, f in enumerate(injfnames):
print("getting injections: " + str(FAR) + ":\t%.1f%%\r" %
(100.0 * cnt / len(injfnames), ),
end=' ',
file=sys.stderr)
working_filename = dbtables.get_connection_filename(f,
tmp_path=None,
verbose=verbose)
connection = sqlite3.connect(working_filename)
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(
dbtables.get_xml(connection))._row_from_cols
# FIXME may not be done correctly if injections are done in timeslides
# FIXME may not be done correctly if injections aren't logarithmic in d
# do we really want d^3 waiting?
# FIXME do we really want injections independent of their FAR
for values in connection.cursor().execute(
"""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.false_alarm_rate < ?
AND coinc_inspiral.ifos == ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (
FAR,
ifos,
)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
connection.close()
dbtables.discard_connection_filename(f,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
print("\nFound = %d Missed = %d" % (len(found), len(missed)),
file=sys.stderr)
return found, missed
(opts, args) = parser.parse_args()
inj_files = glob.glob(opts.injections)
fulldata_files = glob.glob(opts.fulldata)
timeslide_likelihood = []
timeslide_snr = []
zerolag_likelihood = []
zerolag_snr = []
for filename in fulldata_files:
local_disk = None #"/tmp"
working_filename = dbtables.get_connection_filename(filename,
tmp_path=local_disk,
verbose=True)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
cursor = connection.cursor()
for likelihood, snr, is_background in connection.cursor().execute("""
SELECT
insp_coinc_event.likelihood,
coinc_inspiral.snr,
EXISTS (
SELECT
*
FROM
time_slide
WHERE
time_slide.time_slide_id == insp_coinc_event.time_slide_id
AND time_slide.offset != 0
)
