def get_injections(injfnames, FAR, zero_lag_segments, verbose = False):
"""
"""
def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments
found = []
missed = []
print >>sys.stderr, ""
for cnt, f in enumerate(injfnames):
print >>sys.stderr, "getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),),
working_filename = dbtables.get_connection_filename(f, tmp_path = None, verbose = verbose)
connection = sqlite3.connect(working_filename)
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.table.get_table(dbtables.get_xml(connection), lsctables.SimInspiralTable.tableName)._row_from_cols
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
print >>sys.stderr, "\nFound = %d Missed = %d" % (len(found), len(missed))
return found, missed
def get_injections(injfnames, FAR, zero_lag_segments, verbose = False):
"""
"""
def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments
found = []
missed = []
print("", file=sys.stderr)
for cnt, f in enumerate(injfnames):
print("getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),), end=' ', file=sys.stderr)
working_filename = dbtables.get_connection_filename(f, tmp_path = None, verbose = verbose)
connection = sqlite3.connect(working_filename)
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(dbtables.get_xml(connection))._row_from_cols
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
print("\nFound = %d Missed = %d" % (len(found), len(missed)), file=sys.stderr)
return found, missed
def create_string_sngl_is_vetoed_function(connection, veto_segments_name = None):
"""
Creates a function named string_sngl_is_vetoed in the database at
connection. The function accepts three parameters --- the
instrument name, and the integer and integer nanoseconds components
of a time --- and returns true if the instrument is vetoed at that
time or false otherwise. veto_segments_name sets the name of the
segment lists used to define the vetoes.
If veto_segments_name is None then a no-op function is created that
always returns False.
Note: this funtion requires glue.ligolw.dbtables and
glue.ligolw.utils.segments to be imported as dbtables and
ligolwsegments respectively.
"""
if veto_segments_name is None:
connection.create_function("string_sngl_is_vetoed", 3, lambda instrument, peak_time, peak_time_ns: False)
return
xmldoc = dbtables.get_xml(connection)
seglists = ligolwsegments.segmenttable_get_by_name(xmldoc, options.vetoes_name).coalesce()
xmldoc.unlink()
def is_vetoed(instrument, peak_time, peak_time_ns, seglists = seglists):
return instrument in seglists and dbtables.lsctables.LIGOTimeGPS(peak_time, peak_time_ns) in seglists[instrument]
connection.create_function("string_sngl_is_vetoed", 3, is_vetoed)
def add_livetime(self,
connection,
live_time_program,
veto_segments_name=None,
verbose=False):
if verbose:
print >> sys.stderr, "\tcomputing livetimes:",
xmldoc = dbtables.get_xml(connection)
veto_segments = farutils.get_veto_segments(
connection,
live_time_program,
xmldoc,
veto_segments_name=veto_segments_name)
segs = farutils.get_segments(connection, xmldoc,
"lalapps_ring").coalesce()
bglivetime = farutils.add_background_livetime(connection,
live_time_program,
segs,
veto_segments,
coinc_segments=None,
verbose=verbose)
if verbose:
print >> sys.stderr, "\n\tbackground livetime %s\n" % (
str(bglivetime))
for key in bglivetime.keys():
try:
self.cached_livetime[key] += bglivetime[key]
except KeyError:
self.cached_livetime[key] = bglivetime[key]
def make_sim_inspiral_row_from_columns_in_db(connection):
"""
get the unique mapping of a sim inspiral row from columns in this
database
"""
return lsctables.SimInspiralTable.get_table(
dbtables.get_xml(connection)).row_from_cols
def create_string_sngl_is_vetoed_function(connection, veto_segments_name=None):
"""
Creates a function named string_sngl_is_vetoed in the database at
connection. The function accepts three parameters --- the
instrument name, and the integer and integer nanoseconds components
of a time --- and returns true if the instrument is vetoed at that
time or false otherwise. veto_segments_name sets the name of the
segment lists used to define the vetoes.
If veto_segments_name is None then a no-op function is created that
always returns False.
Note: this funtion requires glue.ligolw.dbtables and
glue.ligolw.utils.segments to be imported as dbtables and
ligolwsegments respectively.
"""
if veto_segments_name is None:
connection.create_function(
"string_sngl_is_vetoed", 3,
lambda instrument, peak_time, peak_time_ns: False)
return
xmldoc = dbtables.get_xml(connection)
seglists = ligolwsegments.segmenttable_get_by_name(
xmldoc, options.vetoes_name).coalesce()
xmldoc.unlink()
def is_vetoed(instrument, peak_time, peak_time_ns, seglists=seglists):
return instrument in seglists and dbtables.lsctables.LIGOTimeGPS(
peak_time, peak_time_ns) in seglists[instrument]
connection.create_function("string_sngl_is_vetoed", 3, is_vetoed)
def get_veto_segments(connection, name): """ Return a coalesced glue.segments.segmentlistdict object containing the segments of the given name extracted from the database at the given connection. """ xmldoc = dbtables.get_xml(connection) seglists = ligolw_segments.segmenttable_get_by_name(xmldoc, name).coalesce() xmldoc.unlink() return seglists
def get_background_offset_vectors(connection): """ Return a list of the non-zero offset vectors extracted from the database at the given connection. Each offset vector is returned as a dictionary mapping instrument name to offset. """ xmldoc = dbtables.get_xml(connection) offset_vectors = [offsetvector for offsetvector in dbtables.table.get_table(xmldoc, lsctables.TimeSlideTable.tableName).as_dict().values() if any(offsetvector.values())] xmldoc.unlink() return offset_vectors
def get_veto_segments(connection, name):
"""
Return a coalesced glue.segments.segmentlistdict object containing the
segments of the given name extracted from the database at the given
connection.
"""
xmldoc = dbtables.get_xml(connection)
seglists = ligolw_segments.segmenttable_get_by_name(xmldoc,
name).coalesce()
xmldoc.unlink()
return seglists
def load_segments(filename, name, verbose = False): if verbose: print >>sys.stderr, "loading \"%s\" segments ... " % name, connection = sqlite3.connect(filename) segs = ligolw_segments.segmenttable_get_by_name(dbtables.get_xml(connection), name).coalesce() connection.close() if verbose: print >>sys.stderr, "done." for ifo in segs: print >>sys.stderr, "loaded %d veto segment(s) for %s totalling %g s" % (len(segs[ifo]), ifo, float(abs(segs[ifo]))) return segs
def __init__(self, f, *args, **kwargs):
if isinstance(f, sqlite3.Connection):
db = f
filename = sqlite.get_filename(f)
else:
if hasattr(f, 'read'):
filename = f.name
f.close()
else:
filename = f
db = sqlite.open(filename, 'r')
super().__init__(dbtables.get_xml(db), *args, **kwargs)
self._fallbackpath = os.path.dirname(filename) if filename else None
def load_segments(filename, name, verbose=False):
if verbose:
print >> sys.stderr, "loading \"%s\" segments ... " % name,
connection = sqlite3.connect(filename)
segs = ligolw_segments.segmenttable_get_by_name(
dbtables.get_xml(connection), name).coalesce()
connection.close()
if verbose:
print >> sys.stderr, "done."
for ifo in segs:
print >> sys.stderr, "loaded %d veto segment(s) for %s totalling %g s" % (
len(segs[ifo]), ifo, float(abs(segs[ifo])))
return segs
def get_background_offset_vectors(connection):
"""
Return a list of the non-zero offset vectors extracted from the database
at the given connection. Each offset vector is returned as a dictionary
mapping instrument name to offset.
"""
xmldoc = dbtables.get_xml(connection)
offset_vectors = [
offsetvector for offsetvector in lsctables.TimeSlideTable.get_table(
xmldoc).as_dict().values() if any(offsetvector.values())
]
xmldoc.unlink()
return offset_vectors
def add_livetime(self, connection, live_time_program, veto_segments_name = None, verbose = False): if verbose: print >>sys.stderr, "\tcomputing livetimes:", xmldoc = dbtables.get_xml(connection) veto_segments = farutils.get_veto_segments(connection, live_time_program, xmldoc, veto_segments_name=veto_segments_name) segs = farutils.get_segments(connection, xmldoc, "lalapps_ring").coalesce() bglivetime=farutils.add_background_livetime(connection, live_time_program, segs, veto_segments, coinc_segments=None, verbose=verbose) if verbose: print >>sys.stderr, "\n\tbackground livetime %s\n" % (str(bglivetime)) for key in bglivetime.keys(): try: self.cached_livetime[key] += bglivetime[key] except KeyError: self.cached_livetime[key] = bglivetime[key]
def get_single_ifo_segments(connection, program_name="inspiral", usertag=None):
"""
Return a glue.segments.segmentlistdict object containing coalesced single-ifo
segments obtained from the search_summary table.
@param connection: sqlite connection for the input database
@param usertag: the usertag for the desired filter jobs e.g. ("FULL_DATA","PLAYGROUND")
"""
xmldoc = dbtables.get_xml(connection)
seglist_dict = segments.segmentlistdict()
# extract segments indexed by available instrument
for row in map(
dbtables.table.get_table(xmldoc, lsctables.SearchSummaryTable.tableName).row_from_cols,
connection.cursor().execute(
"""
SELECT search_summary.*
FROM search_summary
JOIN process_params ON (
process_params.process_id == search_summary.process_id)
WHERE
process_params.value == :1
AND process_params.program == :2
""",
(usertag, program_name),
),
):
instrument = row.get_ifos().pop()
if usertag == "FULL_DATA" or usertag == "PLAYGROUND":
try:
seglist_dict[instrument].append(row.get_out())
except KeyError:
seglist_dict[instrument] = [row.get_out()]
else:
buffer_time = 72
filtered_segment = segments.segment(row.get_in()[0] + buffer_time, row.get_in()[1] - buffer_time)
try:
seglist_dict[instrument].append(filtered_segment)
except KeyError:
seglist_dict[instrument] = [filtered_segment]
xmldoc.unlink()
seglist_dict = segments.segmentlistdict(
(key, segments.segmentlist(sorted(set(value)))) for key, value in seglist_dict.items()
)
return seglist_dict
def get_single_ifo_segments(connection, program_name="inspiral", usertag=None):
"""
Return a glue.segments.segmentlistdict object containing coalesced single-ifo
segments obtained from the search_summary table.
@param connection: sqlite connection for the input database
@param usertag: the usertag for the desired filter jobs e.g. ("FULL_DATA","PLAYGROUND")
"""
xmldoc = dbtables.get_xml(connection)
seglist_dict = segments.segmentlistdict()
# extract segments indexed by available instrument
for row in map(
dbtables.table.get_table(
xmldoc, lsctables.SearchSummaryTable.tableName).row_from_cols,
connection.cursor().execute(
"""
SELECT search_summary.*
FROM search_summary
JOIN process_params ON (
process_params.process_id == search_summary.process_id)
WHERE
process_params.value == :1
AND process_params.program == :2
""", (usertag, program_name))):
instrument = row.get_ifos().pop()
if usertag == "FULL_DATA" or usertag == "PLAYGROUND":
try:
seglist_dict[instrument].append(row.get_out())
except KeyError:
seglist_dict[instrument] = [row.get_out()]
else:
buffer_time = 72
filtered_segment = segments.segment(row.get_in()[0] + buffer_time,
row.get_in()[1] - buffer_time)
try:
seglist_dict[instrument].append(filtered_segment)
except KeyError:
seglist_dict[instrument] = [filtered_segment]
xmldoc.unlink()
seglist_dict = segments.segmentlistdict(
(key, segments.segmentlist(sorted(set(value))))
for key, value in seglist_dict.items())
return seglist_dict
def get_singles_times( connection, verbose = False ):
# get single-ifo filtered segments
ifo_segments = compute_dur.get_single_ifo_segments(
connection, program_name = "inspiral", usertag = "FULL_DATA")
# get all veto segments
xmldoc = dbtables.get_xml(connection)
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
sngls_durations = {}
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
for ifo, segs_list in post_vetoes_ifosegs.items():
sngls_durations[ifo] = float( abs(segs_list) )
return sngls_durations
def get_thinca_rings_by_available_instruments(connection,
program_name="thinca"):
"""
Return the thinca rings from the database at the given connection. The
rings are returned as a glue.segments.segmentlistdict indexed by the
set of instruments that were analyzed in that ring.
Example:
>>> seglists = get_thinca_rings_by_available_instruments(connection)
>>> print seglists.keys()
[frozenset(['H1', 'L1'])]
"""
# extract raw rings indexed by available instrument set
xmldoc = dbtables.get_xml(connection)
seglists = segments.segmentlistdict()
for row in map(
dbtables.table.get_table(
xmldoc, lsctables.SearchSummaryTable.tableName).row_from_cols,
connection.cursor().execute(
"""
SELECT
search_summary.*
FROM
search_summary
JOIN process ON (
process.process_id == search_summary.process_id
)
WHERE
process.program == ?
""", (program_name, ))):
available_instruments = frozenset(row.get_ifos())
try:
seglists[available_instruments].append(row.get_out())
except KeyError:
seglists[available_instruments] = [row.get_out()]
xmldoc.unlink()
# remove rings that are exact duplicates on the assumption that there are
# zero-lag and time-slide thinca jobs represented in the same document
return segments.segmentlistdict(
(key, segments.segmentlist(sorted(set(value))))
for key, value in seglists.items())
def get_singles_times(connection, verbose=False):
# get single-ifo filtered segments
ifo_segments = compute_dur.get_single_ifo_segments(connection,
program_name="inspiral",
usertag="FULL_DATA")
# get all veto segments
xmldoc = dbtables.get_xml(connection)
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
sngls_durations = {}
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
for ifo, segs_list in post_vetoes_ifosegs.items():
sngls_durations[ifo] = float(abs(segs_list))
return sngls_durations
def get_rinca_zero_lag_segments(connection, program_name="rinca"):
"""
Return the rinca rings from the database at the given connection. The
rings are returned as a coalesced glue.segments.segmentlistdict indexed
by instrument.
Example:
>>> seglists = get_rinca_zero_lag_segments(connection)
>>> print seglists.keys()
['H1', 'L1']
This function is most useful if only zero-lag segments are desired
because it allows for more convenient manipulation of the segment lists
using the methods in glue.segments. If information about background
segments or the original ring boundaries is desired the data returned by
get_rinca_rings_by_available_instruments() is required.
"""
# extract the raw rings indexed by instrument
xmldoc = dbtables.get_xml(connection)
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program_name)
xmldoc.unlink()
# remove rings that are exact duplicates on the assumption that there are
# zero-lag and time-slide rinca jobs represented in the same document
seglists = segments.segmentlistdict((key, segments.segmentlist(set(value)))
for key, value in seglists.items())
# coalesce the remaining segments making sure we don't loose livetime in
# the process
durations_before = abs(seglists)
seglists.coalesce()
if abs(seglists) != durations_before:
raise ValueError, "detected overlapping rinca rings"
# done
return seglists
def get_thinca_zero_lag_segments(connection, program_name = "thinca"):
"""
Return the thinca rings from the database at the given connection. The
rings are returned as a coalesced glue.segments.segmentlistdict indexed
by instrument.
Example:
>>> seglists = get_thinca_zero_lag_segments(connection)
>>> print seglists.keys()
['H1', 'L1']
This function is most useful if only zero-lag segments are desired
because it allows for more convenient manipulation of the segment lists
using the methods in glue.segments. If information about background
segments or the original ring boundaries is desired the data returned by
get_thinca_rings_by_available_instruments() is required.
"""
# extract the raw rings indexed by instrument
xmldoc = dbtables.get_xml(connection)
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program_name)
xmldoc.unlink()
# remove rings that are exact duplicates on the assumption that there are
# zero-lag and time-slide thinca jobs represented in the same document
seglists = segments.segmentlistdict((key, segments.segmentlist(set(value))) for key, value in seglists.items())
# coalesce the remaining segments making sure we don't loose livetime in
# the process
durations_before = abs(seglists)
seglists.coalesce()
if abs(seglists) != durations_before:
raise ValueError, "detected overlapping thinca rings"
# done
return seglists
def get_thinca_rings_by_available_instruments(connection, program_name = "thinca"):
"""
Return the thinca rings from the database at the given connection. The
rings are returned as a glue.segments.segmentlistdict indexed by the
set of instruments that were analyzed in that ring.
Example:
>>> seglists = get_thinca_rings_by_available_instruments(connection)
>>> print seglists.keys()
[frozenset(['H1', 'L1'])]
"""
# extract raw rings indexed by available instrument set
xmldoc = dbtables.get_xml(connection)
seglists = segments.segmentlistdict()
for row in map(dbtables.table.get_table(xmldoc, lsctables.SearchSummaryTable.tableName).row_from_cols, connection.cursor().execute("""
SELECT
search_summary.*
FROM
search_summary
JOIN process ON (
process.process_id == search_summary.process_id
)
WHERE
process.program == ?
""", (program_name,))):
available_instruments = frozenset(row.get_ifos())
try:
seglists[available_instruments].append(row.get_out())
except KeyError:
seglists[available_instruments] = [row.get_out()]
xmldoc.unlink()
# remove rings that are exact duplicates on the assumption that there are
# zero-lag and time-slide thinca jobs represented in the same document
return segments.segmentlistdict((key, segments.segmentlist(sorted(set(value)))) for key, value in seglists.items())
def successful_injections(
connection,
tag,
on_ifos,
veto_cat,
dist_type = "distance",
weight_dist = False,
verbose = False):
"""
My attempt to get a list of the simulations that actually made
it into some level of coincident time
"""
xmldoc = dbtables.get_xml(connection)
connection.create_function('end_time_with_ns', 2, end_time_with_ns)
# Get the veto segments as dictionaries, keyed by veto category
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# ------------------------ Get List of Injections ------------------------ #
sql_params_dict = {}
sqlquery = """
SELECT DISTINCT
simulation_id,
end_time_with_ns(geocent_end_time, geocent_end_time_ns),"""
# add the desired distance measure to the SQL query
if dist_type == "distance":
connection.create_function('distance_func', 2, chirp_dist)
sqlquery += """
distance_func(distance, sim_inspiral.mchirp)
FROM sim_inspiral """
elif dist_type == "decisive_distance":
connection.create_function('decisive_dist_func', 6, decisive_dist)
sql_params_dict['ifos'] = on_ifos
sql_params_dict['weight_dist'] = weight_dist
sqlquery += """
decisive_dist_func(
eff_dist_h, eff_dist_l, eff_dist_v,
sim_inspiral.mchirp, :weight_dist, :ifos)
FROM sim_inspiral """
if tag != 'ALL_INJ':
# if a specific injection set is wanted
sqlquery += """
JOIN process_params ON (
process_params.process_id == sim_inspiral.process_id)
WHERE process_params.value = :usertag) """
sql_params_dict["usertag"] = tag
else:
# for all injections
tag = 'FULL_DATA'
# Get segments that define which time was filtered
ifo_segments = compute_dur.get_single_ifo_segments(
connection,
program_name = "inspiral",
usertag = tag)
zero_lag_dict = dict([(ifo, 0.0) for ifo in ifo_segments])
successful_inj = []
# determine coincident segments for that veto category
coinc_segs = compute_dur.get_coinc_segments(
ifo_segments - veto_segments[veto_cat],
zero_lag_dict)
# Apply vetoes to single-ifo filter segments
for injection in connection.execute(sqlquery, sql_params_dict):
inj_segment = segments.segment(injection[1], injection[1])
if coinc_segs[on_ifos].intersects_segment( inj_segment ):
successful_inj.append( injection )
return successful_inj
"""
, version='%prog')
(opts,files)=parser.parse_args()
timeslide_likelihood = []
timeslide_snr = []
zerolag_likelihood = []
zerolag_snr = []
injection_likelihood = []
injection_snr = []
for filename in files:
local_disk = None #"/tmp"
working_filename = dbtables.get_connection_filename(filename, tmp_path = local_disk, verbose = True)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
cursor = connection.cursor()
for likelihood, snr, is_background in connection.cursor().execute("""
SELECT
insp_coinc_event.likelihood,
coinc_inspiral.snr,
EXISTS (
SELECT
*
FROM
time_slide
WHERE
time_slide.time_slide_id == insp_coinc_event.time_slide_id
AND time_slide.offset != 0
)
FROM
def __init__(self, flist, opts):
self.far = {}
self.segments = segments.segmentlistdict()
self.non_inj_fnames = []
self.inj_fnames = []
#self.der_fit = None
self.twoDMassBins = None
#self.dBin = {}
self.gw = None
self.found = {}
self.missed = {}
self.wnfunc = None
self.opts = opts
if opts.bootstrap_iterations: self.bootnum = int(opts.bootstrap_iterations)
else: self.bootnum = 100
self.veto_segments = segments.segmentlistdict()
self.zero_lag_segments = {}
self.instruments = []
self.livetime = {}
self.minmass = None
self.maxmass = None
self.mintotal = None
self.maxtotal = None
for f in flist:
if opts.verbose: print >> sys.stderr, "Gathering stats from: %s...." % (f,)
working_filename = dbtables.get_connection_filename(f, verbose = opts.verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# look for a sim table
try:
sim_inspiral_table = table.get_table(xmldoc, dbtables.lsctables.SimInspiralTable.tableName)
self.inj_fnames.append(f)
sim = True
except ValueError:
self.non_inj_fnames.append(f)
sim = False
if not sim:
if opts.veto_segments_name is not None: self.veto_segments = db_thinca_rings.get_veto_segments(connection, opts.veto_segments_name)
self.get_instruments(connection)
self.segments += db_thinca_rings.get_thinca_zero_lag_segments(connection, program_name = opts.live_time_program)
self.get_far_thresholds(connection)
else:
self.get_mass_ranges(connection)
#connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = opts.verbose)
dbtables.DBTable_set_connection(None)
# FIXME Do these have to be done by instruments?
self.segments -= self.veto_segments
# compute far, segments and livetime by instruments
for i in self.instruments:
self.far[i] = min(self.far[i])
# FIXME this bombs if any of the FARS are zero. maybe it should continue
# and just remove that instrument combo from the calculation
if self.far[i] == 0:
print >> sys.stderr, "Encountered 0 FAR in %s, ABORTING" % (i,)
sys.exit(1)
self.zero_lag_segments[i] = self.segments.intersection(i) - self.segments.union(set(self.segments.keys()) - i)
# Livetime must have playground removed
self.livetime[i] = float(abs(self.zero_lag_segments[i] - segmentsUtils.S2playground(self.segments.extent_all())))
if opts.verbose: print >> sys.stderr, "%s FAR %e, livetime %f" % (",".join(sorted(list(i))), self.far[i], self.livetime[i])
# get a 2D mass binning
self.twoDMassBins = self.get_2d_mass_bins(self.minmass, self.maxmass, opts.mass_bins)
def get_injections(injfnames,
zero_lag_segments,
ifos="H1,H2,L1",
FAR=1.0,
verbose=True):
"""
The LV stat uses simulations above threshold, not some IFAR of the loudest event, so the default should be "inf"
"""
def injection_was_made(geocent_end_time,
geocent_end_time_ns,
zero_lag_segments=zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time,
geocent_end_time_ns) in zero_lag_segments
found = []
missed = []
print("", file=sys.stderr)
for cnt, f in enumerate(injfnames):
print("getting injections: " + str(FAR) + ":\t%.1f%%\r" %
(100.0 * cnt / len(injfnames), ),
end=' ',
file=sys.stderr)
working_filename = dbtables.get_connection_filename(f,
tmp_path=None,
verbose=verbose)
connection = sqlite3.connect(working_filename)
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(
dbtables.get_xml(connection))._row_from_cols
# FIXME may not be done correctly if injections are done in timeslides
# FIXME may not be done correctly if injections aren't logarithmic in d
# do we really want d^3 waiting?
# FIXME do we really want injections independent of their FAR
for values in connection.cursor().execute(
"""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.false_alarm_rate < ?
AND coinc_inspiral.ifos == ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (
FAR,
ifos,
)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
connection.close()
dbtables.discard_connection_filename(f,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
print("\nFound = %d Missed = %d" % (len(found), len(missed)),
file=sys.stderr)
return found, missed
def sngl_snr_hist(
connection,
ifo,
mchirp,
eta,
min_snr,
snr_stat = None,
sngls_width = None,
usertag = "FULL_DATA",
datatype = None,
sngls_bins = None):
"""
Creates a histogram of sngl_inspiral triggers and returns a list of counts
and the associated snr bins.
@connection: connection to a SQLite database with lsctables
@ifo: the instrument one desires triggers from
@mchirp: the chirp mass from the desired template
@eta: the symmetric mass ratio from the desired template
@min_snr: a lower threshold on the value of the snr_stat
@sngls_width: the bin width for the histogram
@usertag: the usertag for the triggers. The default is "FULL_DATA".
@datatype: the datatype (all_data, slide, ...) if single-ifo triggers from
coincident events is desired. The default is to collect all triggers.
@sngls_bins: a list of bin edges for the snr-histogram
"""
# create function for the desired snr statistic
set_getsnr_function(connection, snr_stat)
connection.create_function('end_time_w_ns', 2, end_time_w_ns)
# set SQL statement parameters
sql_params_dict = {
"mchirp": mchirp, "eta": eta,
"min_snr": min_snr, "ifo": ifo,
"usertag": usertag}
# SQLite query to get a list of (snr, gps-time) tuples for inspiral triggers
sqlquery = """
SELECT DISTINCT
get_snr(snr, chisq, chisq_dof) as snr_stat,
end_time_w_ns(end_time, end_time_ns)
FROM sngl_inspiral
JOIN process_params ON (
process_params.process_id == sngl_inspiral.process_id) """
# if a datatype is given, get only the inspiral triggers from coincs of that type
if datatype:
sqlquery += """
JOIN coinc_event_map, experiment_map, experiment_summary ON (
coinc_event_map.event_id == sngl_inspiral.event_id
AND experiment_map.coinc_event_id == coinc_event_map.coinc_event_id
AND experiment_summary.experiment_summ_id == experiment_map.experiment_summ_id) """
sqlquery += """
WHERE
sngl_inspiral.ifo == :ifo
AND snr_stat >= :min_snr
AND sngl_inspiral.mchirp == :mchirp
AND sngl_inspiral.eta == :eta
AND process_params.value == :usertag """
if datatype:
sqlquery += """
AND experiment_summary.datatype == :type
"""
sql_params_dict["type"] = datatype
# get dq-veto segments
xmldoc = dbtables.get_xml(connection)
veto_segments = compute_dur.get_veto_segments(xmldoc, False)
veto_segments = veto_segments[ veto_segments.keys()[0] ]
snr_array = np.array([])
# apply vetoes to the list of trigger times
for snr, trig_time in connection.execute( sqlquery, sql_params_dict ):
trig_segment = segments.segment(trig_time, trig_time)
if not veto_segments[ifo].intersects_segment( trig_segment ):
snr_array = np.append( snr_array, snr )
if sngls_bins is None:
sngls_bins = np.arange(min_snr, np.max(snr_array) + sngls_width, sngls_width)
# make the binned snr histogram
sngls_hist, _ = np.histogram(snr_array, bins=sngls_bins)
return sngls_hist, sngls_bins
def get_injections(self, instruments, FAR=float("inf")):
injfnames = self.inj_fnames
zero_lag_segments = self.zero_lag_segments[instruments]
verbose = self.opts.verbose
found = []
missed = []
print >>sys.stderr, ""
for cnt, f in enumerate(injfnames):
print >>sys.stderr, "getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),),
working_filename = dbtables.get_connection_filename(f, tmp_path = opts.tmp_space, verbose = verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# DON'T BOTHER CONTINUING IF THE INSTRUMENTS OF INTEREST ARE NOT HERE
instruments_in_this_file = []
for i in connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event'):
if i[0]: instruments_in_this_file.append(frozenset(lsctables.instrument_set_from_ifos(i[0])))
if instruments not in instruments_in_this_file:
connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
continue
# WORK OUT CORRECT SEGMENTS FOR THIS FILE WHERE WE SHOULD SEE INJECTIONS
segments = self.get_segments(connection, xmldoc)
segments -= self.veto_segments
#print thincasegments
zero_lag_segments = segments.intersection(instruments) - segments.union(set(segments.keys()) - instruments)
###############
# DEFINE THE INJECTION WAS MADE FUNCTION
def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(dbtables.get_xml(connection)).row_from_cols
# INSPIRAL
if self.coinc_inspiral_table:
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# BURSTS
if self.multi_burst_table:
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN multi_burst ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == multi_burst.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND multi_burst.false_alarm_rate < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
print >>sys.stderr, "\nFound = %d Missed = %d" % (len(found), len(missed))
return found, missed
def __init__(self, filelist, live_time_program = None, veto_segments_name = None, data_segments_name = "datasegments", tmp_path = None, verbose = False):
self.segments = segments.segmentlistdict()
self.instruments = set()
self.table_name = None
self.found_injections_by_instrument_set = {}
self.missed_injections_by_instrument_set = {}
self.total_injections_by_instrument_set = {}
self.zerolag_fars_by_instrument_set = {}
self.ts_fars_by_instrument_set = {}
self.numslides = set()
for f in filelist:
if verbose:
print >> sys.stderr, "Gathering stats from: %s...." % (f,)
working_filename = dbtables.get_connection_filename(f, tmp_path = tmp_path, verbose = verbose)
connection = sqlite3.connect(working_filename)
xmldoc = dbtables.get_xml(connection)
sim = False
# look for a sim inspiral table. This is IMR work we have to have one of these :)
try:
sim_inspiral_table = table.get_table(xmldoc, dbtables.lsctables.SimInspiralTable.tableName)
sim = True
except ValueError:
pass
# look for the relevant table for analyses
for table_name in allowed_analysis_table_names():
try:
setattr(self, table_name, table.get_table(xmldoc, table_name))
if self.table_name is None or self.table_name == table_name:
self.table_name = table_name
else:
raise ValueError("detected more than one table type out of " + " ".join(allowed_analysis_table_names()))
except ValueError:
setattr(self, table_name, None)
# the non simulation databases are where we get information about segments
if not sim:
self.numslides.add(connection.cursor().execute('SELECT count(DISTINCT(time_slide_id)) FROM time_slide').fetchone()[0])
[self.instruments.add(ifos) for ifos in get_instruments_from_coinc_event_table(connection)]
# save a reference to the segments for this file, needed to figure out the missed and found injections
self.this_segments = get_segments(connection, xmldoc, self.table_name, live_time_program, veto_segments_name, data_segments_name = data_segments_name)
# FIXME we don't really have any reason to use playground segments, but I put this here as a reminder
# self.this_playground_segments = segmentsUtils.S2playground(self.this_segments.extent_all())
self.segments += self.this_segments
# get the far thresholds for the loudest events in these databases
for (instruments_set, far, ts) in get_event_fars(connection, self.table_name):
if not ts:
self.zerolag_fars_by_instrument_set.setdefault(instruments_set, []).append(far)
else:
self.ts_fars_by_instrument_set.setdefault(instruments_set, []).append(far)
# get the injections
else:
# We need to know the segments in this file to determine which injections are found
self.this_injection_segments = get_segments(connection, xmldoc, self.table_name, live_time_program, veto_segments_name, data_segments_name = data_segments_name)
self.this_injection_instruments = []
distinct_instruments = connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event WHERE instruments!=""').fetchall()
for instruments, in distinct_instruments:
instruments_set = frozenset(lsctables.instrument_set_from_ifos(instruments))
self.this_injection_instruments.append(instruments_set)
segments_to_consider_for_these_injections = self.this_injection_segments.intersection(instruments_set) - self.this_injection_segments.union(set(self.this_injection_segments.keys()) - instruments_set)
found, total, missed = get_min_far_inspiral_injections(connection, segments = segments_to_consider_for_these_injections, table_name = self.table_name)
if verbose:
print >> sys.stderr, "%s total injections: %d; Found injections %d: Missed injections %d" % (instruments, len(total), len(found), len(missed))
self.found_injections_by_instrument_set.setdefault(instruments_set, []).extend(found)
self.total_injections_by_instrument_set.setdefault(instruments_set, []).extend(total)
self.missed_injections_by_instrument_set.setdefault(instruments_set, []).extend(missed)
# All done
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
if len(self.numslides) > 1:
raise ValueError('number of slides differs between input files')
elif self.numslides:
self.numslides = min(self.numslides)
else:
self.numslides = 0
def successful_injections(connection,
tag,
on_ifos,
veto_cat,
dist_type="distance",
weight_dist=False,
verbose=False):
"""
My attempt to get a list of the simulations that actually made
it into some level of coincident time
"""
xmldoc = dbtables.get_xml(connection)
connection.create_function('end_time_with_ns', 2, end_time_with_ns)
# Get the veto segments as dictionaries, keyed by veto category
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# ------------------------ Get List of Injections ------------------------ #
sql_params_dict = {}
sqlquery = """
SELECT DISTINCT
simulation_id,
end_time_with_ns(geocent_end_time, geocent_end_time_ns),"""
# add the desired distance measure to the SQL query
if dist_type == "distance":
connection.create_function('distance_func', 2, chirp_dist)
sqlquery += """
distance_func(distance, sim_inspiral.mchirp)
FROM sim_inspiral """
elif dist_type == "decisive_distance":
connection.create_function('decisive_dist_func', 6, decisive_dist)
sql_params_dict['ifos'] = on_ifos
sql_params_dict['weight_dist'] = weight_dist
sqlquery += """
decisive_dist_func(
eff_dist_h, eff_dist_l, eff_dist_v,
sim_inspiral.mchirp, :weight_dist, :ifos)
FROM sim_inspiral """
if tag != 'ALL_INJ':
# if a specific injection set is wanted
sqlquery += """
JOIN process_params ON (
process_params.process_id == sim_inspiral.process_id)
WHERE process_params.value = :usertag) """
sql_params_dict["usertag"] = tag
else:
# for all injections
tag = 'FULL_DATA'
# Get segments that define which time was filtered
ifo_segments = compute_dur.get_single_ifo_segments(connection,
program_name="inspiral",
usertag=tag)
zero_lag_dict = dict([(ifo, 0.0) for ifo in ifo_segments])
successful_inj = []
# determine coincident segments for that veto category
coinc_segs = compute_dur.get_coinc_segments(
ifo_segments - veto_segments[veto_cat], zero_lag_dict)
# Apply vetoes to single-ifo filter segments
for injection in connection.execute(sqlquery, sql_params_dict):
inj_segment = segments.segment(injection[1], injection[1])
if coinc_segs[on_ifos].intersects_segment(inj_segment):
successful_inj.append(injection)
return successful_inj
inj_files = glob.glob(opts.injections)
fulldata_files = glob.glob(opts.fulldata)
timeslide_likelihood = []
timeslide_snr = []
zerolag_likelihood = []
zerolag_snr = []
for filename in fulldata_files:
local_disk = None #"/tmp"
working_filename = dbtables.get_connection_filename(filename,
tmp_path=local_disk,
verbose=True)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
cursor = connection.cursor()
for likelihood, snr, is_background in connection.cursor().execute("""
SELECT
insp_coinc_event.likelihood,
coinc_inspiral.snr,
EXISTS (
SELECT
*
FROM
time_slide
WHERE
time_slide.time_slide_id == insp_coinc_event.time_slide_id
AND time_slide.offset != 0
)
FROM
def calc_effective_snr(snr, chisq, chisq_dof, fac=250.0):
return snr/ (1 + snr**2/fac)**(0.25) / (chisq/(2*chisq_dof - 2) )**(0.25)
injections = []
injections_info = []
zerolag = []
zerolag_info = []
timeslides = []
timeslides_info = []
for database in databases:
local_disk = None #"/tmp"
working_filename = dbtables.get_connection_filename(database, tmp_path = local_disk, verbose = True)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
cursor = connection.cursor()
# to determine whether or not the database is full of timeslides/zerolag or injections, check if there exists a sim_inspiral table
try:
sim_inspiral_table = table.get_table(xmldoc, dbtables.lsctables.SimInspiralTable.tableName)
is_injections = True
except ValueError:
is_injections = False
if is_injections:
def calc_delta_t_inj(trigger1_end_time, trigger1_end_time_ns, trigger2_end_time, trigger2_end_time_ns):
try:
return abs((trigger1_end_time - trigger2_end_time) + (trigger1_end_time_ns - trigger2_end_time)*1e-9)
except: print "calc_delta_t_inj() failed"
connection.create_function("calc_delta_t_inj", 4, calc_delta_t_inj)
connection.create_function("calc_effective_snr", 3, calc_effective_snr)
def __init__(self, connection, live_time_program, search = "excesspower", veto_segments_name = None): """ Compute and record some summary information about the database. Call this after all the data has been inserted, and before you want any of this information. """ self.connection = connection self.xmldoc = dbtables.get_xml(connection) # find the tables try: self.sngl_burst_table = lsctables.SnglBurstTable.get_table(self.xmldoc) except ValueError: self.sngl_burst_table = None try: self.sim_burst_table = lsctables.SimBurstTable.get_table(self.xmldoc) except ValueError: self.sim_burst_table = None try: self.coinc_def_table = lsctables.CoincDefTable.get_table(self.xmldoc) self.coinc_table = lsctables.CoincTable.get_table(self.xmldoc) self.time_slide_table = lsctables.TimeSlideTable.get_table(self.xmldoc) except ValueError: self.coinc_def_table = None self.coinc_table = None self.time_slide_table = None try: self.multi_burst_table = lsctables.MultiBurstTable.get_table(self.xmldoc) except ValueError: self.multi_burst_table = None # get the segment lists self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(self.xmldoc, live_time_program).coalesce() self.instruments = set(self.seglists.keys()) if veto_segments_name is not None: self.vetoseglists = ligolw_segments.segmenttable_get_by_name(self.xmldoc, veto_segments_name).coalesce() else: self.vetoseglists = ligolw_segments.segments.segmentlistdict() # determine a few coinc_definer IDs # FIXME: don't hard-code the numbers if self.coinc_def_table is not None: try: self.bb_definer_id = self.coinc_def_table.get_coinc_def_id(search, 0, create_new = False) except KeyError: self.bb_definer_id = None try: self.sb_definer_id = self.coinc_def_table.get_coinc_def_id(search, 1, create_new = False) except KeyError: self.sb_definer_id = None try: self.sce_definer_id = self.coinc_def_table.get_coinc_def_id(search, 2, create_new = False) except KeyError: self.sce_definer_id = None try: self.scn_definer_id = self.coinc_def_table.get_coinc_def_id(search, 3, create_new = False) except KeyError: self.scn_definer_id = None else: self.bb_definer_id = None self.sb_definer_id = None self.sce_definer_id = None self.scn_definer_id = None
def printmissed(connection, simulation_table, recovery_table, map_label, livetime_program,
param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ',
limit = None, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
from pylal import ligolw_cbc_compute_durations as compute_dur
from glue import segments
from glue.ligolw import dbtables
# Get simulation/recovery tables
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
#
# Create and prepare the CloseMissedTable to store summary information
#
# Get simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
column_names = simulation_table_columns + \
['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag']
# define needed tables
class CloseMissedTable(table.Table):
tableName = "close_missed_injections:table"
validcolumns = {}
for col_name in column_names:
if 'rank' in col_name:
validcolumns[col_name] = "int_4u"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif 'veto_def_name' == col_name:
validcolumns[col_name] = lsctables.ExperimentSummaryTable.validcolumns['veto_def_name']
elif 'decisive_distance' == col_name:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, 'eff_dist_h')
elif 'gps_time' == col_name or 'gps_time_ns' == col_name:
validcolumns[col_name] = "int_4s"
elif 'sim_tag' == col_name:
validcolumns[col_name] = "lstring"
else:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, col_name, default = 'lstring')
class CloseMissed(object):
__slots__ = CloseMissedTable.validcolumns.keys()
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
CloseMissedTable.RowType = CloseMissed
# create the table
cmtable = lsctables.New(CloseMissedTable)
# set up sim_rec_map table
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, None)
#
# Set table filters
#
# we force the include/exclude filters to None; will check for excluded/included ifo time
# when cycling through the ifo times
filter = """
WHERE
simulation_id NOT IN (
SELECT
sim_id
FROM
sim_rec_map )"""
af = create_filter( connection, simulation_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = None, include_only_coincs = None, sim_tag = sim_tag, verbose = verbose)
af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af)
if af != '':
filter = '\n'.join([ filter, """
AND""", af])
# get desired instrument times
if include_only_coincs is not None:
include_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( include_only_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
if exclude_coincs is not None:
exclude_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( exclude_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
# get the usertags of inspiral jobs in this database
sqlquery = """
SELECT value
FROM process_params
WHERE param == "-userTag"
GROUP BY value
"""
usertags = set(usertag[0] for usertag in connection.cursor().execute(sqlquery) )
# Get the single-ifo science segments after CAT-1 vetoes
try:
if "FULL_DATA" in usertags:
tag = "FULL_DATA"
else:
tag = list(usertags)[0]
except IndexError:
# This is hacky anyway, so let's just take a guess
tag = "FULL_DATA"
ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name = livetime_program, usertag = tag)
if ifo_segments == {}:
raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program
if verbose:
print >> sys.stderr, "Getting all veto category names from the experiment_summary table..."
xmldoc = dbtables.get_xml(connection)
# get veto_segments
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# make a dictionary of zerolag "shifts" needed for the get_coinc_segments function
zerolag_dict = {}
for ifo in ifo_segments:
zerolag_dict[ifo] = 0.0
# Cycle over available veto categories
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
# make a dictionary of coincident segments by exclusive on-ifos
coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs, zerolag_dict)
#
# Get all the on_instrument times and cycle over them
#
sqlquery = """
SELECT DISTINCT experiment.instruments
FROM experiment
JOIN experiment_summary ON (
experiment.experiment_id == experiment_summary.experiment_id )
WHERE experiment_summary.veto_def_name == :1
"""
for on_instruments in connection.cursor().execute(sqlquery, (veto_def_name,)).fetchall():
on_instruments = lsctables.instrument_set_from_ifos(on_instruments[0])
# check if this on_instruments is desired; if not, skip
if include_only_coincs is not None and frozenset(on_instruments) not in include_times:
continue
if exclude_coincs is not None and frozenset(on_instruments) in exclude_times:
continue
on_times = coinc_segs[','.join(sorted(on_instruments))]
def is_in_on_time(gps_time, gps_time_ns):
return gps_time + (1e-9 * gps_time_ns) in on_times
connection.create_function('is_in_on_time', 2, is_in_on_time)
# add the check for on time to the filter
in_this_filter = filter
# figure out if simulation_table has end_times or start_times
end_or_start = any('end_time' in c for c in simulation_table_columns) and '_end_time' or '_start_time'
for instrument in on_instruments:
inst_time = instrument.lower()[0] + end_or_start
inst_time_ns = inst_time + '_ns'
in_this_filter = ''.join([ in_this_filter,
'\n\tAND is_in_on_time(', inst_time, ',', inst_time_ns, ')' ])
#
# Set up decisive distance argument
#
def get_decisive_distance( *args ):
return sorted(args)[1]
connection.create_function('get_decisive_distance', len(on_instruments), get_decisive_distance)
decisive_distance = ''.join(['get_decisive_distance(', ','.join(['eff_dist_'+inst.lower()[0] for inst in on_instruments]), ')' ])
#
# Initialize ranking. Statistics for ranking are based on decisive distance
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(simulation_table, decisive_distance, 'ASC')
# add requirement that stats not be found in the sim_rec_table to in_this_filter
ranker.populate_stats_list(connection, limit = limit, filter = in_this_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Get the Data
#
sqlquery = ''.join(["""
SELECT
*,
get_sim_tag(process_id),
""", decisive_distance, """,
rank(""", decisive_distance, """)
FROM
""", simulation_table, """
""", in_this_filter, """
%s""" % (limit is not None and ''.join(['AND rank(', decisive_distance, ') <= ', str(limit)]) or ''), """
ORDER BY
rank(""", decisive_distance, """) ASC
"""])
if verbose:
print >> sys.stderr, "Getting injections..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
cmrow = CloseMissed()
[ setattr(cmrow, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
cmrow.decisive_distance = values[-2]
cmrow.rank = values[-1]
cmrow.instruments_on = lsctables.ifos_from_instrument_set(on_instruments)
cmrow.veto_def_name = veto_def_name
cmrow.sim_tag = values[-3]
cmrow.mini_followup = None
cmrow.omega_scan = None
cmrow.gps_time = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start)
cmrow.gps_time_ns = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start + '_ns')
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time)) for ifo in on_instruments]
cmrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
injection_time_utc = format_end_time_in_utc( cmrow.gps_time )
daily_ihope_address = get_daily_ihope_page(cmrow.gps_time, pages_location = daily_ihope_pages_location)
cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, injection_time_utc )
# add the row
cmtable.append(cmrow)
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return cmtable
def __init__(self, opts, flist):
self.segments = segments.segmentlistdict()
self.non_inj_fnames = []
self.inj_fnames = []
self.found = {}
self.missed = {}
self.opts = opts
self.veto_segments = segments.segmentlistdict()
self.zero_lag_segments = {}
self.instruments = []
self.livetime = {}
self.multi_burst_table = None
self.coinc_inspiral_table = None
for f in flist:
if opts.verbose: print >> sys.stderr, "Gathering stats from: %s...." % (f,)
working_filename = dbtables.get_connection_filename(f, tmp_path=opts.tmp_space, verbose = opts.verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# look for a sim table
try:
sim_inspiral_table = dbtables.lsctables.SimInspiralTable.get_table(xmldoc)
self.inj_fnames.append(f)
sim = True
except ValueError:
self.non_inj_fnames.append(f)
sim = False
# FIGURE OUT IF IT IS A BURST OR INSPIRAL RUN
try:
self.multi_burst_table = dbtables.lsctables.MultiBurstTable.get_table(xmldoc)
except ValueError:
self.multi_burst_table = None
try:
self.coinc_inspiral_table = dbtables.lsctables.CoincInspiralTable.get_table(xmldoc)
except ValueError:
self.coinc_inspiral_table = None
if self.multi_burst_table and self.coinc_inspiral_table:
print >>sys.stderr, "both burst and inspiral tables found. Aborting"
raise ValueError
if not sim:
self.get_instruments(connection)
self.segments += self.get_segments(connection,xmldoc)
#FIXME, don't assume veto segments are the same in every file!
self.veto_segments = self.get_veto_segments(connection)
dbtables.discard_connection_filename(f, working_filename, verbose = opts.verbose)
dbtables.DBTable_set_connection(None)
# remove redundant instruments
self.instruments = list(set(self.instruments))
# FIXME Do these have to be done by instruments?
self.segments -= self.veto_segments
# segments and livetime by instruments
for i in self.instruments:
self.zero_lag_segments[i] = self.segments.intersection(i) - self.segments.union(set(self.segments.keys()) - i)
self.livetime[i] = float(abs(self.zero_lag_segments[i]))
def sngl_snr_hist(connection,
ifo,
mchirp,
eta,
min_snr,
snr_stat=None,
sngls_width=None,
usertag="FULL_DATA",
datatype=None,
sngls_bins=None):
"""
Creates a histogram of sngl_inspiral triggers and returns a list of counts
and the associated snr bins.
@connection: connection to a SQLite database with lsctables
@ifo: the instrument one desires triggers from
@mchirp: the chirp mass from the desired template
@eta: the symmetric mass ratio from the desired template
@min_snr: a lower threshold on the value of the snr_stat
@sngls_width: the bin width for the histogram
@usertag: the usertag for the triggers. The default is "FULL_DATA".
@datatype: the datatype (all_data, slide, ...) if single-ifo triggers from
coincident events is desired. The default is to collect all triggers.
@sngls_bins: a list of bin edges for the snr-histogram
"""
# create function for the desired snr statistic
set_getsnr_function(connection, snr_stat)
connection.create_function('end_time_w_ns', 2, end_time_w_ns)
# set SQL statement parameters
sql_params_dict = {
"mchirp": mchirp,
"eta": eta,
"min_snr": min_snr,
"ifo": ifo,
"usertag": usertag
}
# SQLite query to get a list of (snr, gps-time) tuples for inspiral triggers
sqlquery = """
SELECT DISTINCT
get_snr(snr, chisq, chisq_dof) as snr_stat,
end_time_w_ns(end_time, end_time_ns)
FROM sngl_inspiral
JOIN process_params ON (
process_params.process_id == sngl_inspiral.process_id) """
# if a datatype is given, get only the inspiral triggers from coincs of that type
if datatype:
sqlquery += """
JOIN coinc_event_map, experiment_map, experiment_summary ON (
coinc_event_map.event_id == sngl_inspiral.event_id
AND experiment_map.coinc_event_id == coinc_event_map.coinc_event_id
AND experiment_summary.experiment_summ_id == experiment_map.experiment_summ_id) """
sqlquery += """
WHERE
sngl_inspiral.ifo == :ifo
AND snr_stat >= :min_snr
AND sngl_inspiral.mchirp == :mchirp
AND sngl_inspiral.eta == :eta
AND process_params.value == :usertag """
if datatype:
sqlquery += """
AND experiment_summary.datatype == :type
"""
sql_params_dict["type"] = datatype
# get dq-veto segments
xmldoc = dbtables.get_xml(connection)
veto_segments = compute_dur.get_veto_segments(xmldoc, False)
veto_segments = veto_segments[veto_segments.keys()[0]]
snr_array = np.array([])
# apply vetoes to the list of trigger times
for snr, trig_time in connection.execute(sqlquery, sql_params_dict):
trig_segment = segments.segment(trig_time, trig_time)
if not veto_segments[ifo].intersects_segment(trig_segment):
snr_array = np.append(snr_array, snr)
if sngls_bins is None:
sngls_bins = np.arange(min_snr,
np.max(snr_array) + sngls_width, sngls_width)
# make the binned snr histogram
sngls_hist, _ = np.histogram(snr_array, bins=sngls_bins)
return sngls_hist, sngls_bins
def make_sim_inspiral_row_from_columns_in_db(connection): """ get the unique mapping of a sim inspiral row from columns in this database """ return lsctables.table.get_table(dbtables.get_xml(connection), lsctables.SimInspiralTable.tableName).row_from_cols
injections = []
injections_info = []
zerolag = []
zerolag_info = []
timeslides = []
timeslides_info = []
for database in databases:
local_disk = None #"/tmp"
working_filename = dbtables.get_connection_filename(database,
tmp_path=local_disk,
verbose=True)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
cursor = connection.cursor()
# to determine whether or not the database is full of timeslides/zerolag or injections, check if there exists a sim_inspiral table
try:
sim_inspiral_table = dbtables.lsctables.SimInspiralTable.get_table(
xmldoc)
is_injections = True
except ValueError:
is_injections = False
if is_injections:
def calc_delta_t_inj(trigger1_end_time, trigger1_end_time_ns,
trigger2_end_time, trigger2_end_time_ns):
try:
return abs((trigger1_end_time - trigger2_end_time) +
def __init__(self, filelist, live_time_program = None, veto_segments_name = None, data_segments_name = "datasegments", tmp_path = None, verbose = False):
self.segments = segments.segmentlistdict()
self.instruments = set()
self.table_name = None
self.found_injections_by_instrument_set = {}
self.missed_injections_by_instrument_set = {}
self.total_injections_by_instrument_set = {}
self.zerolag_fars_by_instrument_set = {}
self.ts_fars_by_instrument_set = {}
self.numslides = set()
for f in filelist:
if verbose:
print >> sys.stderr, "Gathering stats from: %s...." % (f,)
working_filename = dbtables.get_connection_filename(f, tmp_path = tmp_path, verbose = verbose)
connection = sqlite3.connect(working_filename)
xmldoc = dbtables.get_xml(connection)
sim = False
# look for a sim inspiral table. This is IMR work we have to have one of these :)
try:
sim_inspiral_table = table.get_table(xmldoc, dbtables.lsctables.SimInspiralTable.tableName)
sim = True
except ValueError:
pass
# look for the relevant table for analyses
for table_name in allowed_analysis_table_names():
try:
setattr(self, table_name, table.get_table(xmldoc, table_name))
if self.table_name is None or self.table_name == table_name:
self.table_name = table_name
else:
raise ValueError("detected more than one table type out of " + " ".join(allowed_analysis_table_names()))
except ValueError:
setattr(self, table_name, None)
# the non simulation databases are where we get information about segments
if not sim:
self.numslides.add(connection.cursor().execute('SELECT count(DISTINCT(time_slide_id)) FROM time_slide').fetchone()[0])
[self.instruments.add(ifos) for ifos in get_instruments_from_coinc_event_table(connection)]
# save a reference to the segments for this file, needed to figure out the missed and found injections
self.this_segments = get_segments(connection, xmldoc, self.table_name, live_time_program, veto_segments_name, data_segments_name = data_segments_name)
# FIXME we don't really have any reason to use playground segments, but I put this here as a reminder
# self.this_playground_segments = segmentsUtils.S2playground(self.this_segments.extent_all())
self.segments += self.this_segments
# get the far thresholds for the loudest events in these databases
for (instruments_set, far, ts) in get_event_fars(connection, self.table_name):
if not ts:
self.zerolag_fars_by_instrument_set.setdefault(instruments_set, []).append(far)
else:
self.ts_fars_by_instrument_set.setdefault(instruments_set, []).append(far)
# get the injections
else:
# We need to know the segments in this file to determine which injections are found
self.this_injection_segments = get_segments(connection, xmldoc, self.table_name, live_time_program, veto_segments_name, data_segments_name = data_segments_name)
self.this_injection_instruments = []
distinct_instruments = connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event WHERE instruments!=""').fetchall()
for instruments, in distinct_instruments:
instruments_set = frozenset(lsctables.instrument_set_from_ifos(instruments))
self.this_injection_instruments.append(instruments_set)
segments_to_consider_for_these_injections = self.this_injection_segments.intersection(instruments_set) - self.this_injection_segments.union(set(self.this_injection_segments.keys()) - instruments_set)
found, total, missed = get_min_far_inspiral_injections(connection, segments = segments_to_consider_for_these_injections, table_name = self.table_name)
if verbose:
print >> sys.stderr, "%s total injections: %d; Found injections %d: Missed injections %d" % (instruments, len(total), len(found), len(missed))
self.found_injections_by_instrument_set.setdefault(instruments_set, []).extend(found)
self.total_injections_by_instrument_set.setdefault(instruments_set, []).extend(total)
self.missed_injections_by_instrument_set.setdefault(instruments_set, []).extend(missed)
# All done
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
if len(self.numslides) > 1:
raise ValueError('number of slides differs between input files')
elif self.numslides:
self.numslides = min(self.numslides)
else:
self.numslides = 0
def get_injections(injfnames, zero_lag_segments, ifos="H1,H2,L1", FAR=1.0, verbose = True):
"""
The LV stat uses simulations above threshold, not some IFAR of the loudest event, so the default should be "inf"
"""
def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments
found = []
missed = []
print >>sys.stderr, ""
for cnt, f in enumerate(injfnames):
print >>sys.stderr, "getting injections: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),),
working_filename = dbtables.get_connection_filename(f, tmp_path = None, verbose = verbose)
connection = sqlite3.connect(working_filename)
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.table.get_table(dbtables.get_xml(connection), lsctables.SimInspiralTable.tableName)._row_from_cols
# FIXME may not be done correctly if injections are done in timeslides
# FIXME may not be done correctly if injections aren't logarithmic in d
# do we really want d^3 waiting?
# FIXME do we really want injections independent of their FAR
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.false_alarm_rate < ?
AND coinc_inspiral.ifos == ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,ifos,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
print >>sys.stderr, "\nFound = %d Missed = %d" % (len(found), len(missed))
return found, missed
def get_injections(self, instruments, FAR=float("inf")):
injfnames = self.inj_fnames
zero_lag_segments = self.zero_lag_segments[instruments]
verbose = self.opts.verbose
found = []
missed = []
print("", file=sys.stderr)
for cnt, f in enumerate(injfnames):
print("getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" %
(100.0 * cnt / len(injfnames), ),
end=' ',
file=sys.stderr)
working_filename = dbtables.get_connection_filename(
f, tmp_path=opts.tmp_space, verbose=verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# DON'T BOTHER CONTINUING IF THE INSTRUMENTS OF INTEREST ARE NOT HERE
instruments_in_this_file = []
for i in connection.cursor().execute(
'SELECT DISTINCT(instruments) FROM coinc_event'):
if i[0]:
instruments_in_this_file.append(
frozenset(lsctables.instrument_set_from_ifos(i[0])))
if instruments not in instruments_in_this_file:
connection.close()
dbtables.discard_connection_filename(f,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
continue
# WORK OUT CORRECT SEGMENTS FOR THIS FILE WHERE WE SHOULD SEE INJECTIONS
segments = self.get_segments(connection, xmldoc)
segments -= self.veto_segments
#print thincasegments
zero_lag_segments = segments.intersection(
instruments) - segments.union(
set(segments.keys()) - instruments)
###############
# DEFINE THE INJECTION WAS MADE FUNCTION
def injection_was_made(geocent_end_time,
geocent_end_time_ns,
zero_lag_segments=zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(
geocent_end_time, geocent_end_time_ns) in zero_lag_segments
connection.create_function("injection_was_made", 2,
injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(
dbtables.get_xml(connection)).row_from_cols
# INSPIRAL
if self.coinc_inspiral_table:
for values in connection.cursor().execute(
"""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR, )):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# BURSTS
if self.multi_burst_table:
for values in connection.cursor().execute(
"""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN multi_burst ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == multi_burst.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND multi_burst.false_alarm_rate < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR, )):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
dbtables.discard_connection_filename(f,
working_filename,
verbose=verbose)
dbtables.DBTable_set_connection(None)
print("\nFound = %d Missed = %d" % (len(found), len(missed)),
file=sys.stderr)
return found, missed
def __init__(self, opts, flist):
self.segments = segments.segmentlistdict()
self.non_inj_fnames = []
self.inj_fnames = []
self.found = {}
self.missed = {}
self.opts = opts
self.veto_segments = segments.segmentlistdict()
self.zero_lag_segments = {}
self.instruments = []
self.livetime = {}
self.multi_burst_table = None
self.coinc_inspiral_table = None
for f in flist:
if opts.verbose:
print("Gathering stats from: %s...." % (f, ), file=sys.stderr)
working_filename = dbtables.get_connection_filename(
f, tmp_path=opts.tmp_space, verbose=opts.verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# look for a sim table
try:
sim_inspiral_table = dbtables.lsctables.SimInspiralTable.get_table(
xmldoc)
self.inj_fnames.append(f)
sim = True
except ValueError:
self.non_inj_fnames.append(f)
sim = False
# FIGURE OUT IF IT IS A BURST OR INSPIRAL RUN
try:
self.multi_burst_table = dbtables.lsctables.MultiBurstTable.get_table(
xmldoc)
except ValueError:
self.multi_burst_table = None
try:
self.coinc_inspiral_table = dbtables.lsctables.CoincInspiralTable.get_table(
xmldoc)
except ValueError:
self.coinc_inspiral_table = None
if self.multi_burst_table and self.coinc_inspiral_table:
print("both burst and inspiral tables found. Aborting",
file=sys.stderr)
raise ValueError
if not sim:
self.get_instruments(connection)
self.segments += self.get_segments(connection, xmldoc)
#FIXME, don't assume veto segments are the same in every file!
self.veto_segments = self.get_veto_segments(connection)
dbtables.discard_connection_filename(f,
working_filename,
verbose=opts.verbose)
dbtables.DBTable_set_connection(None)
# remove redundant instruments
self.instruments = list(set(self.instruments))
# FIXME Do these have to be done by instruments?
self.segments -= self.veto_segments
# segments and livetime by instruments
for i in self.instruments:
self.zero_lag_segments[i] = self.segments.intersection(
i) - self.segments.union(set(self.segments.keys()) - i)
self.livetime[i] = float(abs(self.zero_lag_segments[i]))
def __init__(self,
connection,
live_time_program,
search="excesspower",
veto_segments_name=None):
"""
Compute and record some summary information about the
database. Call this after all the data has been inserted,
and before you want any of this information.
"""
self.connection = connection
self.xmldoc = dbtables.get_xml(connection)
# find the tables
try:
self.sngl_burst_table = lsctables.SnglBurstTable.get_table(
self.xmldoc)
except ValueError:
self.sngl_burst_table = None
try:
self.sim_burst_table = lsctables.SimBurstTable.get_table(
self.xmldoc)
except ValueError:
self.sim_burst_table = None
try:
self.coinc_def_table = lsctables.CoincDefTable.get_table(
self.xmldoc)
self.coinc_table = lsctables.CoincTable.get_table(self.xmldoc)
self.time_slide_table = lsctables.TimeSlideTable.get_table(
self.xmldoc)
except ValueError:
self.coinc_def_table = None
self.coinc_table = None
self.time_slide_table = None
try:
self.multi_burst_table = lsctables.MultiBurstTable.get_table(
self.xmldoc)
except ValueError:
self.multi_burst_table = None
# get the segment lists
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
self.xmldoc, live_time_program).coalesce()
self.instruments = set(self.seglists.keys())
if veto_segments_name is not None:
self.vetoseglists = ligolw_segments.segmenttable_get_by_name(
self.xmldoc, veto_segments_name).coalesce()
else:
self.vetoseglists = ligolw_segments.segments.segmentlistdict()
# determine a few coinc_definer IDs
# FIXME: don't hard-code the numbers
if self.coinc_def_table is not None:
try:
self.bb_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 0, create_new=False)
except KeyError:
self.bb_definer_id = None
try:
self.sb_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 1, create_new=False)
except KeyError:
self.sb_definer_id = None
try:
self.sce_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 2, create_new=False)
except KeyError:
self.sce_definer_id = None
try:
self.scn_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 3, create_new=False)
except KeyError:
self.scn_definer_id = None
else:
self.bb_definer_id = None
self.sb_definer_id = None
self.sce_definer_id = None
self.scn_definer_id = None
