def rate(self, on_instruments, participating_instruments, frequency, snr, uncombined_ifar, likelihood): # # retrieve the appropriate reputation list (create an empty # one if there are no reputations for this category) # category = self.category_func(frozenset(lsctables.instrument_set_from_ifos(on_instruments)), frozenset(lsctables.instrument_set_from_ifos(participating_instruments)), frequency) reputations = self.reputations.setdefault(category, []) # # len(x) - bisect.bisect_left(x, reputation) = number of # elements in list >= reputation # n = len(reputations) - bisect.bisect_left(reputations, self.reputation_func(snr, uncombined_ifar, likelihood)) # # retrieve the livetime # t = self.cached_livetime.setdefault(frozenset(lsctables.instrument_set_from_ifos(on_instruments)), 0.0) if self.extrapolate: if n < self.extrapolation_coeffs[category][0]: n = extrapolate_reputation(self.rank_by, self.extrapolation_coeffs[category][1], self.reputation_func(snr, uncombined_ifar, likelihood), t) # # return the rate of events above the given reputation # try: return n / t except ZeroDivisionError, e: print >>sys.stderr, "found an event in category %s that has a livetime of 0 s" % repr(self.category_func(on_instruments, participating_instruments, frequency)) raise e
def set_instruments_to_calculate(self):
if not self.opts.instruments:
return self.instruments
if self.opts.instruments in self.instruments:
return frozenset(lsctables.instrument_set_from_ifos(i[0]))
else:
print >>sys.stderr, "Instruments %s do not exist in DB, nothing will be calculated" % (
str(frozenset(lsctables.instrument_set_from_ifos(i[0]))),
)
return []
def get_event_fars(connection, table_name, segments = None):
"""
return the false alarm rate of the most rare zero-lag coinc by instruments
"""
def event_in_requested_segments(end_time, end_time_ns, segments = segments):
return time_within_segments(end_time, end_time_ns, segments)
connection.create_function("event_in_requested_segments", 2, event_in_requested_segments)
if table_name == dbtables.lsctables.CoincInspiralTable.tableName:
query = 'SELECT coinc_event.instruments, coinc_inspiral.combined_far AS combined_far, EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0) FROM coinc_inspiral JOIN coinc_event ON (coinc_inspiral.coinc_event_id == coinc_event.coinc_event_id) WHERE event_in_requested_segments(coinc_inspiral.end_time, coinc_inspiral.end_time_ns);'
elif table_name == dbtables.lsctables.MultiBurstTable.tableName:
query = 'SELECT coinc_event.instruments, multi_burst.false_alarm_rate AS combined_far, EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0) FROM multi_burst JOIN coinc_event ON (multi_burst.coinc_event_id == coinc_event.coinc_event_id) WHERE event_in_requested_segments(multi_burst.peak_time, multi_burst.peak_time_ns);'
elif table_name == dbtables.lsctables.CoincRingdownTable.tableName:
query = 'SELECT coinc_event.instruments, coinc_ringdown.false_alarm_rate AS combined_far, EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0) FROM coinc_ringdown JOIN coinc_event ON (coinc_ringdown.coinc_event_id == coinc_event.coinc_event_id) WHERE event_in_requested_segments(coinc_ringdown.start_time, coinc_ringdown.start_time_ns);'
else:
raise ValueError("table must be in " + " ".join(allowed_analysis_table_names()))
for inst, far, ts in connection.cursor().execute(query):
inst = frozenset(lsctables.instrument_set_from_ifos(inst))
yield (inst, far, ts)
def get_far_thresholds(self,connection):
"""
return the false alarm rate of the most rare zero-lag coinc, and a
dictionary of the thinca segments indexed by instrument.
"""
live_time_program = opts.live_time_program
verbose = opts.verbose
if self.opts.verbose: print >>sys.stderr, "getting FAR thresholds..."
# extract false alarm rate threshold
query = 'CREATE TEMPORARY TABLE distinct_instruments AS SELECT DISTINCT(instruments) as instruments FROM coinc_event;'
connection.cursor().execute(query)
def create_is_playground_func( connection, playground_segs = segmentsUtils.S2playground(self.segments.extent_all()) ):
"""
Construct the is_playground() SQL function.
"""
connection.create_function("is_playground", 2, lambda seconds, nanoseconds: lsctables.LIGOTimeGPS(seconds, nanoseconds) in playground_segs)
create_is_playground_func(connection)
query = 'SELECT distinct_instruments.instruments, (SELECT MIN(coinc_inspiral.combined_far) AS combined_far FROM coinc_inspiral JOIN coinc_event ON (coinc_inspiral.coinc_event_id == coinc_event.coinc_event_id) WHERE coinc_event.instruments == distinct_instruments.instruments AND NOT EXISTS(SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0) AND NOT is_playground(coinc_inspiral.end_time, coinc_inspiral.end_time_ns) ) FROM distinct_instruments;'
for inst, far in connection.cursor().execute(query):
inst = frozenset(lsctables.instrument_set_from_ifos(inst))
self.far.setdefault(inst,[])
self.far[inst].append(far)
query = 'DROP TABLE distinct_instruments'
connection.cursor().execute(query)
def parse_command_line():
parser = OptionParser(
version="%prog CVS $Id$", usage="%prog [options] [file ...]", description="%prog computes mass/mass upperlimit"
)
parser.add_option(
"--instruments", metavar="name[,name,...]", help='Set the list of instruments. Required. Example "H1,H2,L1"'
)
parser.add_option(
"--live-time-program",
default="thinca",
metavar="name",
help='Set the name of the program whose rings will be extracted from the search_summary table. Default = "thinca".',
)
parser.add_option("--far", help="FAR to use for injection finding instead of loudest event")
parser.add_option(
"--veto-segments-name", default="vetoes", help="Set the name of the veto segments to use from the XML document."
)
parser.add_option(
"-t",
"--tmp-space",
metavar="path",
help="Path to a directory suitable for use as a work area while manipulating the database file. The database file will be worked on in this directory, and then moved to the final location when complete. This option is intended to improve performance when running in a networked environment, where there might be a local disk with higher bandwidth than is available to the filesystem on which the final output will reside.",
)
parser.add_option("--verbose", action="store_true", help="Be verbose.")
opts, filenames = parser.parse_args()
if opts.instruments:
opts.instruments = lsctables.instrument_set_from_ifos(opts.instruments)
if not filenames:
print >>sys.stderr, "must specify at least one database file"
sys.exit(1)
return opts, filenames
def segmentlistdict(self):
"""
A segmentlistdict object describing the instruments and
time spanned by this CacheEntry. A new object is
constructed each time this attribute is accessed (segments
are immutable so there is no reason to try to share a
reference to the CacheEntry's internal segment;
modifications of one would not be reflected in the other
anyway).
Example:
>>> c = CacheEntry(u"H1 S5 815901601 576.5 file://localhost/home/kipp/tmp/1/H1-815901601-576.xml")
>>> c.segmentlistdict
{u'H1': [segment(LIGOTimeGPS(815901601, 0), LIGOTimeGPS(815902177, 500000000))]}
The \"observatory\" column of the cache entry, which is
frequently used to store instrument names, is parsed into
instrument names for the dictionary keys using the same
rules as glue.ligolw.lsctables.instrument_set_from_ifos().
Example:
>>> c = CacheEntry(u"H1H2, S5 815901601 576.5 file://localhost/home/kipp/tmp/1/H1H2-815901601-576.xml")
>>> c.segmentlistdict
{u'H1H2': [segment(LIGOTimeGPS(815901601, 0), LIGOTimeGPS(815902177, 500000000))]}
"""
# the import has to be done here to break the cyclic
# dependancy
from glue.ligolw.lsctables import instrument_set_from_ifos
instruments = instrument_set_from_ifos(self.observatory) or (None,)
return segments.segmentlistdict((instrument, segments.segmentlist(self.segment is not None and [self.segment] or [])) for instrument in instruments)
def get_instruments_from_coinc_event_table(connection):
"""
This function returns a list of the instruments analyzed according to the coinc_event_table
"""
instruments = []
for ifos in connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event WHERE instruments!=""'):
# ignore null columns
if ifos[0]:
instruments.append(frozenset(lsctables.instrument_set_from_ifos(ifos[0])))
return instruments
def get_loudest_event(connection, coinc_table="coinc_inspiral", datatype="exclude_play"):
far_threshold_query = """
SELECT coinc_event.instruments, MIN(combined_far) FROM %s JOIN coinc_event ON (%s.coinc_event_id == coinc_event.coinc_event_id) JOIN experiment_map ON (coinc_event.coinc_event_id == experiment_map.coinc_event_id) JOIN experiment_summary ON ( experiment_summary.experiment_summ_id == experiment_map.experiment_summ_id) WHERE experiment_summary.datatype == "%s" GROUP BY coinc_event.instruments;
""" % (coinc_table, coinc_table, datatype)
for inst, far in connection.cursor().execute(far_threshold_query):
inst = frozenset(lsctables.instrument_set_from_ifos(inst))
yield inst, far
return
def apply_sngl_snr_veto(mi_table, snrs=[4.0, 4.0], return_index=False):
"""Veto events in a MultiInspiralTable based on their single-detector
snr in the most sensitive detectors.
@param mi_table
a MultiInspiralTable from which to veto events
@param snrs
an X-element list of single-detector SNRs on which to threshold
for the X most sensitive detectors (in sensitivity order)
@param return_index
boolean to return the index array of non-vetoed elements rather
than a new table containing the elements themselves
@returns
a new MultiInspiralTable with those events not vetoed OR
the indices of the original mi_table not vetoed if return_index=True
"""
if len(mi_table) == 0:
if return_index:
return numpy.zeros(0).astype(bool)
else:
return mi_table
# parse table
ifos = lsctables.instrument_set_from_ifos(mi_table[0].ifos)
if "H1" in ifos and "H2" in ifos:
ifos.remove("H2")
mi_sngl_snr = numpy.asarray([numpy.asarray(mi_table.get_sngl_snr(ifo))
for ifo in ifos])
if len(snrs) > len(ifos):
raise ValueError("%s single-detector thresholds given, but only %d "
"detectors found." % (len(snrs), len(ifos)))
# set snrs
snr_array = numpy.zeros(len(ifos))
snr_array[:len(snrs)] = snrs
snr_array.sort()
# order sngl_snrs for each event
mi_sngl_snr.sort(axis=0)
# test thresholds
keep = (mi_sngl_snr.T > snr_array).all(axis=1)
if return_index:
return keep
else:
out = table.new_from_template(mi_table)
out.extend(numpy.asarray(mi_table)[keep])
return out
def parse_command_line():
parser = OptionParser(version = "%prog CVS $Id$", usage = "%prog [options] [file ...]", description = "%prog computes mass/mass upperlimit")
parser.add_option("--instruments", metavar = "name[,name,...]", help = "Set the list of instruments. Required. Example \"H1,H2,L1\"")
parser.add_option("--live-time-program", default = "thinca", metavar = "name", help = "Set the name of the program whose rings will be extracted from the search_summary table. Default = \"thinca\".")
parser.add_option("--output-name-tag", default = "", metavar = "name", help = "Set the file output name tag, real name is 2Dsearchvolume-<tag>-<ifos>.xml")
parser.add_option("--bootstrap-iterations", default = 1000, metavar = "integer", type = "int", help = "Number of iterations to compute mean and variance of volume MUST BE GREATER THAN 1 TO GET USABLE NUMBERS, a good number is 10000")
parser.add_option("--dist-bins", default = 50, metavar = "integer", type = "int", help = "Number of mass bins")
parser.add_option("--d-err", default = 0.197, metavar = "float", type = "int", help = "random calibration error on distance")
parser.add_option("--d-sys-err", default = 0.074, metavar = "float", type = "int", help = "systematic calibration error on distance (should use worst)")
parser.add_option("--mass-bins", default = 11, metavar = "integer", type = "int", help = "Number of mass bins along 1 dimension (Note the total number of mass bins will generally be less than --mass-bins * --mass-bins once the actual parameter space is carved out)")
parser.add_option("--veto-segments-name", default = "vetoes", help = "Set the name of the veto segments to use from the XML document.")
parser.add_option("--verbose", action = "store_true", help = "Be verbose.")
opts, filenames = parser.parse_args()
if opts.instruments: opts.instruments = lsctables.instrument_set_from_ifos(opts.instruments)
if not filenames:
print >>sys.stderr, "must specify at least one database file"
sys.exit(1)
return opts, filenames
def add_sim_inspiral_coinc(contents, sim, inspirals):
"""Create a coinc_event in the coinc table, and add arcs in the
coinc_event_map table linking the sim_inspiral row and the list of
multi_inspiral rows to the new coinc_event row.
"""
coinc = contents.new_coinc(contents.sb_coinc_def_id)
if inspirals:
coinc.set_instruments(
lsctables.instrument_set_from_ifos(inspirals[0].ifos))
coinc.nevents = len(inspirals)
coincmap = lsctables.CoincMap()
coincmap.coinc_event_id = coinc.coinc_event_id
coincmap.table_name = sim.simulation_id.table_name
coincmap.event_id = sim.simulation_id
contents.coincmaptable.append(coincmap)
for event in inspirals:
coincmap = lsctables.CoincMap()
coincmap.coinc_event_id = coinc.coinc_event_id
coincmap.table_name = event.event_id.table_name
coincmap.event_id = event.event_id
contents.coincmaptable.append(coincmap)
def parse_command_line():
parser = OptionParser(version = "%prog CVS $Id$", usage = "%prog [options] [file ...]", description = "%prog computes mass/mass upperlimit")
parser.add_option("-i", "--instruments", metavar = "name[,name,...]", help = "Set the list of instruments. Required. Example \"H1,H2,L1\"")
parser.add_option("--live-time-program", default = "thinca", metavar = "name", help = "Set the name of the program whose rings will be extracted from the search_summary table. Default = \"thinca\".")
parser.add_option("-t", "--output-name-tag", default = "", metavar = "name", help = "Set the file output name tag, real name is 2Dsearchvolume-<tag>-<ifos>.xml")
parser.add_option("-f", "--full-data-file", default = "FULL_DATACAT_3.sqlite", metavar = "pattern", help = "File in which to find the full data, example FULL_DATACAT_3.sqlite")
parser.add_option("-s", "--inj-data-glob", default = "*INJCAT_3.sqlite", metavar = "pattern", help = "Glob for files to find the inj data, example *INJCAT_3.sqlite")
parser.add_option("-b", "--bootstrap-iterations", default = 1, metavar = "integer", type = "int", help = "Number of iterations to compute mean and variance of volume MUST BE GREATER THAN 1 TO GET USABLE NUMBERS, a good number is 10000")
parser.add_option("--veto-segments-name", help = "Set the name of the veto segments to use from the XML document.")
parser.add_option("--verbose", action = "store_true", help = "Be verbose.")
parser.add_option("--burst-found", default=None, help = "use a burst input file")
parser.add_option("--burst-missed", default=None, help = "use a burst input file")
opts, filenames = parser.parse_args()
if opts.instruments is None:
raise ValueError("missing required argument --instruments")
opts.instruments = lsctables.instrument_set_from_ifos(opts.instruments)
opts.injfnames = glob.glob(opts.inj_data_glob)
return opts, filenames
def printsims(connection, simulation_table, recovery_table, map_label, ranking_stat, rank_by, comparison_datatype,
sort_by = 'rank', param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None,
sim_tag = 'ALLINJ', rank_range = None, convert_durations = 's',
daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
# check and format options appropriately
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
ranking_stat = sqlutils.validate_option(ranking_stat)
rank_by = sqlutils.validate_option(rank_by, lower = False).upper()
comparison_datatype = sqlutils.validate_option(comparison_datatype)
convert_durations = sqlutils.validate_option(convert_durations)
if rank_by == 'MIN':
rank_by = 'ASC'
elif rank_by == 'MAX':
rank_by = 'DESC'
elif rank_by != 'ASC' or rank_by != 'DESC':
raise ValueError, 'rank_by must be MAX (or DESC) or MIN (or ASC)'
if not ranking_stat.startswith(recovery_table):
ranking_stat = '.'.join([recovery_table, ranking_stat])
#
# Set up sim_rec_map table
#
if 'sim_rec_map' not in sqlutils.get_tables_in_database( connection ):
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, ranking_stat)
#
# Initialize ranking. Statistics for ranking are collected from non-injections
# in the recovery table.
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(recovery_table, ranking_stat, rank_by)
# add requirement that stats not be found in the sim_rec_table to in_this_filter
rank_filter = ''.join([
recovery_table, '''.coinc_event_id NOT IN (
SELECT
rec_id
FROM
sim_rec_map)
AND
experiment_summary.datatype == "''', comparison_datatype, '"'])
# add to the rank filter any other desired filters; note that sim-tag is forced to ALLINJ if not comparing
# to simulation datatype
in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs,
sim_tag = comparison_datatype == 'simulation' and sim_tag or 'ALLINJ',
verbose = False)
if in_this_filter != '':
rank_filter = '\n\tAND '.join([ in_this_filter, rank_filter ])
rank_filter = '\n\t'.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), 'WHERE', rank_filter ])
ranker.populate_stats_list(connection, limit = None, filter = rank_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Set recovery table filters
#
in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs, sim_tag = sim_tag,
verbose = verbose)
# Now apply the filter to the sim_rec_map table: this will delete all sim/rec maps where the simulation id is
# mapped to a recovered event that falls outside the filter, even if that particular sim/rec map is in the
# filter. For example, if the filter is recovery_table.combined_far != 0., and there are two entries in the
# sim_rec_map table sharing the same sim_id:
# sim_id:0 | rec_id:0 | rec_id's combined_far = 0.
# sim_id:0 | rec_id:1 | rec_id's combined_far = 1.2
# both entries will get deleted even though the second entry's combined_far is not 0.
if in_this_filter != '':
# join the needed tables to in_this_filter
in_this_filter = ''.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), "\n WHERE\n\t", in_this_filter ])
sqlscript = ''.join([ """
CREATE TEMP TABLE del_ids AS
SELECT
sim_id AS del_id
FROM
sim_rec_map
WHERE
rec_id NOT IN (
SELECT
""", recovery_table, """.coinc_event_id
FROM
""", recovery_table, """
""", in_this_filter, """
AND experiment_summary.datatype == "simulation"
);
DELETE FROM
sim_rec_map
WHERE
sim_id IN (
SELECT
del_id
FROM
del_ids );
DROP TABLE del_ids;""" ])
connection.cursor().executescript(sqlscript)
#
# Set other needed functions
#
# establish what units will be converting duration to
def convert_duration( duration ):
return sqlutils.convert_duration( duration, convert_durations )
connection.create_function( 'convert_duration', 1, convert_duration )
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
# Get range ranks
if rank_range is not None:
rank_range_parser = sqlutils.parse_param_ranges( 'rank(sim_rec_map', 'ranking_stat)',
rank_range, verbose = verbose )
#
# Create and prepare the SelectedFoundTable to store summary information
#
# Get recovery table and simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
recovery_table_columns = sqlutils.get_column_names_from_table( connection, recovery_table )
# Get list of column name for injected parameters
injected_cols = []
for col in simulation_table_columns:
if col == 'simulation_id':
injected_cols.append('simulation_id')
else:
injected_cols.append('injected_'+col)
injected_cols.extend(['injected_decisive_distance','injected_gps_time', 'injected_gps_time_ns', 'injected_event_time_utc__Px_click_for_daily_ihope_xP_'])
# Get list of column names from the recovery table
recovered_cols = []
for col in recovery_table_columns:
if col == 'coinc_event_id':
recovered_cols.append(u'coinc_event_id')
else:
recovered_cols.append(u'recovered_'+col)
# generate list of column names for the summary table
column_names = injected_cols + recovered_cols
# add instruments on, duration, mini_followups
rankname = 'rank_in_' + comparison_datatype.strip().lower() + '_using_' + ranking_stat.split('.')[-1]
durname = ''.join([ 'simulation', u'_duration__Px_', convert_durations, '_xP_' ])
column_names.extend( [ rankname, 'recovered_match_rank', 'instruments_on', 'elogs', durname, 'mini_followup','omega_scan', 'sim_tag' ] )
#
# define needed tables
#
class tmpSimTable(table.Table):
tableName = "tmp_sim_table:table"
validcolumns = dict([ [col, sqlutils.get_col_type(simulation_table, col)] for col in simulation_table_columns ])
class tmpSim(object):
__slots__ = tmpSimTable.validcolumns.keys()
def get_gps_time(self, site):
if '%s_start_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, '%s_start_time' % site ), getattr(self, '%s_start_time_ns' % site))
elif '%s_end_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, '%s_end_time' % site ), getattr(self, '%s_end_time_ns' % site))
else:
raise AttributeError, "could not find an injected %s_start_time nor an injected %s_end_time" %(site,site)
def get_pyvalue(self):
return generic_get_pyvalue(self)
class tmpRecTable(table.Table):
tableName = "tmp_rec_table:table"
validcolumns = dict([ [col, sqlutils.get_col_type(recovery_table, col)] for col in recovery_table_columns ])
class tmpRec(object):
__slots__ = tmpRecTable.validcolumns.keys()
def get_gps_time(self):
if 'start_time' in self.__slots__:
return LIGOTimeGPS(self.start_time, self.start_time_ns)
elif 'end_time' in self.__slots__:
return LIGOTimeGPS(self.end_time, self.end_time_ns)
else:
raise AttributeError, "could not find a recovered start_time or recovered end_time"
def get_pyvalue(self):
return generic_get_pyvalue(self)
class SelectedFoundTable(table.Table):
tableName = "selected_found_injections:table"
validcolumns = {}
for col_name in column_names:
if 'rank_in_' in col_name:
validcolumns[col_name] = "int_4u"
elif '_duration_' in col_name:
validcolumns[col_name] = "real_8"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif col_name == 'injected_start_time' or col_name == 'injected_start_time_ns':
validcolumns[col_name] = "int_4s"
elif col_name == 'injected_gps_time' or col_name == 'injected_gps_time_ns':
validcolumns[col_name] = "int_4s"
elif col_name == 'injected_decisive_distance':
validcolumns[col_name] = "real_8"
elif 'injected_' in col_name or col_name == 'simulation_id':
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, re.sub('injected_', '', col_name))
elif 'recovered_' in col_name or col_name == 'coinc_event_id':
validcolumns[col_name] = sqlutils.get_col_type(recovery_table, re.sub('recovered_', '', col_name))
# if custom columns exist in the database, just set them to lstrings
else:
validcolumns[col_name] = "lstring"
# add FAP columns
validcolumns['recovered_fap'] = "real_8"
validcolumns['recovered_fap_1yr'] = "real_8"
class SelectedFound(object):
__slots__ = SelectedFoundTable.validcolumns.keys()
def set_recovered_gps_time(self, gps_time):
if 'recovered_start_time' in self.__slots__:
self.recovered_start_time = gps_time.seconds
self.recovered_start_time_ns = gps_time.nanoseconds
elif 'recovered_end_time' in self.__slots__:
self.recovered_end_time = gps_time.seconds
self.recovered_end_time_ns = gps_time.nanoseconds
else:
raise AttributeError, "could not find a recovered_start_time or recovered_end_time"
def get_recovered_gps_time(self):
if 'recovered_start_time' in self.__slots__:
return LIGOTimeGPS(self.recovered_start_time, self.recovered_start_time_ns)
elif 'recovered_end_time' in self.__slots__:
return LIGOTimeGPS(self.recovered_end_time, self.recovered_end_time_ns)
else:
raise AttributeError, "could not find a recovered_start_time or recovered_end_time"
def set_injected_gps_time(self, site, gps_time):
if 'injected_%s_start_time' % site in self.__slots__:
setattr(self, 'injected_%s_start_time' % site, gps_time.seconds)
setattr(self, 'injected_%s_start_time_ns' % site, gps_time.nanoseconds)
elif 'injected_%s_end_time' % site in self.__slots__:
setattr(self, 'injected_%s_end_time' % site, gps_time.seconds)
setattr(self, 'injected_%s_end_time_ns' % site, gps_time.nanoseconds)
else:
raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site)
def get_injected_gps_time(self, site):
if 'injected_%s_start_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, 'injected_%s_start_time' % site ), getattr(self, 'injected_%s_start_time_ns' % site))
elif 'injected_%s_end_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, 'injected_%s_end_time' % site ), getattr(self, 'injected_%s_end_time_ns' % site))
else:
raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site)
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
tmpSimTable.RowType = tmpSim
tmpRecTable.RowType = tmpRec
SelectedFoundTable.RowType = SelectedFound
#
# Get the Data
#
tmp_sim_table = lsctables.New(tmpSimTable)
tmp_rec_table = lsctables.New(tmpRecTable)
tmp_sftable = lsctables.New(SelectedFoundTable)
prior_sim_id = ''
group_rank = None
current_match_rank = 1
sqlquery = ''.join(["""
SELECT
""", simulation_table, """.*,
""", recovery_table, """.*,
get_sim_tag(experiment_summary.sim_proc_id),
rank(sim_rec_map.ranking_stat),
NULL AS match_rank,
experiment.instruments,
convert_duration(experiment_summary.duration),
NULL AS mini_followup
FROM
sim_rec_map
JOIN
""", ', '.join([simulation_table, recovery_table]), """, experiment, experiment_summary, experiment_map ON (
sim_rec_map.sim_id == """, simulation_table, """.simulation_id AND
sim_rec_map.rec_id == """, recovery_table, """.coinc_event_id AND
sim_rec_map.rec_id == experiment_map.coinc_event_id AND
experiment_map.experiment_summ_id == experiment_summary.experiment_summ_id AND
experiment_summary.experiment_id == experiment.experiment_id)
ORDER BY
sim_rec_map.sim_id, sim_rec_map.ranking_stat """, rank_by])
if verbose:
print >> sys.stderr, "Getting coincs..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
# sort the data
tmp_sim_row = tmpSim()
tmp_rec_row = tmpRec()
[ setattr(tmp_sim_row, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
[ setattr(tmp_rec_row, column, values[ii+1+jj]) for jj, column in enumerate(recovery_table_columns) ]
# figure out the rank
this_inj_rank = values[-5]
this_sim_id = tmp_sim_row.simulation_id
this_ranking_stat = getattr(tmp_rec_row, ranking_stat.split('.')[-1])
if this_sim_id == prior_sim_id and this_ranking_stat != prior_ranking_stat:
current_match_rank += 1
elif this_sim_id != prior_sim_id:
current_match_rank = 1
group_rank = this_inj_rank
prior_sim_id = this_sim_id
prior_ranking_stat = this_ranking_stat
# only store data if the group_rank falls in the desired rank ranges
if rank_range and rank_range_parser.group_by_param_range(group_rank) is None:
continue
on_instruments = lsctables.instrument_set_from_ifos(values[-3])
duration = values[-2]
# now that have all the information from this row, create a row for the selected found table
sfrow = SelectedFound()
# set the ranks
setattr(sfrow, rankname, group_rank)
sfrow.recovered_match_rank = current_match_rank
# set the injected parameters
use_this_site = sorted(on_instruments)[0][0].lower()
injected_time = tmp_sim_row.get_gps_time( use_this_site )
sfrow.injected_gps_time = injected_time.seconds
sfrow.injected_gps_time_ns = injected_time.nanoseconds
for col in simulation_table_columns:
if col == "simulation_id":
sfrow.simulation_id = tmp_sim_row.simulation_id
else:
setattr(sfrow, 'injected_'+col, getattr( tmp_sim_row, col) )
# set the recovered parameters
for col in recovery_table_columns:
if col == "coinc_event_id":
sfrow.coinc_event_id = tmp_rec_row.coinc_event_id
else:
setattr(sfrow, 'recovered_'+col, getattr( tmp_rec_row, col) )
# calculate and add faps
if sfrow.recovered_combined_far is not None:
t_in_s = float(values[-2]) / sqlutils.convert_duration(1, convert_durations)
sfrow.recovered_fap = 1 - math.exp(-sqlutils.convert_duration(t_in_s, 'yr') * sfrow.recovered_combined_far)
sfrow.recovered_fap_1yr = 1 - math.exp(-sfrow.recovered_combined_far)
else:
sfrow.recovered_fap = None
sfrow.recovered_fap_1yr = None
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, sfrow.injected_gps_time)) for ifo in on_instruments]
sfrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
event_time_utc = format_end_time_in_utc( sfrow.injected_gps_time )
daily_ihope_address = get_daily_ihope_page(sfrow.injected_gps_time, pages_location = daily_ihope_pages_location)
sfrow.injected_event_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, event_time_utc )
# set any other info
sfrow.instruments_on = ','.join(sorted(on_instruments))
sfrow.injected_decisive_distance = sorted([getattr(sfrow, 'injected_eff_dist_%s' % ifo[0].lower()) for ifo in on_instruments])[1]
sfrow.mini_followup = None
sfrow.omega_scan = None
sfrow.sim_tag = values[-6]
setattr(sfrow, durname, duration)
# add the row
tmp_sftable.append(sfrow)
# Re-sort the sftable by rank, recovered_match_rank
sftable = lsctables.New(SelectedFoundTable)
for sfrow in sorted([ row for row in tmp_sftable ], key = lambda row: getattr(row, sort_by == 'rank' and rankname or sort_by) ):
if sfrow.simulation_id not in [row.simulation_id for row in sftable]:
sftable.append(sfrow)
sftable.extend(sub_row for sub_row in sorted([row for row in tmp_sftable
if row.simulation_id == sfrow.simulation_id
and row.coinc_event_id != sfrow.coinc_event_id],
key = lambda row: row.recovered_match_rank))
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return sftable
# get chunk lengths from the values in the ini file
#
short_segment_duration = config_parser.getint('lalapps_StringSearch',
'short-segment-duration')
pad = config_parser.getint('lalapps_StringSearch', 'pad')
min_segment_length = config_parser.getint(
'pipeline', 'segment-length') # not including pad at each end
trig_overlap = config_parser.getint('pipeline', 'trig_overlap')
overlap = short_segment_duration / 2 + 2 * pad # FIXME: correct?
#
# get the instruments and raw segments
#
instruments = lsctables.instrument_set_from_ifos(
config_parser.get('pipeline', 'ifos'))
seglists = ligolwsegments.segmenttable_get_by_name(
utils.load_filename(options.segments_file,
gz=(options.segments_file or "stdin").endswith(".gz"),
verbose=options.verbose),
options.segments_name).coalesce()
# remove extra instruments
for instrument in set(seglists) - instruments:
if options.verbose:
print >> sys.stderr, "warning: ignoring segments for '%s' found in '%s'" % (
instrument, options.segments_file)
del seglists[instrument]
# check for missing instruments
if not instruments.issubset(set(seglists)):
raise ValueError, "segment lists retrieved from '%s' missing segments for instruments %s" % (
options.segments_file, ", ".join(instruments - set(seglists)))
############################################################################
# Creating the thinca_user_tag cache file and writing it to disk
print("\tcreating the thinca_usertag cache file...")
# sieve thinca_cache for THINCA files with this tag
file_sieve = '*' + tag
thinca_usertag_cache = thinca_cache.sieve( description = file_sieve, exact_match = True )
# get distinct on_instruments in the thinca_usertag cache
distinct_instrument_sets = set([ entry.observatory for entry in thinca_usertag_cache ])
# from the distinct_instrument_sets figure out what distinct ifos there are
distinct_ifo_set = set()
for on_instruments in distinct_instrument_sets:
distinct_ifo_set |= lsctables.instrument_set_from_ifos(on_instruments)
distinct_ifos = ''.join(sorted(distinct_ifo_set))
# determine location of veto-file relevant to tag
cat_num = str(get_veto_cat_from_tag( tag ))
veto_file_path = cp.get('input', 'ihope-segments-directory')
veto_file_name = ''.join([
distinct_ifos, '-VETOTIME_CAT_', cat_num, '-', experiment_start, '-', experiment_duration,
'.xml' ])
veto_file = '/'.join([ veto_file_path, veto_file_name ])
if not os.path.exists( veto_file ):
raise ValueError("Veto file %s could not be found." % veto_file)
# store the veto file for additional later use
veto_cat = '_'.join(['CAT', cat_num, 'VETO'])
veto_files[veto_cat] = veto_file
def read_sky_map(filename, nest=False):
"""
Read a LIGO/Virgo-type sky map and return a tuple of the HEALPix array
and a dictionary of metadata from the header.
Parameters
----------
filename: string
Path to the optionally gzip-compressed FITS file.
nest: bool, optional
If omitted or False, then detect the pixel ordering in the FITS file
and rearrange if necessary to RING indexing before returning.
If True, then detect the pixel ordering and rearrange if necessary to
NESTED indexing before returning.
If None, then preserve the ordering from the FITS file.
Regardless of the value of this option, the ordering used in the FITS
file is indicated as the value of the 'nest' key in the metadata
dictionary.
"""
prob, header = hp.read_map(filename, h=True, verbose=False, nest=nest)
header = dict(header)
metadata = {}
ordering = header["ORDERING"]
if ordering == "RING":
metadata["nest"] = False
elif ordering == "NESTED":
metadata["nest"] = True
else:
raise ValueError("ORDERING card in header has unknown value: {0}".format(ordering))
try:
value = header["OBJECT"]
except KeyError:
pass
else:
metadata["objid"] = value
try:
value = header["REFERENC"]
except KeyError:
pass
else:
metadata["url"] = value
try:
value = header["INSTRUME"]
except KeyError:
pass
else:
value = set(str(ifo) for ifo in lsctables.instrument_set_from_ifos(value))
metadata["instruments"] = value
try:
value = header["DATE-OBS"]
except KeyError:
pass
else:
metadata["gps_time"] = iso8601_to_gps(value)
try:
value = header["DATE"]
except KeyError:
pass
else:
metadata["gps_creation_time"] = iso8601_to_gps(value)
try:
value = header["CREATOR"]
except KeyError:
pass
else:
metadata["creator"] = value
try:
value = header["ORIGIN"]
except KeyError:
pass
else:
metadata["origin"] = value
try:
value = header["RUNTIME"]
except KeyError:
pass
else:
metadata["runtime"] = value
return prob, metadata
JOIN coinc_ringdown ON ( coinc_ringdown.coinc_event_id == coinc_event.coinc_event_id ) WHERE -- require coinc to be background (= at least one of its time slide offsets is non-zero) EXISTS ( SELECT * FROM time_slide WHERE time_slide.time_slide_id == coinc_event.time_slide_id AND time_slide.offset != 0 ) """): on_instruments = frozenset(lsctables.instrument_set_from_ifos(on_instruments)) participating_instruments = frozenset(lsctables.instrument_set_from_ifos(participating_instruments)) background.add_coinc(on_instruments, participating_instruments, frequency, snr, uncombined_ifar, likelihood) # # close the database # connection.close() dbtables.discard_connection_filename(filename, working_filename, verbose = options.verbose) background.index() if options.extrapolation_num: if options.verbose: print >>sys.stderr, "\tcalculating FAR extrapolation coefficients ..."
def read_sky_map(filename, nest=False, distances=False):
"""
Read a LIGO/Virgo-type sky map and return a tuple of the HEALPix array
and a dictionary of metadata from the header.
Parameters
----------
filename: string
Path to the optionally gzip-compressed FITS file.
nest: bool, optional
If omitted or False, then detect the pixel ordering in the FITS file
and rearrange if necessary to RING indexing before returning.
If True, then detect the pixel ordering and rearrange if necessary to
NESTED indexing before returning.
If None, then preserve the ordering from the FITS file.
Regardless of the value of this option, the ordering used in the FITS
file is indicated as the value of the 'nest' key in the metadata
dictionary.
distances: bool, optional
If true, then read also read the additional HEALPix layers representing
the conditional mean and standard deviation of distance as a function
of sky location.
"""
out = hp.read_map(
filename, h=True, verbose=False, nest=nest,
field=(list(range(4)) if distances else 0))
prob = out[:-1] if distances else out[0]
header = dict(out[-1])
metadata = {}
ordering = header['ORDERING']
if ordering == 'RING':
metadata['nest'] = False
elif ordering == 'NESTED':
metadata['nest'] = True
else:
raise ValueError(
'ORDERING card in header has unknown value: {0}'.format(ordering))
try:
value = header['OBJECT']
except KeyError:
pass
else:
metadata['objid'] = value
try:
value = header['REFERENC']
except KeyError:
pass
else:
metadata['url'] = value
try:
value = header['INSTRUME']
except KeyError:
pass
else:
value = {str(ifo) for ifo in lsctables.instrument_set_from_ifos(value)}
metadata['instruments'] = value
try:
value = header['DATE-OBS']
except KeyError:
pass
else:
metadata['gps_time'] = iso8601_to_gps(value)
try:
value = header['DATE']
except KeyError:
pass
else:
metadata['gps_creation_time'] = iso8601_to_gps(value)
try:
value = header['CREATOR']
except KeyError:
pass
else:
metadata['creator'] = value
try:
value = header['ORIGIN']
except KeyError:
pass
else:
metadata['origin'] = value
try:
value = header['RUNTIME']
except KeyError:
pass
else:
metadata['runtime'] = value
try:
value = header['DISTMEAN']
except KeyError:
pass
else:
metadata['distmean'] = value
try:
value = header['DISTSTD']
except KeyError:
pass
else:
metadata['diststd'] = value
return prob, metadata
def __init__(self,
filelist,
live_time_program=None,
veto_segments_name=None,
data_segments_name="datasegments",
tmp_path=None,
verbose=False):
self.segments = segments.segmentlistdict()
self.instruments = set()
self.table_name = None
self.found_injections_by_instrument_set = {}
self.missed_injections_by_instrument_set = {}
self.total_injections_by_instrument_set = {}
self.zerolag_fars_by_instrument_set = {}
self.ts_fars_by_instrument_set = {}
self.numslides = set()
for f in filelist:
if verbose:
print >> sys.stderr, "Gathering stats from: %s...." % (f, )
working_filename = dbtables.get_connection_filename(
f, tmp_path=tmp_path, verbose=verbose)
connection = sqlite3.connect(working_filename)
xmldoc = dbtables.get_xml(connection)
sim = False
# look for a sim inspiral table. This is IMR work we have to have one of these :)
try:
sim_inspiral_table = table.get_table(
xmldoc, dbtables.lsctables.SimInspiralTable.tableName)
sim = True
except ValueError:
pass
# look for the relevant table for analyses
for table_name in allowed_analysis_table_names():
try:
setattr(self, table_name,
table.get_table(xmldoc, table_name))
if self.table_name is None or self.table_name == table_name:
self.table_name = table_name
else:
raise ValueError(
"detected more than one table type out of " +
" ".join(allowed_analysis_table_names()))
except ValueError:
setattr(self, table_name, None)
# the non simulation databases are where we get information about segments
if not sim:
self.numslides.add(connection.cursor().execute(
'SELECT count(DISTINCT(time_slide_id)) FROM time_slide').
fetchone()[0])
[
self.instruments.add(ifos) for ifos in
get_instruments_from_coinc_event_table(connection)
]
# save a reference to the segments for this file, needed to figure out the missed and found injections
self.this_segments = get_segments(
connection,
xmldoc,
self.table_name,
live_time_program,
veto_segments_name,
data_segments_name=data_segments_name)
# FIXME we don't really have any reason to use playground segments, but I put this here as a reminder
# self.this_playground_segments = segmentsUtils.S2playground(self.this_segments.extent_all())
self.segments += self.this_segments
# get the far thresholds for the loudest events in these databases
for (instruments_set, far,
ts) in get_event_fars(connection, self.table_name):
if not ts:
self.zerolag_fars_by_instrument_set.setdefault(
instruments_set, []).append(far)
else:
self.ts_fars_by_instrument_set.setdefault(
instruments_set, []).append(far)
# get the injections
else:
# We need to know the segments in this file to determine which injections are found
self.this_injection_segments = get_segments(
connection,
xmldoc,
self.table_name,
live_time_program,
veto_segments_name,
data_segments_name=data_segments_name)
self.this_injection_instruments = []
distinct_instruments = connection.cursor().execute(
'SELECT DISTINCT(instruments) FROM coinc_event WHERE instruments!=""'
).fetchall()
for instruments, in distinct_instruments:
instruments_set = frozenset(
lsctables.instrument_set_from_ifos(instruments))
self.this_injection_instruments.append(instruments_set)
segments_to_consider_for_these_injections = self.this_injection_segments.intersection(
instruments_set) - self.this_injection_segments.union(
set(self.this_injection_segments.keys()) -
instruments_set)
found, total, missed = get_min_far_inspiral_injections(
connection,
segments=segments_to_consider_for_these_injections,
table_name=self.table_name)
if verbose:
print >> sys.stderr, "%s total injections: %d; Found injections %d: Missed injections %d" % (
instruments, len(total), len(found), len(missed))
self.found_injections_by_instrument_set.setdefault(
instruments_set, []).extend(found)
self.total_injections_by_instrument_set.setdefault(
instruments_set, []).extend(total)
self.missed_injections_by_instrument_set.setdefault(
instruments_set, []).extend(missed)
# All done
dbtables.discard_connection_filename(f,
working_filename,
verbose=verbose)
if len(self.numslides) > 1:
raise ValueError('number of slides differs between input files')
elif self.numslides:
self.numslides = min(self.numslides)
else:
self.numslides = 0
############################################################################
# Creating the thinca_user_tag cache file and writing it to disk
print "\tcreating the thinca_usertag cache file..."
# sieve thinca_cache for THINCA files with this tag
file_sieve = '*' + tag
thinca_usertag_cache = thinca_cache.sieve( description = file_sieve, exact_match = True )
# get distinct on_instruments in the thinca_usertag cache
distinct_instrument_sets = set([ entry.observatory for entry in thinca_usertag_cache ])
# from the distinct_instrument_sets figure out what distinct ifos there are
distinct_ifo_set = set()
for on_instruments in distinct_instrument_sets:
distinct_ifo_set |= lsctables.instrument_set_from_ifos(on_instruments)
distinct_ifos = ''.join(sorted(distinct_ifo_set))
# determine location of veto-file relevant to tag
cat_num = str(get_veto_cat_from_tag( tag ))
veto_file_path = cp.get('input', 'ihope-segments-directory')
veto_file_name = ''.join([
distinct_ifos, '-VETOTIME_CAT_', cat_num, '-', experiment_start, '-', experiment_duration,
'.xml' ])
veto_file = '/'.join([ veto_file_path, veto_file_name ])
if not os.path.exists( veto_file ):
raise ValueError, "Veto file %s could not be found." % veto_file
# store the veto file for additional later use
veto_cat = '_'.join(['CAT', cat_num, 'VETO'])
veto_files[veto_cat] = veto_file
def parse_command_line():
parser = OptionParser(version="%prog CVS $Id$",
usage="%prog [options] [file ...]",
description="%prog computes mass/mass upperlimit")
parser.add_option(
"-i",
"--instruments",
metavar="name[,name,...]",
help="Set the list of instruments. Required. Example \"H1,H2,L1\"")
parser.add_option(
"--live-time-program",
default="thinca",
metavar="name",
help=
"Set the name of the program whose rings will be extracted from the search_summary table. Default = \"thinca\"."
)
parser.add_option(
"-t",
"--output-name-tag",
default="",
metavar="name",
help=
"Set the file output name tag, real name is 2Dsearchvolume-<tag>-<ifos>.xml"
)
parser.add_option(
"-f",
"--full-data-file",
default="FULL_DATACAT_3.sqlite",
metavar="pattern",
help=
"File in which to find the full data, example FULL_DATACAT_3.sqlite")
parser.add_option(
"-s",
"--inj-data-glob",
default="*INJCAT_3.sqlite",
metavar="pattern",
help="Glob for files to find the inj data, example *INJCAT_3.sqlite")
parser.add_option(
"-b",
"--bootstrap-iterations",
default=1,
metavar="integer",
type="int",
help=
"Number of iterations to compute mean and variance of volume MUST BE GREATER THAN 1 TO GET USABLE NUMBERS, a good number is 10000"
)
parser.add_option(
"--veto-segments-name",
help="Set the name of the veto segments to use from the XML document.")
parser.add_option("--verbose", action="store_true", help="Be verbose.")
parser.add_option("--burst-found",
default=None,
help="use a burst input file")
parser.add_option("--burst-missed",
default=None,
help="use a burst input file")
opts, filenames = parser.parse_args()
if opts.instruments is None:
raise ValueError("missing required argument --instruments")
opts.instruments = lsctables.instrument_set_from_ifos(opts.instruments)
opts.injfnames = glob.glob(opts.inj_data_glob)
return opts, filenames
def printmissed(connection, simulation_table, recovery_table, map_label, livetime_program,
param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ',
limit = None, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
from pylal import ligolw_cbc_compute_durations as compute_dur
from glue import segments
from glue.ligolw import dbtables
# Get simulation/recovery tables
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
#
# Create and prepare the CloseMissedTable to store summary information
#
# Get simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
column_names = simulation_table_columns + \
['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag']
# define needed tables
class CloseMissedTable(table.Table):
tableName = "close_missed_injections"
validcolumns = {}
for col_name in column_names:
if 'rank' in col_name:
validcolumns[col_name] = "int_4u"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif 'veto_def_name' == col_name:
validcolumns[col_name] = lsctables.ExperimentSummaryTable.validcolumns['veto_def_name']
elif 'decisive_distance' == col_name:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, 'eff_dist_h')
elif 'gps_time' == col_name or 'gps_time_ns' == col_name:
validcolumns[col_name] = "int_4s"
elif 'sim_tag' == col_name:
validcolumns[col_name] = "lstring"
else:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, col_name, default = 'lstring')
class CloseMissed(object):
__slots__ = CloseMissedTable.validcolumns.keys()
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
CloseMissedTable.RowType = CloseMissed
# create the table
cmtable = lsctables.New(CloseMissedTable)
# set up sim_rec_map table
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, None)
#
# Set table filters
#
# we force the include/exclude filters to None; will check for excluded/included ifo time
# when cycling through the ifo times
filter = """
WHERE
simulation_id NOT IN (
SELECT
sim_id
FROM
sim_rec_map )"""
af = create_filter( connection, simulation_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = None, include_only_coincs = None, sim_tag = sim_tag, verbose = verbose)
af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af)
if af != '':
filter = '\n'.join([ filter, """
AND""", af])
# get desired instrument times
if include_only_coincs is not None:
include_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( include_only_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
if exclude_coincs is not None:
exclude_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( exclude_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
# get the usertags of inspiral jobs in this database
sqlquery = """
SELECT value
FROM process_params
WHERE param == "-userTag"
GROUP BY value
"""
usertags = set(usertag[0] for usertag in connection.cursor().execute(sqlquery) )
# Get the single-ifo science segments after CAT-1 vetoes
try:
if "FULL_DATA" in usertags:
tag = "FULL_DATA"
else:
tag = list(usertags)[0]
except IndexError:
# This is hacky anyway, so let's just take a guess
tag = "FULL_DATA"
ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name = livetime_program, usertag = tag)
if ifo_segments == {}:
raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program
if verbose:
print >> sys.stderr, "Getting all veto category names from the experiment_summary table..."
xmldoc = dbtables.get_xml(connection)
# get veto_segments
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# make a dictionary of zerolag "shifts" needed for the get_coinc_segments function
zerolag_dict = {}
for ifo in ifo_segments:
zerolag_dict[ifo] = 0.0
# Cycle over available veto categories
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
# make a dictionary of coincident segments by exclusive on-ifos
coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs, zerolag_dict)
#
# Get all the on_instrument times and cycle over them
#
sqlquery = """
SELECT DISTINCT experiment.instruments
FROM experiment
JOIN experiment_summary ON (
experiment.experiment_id == experiment_summary.experiment_id )
WHERE experiment_summary.veto_def_name == :1
"""
for on_instruments in connection.cursor().execute(sqlquery, (veto_def_name,)).fetchall():
on_instruments = lsctables.instrument_set_from_ifos(on_instruments[0])
# check if this on_instruments is desired; if not, skip
if include_only_coincs is not None and frozenset(on_instruments) not in include_times:
continue
if exclude_coincs is not None and frozenset(on_instruments) in exclude_times:
continue
on_times = coinc_segs[','.join(sorted(on_instruments))]
def is_in_on_time(gps_time, gps_time_ns):
return LIGOTimeGPS(gps_time, gps_time_ns) in on_times
connection.create_function('is_in_on_time', 2, is_in_on_time)
# add the check for on time to the filter
in_this_filter = filter
# figure out if simulation_table has end_times or start_times
end_or_start = any('end_time' in c for c in simulation_table_columns) and '_end_time' or '_start_time'
for instrument in on_instruments:
inst_time = instrument.lower()[0] + end_or_start
inst_time_ns = inst_time + '_ns'
in_this_filter = ''.join([ in_this_filter,
'\n\tAND is_in_on_time(', inst_time, ',', inst_time_ns, ')' ])
#
# Set up decisive distance argument
#
def get_decisive_distance( *args ):
return sorted(args)[1]
connection.create_function('get_decisive_distance', len(on_instruments), get_decisive_distance)
decisive_distance = ''.join(['get_decisive_distance(', ','.join(['eff_dist_'+inst.lower()[0] for inst in on_instruments]), ')' ])
#
# Initialize ranking. Statistics for ranking are based on decisive distance
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(simulation_table, decisive_distance, 'ASC')
# add requirement that stats not be found in the sim_rec_table to in_this_filter
ranker.populate_stats_list(connection, limit = limit, filter = in_this_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Get the Data
#
sqlquery = ''.join(["""
SELECT
*,
get_sim_tag(process_id),
""", decisive_distance, """,
rank(""", decisive_distance, """)
FROM
""", simulation_table, """
""", in_this_filter, """
%s""" % (limit is not None and ''.join(['AND rank(', decisive_distance, ') <= ', str(limit)]) or ''), """
ORDER BY
rank(""", decisive_distance, """) ASC
"""])
if verbose:
print >> sys.stderr, "Getting injections..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
cmrow = CloseMissed()
[ setattr(cmrow, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
cmrow.decisive_distance = values[-2]
cmrow.rank = values[-1]
cmrow.instruments_on = lsctables.ifos_from_instrument_set(on_instruments)
cmrow.veto_def_name = veto_def_name
cmrow.sim_tag = values[-3]
cmrow.mini_followup = None
cmrow.omega_scan = None
cmrow.gps_time = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start)
cmrow.gps_time_ns = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start + '_ns')
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time)) for ifo in on_instruments]
cmrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
injection_time_utc = format_end_time_in_utc( cmrow.gps_time )
daily_ihope_address = get_daily_ihope_page(cmrow.gps_time, pages_location = daily_ihope_pages_location)
cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, injection_time_utc )
# add the row
cmtable.append(cmrow)
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return cmtable
def printsims(connection, simulation_table, recovery_table, map_label, ranking_stat, rank_by, comparison_datatype,
sort_by = 'rank', param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None,
sim_tag = 'ALLINJ', rank_range = None, convert_durations = 's',
daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
# check and format options appropriately
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
ranking_stat = sqlutils.validate_option(ranking_stat)
rank_by = sqlutils.validate_option(rank_by, lower = False).upper()
comparison_datatype = sqlutils.validate_option(comparison_datatype)
convert_durations = sqlutils.validate_option(convert_durations)
if rank_by == 'MIN':
rank_by = 'ASC'
elif rank_by == 'MAX':
rank_by = 'DESC'
elif rank_by != 'ASC' or rank_by != 'DESC':
raise ValueError, 'rank_by must be MAX (or DESC) or MIN (or ASC)'
if not ranking_stat.startswith(recovery_table):
ranking_stat = '.'.join([recovery_table, ranking_stat])
#
# Set up sim_rec_map table
#
if 'sim_rec_map' not in sqlutils.get_tables_in_database( connection ):
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, ranking_stat)
#
# Initialize ranking. Statistics for ranking are collected from non-injections
# in the recovery table.
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(recovery_table, ranking_stat, rank_by)
# add requirement that stats not be found in the sim_rec_table to in_this_filter
rank_filter = ''.join([
recovery_table, '''.coinc_event_id NOT IN (
SELECT
rec_id
FROM
sim_rec_map)
AND
experiment_summary.datatype == "''', comparison_datatype, '"'])
# add to the rank filter any other desired filters; note that sim-tag is forced to ALLINJ if not comparing
# to simulation datatype
in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs,
sim_tag = comparison_datatype == 'simulation' and sim_tag or 'ALLINJ',
verbose = False)
if in_this_filter != '':
rank_filter = '\n\tAND '.join([ in_this_filter, rank_filter ])
rank_filter = '\n\t'.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), 'WHERE', rank_filter ])
ranker.populate_stats_list(connection, limit = None, filter = rank_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Set recovery table filters
#
in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs, sim_tag = sim_tag,
verbose = verbose)
# Now apply the filter to the sim_rec_map table: this will delete all sim/rec maps where the simulation id is
# mapped to a recovered event that falls outside the filter, even if that particular sim/rec map is in the
# filter. For example, if the filter is recovery_table.combined_far != 0., and there are two entries in the
# sim_rec_map table sharing the same sim_id:
# sim_id:0 | rec_id:0 | rec_id's combined_far = 0.
# sim_id:0 | rec_id:1 | rec_id's combined_far = 1.2
# both entries will get deleted even though the second entry's combined_far is not 0.
if in_this_filter != '':
# join the needed tables to in_this_filter
in_this_filter = ''.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), "\n WHERE\n\t", in_this_filter ])
sqlscript = ''.join([ """
CREATE TEMP TABLE del_ids AS
SELECT
sim_id AS del_id
FROM
sim_rec_map
WHERE
rec_id NOT IN (
SELECT
""", recovery_table, """.coinc_event_id
FROM
""", recovery_table, """
""", in_this_filter, """
AND experiment_summary.datatype == "simulation"
);
DELETE FROM
sim_rec_map
WHERE
sim_id IN (
SELECT
del_id
FROM
del_ids );
DROP TABLE del_ids;""" ])
connection.cursor().executescript(sqlscript)
#
# Set other needed functions
#
# establish what units will be converting duration to
def convert_duration( duration ):
return sqlutils.convert_duration( duration, convert_durations )
connection.create_function( 'convert_duration', 1, convert_duration )
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
# Get range ranks
if rank_range is not None:
rank_range_parser = sqlutils.parse_param_ranges( 'rank(sim_rec_map', 'ranking_stat)',
rank_range, verbose = verbose )
#
# Create and prepare the SelectedFoundTable to store summary information
#
# Get recovery table and simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
recovery_table_columns = sqlutils.get_column_names_from_table( connection, recovery_table )
# Get list of column name for injected parameters
injected_cols = []
for col in simulation_table_columns:
if col == 'simulation_id':
injected_cols.append('simulation_id')
else:
injected_cols.append('injected_'+col)
injected_cols.extend(['injected_decisive_distance','injected_gps_time', 'injected_gps_time_ns', 'injected_event_time_utc__Px_click_for_daily_ihope_xP_'])
# Get list of column names from the recovery table
recovered_cols = []
for col in recovery_table_columns:
if col == 'coinc_event_id':
recovered_cols.append(u'coinc_event_id')
else:
recovered_cols.append(u'recovered_'+col)
# generate list of column names for the summary table
column_names = injected_cols + recovered_cols
# add instruments on, duration, mini_followups
rankname = 'rank_in_' + comparison_datatype.strip().lower() + '_using_' + ranking_stat.split('.')[-1]
durname = ''.join([ 'simulation', u'_duration__Px_', convert_durations, '_xP_' ])
column_names.extend( [ rankname, 'recovered_match_rank', 'instruments_on', 'elogs', durname, 'mini_followup','omega_scan', 'sim_tag' ] )
#
# define needed tables
#
class tmpSimTable(table.Table):
tableName = "tmp_sim_table"
validcolumns = dict([ [col, sqlutils.get_col_type(simulation_table, col)] for col in simulation_table_columns ])
class tmpSim(object):
__slots__ = tmpSimTable.validcolumns.keys()
def get_gps_time(self, site):
if '%s_start_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, '%s_start_time' % site ), getattr(self, '%s_start_time_ns' % site))
elif '%s_end_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, '%s_end_time' % site ), getattr(self, '%s_end_time_ns' % site))
else:
raise AttributeError, "could not find an injected %s_start_time nor an injected %s_end_time" %(site,site)
def get_pyvalue(self):
return generic_get_pyvalue(self)
class tmpRecTable(table.Table):
tableName = "tmp_rec_table"
validcolumns = dict([ [col, sqlutils.get_col_type(recovery_table, col)] for col in recovery_table_columns ])
class tmpRec(object):
__slots__ = tmpRecTable.validcolumns.keys()
def get_gps_time(self):
if 'start_time' in self.__slots__:
return LIGOTimeGPS(self.start_time, self.start_time_ns)
elif 'end_time' in self.__slots__:
return LIGOTimeGPS(self.end_time, self.end_time_ns)
else:
raise AttributeError, "could not find a recovered start_time or recovered end_time"
def get_pyvalue(self):
return generic_get_pyvalue(self)
class SelectedFoundTable(table.Table):
tableName = "selected_found_injections"
validcolumns = {}
for col_name in column_names:
if 'rank_in_' in col_name:
validcolumns[col_name] = "int_4u"
elif '_duration_' in col_name:
validcolumns[col_name] = "real_8"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif col_name == 'injected_start_time' or col_name == 'injected_start_time_ns':
validcolumns[col_name] = "int_4s"
elif col_name == 'injected_gps_time' or col_name == 'injected_gps_time_ns':
validcolumns[col_name] = "int_4s"
elif col_name == 'injected_decisive_distance':
validcolumns[col_name] = "real_8"
elif 'injected_' in col_name or col_name == 'simulation_id':
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, re.sub('injected_', '', col_name))
elif 'recovered_' in col_name or col_name == 'coinc_event_id':
validcolumns[col_name] = sqlutils.get_col_type(recovery_table, re.sub('recovered_', '', col_name))
# if custom columns exist in the database, just set them to lstrings
else:
validcolumns[col_name] = "lstring"
# add FAP columns
validcolumns['recovered_fap'] = "real_8"
validcolumns['recovered_fap_1yr'] = "real_8"
class SelectedFound(object):
__slots__ = SelectedFoundTable.validcolumns.keys()
def set_recovered_gps_time(self, gps_time):
if 'recovered_start_time' in self.__slots__:
self.recovered_start_time = gps_time.seconds
self.recovered_start_time_ns = gps_time.nanoseconds
elif 'recovered_end_time' in self.__slots__:
self.recovered_end_time = gps_time.seconds
self.recovered_end_time_ns = gps_time.nanoseconds
else:
raise AttributeError, "could not find a recovered_start_time or recovered_end_time"
def get_recovered_gps_time(self):
if 'recovered_start_time' in self.__slots__:
return LIGOTimeGPS(self.recovered_start_time, self.recovered_start_time_ns)
elif 'recovered_end_time' in self.__slots__:
return LIGOTimeGPS(self.recovered_end_time, self.recovered_end_time_ns)
else:
raise AttributeError, "could not find a recovered_start_time or recovered_end_time"
def set_injected_gps_time(self, site, gps_time):
if 'injected_%s_start_time' % site in self.__slots__:
setattr(self, 'injected_%s_start_time' % site, gps_time.seconds)
setattr(self, 'injected_%s_start_time_ns' % site, gps_time.nanoseconds)
elif 'injected_%s_end_time' % site in self.__slots__:
setattr(self, 'injected_%s_end_time' % site, gps_time.seconds)
setattr(self, 'injected_%s_end_time_ns' % site, gps_time.nanoseconds)
else:
raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site)
def get_injected_gps_time(self, site):
if 'injected_%s_start_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, 'injected_%s_start_time' % site ), getattr(self, 'injected_%s_start_time_ns' % site))
elif 'injected_%s_end_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, 'injected_%s_end_time' % site ), getattr(self, 'injected_%s_end_time_ns' % site))
else:
raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site)
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
tmpSimTable.RowType = tmpSim
tmpRecTable.RowType = tmpRec
SelectedFoundTable.RowType = SelectedFound
#
# Get the Data
#
tmp_sim_table = lsctables.New(tmpSimTable)
tmp_rec_table = lsctables.New(tmpRecTable)
tmp_sftable = lsctables.New(SelectedFoundTable)
prior_sim_id = ''
group_rank = None
current_match_rank = 1
sqlquery = ''.join(["""
SELECT
""", simulation_table, """.*,
""", recovery_table, """.*,
get_sim_tag(experiment_summary.sim_proc_id),
rank(sim_rec_map.ranking_stat),
NULL AS match_rank,
experiment.instruments,
convert_duration(experiment_summary.duration),
NULL AS mini_followup
FROM
sim_rec_map
JOIN
""", ', '.join([simulation_table, recovery_table]), """, experiment, experiment_summary, experiment_map ON (
sim_rec_map.sim_id == """, simulation_table, """.simulation_id AND
sim_rec_map.rec_id == """, recovery_table, """.coinc_event_id AND
sim_rec_map.rec_id == experiment_map.coinc_event_id AND
experiment_map.experiment_summ_id == experiment_summary.experiment_summ_id AND
experiment_summary.experiment_id == experiment.experiment_id)
ORDER BY
sim_rec_map.sim_id, sim_rec_map.ranking_stat """, rank_by])
if verbose:
print >> sys.stderr, "Getting coincs..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
# sort the data
tmp_sim_row = tmpSim()
tmp_rec_row = tmpRec()
[ setattr(tmp_sim_row, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
[ setattr(tmp_rec_row, column, values[ii+1+jj]) for jj, column in enumerate(recovery_table_columns) ]
# figure out the rank
this_inj_rank = values[-5]
this_sim_id = tmp_sim_row.simulation_id
this_ranking_stat = getattr(tmp_rec_row, ranking_stat.split('.')[-1])
if this_sim_id == prior_sim_id and this_ranking_stat != prior_ranking_stat:
current_match_rank += 1
elif this_sim_id != prior_sim_id:
current_match_rank = 1
group_rank = this_inj_rank
prior_sim_id = this_sim_id
prior_ranking_stat = this_ranking_stat
# only store data if the group_rank falls in the desired rank ranges
if rank_range and rank_range_parser.group_by_param_range(group_rank) is None:
continue
on_instruments = lsctables.instrument_set_from_ifos(values[-3])
duration = values[-2]
# now that have all the information from this row, create a row for the selected found table
sfrow = SelectedFound()
# set the ranks
setattr(sfrow, rankname, group_rank)
sfrow.recovered_match_rank = current_match_rank
# set the injected parameters
use_this_site = sorted(on_instruments)[0][0].lower()
injected_time = tmp_sim_row.get_gps_time( use_this_site )
sfrow.injected_gps_time = injected_time.seconds
sfrow.injected_gps_time_ns = injected_time.nanoseconds
for col in simulation_table_columns:
if col == "simulation_id":
sfrow.simulation_id = tmp_sim_row.simulation_id
else:
setattr(sfrow, 'injected_'+col, getattr( tmp_sim_row, col) )
# set the recovered parameters
for col in recovery_table_columns:
if col == "coinc_event_id":
sfrow.coinc_event_id = tmp_rec_row.coinc_event_id
else:
setattr(sfrow, 'recovered_'+col, getattr( tmp_rec_row, col) )
# calculate and add faps
if sfrow.recovered_combined_far is not None:
t_in_s = float(values[-2]) / sqlutils.convert_duration(1, convert_durations)
sfrow.recovered_fap = 1 - math.exp(-sqlutils.convert_duration(t_in_s, 'yr') * sfrow.recovered_combined_far)
sfrow.recovered_fap_1yr = 1 - math.exp(-sfrow.recovered_combined_far)
else:
sfrow.recovered_fap = None
sfrow.recovered_fap_1yr = None
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, sfrow.injected_gps_time)) for ifo in on_instruments]
sfrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
event_time_utc = format_end_time_in_utc( sfrow.injected_gps_time )
daily_ihope_address = get_daily_ihope_page(sfrow.injected_gps_time, pages_location = daily_ihope_pages_location)
sfrow.injected_event_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, event_time_utc )
# set any other info
sfrow.instruments_on = ','.join(sorted(on_instruments))
sfrow.injected_decisive_distance = sorted([getattr(sfrow, 'injected_eff_dist_%s' % ifo[0].lower()) for ifo in on_instruments])[1]
sfrow.mini_followup = None
sfrow.omega_scan = None
sfrow.sim_tag = values[-6]
setattr(sfrow, durname, duration)
# add the row
tmp_sftable.append(sfrow)
# Re-sort the sftable by rank, recovered_match_rank
sftable = lsctables.New(SelectedFoundTable)
for sfrow in sorted([ row for row in tmp_sftable ], key = lambda row: getattr(row, sort_by == 'rank' and rankname or sort_by) ):
if sfrow.simulation_id not in [row.simulation_id for row in sftable]:
sftable.append(sfrow)
sftable.extend(sub_row for sub_row in sorted([row for row in tmp_sftable
if row.simulation_id == sfrow.simulation_id
and row.coinc_event_id != sfrow.coinc_event_id],
key = lambda row: row.recovered_match_rank))
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return sftable
def printmissed(connection, simulation_table, recovery_table, map_label, livetime_program,
param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ',
limit = None, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
from pylal import ligolw_cbc_compute_durations as compute_dur
from glue import segments
from glue.ligolw import dbtables
# Get simulation/recovery tables
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
#
# Create and prepare the CloseMissedTable to store summary information
#
# Get simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
column_names = simulation_table_columns + \
['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag']
# define needed tables
class CloseMissedTable(table.Table):
tableName = "close_missed_injections:table"
validcolumns = {}
for col_name in column_names:
if 'rank' in col_name:
validcolumns[col_name] = "int_4u"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif 'veto_def_name' == col_name:
validcolumns[col_name] = lsctables.ExperimentSummaryTable.validcolumns['veto_def_name']
elif 'decisive_distance' == col_name:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, 'eff_dist_h')
elif 'gps_time' == col_name or 'gps_time_ns' == col_name:
validcolumns[col_name] = "int_4s"
elif 'sim_tag' == col_name:
validcolumns[col_name] = "lstring"
else:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, col_name, default = 'lstring')
class CloseMissed(object):
__slots__ = CloseMissedTable.validcolumns.keys()
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
CloseMissedTable.RowType = CloseMissed
# create the table
cmtable = lsctables.New(CloseMissedTable)
# set up sim_rec_map table
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, None)
#
# Set table filters
#
# we force the include/exclude filters to None; will check for excluded/included ifo time
# when cycling through the ifo times
filter = """
WHERE
simulation_id NOT IN (
SELECT
sim_id
FROM
sim_rec_map )"""
af = create_filter( connection, simulation_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = None, include_only_coincs = None, sim_tag = sim_tag, verbose = verbose)
af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af)
if af != '':
filter = '\n'.join([ filter, """
AND""", af])
# get desired instrument times
if include_only_coincs is not None:
include_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( include_only_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
if exclude_coincs is not None:
exclude_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( exclude_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
# get the usertags of inspiral jobs in this database
sqlquery = """
SELECT value
FROM process_params
WHERE param == "-userTag"
GROUP BY value
"""
usertags = set(usertag[0] for usertag in connection.cursor().execute(sqlquery) )
# Get the single-ifo science segments after CAT-1 vetoes
try:
if "FULL_DATA" in usertags:
tag = "FULL_DATA"
else:
tag = list(usertags)[0]
except IndexError:
# This is hacky anyway, so let's just take a guess
tag = "FULL_DATA"
ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name = livetime_program, usertag = tag)
if ifo_segments == {}:
raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program
if verbose:
print >> sys.stderr, "Getting all veto category names from the experiment_summary table..."
xmldoc = dbtables.get_xml(connection)
# get veto_segments
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# make a dictionary of zerolag "shifts" needed for the get_coinc_segments function
zerolag_dict = {}
for ifo in ifo_segments:
zerolag_dict[ifo] = 0.0
# Cycle over available veto categories
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
# make a dictionary of coincident segments by exclusive on-ifos
coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs, zerolag_dict)
#
# Get all the on_instrument times and cycle over them
#
sqlquery = """
SELECT DISTINCT experiment.instruments
FROM experiment
JOIN experiment_summary ON (
experiment.experiment_id == experiment_summary.experiment_id )
WHERE experiment_summary.veto_def_name == :1
"""
for on_instruments in connection.cursor().execute(sqlquery, (veto_def_name,)).fetchall():
on_instruments = lsctables.instrument_set_from_ifos(on_instruments[0])
# check if this on_instruments is desired; if not, skip
if include_only_coincs is not None and frozenset(on_instruments) not in include_times:
continue
if exclude_coincs is not None and frozenset(on_instruments) in exclude_times:
continue
on_times = coinc_segs[','.join(sorted(on_instruments))]
def is_in_on_time(gps_time, gps_time_ns):
return gps_time + (1e-9 * gps_time_ns) in on_times
connection.create_function('is_in_on_time', 2, is_in_on_time)
# add the check for on time to the filter
in_this_filter = filter
# figure out if simulation_table has end_times or start_times
end_or_start = any('end_time' in c for c in simulation_table_columns) and '_end_time' or '_start_time'
for instrument in on_instruments:
inst_time = instrument.lower()[0] + end_or_start
inst_time_ns = inst_time + '_ns'
in_this_filter = ''.join([ in_this_filter,
'\n\tAND is_in_on_time(', inst_time, ',', inst_time_ns, ')' ])
#
# Set up decisive distance argument
#
def get_decisive_distance( *args ):
return sorted(args)[1]
connection.create_function('get_decisive_distance', len(on_instruments), get_decisive_distance)
decisive_distance = ''.join(['get_decisive_distance(', ','.join(['eff_dist_'+inst.lower()[0] for inst in on_instruments]), ')' ])
#
# Initialize ranking. Statistics for ranking are based on decisive distance
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(simulation_table, decisive_distance, 'ASC')
# add requirement that stats not be found in the sim_rec_table to in_this_filter
ranker.populate_stats_list(connection, limit = limit, filter = in_this_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Get the Data
#
sqlquery = ''.join(["""
SELECT
*,
get_sim_tag(process_id),
""", decisive_distance, """,
rank(""", decisive_distance, """)
FROM
""", simulation_table, """
""", in_this_filter, """
%s""" % (limit is not None and ''.join(['AND rank(', decisive_distance, ') <= ', str(limit)]) or ''), """
ORDER BY
rank(""", decisive_distance, """) ASC
"""])
if verbose:
print >> sys.stderr, "Getting injections..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
cmrow = CloseMissed()
[ setattr(cmrow, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
cmrow.decisive_distance = values[-2]
cmrow.rank = values[-1]
cmrow.instruments_on = lsctables.ifos_from_instrument_set(on_instruments)
cmrow.veto_def_name = veto_def_name
cmrow.sim_tag = values[-3]
cmrow.mini_followup = None
cmrow.omega_scan = None
cmrow.gps_time = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start)
cmrow.gps_time_ns = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start + '_ns')
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time)) for ifo in on_instruments]
cmrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
injection_time_utc = format_end_time_in_utc( cmrow.gps_time )
daily_ihope_address = get_daily_ihope_page(cmrow.gps_time, pages_location = daily_ihope_pages_location)
cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, injection_time_utc )
# add the row
cmtable.append(cmrow)
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return cmtable
def get_injections(self, instruments, FAR=float("inf")):
injfnames = self.inj_fnames
zero_lag_segments = self.zero_lag_segments[instruments]
verbose = self.opts.verbose
found = []
missed = []
print >>sys.stderr, ""
for cnt, f in enumerate(injfnames):
print >>sys.stderr, "getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),),
working_filename = dbtables.get_connection_filename(f, tmp_path = opts.tmp_space, verbose = verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# DON'T BOTHER CONTINUING IF THE INSTRUMENTS OF INTEREST ARE NOT HERE
instruments_in_this_file = []
for i in connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event'):
if i[0]: instruments_in_this_file.append(frozenset(lsctables.instrument_set_from_ifos(i[0])))
if instruments not in instruments_in_this_file:
connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
continue
# WORK OUT CORRECT SEGMENTS FOR THIS FILE WHERE WE SHOULD SEE INJECTIONS
segments = self.get_segments(connection, xmldoc)
segments -= self.veto_segments
#print thincasegments
zero_lag_segments = segments.intersection(instruments) - segments.union(set(segments.keys()) - instruments)
###############
# DEFINE THE INJECTION WAS MADE FUNCTION
def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(dbtables.get_xml(connection)).row_from_cols
# INSPIRAL
if self.coinc_inspiral_table:
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# BURSTS
if self.multi_burst_table:
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN multi_burst ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == multi_burst.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND multi_burst.false_alarm_rate < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
print >>sys.stderr, "\nFound = %d Missed = %d" % (len(found), len(missed))
return found, missed
def get_instruments(self, connection):
for i in connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event'):
if i[0]: self.instruments.append(frozenset(lsctables.instrument_set_from_ifos(i[0])))
def get_ifos(self): """ Return a set of the instruments for this row. """ return lsctables.instrument_set_from_ifos(self.ifos)
def __init__(self, filelist, live_time_program = None, veto_segments_name = None, data_segments_name = "datasegments", tmp_path = None, verbose = False):
self.segments = segments.segmentlistdict()
self.instruments = set()
self.table_name = None
self.found_injections_by_instrument_set = {}
self.missed_injections_by_instrument_set = {}
self.total_injections_by_instrument_set = {}
self.zerolag_fars_by_instrument_set = {}
self.ts_fars_by_instrument_set = {}
self.numslides = set()
for f in filelist:
if verbose:
print >> sys.stderr, "Gathering stats from: %s...." % (f,)
working_filename = dbtables.get_connection_filename(f, tmp_path = tmp_path, verbose = verbose)
connection = sqlite3.connect(working_filename)
xmldoc = dbtables.get_xml(connection)
sim = False
# look for a sim inspiral table. This is IMR work we have to have one of these :)
try:
sim_inspiral_table = table.get_table(xmldoc, dbtables.lsctables.SimInspiralTable.tableName)
sim = True
except ValueError:
pass
# look for the relevant table for analyses
for table_name in allowed_analysis_table_names():
try:
setattr(self, table_name, table.get_table(xmldoc, table_name))
if self.table_name is None or self.table_name == table_name:
self.table_name = table_name
else:
raise ValueError("detected more than one table type out of " + " ".join(allowed_analysis_table_names()))
except ValueError:
setattr(self, table_name, None)
# the non simulation databases are where we get information about segments
if not sim:
self.numslides.add(connection.cursor().execute('SELECT count(DISTINCT(time_slide_id)) FROM time_slide').fetchone()[0])
[self.instruments.add(ifos) for ifos in get_instruments_from_coinc_event_table(connection)]
# save a reference to the segments for this file, needed to figure out the missed and found injections
self.this_segments = get_segments(connection, xmldoc, self.table_name, live_time_program, veto_segments_name, data_segments_name = data_segments_name)
# FIXME we don't really have any reason to use playground segments, but I put this here as a reminder
# self.this_playground_segments = segmentsUtils.S2playground(self.this_segments.extent_all())
self.segments += self.this_segments
# get the far thresholds for the loudest events in these databases
for (instruments_set, far, ts) in get_event_fars(connection, self.table_name):
if not ts:
self.zerolag_fars_by_instrument_set.setdefault(instruments_set, []).append(far)
else:
self.ts_fars_by_instrument_set.setdefault(instruments_set, []).append(far)
# get the injections
else:
# We need to know the segments in this file to determine which injections are found
self.this_injection_segments = get_segments(connection, xmldoc, self.table_name, live_time_program, veto_segments_name, data_segments_name = data_segments_name)
self.this_injection_instruments = []
distinct_instruments = connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event WHERE instruments!=""').fetchall()
for instruments, in distinct_instruments:
instruments_set = frozenset(lsctables.instrument_set_from_ifos(instruments))
self.this_injection_instruments.append(instruments_set)
segments_to_consider_for_these_injections = self.this_injection_segments.intersection(instruments_set) - self.this_injection_segments.union(set(self.this_injection_segments.keys()) - instruments_set)
found, total, missed = get_min_far_inspiral_injections(connection, segments = segments_to_consider_for_these_injections, table_name = self.table_name)
if verbose:
print >> sys.stderr, "%s total injections: %d; Found injections %d: Missed injections %d" % (instruments, len(total), len(found), len(missed))
self.found_injections_by_instrument_set.setdefault(instruments_set, []).extend(found)
self.total_injections_by_instrument_set.setdefault(instruments_set, []).extend(total)
self.missed_injections_by_instrument_set.setdefault(instruments_set, []).extend(missed)
# All done
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
if len(self.numslides) > 1:
raise ValueError('number of slides differs between input files')
elif self.numslides:
self.numslides = min(self.numslides)
else:
self.numslides = 0
def read_sky_map(filename):
prob, header = hp.read_map(filename, h=True, verbose=False)
header = dict(header)
metadata = {}
try:
value = header['OBJECT']
except KeyError:
pass
else:
metadata['objid'] = value
try:
value = header['REFERENC']
except KeyError:
pass
else:
metadata['url'] = value
try:
value = header['INSTRUME']
except KeyError:
pass
else:
value = set(str(ifo) for ifo in lsctables.instrument_set_from_ifos(value))
metadata['instruments'] = value
try:
value = header['DATE-OBS']
except KeyError:
pass
else:
metadata['gps_time'] = iso8601_to_gps(value)
try:
value = header['DATE']
except KeyError:
pass
else:
metadata['gps_creation_time'] = iso8601_to_gps(value)
try:
value = header['CREATOR']
except KeyError:
pass
else:
metadata['creator'] = value
try:
value = header['ORIGIN']
except KeyError:
pass
else:
metadata['origin'] = value
try:
value = header['RUNTIME']
except KeyError:
pass
else:
metadata['runtime'] = value
return prob, metadata
#
# get chunk lengths from the values in the ini file
#
short_segment_duration = config_parser.getint('lalapps_StringSearch', 'short-segment-duration')
pad = config_parser.getint('lalapps_StringSearch', 'pad')
min_segment_length = config_parser.getint('pipeline', 'segment-length') # not including pad at each end
trig_overlap = config_parser.getint('pipeline', 'trig_overlap')
overlap = short_segment_duration / 2 + 2 * pad # FIXME: correct?
#
# get the instruments and raw segments
#
instruments = lsctables.instrument_set_from_ifos(config_parser.get('pipeline','ifos'))
seglists = ligolwsegments.segmenttable_get_by_name(utils.load_filename(options.segments_file, gz = (options.segments_file or "stdin").endswith(".gz"), verbose = options.verbose), options.segments_name).coalesce()
# remove extra instruments
for instrument in set(seglists) - instruments:
if options.verbose:
print >>sys.stderr, "warning: ignoring segments for '%s' found in '%s'" % (instrument, options.segments_file)
del seglists[instrument]
# check for missing instruments
if not instruments.issubset(set(seglists)):
raise ValueError, "segment lists retrieved from '%s' missing segments for instruments %s" % (options.segments_file, ", ".join(instruments - set(seglists)))
# now rely on seglists' keys to provide the instruments
del instruments
#
# Using time slide information, construct segment lists describing times
# requiring trigger construction.
# FIXME this assumes that the first argument is the zero lag / time slide database
zerofname = sys.argv[1]
# FIXME this assumes that subsequent files are injections
injfnames = sys.argv[2:]
vetosegs = get_vetoes(zerofname)
# A class to compute the LVstat
lvs = LVstat()
# FIXME get the on instruments in the zero lag files,
# this assumes that the instruments are same for injection runs
# they probably should be
for on_ifos in get_on_instruments(sys.argv[1]):
#get the correct segment lists and remove vetos
instruments = lsctables.instrument_set_from_ifos(on_ifos[0])
FAR, segs = get_far_threshold_and_segments(zerofname, on_ifos[0], "thinca")
segs -= vetosegs
zero_lag_segments = segs.intersection(instruments) - segs.union(set(segs.keys()) - instruments)
for trigger_ifos in get_ifos(sys.argv[1]):
#get the ifos that participate in coincidence
key = (on_ifos[0], trigger_ifos[0])
f, m = get_injections(sys.argv[2:], zero_lag_segments, trigger_ifos[0])
lvs.calc_eff_factor(on_ifos[0], trigger_ifos[0], m, f)
print "Eff factor: ", lvs.eff_factor[key], " key: ", key
# update the coincs with the effective likelihood
lvs.update_coincs(sys.argv)
def from_xml(cls, name): xml = cls.get_xml_root(xml, name) self = cls(lsctables.instrument_set_from_ifos(ligolw_param.get_pyvalue(xml, "instruments"))) for key in self.densities: self.densities[key] = rate.BinnedLnPDF.from_xml(xml, key) return self
def read_sky_map(filename, nest=False, distances=False):
"""
Read a LIGO/Virgo-type sky map and return a tuple of the HEALPix array
and a dictionary of metadata from the header.
Parameters
----------
filename: string
Path to the optionally gzip-compressed FITS file.
nest: bool, optional
If omitted or False, then detect the pixel ordering in the FITS file
and rearrange if necessary to RING indexing before returning.
If True, then detect the pixel ordering and rearrange if necessary to
NESTED indexing before returning.
If None, then preserve the ordering from the FITS file.
Regardless of the value of this option, the ordering used in the FITS
file is indicated as the value of the 'nest' key in the metadata
dictionary.
distances: bool, optional
If true, then read also read the additional HEALPix layers representing
the conditional mean and standard deviation of distance as a function
of sky location.
"""
out = hp.read_map(
filename, h=True, verbose=False, nest=nest,
field=(list(range(4)) if distances else 0))
prob = out[:-1] if distances else out[0]
header = dict(out[-1])
metadata = {}
ordering = header['ORDERING']
if ordering == 'RING':
metadata['nest'] = False
elif ordering == 'NESTED':
metadata['nest'] = True
else:
raise ValueError(
'ORDERING card in header has unknown value: {0}'.format(ordering))
try:
value = header['OBJECT']
except KeyError:
pass
else:
metadata['objid'] = value
try:
value = header['REFERENC']
except KeyError:
pass
else:
metadata['url'] = value
try:
value = header['INSTRUME']
except KeyError:
pass
else:
value = set(str(ifo) for ifo in lsctables.instrument_set_from_ifos(value))
metadata['instruments'] = value
try:
value = header['DATE-OBS']
except KeyError:
pass
else:
metadata['gps_time'] = iso8601_to_gps(value)
try:
value = header['DATE']
except KeyError:
pass
else:
metadata['gps_creation_time'] = iso8601_to_gps(value)
try:
value = header['CREATOR']
except KeyError:
pass
else:
metadata['creator'] = value
try:
value = header['ORIGIN']
except KeyError:
pass
else:
metadata['origin'] = value
try:
value = header['RUNTIME']
except KeyError:
pass
else:
metadata['runtime'] = value
try:
value = header['DISTMEAN']
except KeyError:
pass
else:
metadata['distmean'] = value
try:
value = header['DISTSTD']
except KeyError:
pass
else:
metadata['diststd'] = value
return prob, metadata
def get_instruments(self, connection):
for i in connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event'):
self.instruments.append(frozenset(lsctables.instrument_set_from_ifos(i[0])))
return self.instruments
def get_injections(self, instruments, FAR=float("inf")):
injfnames = self.inj_fnames
zero_lag_segments = self.zero_lag_segments[instruments]
verbose = self.opts.verbose
found = []
missed = []
print >>sys.stderr, ""
for cnt, f in enumerate(injfnames):
print >>sys.stderr, "getting injections below FAR: " + str(FAR) + ":\t%.1f%%\r" % (100.0 * cnt / len(injfnames),),
working_filename = dbtables.get_connection_filename(f, tmp_path = opts.tmp_space, verbose = verbose)
connection = sqlite3.connect(working_filename)
dbtables.DBTable_set_connection(connection)
xmldoc = dbtables.get_xml(connection)
# DON'T BOTHER CONTINUING IF THE INSTRUMENTS OF INTEREST ARE NOT HERE
instruments_in_this_file = []
for i in connection.cursor().execute('SELECT DISTINCT(instruments) FROM coinc_event'):
if i[0]: instruments_in_this_file.append(frozenset(lsctables.instrument_set_from_ifos(i[0])))
if instruments not in instruments_in_this_file:
connection.close()
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
continue
# WORK OUT CORRECT SEGMENTS FOR THIS FILE WHERE WE SHOULD SEE INJECTIONS
segments = self.get_segments(connection, xmldoc)
segments -= self.veto_segments
#print thincasegments
zero_lag_segments = segments.intersection(instruments) - segments.union(set(segments.keys()) - instruments)
###############
# DEFINE THE INJECTION WAS MADE FUNCTION
def injection_was_made(geocent_end_time, geocent_end_time_ns, zero_lag_segments = zero_lag_segments):
"""
return True if injection was made in the given segmentlist
"""
return lsctables.LIGOTimeGPS(geocent_end_time, geocent_end_time_ns) in zero_lag_segments
connection.create_function("injection_was_made", 2, injection_was_made)
make_sim_inspiral = lsctables.SimInspiralTable.get_table(dbtables.get_xml(connection)).row_from_cols
# INSPIRAL
if self.coinc_inspiral_table:
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN coinc_inspiral ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == coinc_inspiral.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND coinc_inspiral.combined_far < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# BURSTS
if self.multi_burst_table:
for values in connection.cursor().execute("""
SELECT
sim_inspiral.*,
-- true if injection matched a coinc below the false alarm rate threshold
EXISTS (
SELECT
*
FROM
coinc_event_map AS mapa
JOIN coinc_event_map AS mapb ON (
mapa.coinc_event_id == mapb.coinc_event_id
)
JOIN multi_burst ON (
mapb.table_name == "coinc_event"
AND mapb.event_id == multi_burst.coinc_event_id
)
WHERE
mapa.table_name == "sim_inspiral"
AND mapa.event_id == sim_inspiral.simulation_id
AND multi_burst.false_alarm_rate < ?
)
FROM
sim_inspiral
WHERE
-- only interested in injections that were injected
injection_was_made(sim_inspiral.geocent_end_time, sim_inspiral.geocent_end_time_ns)
""", (FAR,)):
sim = make_sim_inspiral(values)
if values[-1]:
found.append(sim)
else:
missed.append(sim)
# done
dbtables.discard_connection_filename(f, working_filename, verbose = verbose)
dbtables.DBTable_set_connection(None)
print >>sys.stderr, "\nFound = %d Missed = %d" % (len(found), len(missed))
return found, missed
def get_ifos(self):
"""
Return a set of the instruments for this row.
"""
return lsctables.instrument_set_from_ifos(self.ifos)
# FIXME this assumes that the first argument is the zero lag / time slide database
zerofname = sys.argv[1]
# FIXME this assumes that subsequent files are injections
injfnames = sys.argv[2:]
vetosegs = get_vetoes(zerofname)
# A class to compute the LVstat
lvs = LVstat()
# FIXME get the on instruments in the zero lag files,
# this assumes that the instruments are same for injection runs
# they probably should be
for on_ifos in get_on_instruments(sys.argv[1]):
#get the correct segment lists and remove vetos
instruments = lsctables.instrument_set_from_ifos(on_ifos[0])
FAR, segs = get_far_threshold_and_segments(zerofname, on_ifos[0], "thinca")
segs -= vetosegs
zero_lag_segments = segs.intersection(instruments) - segs.union(
set(segs.keys()) - instruments)
for trigger_ifos in get_ifos(sys.argv[1]):
#get the ifos that participate in coincidence
key = (on_ifos[0], trigger_ifos[0])
f, m = get_injections(sys.argv[2:], zero_lag_segments, trigger_ifos[0])
lvs.calc_eff_factor(on_ifos[0], trigger_ifos[0], m, f)
print "Eff factor: ", lvs.eff_factor[key], " key: ", key
# update the coincs with the effective likelihood
lvs.update_coincs(sys.argv)
