def get_livetime(self, category, match_criteria = [], time_units = 'yr'):
"""
Returns the sum of all the livetimes of categories that match the given category via the given match_criteria.
"""
if match_criteria == []:
match_criteria = Category.default_match_criteria
return sqlutils.convert_duration(sum([cat.livetime for cat in self if cat.selective_eq(category, match_criteria)]), time_units)
def get_livetime(self, category, match_criteria = [], time_units = 'yr'):
"""
Returns the sum of all the livetimes of categories that match the given category via the given match_criteria.
"""
if match_criteria == []:
match_criteria = Category.default_match_criteria
return sqlutils.convert_duration(sum([cat.livetime for cat in self if cat.selective_eq(category, match_criteria)]), time_units)
def eff_bkgd_time(T_i, tau_ij, ifos, unit):
# numerator is the product of the relevant single-ifo analyzed times
numerator = np.prod([T for (ifo, T) in T_i.items() if ifo in ifos])
# sorted list of the coincidence windows from smallest to largest
taus = sorted(tau_ij.values())
# denominator is a non-trivial combination of the coincident windows
if len(ifos) == 2:
denominator = taus[0]
elif len(ifos) == 3:
denominator = 0.5*tau[0]*(tau[2] + tau[1]) - \
0.25*( (tau[2]-tau[1])**2. + tau[0]**2. )
else:
raise ValueError, "Can only estimate background times for double & triples"
return sqlutils.convert_duration(numerator/denominator, unit)
def eff_bkgd_time(T_i, tau_ij, ifos, unit):
# numerator is the product of the relevant single-ifo analyzed times
numerator = np.prod([T for (ifo, T) in T_i.items() if ifo in ifos])
# sorted list of the coincidence windows from smallest to largest
taus = sorted(tau_ij.values())
# denominator is a non-trivial combination of the coincident windows
if len(ifos) == 2:
denominator = taus[0]
elif len(ifos) == 3:
denominator = 0.5*tau[0]*(tau[2] + tau[1]) - \
0.25*( (tau[2]-tau[1])**2. + tau[0]**2. )
else:
raise ValueError, "Can only estimate background times for double & triples"
return sqlutils.convert_duration(numerator / denominator, unit)
def coinc_time(connection, datatype, inc_ifo_list, unit, tsid=None):
params_dict = {'type': datatype}
cursor = connection.cursor()
sqlquery = """
SELECT duration, instruments
FROM experiment_summary
JOIN experiment ON (
experiment.experiment_id == experiment_summary.experiment_id)
WHERE datatype = :type """
if tsid:
params_dict['tsid'] = tsid
sqlquery += " AND time_slide_id = :tsid"
cursor.execute( sqlquery, params_dict )
# all elements of inclusive ifo_list must be found in ifos for T to be included
livetime = np.sum([ T for T, ifos in cursor if set(ifos.split(',')).issuperset(set(inc_ifo_list)) ])
return sqlutils.convert_duration(livetime, unit)
def coinc_time(connection, datatype, inc_ifo_list, unit, tsid=None):
params_dict = {'type': datatype}
cursor = connection.cursor()
sqlquery = """
SELECT duration, instruments
FROM experiment_summary
JOIN experiment ON (
experiment.experiment_id == experiment_summary.experiment_id)
WHERE datatype = :type """
if tsid:
params_dict['tsid'] = tsid
sqlquery += " AND time_slide_id = :tsid"
cursor.execute(sqlquery, params_dict)
# all elements of inclusive ifo_list must be found in ifos for T to be included
livetime = np.sum([
T for T, ifos in cursor
if set(ifos.split(',')).issuperset(set(inc_ifo_list))
])
return sqlutils.convert_duration(livetime, unit)
def get_livetime(self, time_units = 'yr'):
return sqlutils.convert_duration( self.livetime, time_units )
def convert_duration( duration ):
return sqlutils.convert_duration( duration, convert_durations )
def printsims(connection, simulation_table, recovery_table, map_label, ranking_stat, rank_by, comparison_datatype,
sort_by = 'rank', param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None,
sim_tag = 'ALLINJ', rank_range = None, convert_durations = 's',
daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
# check and format options appropriately
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
ranking_stat = sqlutils.validate_option(ranking_stat)
rank_by = sqlutils.validate_option(rank_by, lower = False).upper()
comparison_datatype = sqlutils.validate_option(comparison_datatype)
convert_durations = sqlutils.validate_option(convert_durations)
if rank_by == 'MIN':
rank_by = 'ASC'
elif rank_by == 'MAX':
rank_by = 'DESC'
elif rank_by != 'ASC' or rank_by != 'DESC':
raise ValueError, 'rank_by must be MAX (or DESC) or MIN (or ASC)'
if not ranking_stat.startswith(recovery_table):
ranking_stat = '.'.join([recovery_table, ranking_stat])
#
# Set up sim_rec_map table
#
if 'sim_rec_map' not in sqlutils.get_tables_in_database( connection ):
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, ranking_stat)
#
# Initialize ranking. Statistics for ranking are collected from non-injections
# in the recovery table.
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(recovery_table, ranking_stat, rank_by)
# add requirement that stats not be found in the sim_rec_table to in_this_filter
rank_filter = ''.join([
recovery_table, '''.coinc_event_id NOT IN (
SELECT
rec_id
FROM
sim_rec_map)
AND
experiment_summary.datatype == "''', comparison_datatype, '"'])
# add to the rank filter any other desired filters; note that sim-tag is forced to ALLINJ if not comparing
# to simulation datatype
in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs,
sim_tag = comparison_datatype == 'simulation' and sim_tag or 'ALLINJ',
verbose = False)
if in_this_filter != '':
rank_filter = '\n\tAND '.join([ in_this_filter, rank_filter ])
rank_filter = '\n\t'.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), 'WHERE', rank_filter ])
ranker.populate_stats_list(connection, limit = None, filter = rank_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Set recovery table filters
#
in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs, sim_tag = sim_tag,
verbose = verbose)
# Now apply the filter to the sim_rec_map table: this will delete all sim/rec maps where the simulation id is
# mapped to a recovered event that falls outside the filter, even if that particular sim/rec map is in the
# filter. For example, if the filter is recovery_table.combined_far != 0., and there are two entries in the
# sim_rec_map table sharing the same sim_id:
# sim_id:0 | rec_id:0 | rec_id's combined_far = 0.
# sim_id:0 | rec_id:1 | rec_id's combined_far = 1.2
# both entries will get deleted even though the second entry's combined_far is not 0.
if in_this_filter != '':
# join the needed tables to in_this_filter
in_this_filter = ''.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), "\n WHERE\n\t", in_this_filter ])
sqlscript = ''.join([ """
CREATE TEMP TABLE del_ids AS
SELECT
sim_id AS del_id
FROM
sim_rec_map
WHERE
rec_id NOT IN (
SELECT
""", recovery_table, """.coinc_event_id
FROM
""", recovery_table, """
""", in_this_filter, """
AND experiment_summary.datatype == "simulation"
);
DELETE FROM
sim_rec_map
WHERE
sim_id IN (
SELECT
del_id
FROM
del_ids );
DROP TABLE del_ids;""" ])
connection.cursor().executescript(sqlscript)
#
# Set other needed functions
#
# establish what units will be converting duration to
def convert_duration( duration ):
return sqlutils.convert_duration( duration, convert_durations )
connection.create_function( 'convert_duration', 1, convert_duration )
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
# Get range ranks
if rank_range is not None:
rank_range_parser = sqlutils.parse_param_ranges( 'rank(sim_rec_map', 'ranking_stat)',
rank_range, verbose = verbose )
#
# Create and prepare the SelectedFoundTable to store summary information
#
# Get recovery table and simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
recovery_table_columns = sqlutils.get_column_names_from_table( connection, recovery_table )
# Get list of column name for injected parameters
injected_cols = []
for col in simulation_table_columns:
if col == 'simulation_id':
injected_cols.append('simulation_id')
else:
injected_cols.append('injected_'+col)
injected_cols.extend(['injected_decisive_distance','injected_gps_time', 'injected_gps_time_ns', 'injected_event_time_utc__Px_click_for_daily_ihope_xP_'])
# Get list of column names from the recovery table
recovered_cols = []
for col in recovery_table_columns:
if col == 'coinc_event_id':
recovered_cols.append(u'coinc_event_id')
else:
recovered_cols.append(u'recovered_'+col)
# generate list of column names for the summary table
column_names = injected_cols + recovered_cols
# add instruments on, duration, mini_followups
rankname = 'rank_in_' + comparison_datatype.strip().lower() + '_using_' + ranking_stat.split('.')[-1]
durname = ''.join([ 'simulation', u'_duration__Px_', convert_durations, '_xP_' ])
column_names.extend( [ rankname, 'recovered_match_rank', 'instruments_on', 'elogs', durname, 'mini_followup','omega_scan', 'sim_tag' ] )
#
# define needed tables
#
class tmpSimTable(table.Table):
tableName = "tmp_sim_table:table"
validcolumns = dict([ [col, sqlutils.get_col_type(simulation_table, col)] for col in simulation_table_columns ])
class tmpSim(object):
__slots__ = tmpSimTable.validcolumns.keys()
def get_gps_time(self, site):
if '%s_start_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, '%s_start_time' % site ), getattr(self, '%s_start_time_ns' % site))
elif '%s_end_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, '%s_end_time' % site ), getattr(self, '%s_end_time_ns' % site))
else:
raise AttributeError, "could not find an injected %s_start_time nor an injected %s_end_time" %(site,site)
def get_pyvalue(self):
return generic_get_pyvalue(self)
class tmpRecTable(table.Table):
tableName = "tmp_rec_table:table"
validcolumns = dict([ [col, sqlutils.get_col_type(recovery_table, col)] for col in recovery_table_columns ])
class tmpRec(object):
__slots__ = tmpRecTable.validcolumns.keys()
def get_gps_time(self):
if 'start_time' in self.__slots__:
return LIGOTimeGPS(self.start_time, self.start_time_ns)
elif 'end_time' in self.__slots__:
return LIGOTimeGPS(self.end_time, self.end_time_ns)
else:
raise AttributeError, "could not find a recovered start_time or recovered end_time"
def get_pyvalue(self):
return generic_get_pyvalue(self)
class SelectedFoundTable(table.Table):
tableName = "selected_found_injections:table"
validcolumns = {}
for col_name in column_names:
if 'rank_in_' in col_name:
validcolumns[col_name] = "int_4u"
elif '_duration_' in col_name:
validcolumns[col_name] = "real_8"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif col_name == 'injected_start_time' or col_name == 'injected_start_time_ns':
validcolumns[col_name] = "int_4s"
elif col_name == 'injected_gps_time' or col_name == 'injected_gps_time_ns':
validcolumns[col_name] = "int_4s"
elif col_name == 'injected_decisive_distance':
validcolumns[col_name] = "real_8"
elif 'injected_' in col_name or col_name == 'simulation_id':
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, re.sub('injected_', '', col_name))
elif 'recovered_' in col_name or col_name == 'coinc_event_id':
validcolumns[col_name] = sqlutils.get_col_type(recovery_table, re.sub('recovered_', '', col_name))
# if custom columns exist in the database, just set them to lstrings
else:
validcolumns[col_name] = "lstring"
# add FAP columns
validcolumns['recovered_fap'] = "real_8"
validcolumns['recovered_fap_1yr'] = "real_8"
class SelectedFound(object):
__slots__ = SelectedFoundTable.validcolumns.keys()
def set_recovered_gps_time(self, gps_time):
if 'recovered_start_time' in self.__slots__:
self.recovered_start_time = gps_time.seconds
self.recovered_start_time_ns = gps_time.nanoseconds
elif 'recovered_end_time' in self.__slots__:
self.recovered_end_time = gps_time.seconds
self.recovered_end_time_ns = gps_time.nanoseconds
else:
raise AttributeError, "could not find a recovered_start_time or recovered_end_time"
def get_recovered_gps_time(self):
if 'recovered_start_time' in self.__slots__:
return LIGOTimeGPS(self.recovered_start_time, self.recovered_start_time_ns)
elif 'recovered_end_time' in self.__slots__:
return LIGOTimeGPS(self.recovered_end_time, self.recovered_end_time_ns)
else:
raise AttributeError, "could not find a recovered_start_time or recovered_end_time"
def set_injected_gps_time(self, site, gps_time):
if 'injected_%s_start_time' % site in self.__slots__:
setattr(self, 'injected_%s_start_time' % site, gps_time.seconds)
setattr(self, 'injected_%s_start_time_ns' % site, gps_time.nanoseconds)
elif 'injected_%s_end_time' % site in self.__slots__:
setattr(self, 'injected_%s_end_time' % site, gps_time.seconds)
setattr(self, 'injected_%s_end_time_ns' % site, gps_time.nanoseconds)
else:
raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site)
def get_injected_gps_time(self, site):
if 'injected_%s_start_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, 'injected_%s_start_time' % site ), getattr(self, 'injected_%s_start_time_ns' % site))
elif 'injected_%s_end_time' % site in self.__slots__:
return LIGOTimeGPS(getattr(self, 'injected_%s_end_time' % site ), getattr(self, 'injected_%s_end_time_ns' % site))
else:
raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site)
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
tmpSimTable.RowType = tmpSim
tmpRecTable.RowType = tmpRec
SelectedFoundTable.RowType = SelectedFound
#
# Get the Data
#
tmp_sim_table = lsctables.New(tmpSimTable)
tmp_rec_table = lsctables.New(tmpRecTable)
tmp_sftable = lsctables.New(SelectedFoundTable)
prior_sim_id = ''
group_rank = None
current_match_rank = 1
sqlquery = ''.join(["""
SELECT
""", simulation_table, """.*,
""", recovery_table, """.*,
get_sim_tag(experiment_summary.sim_proc_id),
rank(sim_rec_map.ranking_stat),
NULL AS match_rank,
experiment.instruments,
convert_duration(experiment_summary.duration),
NULL AS mini_followup
FROM
sim_rec_map
JOIN
""", ', '.join([simulation_table, recovery_table]), """, experiment, experiment_summary, experiment_map ON (
sim_rec_map.sim_id == """, simulation_table, """.simulation_id AND
sim_rec_map.rec_id == """, recovery_table, """.coinc_event_id AND
sim_rec_map.rec_id == experiment_map.coinc_event_id AND
experiment_map.experiment_summ_id == experiment_summary.experiment_summ_id AND
experiment_summary.experiment_id == experiment.experiment_id)
ORDER BY
sim_rec_map.sim_id, sim_rec_map.ranking_stat """, rank_by])
if verbose:
print >> sys.stderr, "Getting coincs..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
# sort the data
tmp_sim_row = tmpSim()
tmp_rec_row = tmpRec()
[ setattr(tmp_sim_row, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
[ setattr(tmp_rec_row, column, values[ii+1+jj]) for jj, column in enumerate(recovery_table_columns) ]
# figure out the rank
this_inj_rank = values[-5]
this_sim_id = tmp_sim_row.simulation_id
this_ranking_stat = getattr(tmp_rec_row, ranking_stat.split('.')[-1])
if this_sim_id == prior_sim_id and this_ranking_stat != prior_ranking_stat:
current_match_rank += 1
elif this_sim_id != prior_sim_id:
current_match_rank = 1
group_rank = this_inj_rank
prior_sim_id = this_sim_id
prior_ranking_stat = this_ranking_stat
# only store data if the group_rank falls in the desired rank ranges
if rank_range and rank_range_parser.group_by_param_range(group_rank) is None:
continue
on_instruments = lsctables.instrument_set_from_ifos(values[-3])
duration = values[-2]
# now that have all the information from this row, create a row for the selected found table
sfrow = SelectedFound()
# set the ranks
setattr(sfrow, rankname, group_rank)
sfrow.recovered_match_rank = current_match_rank
# set the injected parameters
use_this_site = sorted(on_instruments)[0][0].lower()
injected_time = tmp_sim_row.get_gps_time( use_this_site )
sfrow.injected_gps_time = injected_time.seconds
sfrow.injected_gps_time_ns = injected_time.nanoseconds
for col in simulation_table_columns:
if col == "simulation_id":
sfrow.simulation_id = tmp_sim_row.simulation_id
else:
setattr(sfrow, 'injected_'+col, getattr( tmp_sim_row, col) )
# set the recovered parameters
for col in recovery_table_columns:
if col == "coinc_event_id":
sfrow.coinc_event_id = tmp_rec_row.coinc_event_id
else:
setattr(sfrow, 'recovered_'+col, getattr( tmp_rec_row, col) )
# calculate and add faps
if sfrow.recovered_combined_far is not None:
t_in_s = float(values[-2]) / sqlutils.convert_duration(1, convert_durations)
sfrow.recovered_fap = 1 - math.exp(-sqlutils.convert_duration(t_in_s, 'yr') * sfrow.recovered_combined_far)
sfrow.recovered_fap_1yr = 1 - math.exp(-sfrow.recovered_combined_far)
else:
sfrow.recovered_fap = None
sfrow.recovered_fap_1yr = None
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, sfrow.injected_gps_time)) for ifo in on_instruments]
sfrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
event_time_utc = format_end_time_in_utc( sfrow.injected_gps_time )
daily_ihope_address = get_daily_ihope_page(sfrow.injected_gps_time, pages_location = daily_ihope_pages_location)
sfrow.injected_event_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, event_time_utc )
# set any other info
sfrow.instruments_on = ','.join(sorted(on_instruments))
sfrow.injected_decisive_distance = sorted([getattr(sfrow, 'injected_eff_dist_%s' % ifo[0].lower()) for ifo in on_instruments])[1]
sfrow.mini_followup = None
sfrow.omega_scan = None
sfrow.sim_tag = values[-6]
setattr(sfrow, durname, duration)
# add the row
tmp_sftable.append(sfrow)
# Re-sort the sftable by rank, recovered_match_rank
sftable = lsctables.New(SelectedFoundTable)
for sfrow in sorted([ row for row in tmp_sftable ], key = lambda row: getattr(row, sort_by == 'rank' and rankname or sort_by) ):
if sfrow.simulation_id not in [row.simulation_id for row in sftable]:
sftable.append(sfrow)
sftable.extend(sub_row for sub_row in sorted([row for row in tmp_sftable
if row.simulation_id == sfrow.simulation_id
and row.coinc_event_id != sfrow.coinc_event_id],
key = lambda row: row.recovered_match_rank))
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return sftable
def get_livetime(self, time_units = 'yr'):
return sqlutils.convert_duration( self.livetime, time_units )
