def printmissed(connection, simulation_table, recovery_table, map_label, livetime_program,
param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ',
limit = None, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
from pylal import ligolw_cbc_compute_durations as compute_dur
from glue import segments
from glue.ligolw import dbtables
# Get simulation/recovery tables
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
#
# Create and prepare the CloseMissedTable to store summary information
#
# Get simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
column_names = simulation_table_columns + \
['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag']
# define needed tables
class CloseMissedTable(table.Table):
tableName = "close_missed_injections:table"
validcolumns = {}
for col_name in column_names:
if 'rank' in col_name:
validcolumns[col_name] = "int_4u"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif 'veto_def_name' == col_name:
validcolumns[col_name] = lsctables.ExperimentSummaryTable.validcolumns['veto_def_name']
elif 'decisive_distance' == col_name:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, 'eff_dist_h')
elif 'gps_time' == col_name or 'gps_time_ns' == col_name:
validcolumns[col_name] = "int_4s"
elif 'sim_tag' == col_name:
validcolumns[col_name] = "lstring"
else:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, col_name, default = 'lstring')
class CloseMissed(object):
__slots__ = CloseMissedTable.validcolumns.keys()
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
CloseMissedTable.RowType = CloseMissed
# create the table
cmtable = lsctables.New(CloseMissedTable)
# set up sim_rec_map table
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, None)
#
# Set table filters
#
# we force the include/exclude filters to None; will check for excluded/included ifo time
# when cycling through the ifo times
filter = """
WHERE
simulation_id NOT IN (
SELECT
sim_id
FROM
sim_rec_map )"""
af = create_filter( connection, simulation_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = None, include_only_coincs = None, sim_tag = sim_tag, verbose = verbose)
af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af)
if af != '':
filter = '\n'.join([ filter, """
AND""", af])
# get desired instrument times
if include_only_coincs is not None:
include_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( include_only_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
if exclude_coincs is not None:
exclude_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( exclude_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
# get the usertags of inspiral jobs in this database
sqlquery = """
SELECT value
FROM process_params
WHERE param == "-userTag"
GROUP BY value
"""
usertags = set(usertag[0] for usertag in connection.cursor().execute(sqlquery) )
# Get the single-ifo science segments after CAT-1 vetoes
try:
if "FULL_DATA" in usertags:
tag = "FULL_DATA"
else:
tag = list(usertags)[0]
except IndexError:
# This is hacky anyway, so let's just take a guess
tag = "FULL_DATA"
ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name = livetime_program, usertag = tag)
if ifo_segments == {}:
raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program
if verbose:
print >> sys.stderr, "Getting all veto category names from the experiment_summary table..."
xmldoc = dbtables.get_xml(connection)
# get veto_segments
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# make a dictionary of zerolag "shifts" needed for the get_coinc_segments function
zerolag_dict = {}
for ifo in ifo_segments:
zerolag_dict[ifo] = 0.0
# Cycle over available veto categories
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
# make a dictionary of coincident segments by exclusive on-ifos
coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs, zerolag_dict)
#
# Get all the on_instrument times and cycle over them
#
sqlquery = """
SELECT DISTINCT experiment.instruments
FROM experiment
JOIN experiment_summary ON (
experiment.experiment_id == experiment_summary.experiment_id )
WHERE experiment_summary.veto_def_name == :1
"""
for on_instruments in connection.cursor().execute(sqlquery, (veto_def_name,)).fetchall():
on_instruments = lsctables.instrument_set_from_ifos(on_instruments[0])
# check if this on_instruments is desired; if not, skip
if include_only_coincs is not None and frozenset(on_instruments) not in include_times:
continue
if exclude_coincs is not None and frozenset(on_instruments) in exclude_times:
continue
on_times = coinc_segs[','.join(sorted(on_instruments))]
def is_in_on_time(gps_time, gps_time_ns):
return gps_time + (1e-9 * gps_time_ns) in on_times
connection.create_function('is_in_on_time', 2, is_in_on_time)
# add the check for on time to the filter
in_this_filter = filter
# figure out if simulation_table has end_times or start_times
end_or_start = any('end_time' in c for c in simulation_table_columns) and '_end_time' or '_start_time'
for instrument in on_instruments:
inst_time = instrument.lower()[0] + end_or_start
inst_time_ns = inst_time + '_ns'
in_this_filter = ''.join([ in_this_filter,
'\n\tAND is_in_on_time(', inst_time, ',', inst_time_ns, ')' ])
#
# Set up decisive distance argument
#
def get_decisive_distance( *args ):
return sorted(args)[1]
connection.create_function('get_decisive_distance', len(on_instruments), get_decisive_distance)
decisive_distance = ''.join(['get_decisive_distance(', ','.join(['eff_dist_'+inst.lower()[0] for inst in on_instruments]), ')' ])
#
# Initialize ranking. Statistics for ranking are based on decisive distance
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(simulation_table, decisive_distance, 'ASC')
# add requirement that stats not be found in the sim_rec_table to in_this_filter
ranker.populate_stats_list(connection, limit = limit, filter = in_this_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Get the Data
#
sqlquery = ''.join(["""
SELECT
*,
get_sim_tag(process_id),
""", decisive_distance, """,
rank(""", decisive_distance, """)
FROM
""", simulation_table, """
""", in_this_filter, """
%s""" % (limit is not None and ''.join(['AND rank(', decisive_distance, ') <= ', str(limit)]) or ''), """
ORDER BY
rank(""", decisive_distance, """) ASC
"""])
if verbose:
print >> sys.stderr, "Getting injections..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
cmrow = CloseMissed()
[ setattr(cmrow, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
cmrow.decisive_distance = values[-2]
cmrow.rank = values[-1]
cmrow.instruments_on = lsctables.ifos_from_instrument_set(on_instruments)
cmrow.veto_def_name = veto_def_name
cmrow.sim_tag = values[-3]
cmrow.mini_followup = None
cmrow.omega_scan = None
cmrow.gps_time = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start)
cmrow.gps_time_ns = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start + '_ns')
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time)) for ifo in on_instruments]
cmrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
injection_time_utc = format_end_time_in_utc( cmrow.gps_time )
daily_ihope_address = get_daily_ihope_page(cmrow.gps_time, pages_location = daily_ihope_pages_location)
cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, injection_time_utc )
# add the row
cmtable.append(cmrow)
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return cmtable
def create_filter( connection, tableName, param_name = None, param_ranges = None,
exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ', verbose = False):
"""
Strings together param_name, param_ranges, exclude/include_only_coincs, and
sim_tag options into a filter string that can be stuck in a sqlite WHERE clause.
"""
from pylal import ligolw_sqlutils as sqlutils
in_this_filter = ''
# Get param and param-ranges if specified
if param_name is not None:
param_name = sqlutils.validate_option(param_name)
param_filters = sqlutils.parse_param_ranges( tableName, param_name,
param_ranges, verbose = verbose ).get_param_filters()
# since want triggers that fall within all the parameters, concatenate
# all param ranges
param_filters = '\n\t\tOR '.join( param_filters )
in_this_filter = ''.join([ in_this_filter, '\n\tAND (\n\t\t', param_filters, '\n\t)' ])
# Get exclude_coincs list if specified
if exclude_coincs is not None:
exclude_coinc_filters = sqlutils.parse_coinc_options( exclude_coincs,
verbose = verbose ).get_coinc_filters( coinc_instruments_table = tableName )
# concatenate exclude_coinc_filters
exclude_coinc_filters = '\n\t\tOR '.join( exclude_coinc_filters )
# add to in_this_filter
in_this_filter = ''.join([ in_this_filter, '\n\tAND NOT (\n\t\t', exclude_coinc_filters, '\n\t)' ])
# Get include_only_coincs list if specified
if include_only_coincs is not None:
include_coinc_filters = sqlutils.parse_coinc_options( include_only_coincs,
verbose = verbose ).get_coinc_filters( coinc_instruments_table = tableName )
# concatenate include_coinc_filters
include_coinc_filters = '\n\t\tOR '.join( include_coinc_filters )
# add to in_this_filter
in_this_filter = ''.join([ in_this_filter, '\n\tAND (\n\t\t', include_coinc_filters, '\n\t)' ])
# if sim-tag specified add the sim-tag to the filter
if sim_tag != 'ALLINJ':
# create a map between sim_proc_id and sim-tag and a function to parse it
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
# parse the sim_tag for multiple sims; then cycle over and add to the filter
sim_filter = ''
sim_list = sim_tag.split('+')
for sim_tag in sim_list:
# check that sim_tag is in the the map
sim_tag = sqlutils.validate_option(sim_tag, lower = False).upper()
if sim_tag not in sim_map.tag_id_map.keys():
raise ValueError, "sim-tag %s not found in database" % sim_tag
# create the filter
sim_filter = '\n\t\tOR '.join([ sim_filter, ''.join(['get_sim_tag(experiment_summary.sim_proc_id) == "', sim_tag, '"' ]) ])
# strip the leading OR and add to in_this_filter
sim_filter = re.sub(r'OR ', '', sim_filter, 1)
in_this_filter = ''.join([ in_this_filter, '\n\tAND (', sim_filter, '\n\t)' ])
# remove the leading AND
in_this_filter = re.sub(r'\n\tAND', '', in_this_filter, 1)
return in_this_filter