def get_singles_times( connection, verbose = False ): # get single-ifo filtered segments ifo_segments = compute_dur.get_single_ifo_segments( connection, program_name = "inspiral", usertag = "FULL_DATA") # get all veto segments xmldoc = dbtables.get_xml(connection) veto_segments = compute_dur.get_veto_segments(xmldoc, verbose) sngls_durations = {} for veto_def_name, veto_seg_dict in veto_segments.items(): post_vetoes_ifosegs = ifo_segments - veto_seg_dict for ifo, segs_list in post_vetoes_ifosegs.items(): sngls_durations[ifo] = float( abs(segs_list) ) return sngls_durations
def get_singles_times(connection, verbose=False): # get single-ifo filtered segments ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name="inspiral", usertag="FULL_DATA") # get all veto segments xmldoc = dbtables.get_xml(connection) veto_segments = compute_dur.get_veto_segments(xmldoc, verbose) sngls_durations = {} for veto_def_name, veto_seg_dict in veto_segments.items(): post_vetoes_ifosegs = ifo_segments - veto_seg_dict for ifo, segs_list in post_vetoes_ifosegs.items(): sngls_durations[ifo] = float(abs(segs_list)) return sngls_durations
def printmissed(connection, simulation_table, recovery_table, map_label, livetime_program, param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ', limit = None, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False): from pylal import ligolw_sqlutils as sqlutils from pylal import ligolw_cbc_compute_durations as compute_dur from glue import segments from glue.ligolw import dbtables # Get simulation/recovery tables simulation_table = sqlutils.validate_option(simulation_table) recovery_table = sqlutils.validate_option(recovery_table) # create the get_sim_tag function sim_map = sqlutils.sim_tag_proc_id_mapper( connection ) connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag ) # # Create and prepare the CloseMissedTable to store summary information # # Get simulation table column names from database simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table ) column_names = simulation_table_columns + \ ['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag'] # define needed tables class CloseMissedTable(table.Table): tableName = "close_missed_injections:table" validcolumns = {} for col_name in column_names: if 'rank' in col_name: validcolumns[col_name] = "int_4u" elif 'instruments_on' == col_name: validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments'] elif 'veto_def_name' == col_name: validcolumns[col_name] = lsctables.ExperimentSummaryTable.validcolumns['veto_def_name'] elif 'decisive_distance' == col_name: validcolumns[col_name] = sqlutils.get_col_type(simulation_table, 'eff_dist_h') elif 'gps_time' == col_name or 'gps_time_ns' == col_name: validcolumns[col_name] = "int_4s" elif 'sim_tag' == col_name: validcolumns[col_name] = "lstring" else: validcolumns[col_name] = sqlutils.get_col_type(simulation_table, col_name, default = 'lstring') class CloseMissed(object): __slots__ = CloseMissedTable.validcolumns.keys() def get_pyvalue(self): return generic_get_pyvalue(self) # connect the rows to the tables CloseMissedTable.RowType = CloseMissed # create the table cmtable = lsctables.New(CloseMissedTable) # set up sim_rec_map table sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, None) # # Set table filters # # we force the include/exclude filters to None; will check for excluded/included ifo time # when cycling through the ifo times filter = """ WHERE simulation_id NOT IN ( SELECT sim_id FROM sim_rec_map )""" af = create_filter( connection, simulation_table, param_name = param_name, param_ranges = param_ranges, exclude_coincs = None, include_only_coincs = None, sim_tag = sim_tag, verbose = verbose) af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af) if af != '': filter = '\n'.join([ filter, """ AND""", af]) # get desired instrument times if include_only_coincs is not None: include_times = [on_instruments for on_instruments, type in sqlutils.parse_coinc_options( include_only_coincs, verbose = verbose ).get_coinc_types().items() if 'ALL' in type] if exclude_coincs is not None: exclude_times = [on_instruments for on_instruments, type in sqlutils.parse_coinc_options( exclude_coincs, verbose = verbose ).get_coinc_types().items() if 'ALL' in type] # get the usertags of inspiral jobs in this database sqlquery = """ SELECT value FROM process_params WHERE param == "-userTag" GROUP BY value """ usertags = set(usertag[0] for usertag in connection.cursor().execute(sqlquery) ) # Get the single-ifo science segments after CAT-1 vetoes try: if "FULL_DATA" in usertags: tag = "FULL_DATA" else: tag = list(usertags)[0] except IndexError: # This is hacky anyway, so let's just take a guess tag = "FULL_DATA" ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name = livetime_program, usertag = tag) if ifo_segments == {}: raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program if verbose: print >> sys.stderr, "Getting all veto category names from the experiment_summary table..." xmldoc = dbtables.get_xml(connection) # get veto_segments veto_segments = compute_dur.get_veto_segments(xmldoc, verbose) # make a dictionary of zerolag "shifts" needed for the get_coinc_segments function zerolag_dict = {} for ifo in ifo_segments: zerolag_dict[ifo] = 0.0 # Cycle over available veto categories for veto_def_name, veto_seg_dict in veto_segments.items(): post_vetoes_ifosegs = ifo_segments - veto_seg_dict # make a dictionary of coincident segments by exclusive on-ifos coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs, zerolag_dict) # # Get all the on_instrument times and cycle over them # sqlquery = """ SELECT DISTINCT experiment.instruments FROM experiment JOIN experiment_summary ON ( experiment.experiment_id == experiment_summary.experiment_id ) WHERE experiment_summary.veto_def_name == :1 """ for on_instruments in connection.cursor().execute(sqlquery, (veto_def_name,)).fetchall(): on_instruments = lsctables.instrument_set_from_ifos(on_instruments[0]) # check if this on_instruments is desired; if not, skip if include_only_coincs is not None and frozenset(on_instruments) not in include_times: continue if exclude_coincs is not None and frozenset(on_instruments) in exclude_times: continue on_times = coinc_segs[','.join(sorted(on_instruments))] def is_in_on_time(gps_time, gps_time_ns): return gps_time + (1e-9 * gps_time_ns) in on_times connection.create_function('is_in_on_time', 2, is_in_on_time) # add the check for on time to the filter in_this_filter = filter # figure out if simulation_table has end_times or start_times end_or_start = any('end_time' in c for c in simulation_table_columns) and '_end_time' or '_start_time' for instrument in on_instruments: inst_time = instrument.lower()[0] + end_or_start inst_time_ns = inst_time + '_ns' in_this_filter = ''.join([ in_this_filter, '\n\tAND is_in_on_time(', inst_time, ',', inst_time_ns, ')' ]) # # Set up decisive distance argument # def get_decisive_distance( *args ): return sorted(args)[1] connection.create_function('get_decisive_distance', len(on_instruments), get_decisive_distance) decisive_distance = ''.join(['get_decisive_distance(', ','.join(['eff_dist_'+inst.lower()[0] for inst in on_instruments]), ')' ]) # # Initialize ranking. Statistics for ranking are based on decisive distance # if verbose: print >> sys.stderr, "Getting statistics for ranking..." ranker = sqlutils.rank_stats(simulation_table, decisive_distance, 'ASC') # add requirement that stats not be found in the sim_rec_table to in_this_filter ranker.populate_stats_list(connection, limit = limit, filter = in_this_filter) connection.create_function( 'rank', 1, ranker.get_rank ) # # Get the Data # sqlquery = ''.join([""" SELECT *, get_sim_tag(process_id), """, decisive_distance, """, rank(""", decisive_distance, """) FROM """, simulation_table, """ """, in_this_filter, """ %s""" % (limit is not None and ''.join(['AND rank(', decisive_distance, ') <= ', str(limit)]) or ''), """ ORDER BY rank(""", decisive_distance, """) ASC """]) if verbose: print >> sys.stderr, "Getting injections..." print >> sys.stderr, "SQLite query used is:" print >> sys.stderr, sqlquery for values in connection.cursor().execute( sqlquery ).fetchall(): cmrow = CloseMissed() [ setattr(cmrow, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ] cmrow.decisive_distance = values[-2] cmrow.rank = values[-1] cmrow.instruments_on = lsctables.ifos_from_instrument_set(on_instruments) cmrow.veto_def_name = veto_def_name cmrow.sim_tag = values[-3] cmrow.mini_followup = None cmrow.omega_scan = None cmrow.gps_time = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start) cmrow.gps_time_ns = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start + '_ns') # set elog page elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time)) for ifo in on_instruments] cmrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ]) # set daily_ihope page injection_time_utc = format_end_time_in_utc( cmrow.gps_time ) daily_ihope_address = get_daily_ihope_page(cmrow.gps_time, pages_location = daily_ihope_pages_location) cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, injection_time_utc ) # add the row cmtable.append(cmrow) # drop the sim_rec_map table connection.cursor().execute("DROP TABLE sim_rec_map") return cmtable
def successful_injections(connection, tag, on_ifos, veto_cat, dist_type="distance", weight_dist=False, verbose=False): """ My attempt to get a list of the simulations that actually made it into some level of coincident time """ xmldoc = dbtables.get_xml(connection) connection.create_function('end_time_with_ns', 2, end_time_with_ns) # Get the veto segments as dictionaries, keyed by veto category veto_segments = compute_dur.get_veto_segments(xmldoc, verbose) # ------------------------ Get List of Injections ------------------------ # sql_params_dict = {} sqlquery = """ SELECT DISTINCT simulation_id, end_time_with_ns(geocent_end_time, geocent_end_time_ns),""" # add the desired distance measure to the SQL query if dist_type == "distance": connection.create_function('distance_func', 2, chirp_dist) sqlquery += """ distance_func(distance, sim_inspiral.mchirp) FROM sim_inspiral """ elif dist_type == "decisive_distance": connection.create_function('decisive_dist_func', 6, decisive_dist) sql_params_dict['ifos'] = on_ifos sql_params_dict['weight_dist'] = weight_dist sqlquery += """ decisive_dist_func( eff_dist_h, eff_dist_l, eff_dist_v, sim_inspiral.mchirp, :weight_dist, :ifos) FROM sim_inspiral """ if tag != 'ALL_INJ': # if a specific injection set is wanted sqlquery += """ JOIN process_params ON ( process_params.process_id == sim_inspiral.process_id) WHERE process_params.value = :usertag) """ sql_params_dict["usertag"] = tag else: # for all injections tag = 'FULL_DATA' # Get segments that define which time was filtered ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name="inspiral", usertag=tag) zero_lag_dict = dict([(ifo, 0.0) for ifo in ifo_segments]) successful_inj = [] # determine coincident segments for that veto category coinc_segs = compute_dur.get_coinc_segments( ifo_segments - veto_segments[veto_cat], zero_lag_dict) # Apply vetoes to single-ifo filter segments for injection in connection.execute(sqlquery, sql_params_dict): inj_segment = segments.segment(injection[1], injection[1]) if coinc_segs[on_ifos].intersects_segment(inj_segment): successful_inj.append(injection) return successful_inj
def sngl_snr_hist( connection, ifo, mchirp, eta, min_snr, snr_stat = None, sngls_width = None, usertag = "FULL_DATA", datatype = None, sngls_bins = None): """ Creates a histogram of sngl_inspiral triggers and returns a list of counts and the associated snr bins. @connection: connection to a SQLite database with lsctables @ifo: the instrument one desires triggers from @mchirp: the chirp mass from the desired template @eta: the symmetric mass ratio from the desired template @min_snr: a lower threshold on the value of the snr_stat @sngls_width: the bin width for the histogram @usertag: the usertag for the triggers. The default is "FULL_DATA". @datatype: the datatype (all_data, slide, ...) if single-ifo triggers from coincident events is desired. The default is to collect all triggers. @sngls_bins: a list of bin edges for the snr-histogram """ # create function for the desired snr statistic set_getsnr_function(connection, snr_stat) connection.create_function('end_time_w_ns', 2, end_time_w_ns) # set SQL statement parameters sql_params_dict = { "mchirp": mchirp, "eta": eta, "min_snr": min_snr, "ifo": ifo, "usertag": usertag} # SQLite query to get a list of (snr, gps-time) tuples for inspiral triggers sqlquery = """ SELECT DISTINCT get_snr(snr, chisq, chisq_dof) as snr_stat, end_time_w_ns(end_time, end_time_ns) FROM sngl_inspiral JOIN process_params ON ( process_params.process_id == sngl_inspiral.process_id) """ # if a datatype is given, get only the inspiral triggers from coincs of that type if datatype: sqlquery += """ JOIN coinc_event_map, experiment_map, experiment_summary ON ( coinc_event_map.event_id == sngl_inspiral.event_id AND experiment_map.coinc_event_id == coinc_event_map.coinc_event_id AND experiment_summary.experiment_summ_id == experiment_map.experiment_summ_id) """ sqlquery += """ WHERE sngl_inspiral.ifo == :ifo AND snr_stat >= :min_snr AND sngl_inspiral.mchirp == :mchirp AND sngl_inspiral.eta == :eta AND process_params.value == :usertag """ if datatype: sqlquery += """ AND experiment_summary.datatype == :type """ sql_params_dict["type"] = datatype # get dq-veto segments xmldoc = dbtables.get_xml(connection) veto_segments = compute_dur.get_veto_segments(xmldoc, False) veto_segments = veto_segments[ veto_segments.keys()[0] ] snr_array = np.array([]) # apply vetoes to the list of trigger times for snr, trig_time in connection.execute( sqlquery, sql_params_dict ): trig_segment = segments.segment(trig_time, trig_time) if not veto_segments[ifo].intersects_segment( trig_segment ): snr_array = np.append( snr_array, snr ) if sngls_bins is None: sngls_bins = np.arange(min_snr, np.max(snr_array) + sngls_width, sngls_width) # make the binned snr histogram sngls_hist, _ = np.histogram(snr_array, bins=sngls_bins) return sngls_hist, sngls_bins
def successful_injections( connection, tag, on_ifos, veto_cat, dist_type = "distance", weight_dist = False, verbose = False): """ My attempt to get a list of the simulations that actually made it into some level of coincident time """ xmldoc = dbtables.get_xml(connection) connection.create_function('end_time_with_ns', 2, end_time_with_ns) # Get the veto segments as dictionaries, keyed by veto category veto_segments = compute_dur.get_veto_segments(xmldoc, verbose) # ------------------------ Get List of Injections ------------------------ # sql_params_dict = {} sqlquery = """ SELECT DISTINCT simulation_id, end_time_with_ns(geocent_end_time, geocent_end_time_ns),""" # add the desired distance measure to the SQL query if dist_type == "distance": connection.create_function('distance_func', 2, chirp_dist) sqlquery += """ distance_func(distance, sim_inspiral.mchirp) FROM sim_inspiral """ elif dist_type == "decisive_distance": connection.create_function('decisive_dist_func', 6, decisive_dist) sql_params_dict['ifos'] = on_ifos sql_params_dict['weight_dist'] = weight_dist sqlquery += """ decisive_dist_func( eff_dist_h, eff_dist_l, eff_dist_v, sim_inspiral.mchirp, :weight_dist, :ifos) FROM sim_inspiral """ if tag != 'ALL_INJ': # if a specific injection set is wanted sqlquery += """ JOIN process_params ON ( process_params.process_id == sim_inspiral.process_id) WHERE process_params.value = :usertag) """ sql_params_dict["usertag"] = tag else: # for all injections tag = 'FULL_DATA' # Get segments that define which time was filtered ifo_segments = compute_dur.get_single_ifo_segments( connection, program_name = "inspiral", usertag = tag) zero_lag_dict = dict([(ifo, 0.0) for ifo in ifo_segments]) successful_inj = [] # determine coincident segments for that veto category coinc_segs = compute_dur.get_coinc_segments( ifo_segments - veto_segments[veto_cat], zero_lag_dict) # Apply vetoes to single-ifo filter segments for injection in connection.execute(sqlquery, sql_params_dict): inj_segment = segments.segment(injection[1], injection[1]) if coinc_segs[on_ifos].intersects_segment( inj_segment ): successful_inj.append( injection ) return successful_inj
def sngl_snr_hist(connection, ifo, mchirp, eta, min_snr, snr_stat=None, sngls_width=None, usertag="FULL_DATA", datatype=None, sngls_bins=None): """ Creates a histogram of sngl_inspiral triggers and returns a list of counts and the associated snr bins. @connection: connection to a SQLite database with lsctables @ifo: the instrument one desires triggers from @mchirp: the chirp mass from the desired template @eta: the symmetric mass ratio from the desired template @min_snr: a lower threshold on the value of the snr_stat @sngls_width: the bin width for the histogram @usertag: the usertag for the triggers. The default is "FULL_DATA". @datatype: the datatype (all_data, slide, ...) if single-ifo triggers from coincident events is desired. The default is to collect all triggers. @sngls_bins: a list of bin edges for the snr-histogram """ # create function for the desired snr statistic set_getsnr_function(connection, snr_stat) connection.create_function('end_time_w_ns', 2, end_time_w_ns) # set SQL statement parameters sql_params_dict = { "mchirp": mchirp, "eta": eta, "min_snr": min_snr, "ifo": ifo, "usertag": usertag } # SQLite query to get a list of (snr, gps-time) tuples for inspiral triggers sqlquery = """ SELECT DISTINCT get_snr(snr, chisq, chisq_dof) as snr_stat, end_time_w_ns(end_time, end_time_ns) FROM sngl_inspiral JOIN process_params ON ( process_params.process_id == sngl_inspiral.process_id) """ # if a datatype is given, get only the inspiral triggers from coincs of that type if datatype: sqlquery += """ JOIN coinc_event_map, experiment_map, experiment_summary ON ( coinc_event_map.event_id == sngl_inspiral.event_id AND experiment_map.coinc_event_id == coinc_event_map.coinc_event_id AND experiment_summary.experiment_summ_id == experiment_map.experiment_summ_id) """ sqlquery += """ WHERE sngl_inspiral.ifo == :ifo AND snr_stat >= :min_snr AND sngl_inspiral.mchirp == :mchirp AND sngl_inspiral.eta == :eta AND process_params.value == :usertag """ if datatype: sqlquery += """ AND experiment_summary.datatype == :type """ sql_params_dict["type"] = datatype # get dq-veto segments xmldoc = dbtables.get_xml(connection) veto_segments = compute_dur.get_veto_segments(xmldoc, False) veto_segments = veto_segments[veto_segments.keys()[0]] snr_array = np.array([]) # apply vetoes to the list of trigger times for snr, trig_time in connection.execute(sqlquery, sql_params_dict): trig_segment = segments.segment(trig_time, trig_time) if not veto_segments[ifo].intersects_segment(trig_segment): snr_array = np.append(snr_array, snr) if sngls_bins is None: sngls_bins = np.arange(min_snr, np.max(snr_array) + sngls_width, sngls_width) # make the binned snr histogram sngls_hist, _ = np.histogram(snr_array, bins=sngls_bins) return sngls_hist, sngls_bins