def get_livetime(self, category, match_criteria = [], time_units = 'yr'): """ Returns the sum of all the livetimes of categories that match the given category via the given match_criteria. """ if match_criteria == []: match_criteria = Category.default_match_criteria return sqlutils.convert_duration(sum([cat.livetime for cat in self if cat.selective_eq(category, match_criteria)]), time_units)
def eff_bkgd_time(T_i, tau_ij, ifos, unit): # numerator is the product of the relevant single-ifo analyzed times numerator = np.prod([T for (ifo, T) in T_i.items() if ifo in ifos]) # sorted list of the coincidence windows from smallest to largest taus = sorted(tau_ij.values()) # denominator is a non-trivial combination of the coincident windows if len(ifos) == 2: denominator = taus[0] elif len(ifos) == 3: denominator = 0.5*tau[0]*(tau[2] + tau[1]) - \ 0.25*( (tau[2]-tau[1])**2. + tau[0]**2. ) else: raise ValueError, "Can only estimate background times for double & triples" return sqlutils.convert_duration(numerator/denominator, unit)
def eff_bkgd_time(T_i, tau_ij, ifos, unit): # numerator is the product of the relevant single-ifo analyzed times numerator = np.prod([T for (ifo, T) in T_i.items() if ifo in ifos]) # sorted list of the coincidence windows from smallest to largest taus = sorted(tau_ij.values()) # denominator is a non-trivial combination of the coincident windows if len(ifos) == 2: denominator = taus[0] elif len(ifos) == 3: denominator = 0.5*tau[0]*(tau[2] + tau[1]) - \ 0.25*( (tau[2]-tau[1])**2. + tau[0]**2. ) else: raise ValueError, "Can only estimate background times for double & triples" return sqlutils.convert_duration(numerator / denominator, unit)
def coinc_time(connection, datatype, inc_ifo_list, unit, tsid=None): params_dict = {'type': datatype} cursor = connection.cursor() sqlquery = """ SELECT duration, instruments FROM experiment_summary JOIN experiment ON ( experiment.experiment_id == experiment_summary.experiment_id) WHERE datatype = :type """ if tsid: params_dict['tsid'] = tsid sqlquery += " AND time_slide_id = :tsid" cursor.execute( sqlquery, params_dict ) # all elements of inclusive ifo_list must be found in ifos for T to be included livetime = np.sum([ T for T, ifos in cursor if set(ifos.split(',')).issuperset(set(inc_ifo_list)) ]) return sqlutils.convert_duration(livetime, unit)
def coinc_time(connection, datatype, inc_ifo_list, unit, tsid=None): params_dict = {'type': datatype} cursor = connection.cursor() sqlquery = """ SELECT duration, instruments FROM experiment_summary JOIN experiment ON ( experiment.experiment_id == experiment_summary.experiment_id) WHERE datatype = :type """ if tsid: params_dict['tsid'] = tsid sqlquery += " AND time_slide_id = :tsid" cursor.execute(sqlquery, params_dict) # all elements of inclusive ifo_list must be found in ifos for T to be included livetime = np.sum([ T for T, ifos in cursor if set(ifos.split(',')).issuperset(set(inc_ifo_list)) ]) return sqlutils.convert_duration(livetime, unit)
def get_livetime(self, time_units = 'yr'): return sqlutils.convert_duration( self.livetime, time_units )
def convert_duration( duration ): return sqlutils.convert_duration( duration, convert_durations )
def printsims(connection, simulation_table, recovery_table, map_label, ranking_stat, rank_by, comparison_datatype, sort_by = 'rank', param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ', rank_range = None, convert_durations = 's', daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False): from pylal import ligolw_sqlutils as sqlutils # check and format options appropriately simulation_table = sqlutils.validate_option(simulation_table) recovery_table = sqlutils.validate_option(recovery_table) ranking_stat = sqlutils.validate_option(ranking_stat) rank_by = sqlutils.validate_option(rank_by, lower = False).upper() comparison_datatype = sqlutils.validate_option(comparison_datatype) convert_durations = sqlutils.validate_option(convert_durations) if rank_by == 'MIN': rank_by = 'ASC' elif rank_by == 'MAX': rank_by = 'DESC' elif rank_by != 'ASC' or rank_by != 'DESC': raise ValueError, 'rank_by must be MAX (or DESC) or MIN (or ASC)' if not ranking_stat.startswith(recovery_table): ranking_stat = '.'.join([recovery_table, ranking_stat]) # # Set up sim_rec_map table # if 'sim_rec_map' not in sqlutils.get_tables_in_database( connection ): sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, ranking_stat) # # Initialize ranking. Statistics for ranking are collected from non-injections # in the recovery table. # if verbose: print >> sys.stderr, "Getting statistics for ranking..." ranker = sqlutils.rank_stats(recovery_table, ranking_stat, rank_by) # add requirement that stats not be found in the sim_rec_table to in_this_filter rank_filter = ''.join([ recovery_table, '''.coinc_event_id NOT IN ( SELECT rec_id FROM sim_rec_map) AND experiment_summary.datatype == "''', comparison_datatype, '"']) # add to the rank filter any other desired filters; note that sim-tag is forced to ALLINJ if not comparing # to simulation datatype in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges, exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs, sim_tag = comparison_datatype == 'simulation' and sim_tag or 'ALLINJ', verbose = False) if in_this_filter != '': rank_filter = '\n\tAND '.join([ in_this_filter, rank_filter ]) rank_filter = '\n\t'.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), 'WHERE', rank_filter ]) ranker.populate_stats_list(connection, limit = None, filter = rank_filter) connection.create_function( 'rank', 1, ranker.get_rank ) # # Set recovery table filters # in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges, exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs, sim_tag = sim_tag, verbose = verbose) # Now apply the filter to the sim_rec_map table: this will delete all sim/rec maps where the simulation id is # mapped to a recovered event that falls outside the filter, even if that particular sim/rec map is in the # filter. For example, if the filter is recovery_table.combined_far != 0., and there are two entries in the # sim_rec_map table sharing the same sim_id: # sim_id:0 | rec_id:0 | rec_id's combined_far = 0. # sim_id:0 | rec_id:1 | rec_id's combined_far = 1.2 # both entries will get deleted even though the second entry's combined_far is not 0. if in_this_filter != '': # join the needed tables to in_this_filter in_this_filter = ''.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), "\n WHERE\n\t", in_this_filter ]) sqlscript = ''.join([ """ CREATE TEMP TABLE del_ids AS SELECT sim_id AS del_id FROM sim_rec_map WHERE rec_id NOT IN ( SELECT """, recovery_table, """.coinc_event_id FROM """, recovery_table, """ """, in_this_filter, """ AND experiment_summary.datatype == "simulation" ); DELETE FROM sim_rec_map WHERE sim_id IN ( SELECT del_id FROM del_ids ); DROP TABLE del_ids;""" ]) connection.cursor().executescript(sqlscript) # # Set other needed functions # # establish what units will be converting duration to def convert_duration( duration ): return sqlutils.convert_duration( duration, convert_durations ) connection.create_function( 'convert_duration', 1, convert_duration ) # create the get_sim_tag function sim_map = sqlutils.sim_tag_proc_id_mapper( connection ) connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag ) # Get range ranks if rank_range is not None: rank_range_parser = sqlutils.parse_param_ranges( 'rank(sim_rec_map', 'ranking_stat)', rank_range, verbose = verbose ) # # Create and prepare the SelectedFoundTable to store summary information # # Get recovery table and simulation table column names from database simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table ) recovery_table_columns = sqlutils.get_column_names_from_table( connection, recovery_table ) # Get list of column name for injected parameters injected_cols = [] for col in simulation_table_columns: if col == 'simulation_id': injected_cols.append('simulation_id') else: injected_cols.append('injected_'+col) injected_cols.extend(['injected_decisive_distance','injected_gps_time', 'injected_gps_time_ns', 'injected_event_time_utc__Px_click_for_daily_ihope_xP_']) # Get list of column names from the recovery table recovered_cols = [] for col in recovery_table_columns: if col == 'coinc_event_id': recovered_cols.append(u'coinc_event_id') else: recovered_cols.append(u'recovered_'+col) # generate list of column names for the summary table column_names = injected_cols + recovered_cols # add instruments on, duration, mini_followups rankname = 'rank_in_' + comparison_datatype.strip().lower() + '_using_' + ranking_stat.split('.')[-1] durname = ''.join([ 'simulation', u'_duration__Px_', convert_durations, '_xP_' ]) column_names.extend( [ rankname, 'recovered_match_rank', 'instruments_on', 'elogs', durname, 'mini_followup','omega_scan', 'sim_tag' ] ) # # define needed tables # class tmpSimTable(table.Table): tableName = "tmp_sim_table:table" validcolumns = dict([ [col, sqlutils.get_col_type(simulation_table, col)] for col in simulation_table_columns ]) class tmpSim(object): __slots__ = tmpSimTable.validcolumns.keys() def get_gps_time(self, site): if '%s_start_time' % site in self.__slots__: return LIGOTimeGPS(getattr(self, '%s_start_time' % site ), getattr(self, '%s_start_time_ns' % site)) elif '%s_end_time' % site in self.__slots__: return LIGOTimeGPS(getattr(self, '%s_end_time' % site ), getattr(self, '%s_end_time_ns' % site)) else: raise AttributeError, "could not find an injected %s_start_time nor an injected %s_end_time" %(site,site) def get_pyvalue(self): return generic_get_pyvalue(self) class tmpRecTable(table.Table): tableName = "tmp_rec_table:table" validcolumns = dict([ [col, sqlutils.get_col_type(recovery_table, col)] for col in recovery_table_columns ]) class tmpRec(object): __slots__ = tmpRecTable.validcolumns.keys() def get_gps_time(self): if 'start_time' in self.__slots__: return LIGOTimeGPS(self.start_time, self.start_time_ns) elif 'end_time' in self.__slots__: return LIGOTimeGPS(self.end_time, self.end_time_ns) else: raise AttributeError, "could not find a recovered start_time or recovered end_time" def get_pyvalue(self): return generic_get_pyvalue(self) class SelectedFoundTable(table.Table): tableName = "selected_found_injections:table" validcolumns = {} for col_name in column_names: if 'rank_in_' in col_name: validcolumns[col_name] = "int_4u" elif '_duration_' in col_name: validcolumns[col_name] = "real_8" elif 'instruments_on' == col_name: validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments'] elif col_name == 'injected_start_time' or col_name == 'injected_start_time_ns': validcolumns[col_name] = "int_4s" elif col_name == 'injected_gps_time' or col_name == 'injected_gps_time_ns': validcolumns[col_name] = "int_4s" elif col_name == 'injected_decisive_distance': validcolumns[col_name] = "real_8" elif 'injected_' in col_name or col_name == 'simulation_id': validcolumns[col_name] = sqlutils.get_col_type(simulation_table, re.sub('injected_', '', col_name)) elif 'recovered_' in col_name or col_name == 'coinc_event_id': validcolumns[col_name] = sqlutils.get_col_type(recovery_table, re.sub('recovered_', '', col_name)) # if custom columns exist in the database, just set them to lstrings else: validcolumns[col_name] = "lstring" # add FAP columns validcolumns['recovered_fap'] = "real_8" validcolumns['recovered_fap_1yr'] = "real_8" class SelectedFound(object): __slots__ = SelectedFoundTable.validcolumns.keys() def set_recovered_gps_time(self, gps_time): if 'recovered_start_time' in self.__slots__: self.recovered_start_time = gps_time.seconds self.recovered_start_time_ns = gps_time.nanoseconds elif 'recovered_end_time' in self.__slots__: self.recovered_end_time = gps_time.seconds self.recovered_end_time_ns = gps_time.nanoseconds else: raise AttributeError, "could not find a recovered_start_time or recovered_end_time" def get_recovered_gps_time(self): if 'recovered_start_time' in self.__slots__: return LIGOTimeGPS(self.recovered_start_time, self.recovered_start_time_ns) elif 'recovered_end_time' in self.__slots__: return LIGOTimeGPS(self.recovered_end_time, self.recovered_end_time_ns) else: raise AttributeError, "could not find a recovered_start_time or recovered_end_time" def set_injected_gps_time(self, site, gps_time): if 'injected_%s_start_time' % site in self.__slots__: setattr(self, 'injected_%s_start_time' % site, gps_time.seconds) setattr(self, 'injected_%s_start_time_ns' % site, gps_time.nanoseconds) elif 'injected_%s_end_time' % site in self.__slots__: setattr(self, 'injected_%s_end_time' % site, gps_time.seconds) setattr(self, 'injected_%s_end_time_ns' % site, gps_time.nanoseconds) else: raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site) def get_injected_gps_time(self, site): if 'injected_%s_start_time' % site in self.__slots__: return LIGOTimeGPS(getattr(self, 'injected_%s_start_time' % site ), getattr(self, 'injected_%s_start_time_ns' % site)) elif 'injected_%s_end_time' % site in self.__slots__: return LIGOTimeGPS(getattr(self, 'injected_%s_end_time' % site ), getattr(self, 'injected_%s_end_time_ns' % site)) else: raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site) def get_pyvalue(self): return generic_get_pyvalue(self) # connect the rows to the tables tmpSimTable.RowType = tmpSim tmpRecTable.RowType = tmpRec SelectedFoundTable.RowType = SelectedFound # # Get the Data # tmp_sim_table = lsctables.New(tmpSimTable) tmp_rec_table = lsctables.New(tmpRecTable) tmp_sftable = lsctables.New(SelectedFoundTable) prior_sim_id = '' group_rank = None current_match_rank = 1 sqlquery = ''.join([""" SELECT """, simulation_table, """.*, """, recovery_table, """.*, get_sim_tag(experiment_summary.sim_proc_id), rank(sim_rec_map.ranking_stat), NULL AS match_rank, experiment.instruments, convert_duration(experiment_summary.duration), NULL AS mini_followup FROM sim_rec_map JOIN """, ', '.join([simulation_table, recovery_table]), """, experiment, experiment_summary, experiment_map ON ( sim_rec_map.sim_id == """, simulation_table, """.simulation_id AND sim_rec_map.rec_id == """, recovery_table, """.coinc_event_id AND sim_rec_map.rec_id == experiment_map.coinc_event_id AND experiment_map.experiment_summ_id == experiment_summary.experiment_summ_id AND experiment_summary.experiment_id == experiment.experiment_id) ORDER BY sim_rec_map.sim_id, sim_rec_map.ranking_stat """, rank_by]) if verbose: print >> sys.stderr, "Getting coincs..." print >> sys.stderr, "SQLite query used is:" print >> sys.stderr, sqlquery for values in connection.cursor().execute( sqlquery ).fetchall(): # sort the data tmp_sim_row = tmpSim() tmp_rec_row = tmpRec() [ setattr(tmp_sim_row, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ] [ setattr(tmp_rec_row, column, values[ii+1+jj]) for jj, column in enumerate(recovery_table_columns) ] # figure out the rank this_inj_rank = values[-5] this_sim_id = tmp_sim_row.simulation_id this_ranking_stat = getattr(tmp_rec_row, ranking_stat.split('.')[-1]) if this_sim_id == prior_sim_id and this_ranking_stat != prior_ranking_stat: current_match_rank += 1 elif this_sim_id != prior_sim_id: current_match_rank = 1 group_rank = this_inj_rank prior_sim_id = this_sim_id prior_ranking_stat = this_ranking_stat # only store data if the group_rank falls in the desired rank ranges if rank_range and rank_range_parser.group_by_param_range(group_rank) is None: continue on_instruments = lsctables.instrument_set_from_ifos(values[-3]) duration = values[-2] # now that have all the information from this row, create a row for the selected found table sfrow = SelectedFound() # set the ranks setattr(sfrow, rankname, group_rank) sfrow.recovered_match_rank = current_match_rank # set the injected parameters use_this_site = sorted(on_instruments)[0][0].lower() injected_time = tmp_sim_row.get_gps_time( use_this_site ) sfrow.injected_gps_time = injected_time.seconds sfrow.injected_gps_time_ns = injected_time.nanoseconds for col in simulation_table_columns: if col == "simulation_id": sfrow.simulation_id = tmp_sim_row.simulation_id else: setattr(sfrow, 'injected_'+col, getattr( tmp_sim_row, col) ) # set the recovered parameters for col in recovery_table_columns: if col == "coinc_event_id": sfrow.coinc_event_id = tmp_rec_row.coinc_event_id else: setattr(sfrow, 'recovered_'+col, getattr( tmp_rec_row, col) ) # calculate and add faps if sfrow.recovered_combined_far is not None: t_in_s = float(values[-2]) / sqlutils.convert_duration(1, convert_durations) sfrow.recovered_fap = 1 - math.exp(-sqlutils.convert_duration(t_in_s, 'yr') * sfrow.recovered_combined_far) sfrow.recovered_fap_1yr = 1 - math.exp(-sfrow.recovered_combined_far) else: sfrow.recovered_fap = None sfrow.recovered_fap_1yr = None # set elog page elog_pages = [(ifo, get_elog_page(ifo, sfrow.injected_gps_time)) for ifo in on_instruments] sfrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ]) # set daily_ihope page event_time_utc = format_end_time_in_utc( sfrow.injected_gps_time ) daily_ihope_address = get_daily_ihope_page(sfrow.injected_gps_time, pages_location = daily_ihope_pages_location) sfrow.injected_event_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, event_time_utc ) # set any other info sfrow.instruments_on = ','.join(sorted(on_instruments)) sfrow.injected_decisive_distance = sorted([getattr(sfrow, 'injected_eff_dist_%s' % ifo[0].lower()) for ifo in on_instruments])[1] sfrow.mini_followup = None sfrow.omega_scan = None sfrow.sim_tag = values[-6] setattr(sfrow, durname, duration) # add the row tmp_sftable.append(sfrow) # Re-sort the sftable by rank, recovered_match_rank sftable = lsctables.New(SelectedFoundTable) for sfrow in sorted([ row for row in tmp_sftable ], key = lambda row: getattr(row, sort_by == 'rank' and rankname or sort_by) ): if sfrow.simulation_id not in [row.simulation_id for row in sftable]: sftable.append(sfrow) sftable.extend(sub_row for sub_row in sorted([row for row in tmp_sftable if row.simulation_id == sfrow.simulation_id and row.coinc_event_id != sfrow.coinc_event_id], key = lambda row: row.recovered_match_rank)) # drop the sim_rec_map table connection.cursor().execute("DROP TABLE sim_rec_map") return sftable