def get_glitch_times(glitch_xmlfiles): """ Returns list of (gps,gps_ns) tuples for all events contained in the glitch_xmlfiles. """ # load files in database connection, cursor = idq_tables_dbutils.load_xml_files_into_database( glitch_xmlfiles) # get table names from the database tablenames = dbtables.get_table_names(connection) if not tablenames: print "No tables were found in the database." return [] # check if glitch table is present if not table.StripTableName( idq_tables.IDQGlitchTable.tableName) in tablenames: print "No glitch table is found in database." print "Can not perform requested query." return [] data = cursor.execute("""SELECT gps, gps_ns FROM """ + \ table.StripTableName(idq_tables.IDQGlitchTable.tableName)).fetchall() # close database connection.close() return data
def which_table(etg): """Find the correct table to use for a given trigger generator ### Example: ### @code >>> which_table("ExcessPower") 'sngl_burst' @endcode @param etg the name of the trigger generator in question @returns the name of the `LIGO_LW` table appropriate for triggers generated by the given etg """ etg = str(etg).lower() if etg in lsctables.TableByName.keys(): return etg elif MULTI_BURST_REGEX.search(etg): return ligolw_table.StripTableName(lsctables.MultiBurstTable.tableName) elif SNGL_BURST_REGEX.search(etg): return ligolw_table.StripTableName(lsctables.SnglBurstTable.tableName) elif MULTI_INSPIRAL_REGEX.search(etg): return ligolw_table.StripTableName( lsctables.MultiInspiralTable.tableName) elif SNGL_INSPIRAL_REGEX.search(etg): return ligolw_table.StripTableName( lsctables.SnglInspiralTable.tableName) elif SNGL_RING_REGEX.search(etg): return ligolw_table.StripTableName( lsctables.SnglRingdownTable.tableName) else: raise ValueError("No LIGO_LW table mapped for ETG=\'%s\'" % etg)
def time_column(table, ifo=None): """Extract the 'time' column from the given table. This function uses time_func to determine the correct column to use as a proxy for 'time' and returns that column. The following mappings are used: - `sngl_inspiral` -> 'end' time - `sngl_burst` -> 'peak' time - `sngl_ringdown` -> 'start' time @param table any `LIGO_LW` table @param ifo an interferometer prefix if you want single-detector times @returns a numpy array object with a 'time' element for each row in the table """ if hasattr(table, "get_time"): return numpy.asarray(table.get_time()) func_name = time_func(ligolw_table.StripTableName( table.tableName)).__name__ if hasattr(table, func_name): return numpy.asarray(getattr(table, func_name)()) else: return numpy.asarray(map(func_name, table))
def startTable(self, parent, attrs, __orig_startTable=ContentHandler.startTable): name = table.StripTableName(attrs[u"Name"]) if name in TableByName: return TableByName[name](attrs) return __orig_startTable(self, parent, attrs)
def get_glitch_snglburst_data(connection, cursor, glitch_columns, snglburst_columns): """ Connects glitch and snglburst tables, and returns requested data, as a list of tuples with values for requested columns in the order of [(glitch_columns, snglburst_columns), ...]. @param connection UNDOCUMENTED @param cursor UNDOCUMENTED @param glitch_columns is the list of requested glitch table column names, @param snglburst_columns is the list of requested snglburst table column names. """ # get table names from the database tablenames = dbtables.get_table_names(connection) if not tablenames: print "No tables were found in the database." return [] # check if glitch table and ovl tables are present if not ( (table.StripTableName(idq_tables.IDQGlitchTable.tableName) in tablenames)\ and (table.StripTableName(lsctables.SnglBurstTable.tableName) in tablenames) ): print "No glitch or snglburst table is found in database." print "Need both to perform requested query." return [] # construct feature string with requested columns for SELECT statement glitch_features = ', '.join(['g.' + name for name in glitch_columns]) snglburst_features = ', '.join(['b.' + name for name in snglburst_columns]) features_string = glitch_features + ', ' + snglburst_features # do the query data = cursor.execute('''SELECT ''' + features_string + ''' FROM ''' + \ table.StripTableName(idq_tables.IDQGlitchTable.tableName) + \ ''' AS g JOIN coinc_event_map AS c1 ON g.event_id == c1.event_id''' + \ ''' JOIN coinc_event_map AS c2 ON c1.coinc_event_id == c2.coinc_event_id''' + \ ''' JOIN ''' + table.StripTableName(lsctables.SnglBurstTable.tableName) + \ ''' AS b ON b.event_id == c2.event_id''').fetchall() return data
def startTable(self, parent, attrs): name = table.StripTableName(attrs[u"Name"]) if name in dbtables.TableByName: return dbtables.TableByName[name](attrs, connection=self.connection) elif name in IDQTableByName: IDQDBTable = dbtables.DBTable(attrs, connection=self.connection) IDQDBTable.tableName = IDQTableByName[name].tableName IDQDBTable.validcolumns = IDQTableByName[name].validcolumns IDQDBTable.loadcolumns = IDQTableByName[name].loadcolumns IDQDBTable.constraints = IDQTableByName[name].constraints IDQDBTable.next_id = IDQTableByName[name].next_id IDQDBTable.RowType = IDQTableByName[name].RowType IDQDBTable.how_to_index = IDQTableByName[name].how_to_index return IDQDBTable return dbtables.DBTable(attrs, connection=self.connection)
def new_table(tab, *args, **kwargs): """Create a new `~glue.ligolw.Table` This is just a convenience wrapper around `~glue.ligolw.lsctables.New` Parameters ---------- tab : `type`, `str` `~glue.ligolw.Table` subclass, or name of table to create *args, **kwargs other parameters are passed directly to `~glue.ligolw.lsctables.New` Returns ------- table : `~glue.ligolw.Table` a newly-created table with the relevant attributes and structure """ if isinstance(tab, str): tab = lsctables.TableByName[table.StripTableName(tab)] return lsctables.New(tab, *args, **kwargs)
def coinc_to_triggers(xmldoc, trigger_types): """ Function returns list of glitch-trigger(s) coincident events. Coincident event in the list is represented by tuple (glitch_object, [trigger1, trigger2, ...]). trigger_types is the list of trigger type names corresponding to "search" column of the sngl_burst table """ # get necessary tables from xmldoc coinc_def_table = table.get_table(xmldoc, lsctables.CoincDefTable.tableName) coinc_table = table.get_table(xmldoc, lsctables.CoincTable.tableName) coinc_map_table = table.get_table(xmldoc, lsctables.CoincMapTable.tableName) idq_glitch_table = table.get_table(xmldoc, lsctables.IDQGlitchTable.tableName) sngl_burst_table = table.get_table(xmldoc, lsctables.SnglBurstTable.tableName) # get coinc_def_id #coinc_def_id = coinc_def_table.get_coinc_def_id( # search = IDQCoincDef['idq_glitch<-->sngl_burst'][0], # search_coinc_type = IDQCoincDef['idq_glitch<-->sngl_burst'][1], # create_new = False, # description = 'idq_glitch<-->sngl_burst') coinc_def_ids = [ row.coinc_def_id for row in coinc_def_table if row.description == 'idq_glitch<-->sngl_burst' ] # use this id to get all coinc_event ids trig_coinc_ids = [ coinc.coinc_event_id for coinc in coinc_table if coinc.coinc_def_id in coinc_def_ids ] # convert idq_glitch and sngl_burst tables into dictionaries for a quick lookup glitches = dict([(glitch.event_id, glitch) for glitch in idq_glitch_table]) triggers = dict([(row.event_id, row) for row in sngl_burst_table if row.search in trigger_types]) # create dictionary of connected events using coinc_event_map. # We can not assume any specific order of rows in the table. connected_events_dict = {} for row in coinc_map_table: try: connected_events_dict[row.coinc_event_id].append(row) except: connected_events_dict[row.coinc_event_id] = [row] glitch_table_name = table.StripTableName( lsctables.IDQGlitchTable.tableName) sngl_burst_table_name = table.StripTableName( lsctables.SnglBurstTable.tableName) glitch_trig_tuples = [] for coinc_id in trig_coinc_ids: # get connectected events for this id connected_events = connected_events_dict[coinc_id] connected_trigs = [] if len(connected_events) >= 2: for event in connected_events: if event.table_name == glitch_table_name: glitch_event = glitches[event.event_id] elif event.table_name == sngl_burst_table_name: try: connected_trigs.append(triggers[event.event_id]) except: # no trigger with that id, it is probably of different type. pass else: raise ValueError("Event is not the row of either " + \ glitch_table_name + " or " + sngl_burst_table_name ) else: raise Exception("Glitch-Triggers coincidences must contain at least 2 events. " \ + str(len(connected_events))+ " events are found instead." ) glitch_trig_tuples.append((glitch_event, connected_trigs)) return glitch_trig_tuples
def coinc_to_ovl_data(xmldoc): """Function returns list of (idq_glitch_object, ovl_data_object) tuples where objects in the tuple are mapped to each other via coinc tables. """ # get necessary tables from xmldoc coinc_def_table = table.get_table(xmldoc, lsctables.CoincDefTable.tableName) coinc_table = table.get_table(xmldoc, lsctables.CoincTable.tableName) coinc_map_table = table.get_table(xmldoc, lsctables.CoincMapTable.tableName) idq_glitch_table = table.get_table(xmldoc, lsctables.IDQGlitchTable.tableName) ovl_data_table = table.get_table(xmldoc, lsctables.OVLDataTable.tableName) # get coinc_def_ids #coinc_def_id = coinc_def_table.get_coinc_def_id( # search = IDQCoincDef['idq_glitch<-->ovl_data'][0], # search_coinc_type = IDQCoincDef['idq_glitch<-->ovl_data'][1], # create_new = False, # description = 'idq_glitch<-->ovl_data') coinc_def_ids = [ row.coinc_def_id for row in coinc_def_table if row.description == 'idq_glitch<-->ovl_data' ] # use this id to get all coinc_event ids ovl_coinc_ids = [ coinc.coinc_event_id for coinc in coinc_table if coinc.coinc_def_id in coinc_def_ids ] # convert idq_glitch and ovl tables into dictionaries for a quick lookup glitches = dict([(glitch.event_id, glitch) for glitch in idq_glitch_table]) ovl_data = dict([(row.event_id, row) for row in ovl_data_table]) # create dictionary of connected events in coinc_event_map. # We can not assume any specific order of rows in the table. connected_events_dict = {} for row in coinc_map_table: try: connected_events_dict[row.coinc_event_id].append(row) except: connected_events_dict[row.coinc_event_id] = [row] glitch_table_name = table.StripTableName( lsctables.IDQGlitchTable.tableName) ovl_data_table_name = table.StripTableName( lsctables.OVLDataTable.tableName) glitch_ovl_pairs = [] for coinc_id in ovl_coinc_ids: # get connectected events for this id connected_events = connected_events_dict[coinc_id] if len(connected_events) == 2: for event in connected_events: if event.table_name == glitch_table_name: glitch_event = glitches[event.event_id] elif event.table_name == ovl_data_table_name: ovl_event = ovl_data[event.event_id] else: print event.table_name raise ValueError("Event is not the row of either " + \ glitch_table_name + \ " or " + ovl_data_table_name ) else: raise Exception("Glitch-OVL coincidence must contain exactly 2 events. "\ + str(len(connected_events))+ " events are found instead." ) glitch_ovl_pairs.append((glitch_event, ovl_event)) return glitch_ovl_pairs
'idq_glitch<-->ovl_data': [IDQGlitchTable.tableName, OVLDataTable.tableName] } # # Override portions of a ligolw.LIGOLWContentHandler class # # Table name ---> table type mapping # # Extend lsctables.TableByName to include iDQ tables # IDQTableByName = { table.StripTableName(IDQGlitchTable.tableName): IDQGlitchTable, table.StripTableName(OVLDataTable.tableName): OVLDataTable } TableByName = dict(lsctables.TableByName, **IDQTableByName) # # Override portions of a ligolw.LIGOLWContentHandler class # def use_in(ContentHandler): """ Modify ContentHandler, a sub-class of glue.ligolw.LIGOLWContentHandler, to cause it to use the tables from this module as well as from lsctable.py
def remove_redundant_entries(connection, cursor, verbose=False): """ Clean up idq tables by removing duplicate or redundant rows. """ # get table names from the database tablenames = dbtables.get_table_names(connection) if not tablenames: print "No tables were found in the database. Nothing to update." return # clean up process and process params tables if (table.StripTableName(lsctables.ProcessTable.tableName) in tablenames) \ and (table.StripTableName(lsctables.ProcessParamsTable.tableName) in tablenames): if verbose: print "Removing redundant entries from process and process_params tables ..." # get the lowest process_id from process table min_process_id = cursor.execute( '''SELECT MIN(process_id) FROM process''').fetchone()[0] # delete redundant rows from process and process_params table that corresponds to different process ids cursor.execute( '''DELETE FROM ''' + table.StripTableName(lsctables.ProcessParamsTable.tableName) + ''' WHERE process_id != ?''', (min_process_id, )) cursor.execute( '''DELETE FROM ''' + table.StripTableName(lsctables.ProcessTable.tableName) + ''' WHERE process_id != ?''', (min_process_id, )) # get all tables that use process_id prc_id_tables = cursor.execute( "SELECT name FROM sqlite_master WHERE type == 'table' AND sql LIKE '%process_id%'" ).fetchall() if verbose: print "Reassigning process_ids in all tables in the database ..." # loop over these tables and set all process_ids to min_process_id for (table_name, ) in prc_id_tables: cursor.execute( '''UPDATE ''' + table_name + ''' SET process_id = ?''', (min_process_id, )) connection.commit() # clean up coinc_definer and coinc event tables if ( table.StripTableName(lsctables.CoincDefTable.tableName) in tablenames ) and \ (table.StripTableName(lsctables.CoincTable.tableName) in tablenames ): if verbose: print "Removing redundant entries from coinc_definer and coinc_event tables ..." # get lowest (coinc_def_ids, search_coinc_type, description ) from coinc_def table # corresponding to unique search_coinc_type and description min_coinc_def_ids = cursor.execute( ''' SELECT MIN(coinc_def_id), search_coinc_type, description FROM coinc_definer GROUP BY search_coinc_type, description ''' ).fetchall() if verbose: print "Updating coinc_def_id(s) in coinc_event table ..." # loop over them and update coinc_event table replacing redundant coinc_def ids for (min_coinc_def_id, min_search_coinc_type, min_description) in min_coinc_def_ids: cursor.execute( '''UPDATE ''' + table.StripTableName(lsctables.CoincTable.tableName) + ''' SET coinc_def_id = ? WHERE coinc_def_id IN ( SELECT coinc_def_id FROM ''' + table.StripTableName(lsctables.CoincDefTable.tableName) + ''' WHERE ( search_coinc_type = ? ) AND ( description = ? ) )''', ( min_coinc_def_id, min_search_coinc_type, min_description, )) if verbose: print "Removing unused entries from coinc_definer table ..." # remove redundant and already unused records from coinc_def table cursor.execute( '''DELETE FROM ''' + table.StripTableName(lsctables.CoincDefTable.tableName) + ''' WHERE coinc_def_id NOT IN ( SELECT MIN(coinc_def_id) FROM ''' + table.StripTableName(lsctables.CoincDefTable.tableName) + ''' GROUP BY search_coinc_type, description ) ''') connection.commit() if verbose: print "Finished updating tables"
def delete_glitch_events_in_segment(connection, cursor, segment): """ Removes glitch events whose gps times fall within segment. All rows from the other tables in database connected to these glitch events via coinc_map tables are also deteled. @param connection UNDOCUMENTED @param cursor UNDOCUMENTED @param segment is glue segment """ # get table names from the database tablenames = dbtables.get_table_names(connection) if not tablenames: print "No tables were found in the database. Nothing to do." return # check if glitch table is present if not table.StripTableName( idq_tables.IDQGlitchTable.tableName) in tablenames: print "No glitch table is found in database. Nothing to do" return # create sqlite function for testing if glitch gos time is inside the segment("md5", 1, md5sum) connection.create_function('gpstime_inside_segment', 3, gpstime_inside_segment) # convert segment inot segment string segment_string = segmentsUtils.to_range_strings([segment])[0] # check if coinc_def table is present if table.StripTableName(lsctables.CoincDefTable.tableName) in tablenames: # get all distinct coinc types from coinc_def table coinc_descriptions = cursor.execute( '''SELECT DISTINCT description FROM ''' + table.StripTableName(lsctables.CoincDefTable.tableName)).fetchall( ) # loop over coinc descriptions for (description, ) in coinc_descriptions: tables_in_coinc = idq_tables.CoincDefToTableNames[description] # work only with coincs that involve glitch table if idq_tables.IDQGlitchTable.tableName in tables_in_coinc: # remove glitch table, we will work with it later connected_tables = [ t for t in tables_in_coinc if t != idq_tables.IDQGlitchTable.tableName ] # loop over other tables in the coinc for t in connected_tables: # delete entries connected to glitch events that should be removed cursor.execute('''DELETE FROM ''' + table.StripTableName(t) + \ ''' WHERE event_id IN (SELECT t.event_id FROM ''' + table.StripTableName(t) + \ ''' AS t JOIN coinc_event_map AS c1 ON t.event_id == c1.event_id''' + \ ''' JOIN coinc_event_map AS c2 ON c1.coinc_event_id == c2.coinc_event_id''' + \ ''' JOIN ''' + table.StripTableName(idq_tables.IDQGlitchTable.tableName) + \ ''' AS g ON c2.event_id == g.event_id WHERE gpstime_inside_segment(g.gps, g.gps_ns, ?))''',\ (segment_string,)) # delete entries from coinc_event table corresponding to deleted events cursor.execute('''DELETE FROM coinc_event WHERE coinc_event_id IN''' + \ ''' (SELECT c1.coinc_event_id FROM coinc_event_map AS c1 JOIN ''' + \ table.StripTableName(idq_tables.IDQGlitchTable.tableName) + \ ''' AS g ON c1.event_id == g.event_id WHERE gpstime_inside_segment(g.gps, g.gps_ns, ?))''',\ (segment_string,)) # delete entries from coinc_map table corresponding to deleted events cursor.execute('''DELETE FROM coinc_event_map WHERE event_id IN''' + \ ''' (SELECT c1.event_id FROM coinc_event_map AS c1''' + \ ''' JOIN coinc_event_map AS c2 ON c1.coinc_event_id == c2.coinc_event_id ''' + \ ''' JOIN ''' + table.StripTableName(idq_tables.IDQGlitchTable.tableName) +\ ''' AS g ON c2.event_id == g.event_id WHERE gpstime_inside_segment(g.gps, g.gps_ns, ?))''',\ (segment_string,)) # delete entries in glitch table cursor.execute( '''DELETE FROM ''' + table.StripTableName(idq_tables.IDQGlitchTable.tableName) + ''' WHERE gpstime_inside_segment(gps, gps_ns, ?)''', (segment_string, )) # commit everythings connection.commit()
def table_from_file(f, tablename, columns=None, filt=None, contenthandler=GWpyContentHandler, nproc=1, verbose=False): """Read a :class:`~glue.ligolw.table.Table` from a LIGO_LW file. Parameters ---------- f : `file`, `str`, `CacheEntry`, `list`, `Cache` object representing one or more files. One of - an open `file` - a `str` pointing to a file path on disk - a formatted :class:`~glue.lal.CacheEntry` representing one file - a `list` of `str` file paths - a formatted :class:`~glue.lal.Cache` representing many files tablename : `str` name of the table to read. columns : `list`, optional list of column name strings to read, default all. filt : `function`, optional function by which to `filter` events. The callable must accept as input a row of the table event and return `True`/`False`. contenthandler : :class:`~glue.ligolw.ligolw.LIGOLWContentHandler` SAX content handler for parsing LIGO_LW documents. Returns ------- table : :class:`~glue.ligolw.table.Table` `Table` of data with given columns filled """ # find table class tableclass = lsctables.TableByName[table.StripTableName(tablename)] # allow cache multiprocessing if nproc != 1: return tableclass.read(f, columns=columns, contenthandler=contenthandler, nproc=nproc, format='cache') # set columns to read if columns is not None: _oldcols = tableclass.loadcolumns tableclass.loadcolumns = columns # generate Document and populate files = [fp.name if isinstance(fp, (file, GzipFile, AstroGzipFile)) else fp for fp in file_list(f)] xmldoc = Document() ligolw_add(xmldoc, files, non_lsc_tables_ok=True, contenthandler=contenthandler, verbose=verbose) # extract table try: out = tableclass.get_table(xmldoc) except ValueError: out = lsctables.New(tableclass, columns=columns) if verbose: gprint('%d rows found in %s table' % (len(out), out.tableName)) if filt: if verbose: gprint('filtering rows ...', end=' ') try: out_ = out.copy() except AttributeError: out_ = table.new_from_template(out) out_.extend(filter(filt, out)) out = out_ if verbose: gprint('%d rows remaining\n' % len(out)) if columns is not None: tableclass.loadcolumns = _oldcols return out
def sim_and_sngl_inspirals_for_xmldoc(xmldoc): """Retrieve (as a generator) all of the (sim_inspiral, (sngl_inspiral, sngl_inspiral, ... sngl_inspiral) tuples from found coincidences in a LIGO-LW XML document.""" # Look up necessary tables. coinc_table = ligolw_table.get_table(xmldoc, lsctables.CoincTable.tableName) coinc_def_table = ligolw_table.get_table(xmldoc, lsctables.CoincDefTable.tableName) coinc_map_table = ligolw_table.get_table(xmldoc, lsctables.CoincMapTable.tableName) # Look up coinc_def ids. sim_coinc_def_id = coinc_def_table.get_coinc_def_id( ligolw_inspinjfind.InspiralSCExactCoincDef.search, ligolw_inspinjfind.InspiralSCExactCoincDef.search_coinc_type, create_new=False) def events_for_coinc_event_id(coinc_event_id): for coinc_map in coinc_map_table: if coinc_map.coinc_event_id == coinc_event_id: for row in ligolw_table.get_table(xmldoc, coinc_map.table_name): column_name = coinc_map.event_id.column_name if getattr(row, column_name) == coinc_map.event_id: yield coinc_map.event_id, row # Loop over all coinc_event <-> sim_inspiral coincs. for sim_coinc in coinc_table: # If this is not a coinc_event <-> sim_inspiral coinc, skip it. if sim_coinc.coinc_def_id != sim_coinc_def_id: continue # Locate the sim_inspiral and coinc events. sim_inspiral = None coinc = None for event_id, event in events_for_coinc_event_id( sim_coinc.coinc_event_id): if event_id.table_name == ligolw_table.StripTableName( lsctables.SimInspiralTable.tableName): if sim_inspiral is not None: raise RuntimeError( "Found more than one matching sim_inspiral entry") sim_inspiral = event elif event_id.table_name == ligolw_table.StripTableName( lsctables.CoincTable.tableName): if coinc is not None: raise RuntimeError( "Found more than one matching coinc entry") coinc = event else: raise RuntimeError( "Did not expect coincidence to contain an event of type '%s'" % event_id.table_name) sngl_inspirals = tuple(event for event_id, event in events_for_coinc_event_id(coinc.coinc_event_id)) yield sim_inspiral, sngl_inspirals
def ContentHandler(xmldoc): return ligolw.PartialLIGOLWContentHandler(xmldoc, lambda name, attrs:\ (name == ligolw.Table.tagName) and\ (table.StripTableName(attrs["Name"]) in ["segment_definer","segment_summary"]))
def ContentHandler(xmldoc): return ligolw.PartialLIGOLWContentHandler(xmldoc, lambda name, attrs:\ (name == ligolw.Table.tagName) and\ (table.StripTableName(attrs["Name"]) in tables))