def get_glitch_times(glitch_xmlfiles):
"""
Returns list of (gps,gps_ns) tuples for all events contained in the glitch_xmlfiles.
"""
# load files in database
connection, cursor = idq_tables_dbutils.load_xml_files_into_database(
glitch_xmlfiles)
# get table names from the database
tablenames = dbtables.get_table_names(connection)
if not tablenames:
print "No tables were found in the database."
return []
# check if glitch table is present
if not table.StripTableName(
idq_tables.IDQGlitchTable.tableName) in tablenames:
print "No glitch table is found in database."
print "Can not perform requested query."
return []
data = cursor.execute("""SELECT gps, gps_ns FROM """ + \
table.StripTableName(idq_tables.IDQGlitchTable.tableName)).fetchall()
# close database
connection.close()
return data
def which_table(etg):
"""Find the correct table to use for a given trigger generator
### Example: ###
@code
>>> which_table("ExcessPower")
'sngl_burst'
@endcode
@param etg
the name of the trigger generator in question
@returns the name of the `LIGO_LW` table appropriate for triggers
generated by the given etg
"""
etg = str(etg).lower()
if etg in lsctables.TableByName.keys():
return etg
elif MULTI_BURST_REGEX.search(etg):
return ligolw_table.StripTableName(lsctables.MultiBurstTable.tableName)
elif SNGL_BURST_REGEX.search(etg):
return ligolw_table.StripTableName(lsctables.SnglBurstTable.tableName)
elif MULTI_INSPIRAL_REGEX.search(etg):
return ligolw_table.StripTableName(
lsctables.MultiInspiralTable.tableName)
elif SNGL_INSPIRAL_REGEX.search(etg):
return ligolw_table.StripTableName(
lsctables.SnglInspiralTable.tableName)
elif SNGL_RING_REGEX.search(etg):
return ligolw_table.StripTableName(
lsctables.SnglRingdownTable.tableName)
else:
raise ValueError("No LIGO_LW table mapped for ETG=\'%s\'" % etg)
def time_column(table, ifo=None):
"""Extract the 'time' column from the given table.
This function uses time_func to determine the correct column to
use as a proxy for 'time' and returns that column.
The following mappings are used:
- `sngl_inspiral` -> 'end' time
- `sngl_burst` -> 'peak' time
- `sngl_ringdown` -> 'start' time
@param table
any `LIGO_LW` table
@param ifo
an interferometer prefix if you want single-detector times
@returns a numpy array object with a 'time' element for each row in
the table
"""
if hasattr(table, "get_time"):
return numpy.asarray(table.get_time())
func_name = time_func(ligolw_table.StripTableName(
table.tableName)).__name__
if hasattr(table, func_name):
return numpy.asarray(getattr(table, func_name)())
else:
return numpy.asarray(map(func_name, table))
def startTable(self,
parent,
attrs,
__orig_startTable=ContentHandler.startTable):
name = table.StripTableName(attrs[u"Name"])
if name in TableByName:
return TableByName[name](attrs)
return __orig_startTable(self, parent, attrs)
def get_glitch_snglburst_data(connection, cursor, glitch_columns,
snglburst_columns):
"""
Connects glitch and snglburst tables, and returns requested data,
as a list of tuples with values for requested columns in the order of
[(glitch_columns, snglburst_columns), ...].
@param connection UNDOCUMENTED
@param cursor UNDOCUMENTED
@param glitch_columns is the list of requested glitch table column names,
@param snglburst_columns is the list of requested snglburst table column names.
"""
# get table names from the database
tablenames = dbtables.get_table_names(connection)
if not tablenames:
print "No tables were found in the database."
return []
# check if glitch table and ovl tables are present
if not ( (table.StripTableName(idq_tables.IDQGlitchTable.tableName) in tablenames)\
and (table.StripTableName(lsctables.SnglBurstTable.tableName) in tablenames) ):
print "No glitch or snglburst table is found in database."
print "Need both to perform requested query."
return []
# construct feature string with requested columns for SELECT statement
glitch_features = ', '.join(['g.' + name for name in glitch_columns])
snglburst_features = ', '.join(['b.' + name for name in snglburst_columns])
features_string = glitch_features + ', ' + snglburst_features
# do the query
data = cursor.execute('''SELECT ''' + features_string + ''' FROM ''' + \
table.StripTableName(idq_tables.IDQGlitchTable.tableName) + \
''' AS g JOIN coinc_event_map AS c1 ON g.event_id == c1.event_id''' + \
''' JOIN coinc_event_map AS c2 ON c1.coinc_event_id == c2.coinc_event_id''' + \
''' JOIN ''' + table.StripTableName(lsctables.SnglBurstTable.tableName) + \
''' AS b ON b.event_id == c2.event_id''').fetchall()
return data
def startTable(self, parent, attrs):
name = table.StripTableName(attrs[u"Name"])
if name in dbtables.TableByName:
return dbtables.TableByName[name](attrs,
connection=self.connection)
elif name in IDQTableByName:
IDQDBTable = dbtables.DBTable(attrs, connection=self.connection)
IDQDBTable.tableName = IDQTableByName[name].tableName
IDQDBTable.validcolumns = IDQTableByName[name].validcolumns
IDQDBTable.loadcolumns = IDQTableByName[name].loadcolumns
IDQDBTable.constraints = IDQTableByName[name].constraints
IDQDBTable.next_id = IDQTableByName[name].next_id
IDQDBTable.RowType = IDQTableByName[name].RowType
IDQDBTable.how_to_index = IDQTableByName[name].how_to_index
return IDQDBTable
return dbtables.DBTable(attrs, connection=self.connection)
def new_table(tab, *args, **kwargs):
"""Create a new `~glue.ligolw.Table`
This is just a convenience wrapper around `~glue.ligolw.lsctables.New`
Parameters
----------
tab : `type`, `str`
`~glue.ligolw.Table` subclass, or name of table to create
*args, **kwargs
other parameters are passed directly to `~glue.ligolw.lsctables.New`
Returns
-------
table : `~glue.ligolw.Table`
a newly-created table with the relevant attributes and structure
"""
if isinstance(tab, str):
tab = lsctables.TableByName[table.StripTableName(tab)]
return lsctables.New(tab, *args, **kwargs)
def coinc_to_triggers(xmldoc, trigger_types):
""" Function returns list of glitch-trigger(s) coincident events.
Coincident event in the list is represented by tuple (glitch_object, [trigger1, trigger2, ...]).
trigger_types is the list of trigger type names corresponding to "search" column of the sngl_burst table
"""
# get necessary tables from xmldoc
coinc_def_table = table.get_table(xmldoc,
lsctables.CoincDefTable.tableName)
coinc_table = table.get_table(xmldoc, lsctables.CoincTable.tableName)
coinc_map_table = table.get_table(xmldoc,
lsctables.CoincMapTable.tableName)
idq_glitch_table = table.get_table(xmldoc,
lsctables.IDQGlitchTable.tableName)
sngl_burst_table = table.get_table(xmldoc,
lsctables.SnglBurstTable.tableName)
# get coinc_def_id
#coinc_def_id = coinc_def_table.get_coinc_def_id(
# search = IDQCoincDef['idq_glitch<-->sngl_burst'][0],
# search_coinc_type = IDQCoincDef['idq_glitch<-->sngl_burst'][1],
# create_new = False,
# description = 'idq_glitch<-->sngl_burst')
coinc_def_ids = [
row.coinc_def_id for row in coinc_def_table
if row.description == 'idq_glitch<-->sngl_burst'
]
# use this id to get all coinc_event ids
trig_coinc_ids = [
coinc.coinc_event_id for coinc in coinc_table
if coinc.coinc_def_id in coinc_def_ids
]
# convert idq_glitch and sngl_burst tables into dictionaries for a quick lookup
glitches = dict([(glitch.event_id, glitch) for glitch in idq_glitch_table])
triggers = dict([(row.event_id, row) for row in sngl_burst_table
if row.search in trigger_types])
# create dictionary of connected events using coinc_event_map.
# We can not assume any specific order of rows in the table.
connected_events_dict = {}
for row in coinc_map_table:
try:
connected_events_dict[row.coinc_event_id].append(row)
except:
connected_events_dict[row.coinc_event_id] = [row]
glitch_table_name = table.StripTableName(
lsctables.IDQGlitchTable.tableName)
sngl_burst_table_name = table.StripTableName(
lsctables.SnglBurstTable.tableName)
glitch_trig_tuples = []
for coinc_id in trig_coinc_ids:
# get connectected events for this id
connected_events = connected_events_dict[coinc_id]
connected_trigs = []
if len(connected_events) >= 2:
for event in connected_events:
if event.table_name == glitch_table_name:
glitch_event = glitches[event.event_id]
elif event.table_name == sngl_burst_table_name:
try:
connected_trigs.append(triggers[event.event_id])
except: # no trigger with that id, it is probably of different type.
pass
else:
raise ValueError("Event is not the row of either " + \
glitch_table_name + " or " + sngl_burst_table_name
)
else:
raise Exception("Glitch-Triggers coincidences must contain at least 2 events. " \
+ str(len(connected_events))+ " events are found instead."
)
glitch_trig_tuples.append((glitch_event, connected_trigs))
return glitch_trig_tuples
def coinc_to_ovl_data(xmldoc):
"""Function returns list of (idq_glitch_object, ovl_data_object) tuples
where objects in the tuple are mapped to each other via coinc tables.
"""
# get necessary tables from xmldoc
coinc_def_table = table.get_table(xmldoc,
lsctables.CoincDefTable.tableName)
coinc_table = table.get_table(xmldoc, lsctables.CoincTable.tableName)
coinc_map_table = table.get_table(xmldoc,
lsctables.CoincMapTable.tableName)
idq_glitch_table = table.get_table(xmldoc,
lsctables.IDQGlitchTable.tableName)
ovl_data_table = table.get_table(xmldoc, lsctables.OVLDataTable.tableName)
# get coinc_def_ids
#coinc_def_id = coinc_def_table.get_coinc_def_id(
# search = IDQCoincDef['idq_glitch<-->ovl_data'][0],
# search_coinc_type = IDQCoincDef['idq_glitch<-->ovl_data'][1],
# create_new = False,
# description = 'idq_glitch<-->ovl_data')
coinc_def_ids = [
row.coinc_def_id for row in coinc_def_table
if row.description == 'idq_glitch<-->ovl_data'
]
# use this id to get all coinc_event ids
ovl_coinc_ids = [
coinc.coinc_event_id for coinc in coinc_table
if coinc.coinc_def_id in coinc_def_ids
]
# convert idq_glitch and ovl tables into dictionaries for a quick lookup
glitches = dict([(glitch.event_id, glitch) for glitch in idq_glitch_table])
ovl_data = dict([(row.event_id, row) for row in ovl_data_table])
# create dictionary of connected events in coinc_event_map.
# We can not assume any specific order of rows in the table.
connected_events_dict = {}
for row in coinc_map_table:
try:
connected_events_dict[row.coinc_event_id].append(row)
except:
connected_events_dict[row.coinc_event_id] = [row]
glitch_table_name = table.StripTableName(
lsctables.IDQGlitchTable.tableName)
ovl_data_table_name = table.StripTableName(
lsctables.OVLDataTable.tableName)
glitch_ovl_pairs = []
for coinc_id in ovl_coinc_ids:
# get connectected events for this id
connected_events = connected_events_dict[coinc_id]
if len(connected_events) == 2:
for event in connected_events:
if event.table_name == glitch_table_name:
glitch_event = glitches[event.event_id]
elif event.table_name == ovl_data_table_name:
ovl_event = ovl_data[event.event_id]
else:
print event.table_name
raise ValueError("Event is not the row of either " + \
glitch_table_name + \
" or " + ovl_data_table_name
)
else:
raise Exception("Glitch-OVL coincidence must contain exactly 2 events. "\
+ str(len(connected_events))+ " events are found instead."
)
glitch_ovl_pairs.append((glitch_event, ovl_event))
return glitch_ovl_pairs
'idq_glitch<-->ovl_data':
[IDQGlitchTable.tableName, OVLDataTable.tableName]
}
#
# Override portions of a ligolw.LIGOLWContentHandler class
#
# Table name ---> table type mapping
#
# Extend lsctables.TableByName to include iDQ tables
#
IDQTableByName = {
table.StripTableName(IDQGlitchTable.tableName): IDQGlitchTable,
table.StripTableName(OVLDataTable.tableName): OVLDataTable
}
TableByName = dict(lsctables.TableByName, **IDQTableByName)
#
# Override portions of a ligolw.LIGOLWContentHandler class
#
def use_in(ContentHandler):
"""
Modify ContentHandler, a sub-class of
glue.ligolw.LIGOLWContentHandler, to cause it to use the
tables from this module as well as from lsctable.py
def remove_redundant_entries(connection, cursor, verbose=False):
"""
Clean up idq tables by removing duplicate or redundant rows.
"""
# get table names from the database
tablenames = dbtables.get_table_names(connection)
if not tablenames:
print "No tables were found in the database. Nothing to update."
return
# clean up process and process params tables
if (table.StripTableName(lsctables.ProcessTable.tableName) in tablenames) \
and (table.StripTableName(lsctables.ProcessParamsTable.tableName) in tablenames):
if verbose:
print "Removing redundant entries from process and process_params tables ..."
# get the lowest process_id from process table
min_process_id = cursor.execute(
'''SELECT MIN(process_id) FROM process''').fetchone()[0]
# delete redundant rows from process and process_params table that corresponds to different process ids
cursor.execute(
'''DELETE FROM ''' +
table.StripTableName(lsctables.ProcessParamsTable.tableName) +
''' WHERE process_id != ?''', (min_process_id, ))
cursor.execute(
'''DELETE FROM ''' +
table.StripTableName(lsctables.ProcessTable.tableName) +
''' WHERE process_id != ?''', (min_process_id, ))
# get all tables that use process_id
prc_id_tables = cursor.execute(
"SELECT name FROM sqlite_master WHERE type == 'table' AND sql LIKE '%process_id%'"
).fetchall()
if verbose:
print "Reassigning process_ids in all tables in the database ..."
# loop over these tables and set all process_ids to min_process_id
for (table_name, ) in prc_id_tables:
cursor.execute(
'''UPDATE ''' + table_name + ''' SET process_id = ?''',
(min_process_id, ))
connection.commit()
# clean up coinc_definer and coinc event tables
if ( table.StripTableName(lsctables.CoincDefTable.tableName) in tablenames ) and \
(table.StripTableName(lsctables.CoincTable.tableName) in tablenames ):
if verbose:
print "Removing redundant entries from coinc_definer and coinc_event tables ..."
# get lowest (coinc_def_ids, search_coinc_type, description ) from coinc_def table
# corresponding to unique search_coinc_type and description
min_coinc_def_ids = cursor.execute(
''' SELECT MIN(coinc_def_id), search_coinc_type, description
FROM coinc_definer GROUP BY search_coinc_type, description '''
).fetchall()
if verbose:
print "Updating coinc_def_id(s) in coinc_event table ..."
# loop over them and update coinc_event table replacing redundant coinc_def ids
for (min_coinc_def_id, min_search_coinc_type,
min_description) in min_coinc_def_ids:
cursor.execute(
'''UPDATE ''' +
table.StripTableName(lsctables.CoincTable.tableName) +
''' SET coinc_def_id = ?
WHERE coinc_def_id IN
(
SELECT coinc_def_id FROM ''' +
table.StripTableName(lsctables.CoincDefTable.tableName) +
''' WHERE ( search_coinc_type = ? ) AND ( description = ? )
)''', (
min_coinc_def_id,
min_search_coinc_type,
min_description,
))
if verbose:
print "Removing unused entries from coinc_definer table ..."
# remove redundant and already unused records from coinc_def table
cursor.execute(
'''DELETE FROM ''' +
table.StripTableName(lsctables.CoincDefTable.tableName) +
''' WHERE coinc_def_id NOT IN
(
SELECT MIN(coinc_def_id) FROM ''' +
table.StripTableName(lsctables.CoincDefTable.tableName) +
''' GROUP BY search_coinc_type, description
) ''')
connection.commit()
if verbose:
print "Finished updating tables"
def delete_glitch_events_in_segment(connection, cursor, segment):
"""
Removes glitch events whose gps times fall within segment.
All rows from the other tables in database connected to these glitch
events via coinc_map tables are also deteled.
@param connection UNDOCUMENTED
@param cursor UNDOCUMENTED
@param segment is glue segment
"""
# get table names from the database
tablenames = dbtables.get_table_names(connection)
if not tablenames:
print "No tables were found in the database. Nothing to do."
return
# check if glitch table is present
if not table.StripTableName(
idq_tables.IDQGlitchTable.tableName) in tablenames:
print "No glitch table is found in database. Nothing to do"
return
# create sqlite function for testing if glitch gos time is inside the segment("md5", 1, md5sum)
connection.create_function('gpstime_inside_segment', 3,
gpstime_inside_segment)
# convert segment inot segment string
segment_string = segmentsUtils.to_range_strings([segment])[0]
# check if coinc_def table is present
if table.StripTableName(lsctables.CoincDefTable.tableName) in tablenames:
# get all distinct coinc types from coinc_def table
coinc_descriptions = cursor.execute(
'''SELECT DISTINCT description FROM ''' +
table.StripTableName(lsctables.CoincDefTable.tableName)).fetchall(
)
# loop over coinc descriptions
for (description, ) in coinc_descriptions:
tables_in_coinc = idq_tables.CoincDefToTableNames[description]
# work only with coincs that involve glitch table
if idq_tables.IDQGlitchTable.tableName in tables_in_coinc:
# remove glitch table, we will work with it later
connected_tables = [
t for t in tables_in_coinc
if t != idq_tables.IDQGlitchTable.tableName
]
# loop over other tables in the coinc
for t in connected_tables:
# delete entries connected to glitch events that should be removed
cursor.execute('''DELETE FROM ''' + table.StripTableName(t) + \
''' WHERE event_id IN (SELECT t.event_id FROM ''' + table.StripTableName(t) + \
''' AS t JOIN coinc_event_map AS c1 ON t.event_id == c1.event_id''' + \
''' JOIN coinc_event_map AS c2 ON c1.coinc_event_id == c2.coinc_event_id''' + \
''' JOIN ''' + table.StripTableName(idq_tables.IDQGlitchTable.tableName) + \
''' AS g ON c2.event_id == g.event_id WHERE gpstime_inside_segment(g.gps, g.gps_ns, ?))''',\
(segment_string,))
# delete entries from coinc_event table corresponding to deleted events
cursor.execute('''DELETE FROM coinc_event WHERE coinc_event_id IN''' + \
''' (SELECT c1.coinc_event_id FROM coinc_event_map AS c1 JOIN ''' + \
table.StripTableName(idq_tables.IDQGlitchTable.tableName) + \
''' AS g ON c1.event_id == g.event_id WHERE gpstime_inside_segment(g.gps, g.gps_ns, ?))''',\
(segment_string,))
# delete entries from coinc_map table corresponding to deleted events
cursor.execute('''DELETE FROM coinc_event_map WHERE event_id IN''' + \
''' (SELECT c1.event_id FROM coinc_event_map AS c1''' + \
''' JOIN coinc_event_map AS c2 ON c1.coinc_event_id == c2.coinc_event_id ''' + \
''' JOIN ''' + table.StripTableName(idq_tables.IDQGlitchTable.tableName) +\
''' AS g ON c2.event_id == g.event_id WHERE gpstime_inside_segment(g.gps, g.gps_ns, ?))''',\
(segment_string,))
# delete entries in glitch table
cursor.execute(
'''DELETE FROM ''' +
table.StripTableName(idq_tables.IDQGlitchTable.tableName) +
''' WHERE gpstime_inside_segment(gps, gps_ns, ?)''',
(segment_string, ))
# commit everythings
connection.commit()
def table_from_file(f, tablename, columns=None, filt=None,
contenthandler=GWpyContentHandler, nproc=1, verbose=False):
"""Read a :class:`~glue.ligolw.table.Table` from a LIGO_LW file.
Parameters
----------
f : `file`, `str`, `CacheEntry`, `list`, `Cache`
object representing one or more files. One of
- an open `file`
- a `str` pointing to a file path on disk
- a formatted :class:`~glue.lal.CacheEntry` representing one file
- a `list` of `str` file paths
- a formatted :class:`~glue.lal.Cache` representing many files
tablename : `str`
name of the table to read.
columns : `list`, optional
list of column name strings to read, default all.
filt : `function`, optional
function by which to `filter` events. The callable must accept as
input a row of the table event and return `True`/`False`.
contenthandler : :class:`~glue.ligolw.ligolw.LIGOLWContentHandler`
SAX content handler for parsing LIGO_LW documents.
Returns
-------
table : :class:`~glue.ligolw.table.Table`
`Table` of data with given columns filled
"""
# find table class
tableclass = lsctables.TableByName[table.StripTableName(tablename)]
# allow cache multiprocessing
if nproc != 1:
return tableclass.read(f, columns=columns,
contenthandler=contenthandler,
nproc=nproc, format='cache')
# set columns to read
if columns is not None:
_oldcols = tableclass.loadcolumns
tableclass.loadcolumns = columns
# generate Document and populate
files = [fp.name if isinstance(fp, (file, GzipFile, AstroGzipFile)) else
fp for fp in file_list(f)]
xmldoc = Document()
ligolw_add(xmldoc, files, non_lsc_tables_ok=True,
contenthandler=contenthandler, verbose=verbose)
# extract table
try:
out = tableclass.get_table(xmldoc)
except ValueError:
out = lsctables.New(tableclass, columns=columns)
if verbose:
gprint('%d rows found in %s table' % (len(out), out.tableName))
if filt:
if verbose:
gprint('filtering rows ...', end=' ')
try:
out_ = out.copy()
except AttributeError:
out_ = table.new_from_template(out)
out_.extend(filter(filt, out))
out = out_
if verbose:
gprint('%d rows remaining\n' % len(out))
if columns is not None:
tableclass.loadcolumns = _oldcols
return out
def sim_and_sngl_inspirals_for_xmldoc(xmldoc):
"""Retrieve (as a generator) all of the
(sim_inspiral, (sngl_inspiral, sngl_inspiral, ... sngl_inspiral) tuples from
found coincidences in a LIGO-LW XML document."""
# Look up necessary tables.
coinc_table = ligolw_table.get_table(xmldoc,
lsctables.CoincTable.tableName)
coinc_def_table = ligolw_table.get_table(xmldoc,
lsctables.CoincDefTable.tableName)
coinc_map_table = ligolw_table.get_table(xmldoc,
lsctables.CoincMapTable.tableName)
# Look up coinc_def ids.
sim_coinc_def_id = coinc_def_table.get_coinc_def_id(
ligolw_inspinjfind.InspiralSCExactCoincDef.search,
ligolw_inspinjfind.InspiralSCExactCoincDef.search_coinc_type,
create_new=False)
def events_for_coinc_event_id(coinc_event_id):
for coinc_map in coinc_map_table:
if coinc_map.coinc_event_id == coinc_event_id:
for row in ligolw_table.get_table(xmldoc,
coinc_map.table_name):
column_name = coinc_map.event_id.column_name
if getattr(row, column_name) == coinc_map.event_id:
yield coinc_map.event_id, row
# Loop over all coinc_event <-> sim_inspiral coincs.
for sim_coinc in coinc_table:
# If this is not a coinc_event <-> sim_inspiral coinc, skip it.
if sim_coinc.coinc_def_id != sim_coinc_def_id:
continue
# Locate the sim_inspiral and coinc events.
sim_inspiral = None
coinc = None
for event_id, event in events_for_coinc_event_id(
sim_coinc.coinc_event_id):
if event_id.table_name == ligolw_table.StripTableName(
lsctables.SimInspiralTable.tableName):
if sim_inspiral is not None:
raise RuntimeError(
"Found more than one matching sim_inspiral entry")
sim_inspiral = event
elif event_id.table_name == ligolw_table.StripTableName(
lsctables.CoincTable.tableName):
if coinc is not None:
raise RuntimeError(
"Found more than one matching coinc entry")
coinc = event
else:
raise RuntimeError(
"Did not expect coincidence to contain an event of type '%s'"
% event_id.table_name)
sngl_inspirals = tuple(event for event_id, event in
events_for_coinc_event_id(coinc.coinc_event_id))
yield sim_inspiral, sngl_inspirals
def ContentHandler(xmldoc):
return ligolw.PartialLIGOLWContentHandler(xmldoc, lambda name, attrs:\
(name == ligolw.Table.tagName) and\
(table.StripTableName(attrs["Name"]) in ["segment_definer","segment_summary"]))
def ContentHandler(xmldoc):
return ligolw.PartialLIGOLWContentHandler(xmldoc, lambda name, attrs:\
(name == ligolw.Table.tagName) and\
(table.StripTableName(attrs["Name"]) in tables))
