def __init__(self, xmldoc, program=None):
#
# Find the out segments
#
self.outsegs = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program).coalesce()
#
# Find the sngl_burst table
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
#
# Get the list of process IDs we care about
#
self.process_ids = set(
self.snglbursttable.getColumnByName("process_id"))
if program is not None:
self.process_ids &= lsctables.ProcessTable.get_table(
xmldoc).get_ids_by_program(program)
#
# Find the sim_burst table, or make a fake one
#
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except:
self.simbursttable = []
#
# Find the coinc tables, or make fake ones
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
self.multibursttable = lsctables.MultiBurstTable.get_table(xmldoc)
except:
self.coinctable = []
self.coincmaptable = []
self.multibursttable = []
def __init__(self, xmldoc, program = None):
#
# Find the out segments
#
self.outsegs = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program).coalesce()
#
# Find the sngl_burst table
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
#
# Get the list of process IDs we care about
#
self.process_ids = set(self.snglbursttable.getColumnByName("process_id"))
if program is not None:
self.process_ids &= lsctables.ProcessTable.get_table(xmldoc).get_ids_by_program(program)
#
# Find the sim_burst table, or make a fake one
#
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except:
self.simbursttable = []
#
# Find the coinc tables, or make fake ones
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
self.multibursttable = lsctables.MultiBurstTable.get_table(xmldoc)
except:
self.coinctable = []
self.coincmaptable = []
self.multibursttable = []
def __init__(self, xmldoc, b_b_def, sb_b_def, si_b_def, sb_c_e_def,
sb_c_n_def, si_c_e_def, si_c_n_def, process,
livetime_program):
#
# store the process row
#
self.process = process
#
# locate the sngl_burst, time_slide and injection tables
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except ValueError:
self.simbursttable = None
try:
self.siminspiraltable = lsctables.SimInspiralTable.get_table(
xmldoc)
except ValueError:
self.siminspiraltable = None
try:
timeslidetable = lsctables.TimeSlideTable.get_table(xmldoc)
except ValueError:
timeslidetable = None
if timeslidetable is not None:
self.offsetvectors = timeslidetable.as_dict()
else:
self.offsetvectors = {}
#
# store the longest duration of a burst event
#
if self.snglbursttable:
self.longestduration = max(
self.snglbursttable.getColumnByName("duration"))
else:
self.longestduration = 0.0
#
# get a segmentlistdict indicating the times when the jobs
# that produced burst events could have produced output
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, livetime_program).coalesce()
#
# FIXME: in the future, the sim_inspiral table should
# indicate time slide at which the injection was done, like
# the sim_burst table does. for now, fake it by giving
# each one a time_slide_id attribute. this is the only
# reason why a place-holder class is required for the
# SimInspiral row type; that class can be deleted when
# this is no longer needed
#
if self.siminspiraltable is not None:
time_slide_id = ligolw_time_slide.get_time_slide_id(
xmldoc, {}.fromkeys(self.seglists, 0.0),
create_new=process,
superset_ok=True,
nonunique_ok=False)
for sim in self.siminspiraltable:
sim.time_slide_id = time_slide_id
#
# get coinc_definer rows for sim_* <--> sngl_burst coincs
# for whichever sim_* tables are present; this creates a
# coinc_definer table if the document doesn't have one
#
if self.simbursttable is not None:
self.sb_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
sb_b_def.search,
sb_b_def.search_coinc_type,
create_new=True,
description=sb_b_def.description)
else:
self.sb_b_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
si_b_def.search,
si_b_def.search_coinc_type,
create_new=True,
description=si_b_def.description)
else:
self.si_b_coinc_def_id = None
#
# get coinc_def_id's for sngl_burst <--> sngl_burst, and
# both kinds of sim_* <--> coinc_event coincs. set all to
# None if this document does not contain any sngl_burst
# <--> sngl_burst coincs.
#
try:
b_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
b_b_def.search,
b_b_def.search_coinc_type,
create_new=False)
except KeyError:
b_b_coinc_def_id = None
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
else:
if self.simbursttable is not None:
self.sb_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
sb_c_e_def.search,
sb_c_e_def.search_coinc_type,
create_new=True,
description=sb_c_e_def.description)
self.sb_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
sb_c_n_def.search,
sb_c_n_def.search_coinc_type,
create_new=True,
description=sb_c_n_def.description)
else:
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
si_c_e_def.search,
si_c_e_def.search_coinc_type,
create_new=True,
description=si_c_e_def.description)
self.si_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
si_c_n_def.search,
si_c_n_def.search_coinc_type,
create_new=True,
description=si_c_n_def.description)
else:
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
#
# get coinc table, create one if needed
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
except ValueError:
self.coinctable = lsctables.New(lsctables.CoincTable)
xmldoc.childNodes[0].appendChild(self.coinctable)
self.coinctable.sync_next_id()
#
# get coinc_map table, create one if needed
#
try:
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
except ValueError:
self.coincmaptable = lsctables.New(lsctables.CoincMapTable)
xmldoc.childNodes[0].appendChild(self.coincmaptable)
#
# index the document
#
# index sngl_burst table
index = dict((row.event_id, row) for row in self.snglbursttable)
# find IDs of burst<-->burst coincs
self.coincs = dict((row.coinc_event_id, []) for row in self.coinctable
if row.coinc_def_id == b_b_coinc_def_id)
# construct event list for each burst<-->burst coinc
for row in self.coincmaptable:
try:
self.coincs[row.coinc_event_id].append(index[row.event_id])
except KeyError:
pass
del index
# sort the event list for each coin by peak time and
# convert to tuples for speed
for coinc_event_id, events in self.coincs.items():
events.sort(key=lambda event: event.peak)
self.coincs[coinc_event_id] = tuple(events)
# convert dictionary to a list of (coinc_event_id, events)
# tuples and create a coinc_event_id to offset vector
# look-up table
self.coincs = self.coincs.items()
self.coincoffsets = dict(
(row.coinc_event_id, self.offsetvectors[row.time_slide_id])
for row in self.coinctable if row.coinc_def_id == b_b_coinc_def_id)
#
# represent the sngl_burst table as a dictionary indexed by
# instrument whose values are the event lists for those
# instruments sorted by peak time. sort the coincs list by
# the peak time of the first (earliest) event in each coinc
# (recall that the event tuple for each coinc has been
# time-ordered)
#
self.snglbursttable = dict(
(instrument,
sorted((event for event in self.snglbursttable
if event.ifo == instrument),
key=lambda event: event.peak))
for instrument in set(self.snglbursttable.getColumnByName("ifo")))
self.coincs.sort(
key=lambda coinc_id_events: coinc_id_events[1][0].peak)
def __init__(self,
connection,
live_time_program,
search="excesspower",
veto_segments_name=None):
"""
Compute and record some summary information about the
database. Call this after all the data has been inserted,
and before you want any of this information.
"""
self.connection = connection
self.xmldoc = dbtables.get_xml(connection)
# find the tables
try:
self.sngl_burst_table = lsctables.SnglBurstTable.get_table(
self.xmldoc)
except ValueError:
self.sngl_burst_table = None
try:
self.sim_burst_table = lsctables.SimBurstTable.get_table(
self.xmldoc)
except ValueError:
self.sim_burst_table = None
try:
self.coinc_def_table = lsctables.CoincDefTable.get_table(
self.xmldoc)
self.coinc_table = lsctables.CoincTable.get_table(self.xmldoc)
self.time_slide_table = lsctables.TimeSlideTable.get_table(
self.xmldoc)
except ValueError:
self.coinc_def_table = None
self.coinc_table = None
self.time_slide_table = None
try:
self.multi_burst_table = lsctables.MultiBurstTable.get_table(
self.xmldoc)
except ValueError:
self.multi_burst_table = None
# get the segment lists
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
self.xmldoc, live_time_program).coalesce()
self.instruments = set(self.seglists.keys())
if veto_segments_name is not None:
self.vetoseglists = ligolw_segments.segmenttable_get_by_name(
self.xmldoc, veto_segments_name).coalesce()
else:
self.vetoseglists = ligolw_segments.segments.segmentlistdict()
# determine a few coinc_definer IDs
# FIXME: don't hard-code the numbers
if self.coinc_def_table is not None:
try:
self.bb_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 0, create_new=False)
except KeyError:
self.bb_definer_id = None
try:
self.sb_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 1, create_new=False)
except KeyError:
self.sb_definer_id = None
try:
self.sce_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 2, create_new=False)
except KeyError:
self.sce_definer_id = None
try:
self.scn_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 3, create_new=False)
except KeyError:
self.scn_definer_id = None
else:
self.bb_definer_id = None
self.sb_definer_id = None
self.sce_definer_id = None
self.scn_definer_id = None
def bucluster(
xmldoc,
program,
process,
prefunc,
postfunc,
testfunc,
clusterfunc,
sortfunc = None,
bailoutfunc = None,
verbose = False
):
"""
Run the clustering algorithm on the list of burst candidates. The
return value is the tuple (xmldoc, changed), where xmldoc is the
input document, and changed is a boolean that is True if the
contents of the sngl_burst table were altered, and False if the
triggers were not modified by the clustering process.
If the document does not contain a sngl_burst table, then the
document is not modified (including no modifications to the process
metadata tables).
"""
#
# Extract live time segment and sngl_burst table
#
try:
sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc)
except ValueError:
# no-op: document does not contain a sngl_burst table
if verbose:
print("document does not contain a sngl_burst table, skipping ...", file=sys.stderr)
return xmldoc, False
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce()
#
# Preprocess candidates
#
if verbose:
print("pre-processing ...", file=sys.stderr)
preprocess_output = prefunc(sngl_burst_table)
#
# Cluster
#
table_changed = snglcluster.cluster_events(sngl_burst_table, testfunc, clusterfunc, sortfunc = sortfunc, bailoutfunc = bailoutfunc, verbose = verbose)
#
# Postprocess candidates
#
if verbose:
print("post-processing ...", file=sys.stderr)
postfunc(sngl_burst_table, preprocess_output)
#
# Update instrument list in process table and add search summary
# information
#
process.instruments = seglists.keys()
ligolw_search_summary.append_search_summary(xmldoc, process, inseg = seglists.extent_all(), outseg = seglists.extent_all(), nevents = len(sngl_burst_table))
#
# Done
#
return xmldoc, table_changed
#
# Load Data
#
# =============================================================================
#
summary = {}
seglists = segments.segmentlistdict()
for n, filename in enumerate(ligolw_utils.sort_files_by_size(filenames, options.verbose, reverse = True)):
if options.verbose:
print("%d/%d:" % (n + 1, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(filename, verbose = options.verbose, contenthandler = ligolw.LIGOLWContentHandler)
seglists |= ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, options.livetime_program).coalesce()
xmldoc.unlink()
#
# =============================================================================
#
# Plot
#
# =============================================================================
#
def new_plots(ifo, plots):
l = (
RateVsPeakFreq(ifo, segments.segment(options.frequency_range), 4),
#
summary = {}
seglists = segments.segmentlistdict()
for n, filename in enumerate(
ligolw_utils.sort_files_by_size(filenames,
options.verbose,
reverse=True)):
if options.verbose:
print("%d/%d:" % (n + 1, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(
filename,
verbose=options.verbose,
contenthandler=ligolw.LIGOLWContentHandler)
seglists |= ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, options.livetime_program).coalesce()
xmldoc.unlink()
#
# =============================================================================
#
# Plot
#
# =============================================================================
#
def new_plots(ifo, plots):
l = (RateVsPeakFreq(ifo, segments.segment(options.frequency_range),
4), Durations(ifo), Delays(ifo, 0.25, 20),
RateVsSNR(ifo), RateVsConfidence(ifo), ConfidenceVsTime(ifo),
def bucluster(xmldoc,
program,
process,
prefunc,
postfunc,
testfunc,
clusterfunc,
sortfunc=None,
bailoutfunc=None,
verbose=False):
"""
Run the clustering algorithm on the list of burst candidates. The
return value is the tuple (xmldoc, changed), where xmldoc is the
input document, and changed is a boolean that is True if the
contents of the sngl_burst table were altered, and False if the
triggers were not modified by the clustering process.
If the document does not contain a sngl_burst table, then the
document is not modified (including no modifications to the process
metadata tables).
"""
#
# Extract live time segment and sngl_burst table
#
try:
sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc)
except ValueError:
# no-op: document does not contain a sngl_burst table
if verbose:
print("document does not contain a sngl_burst table, skipping ...",
file=sys.stderr)
return xmldoc, False
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program=program).coalesce()
#
# Preprocess candidates
#
if verbose:
print("pre-processing ...", file=sys.stderr)
preprocess_output = prefunc(sngl_burst_table)
#
# Cluster
#
table_changed = snglcluster.cluster_events(sngl_burst_table,
testfunc,
clusterfunc,
sortfunc=sortfunc,
bailoutfunc=bailoutfunc,
verbose=verbose)
#
# Postprocess candidates
#
if verbose:
print("post-processing ...", file=sys.stderr)
postfunc(sngl_burst_table, preprocess_output)
#
# Update instrument list in process table and add search summary
# information
#
process.instruments = seglists.keys()
ligolw_search_summary.append_search_summary(xmldoc,
process,
inseg=seglists.extent_all(),
outseg=seglists.extent_all(),
nevents=len(sngl_burst_table))
#
# Done
#
return xmldoc, table_changed
def __init__(self, xmldoc, b_b_def, sb_b_def, si_b_def, sb_c_e_def, sb_c_n_def, si_c_e_def, si_c_n_def, process, livetime_program):
#
# store the process row
#
self.process = process
#
# locate the sngl_burst, time_slide and injection tables
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except ValueError:
self.simbursttable = None
try:
self.siminspiraltable = lsctables.SimInspiralTable.get_table(xmldoc)
except ValueError:
self.siminspiraltable = None
try:
timeslidetable = lsctables.TimeSlideTable.get_table(xmldoc)
except ValueError:
timeslidetable = None
if timeslidetable is not None:
self.offsetvectors = timeslidetable.as_dict()
else:
self.offsetvectors = {}
#
# store the longest duration of a burst event
#
if self.snglbursttable:
self.longestduration = max(self.snglbursttable.getColumnByName("duration"))
else:
self.longestduration = 0.0
#
# get a segmentlistdict indicating the times when the jobs
# that produced burst events could have produced output
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, livetime_program).coalesce()
#
# FIXME: in the future, the sim_inspiral table should
# indicate time slide at which the injection was done, like
# the sim_burst table does. for now, fake it by giving
# each one a time_slide_id attribute. this is the only
# reason why a place-holder class is required for the
# SimInspiral row type; that class can be deleted when
# this is no longer needed
#
if self.siminspiraltable is not None:
time_slide_id = ligolw_time_slide.get_time_slide_id(xmldoc, {}.fromkeys(self.seglists, 0.0), create_new = process, superset_ok = True, nonunique_ok = False)
for sim in self.siminspiraltable:
sim.time_slide_id = time_slide_id
#
# get coinc_definer rows for sim_* <--> sngl_burst coincs
# for whichever sim_* tables are present; this creates a
# coinc_definer table if the document doesn't have one
#
if self.simbursttable is not None:
self.sb_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, sb_b_def.search, sb_b_def.search_coinc_type, create_new = True, description = sb_b_def.description)
else:
self.sb_b_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, si_b_def.search, si_b_def.search_coinc_type, create_new = True, description = si_b_def.description)
else:
self.si_b_coinc_def_id = None
#
# get coinc_def_id's for sngl_burst <--> sngl_burst, and
# both kinds of sim_* <--> coinc_event coincs. set all to
# None if this document does not contain any sngl_burst
# <--> sngl_burst coincs.
#
try:
b_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, b_b_def.search, b_b_def.search_coinc_type, create_new = False)
except KeyError:
b_b_coinc_def_id = None
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
else:
if self.simbursttable is not None:
self.sb_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, sb_c_e_def.search, sb_c_e_def.search_coinc_type, create_new = True, description = sb_c_e_def.description)
self.sb_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, sb_c_n_def.search, sb_c_n_def.search_coinc_type, create_new = True, description = sb_c_n_def.description)
else:
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, si_c_e_def.search, si_c_e_def.search_coinc_type, create_new = True, description = si_c_e_def.description)
self.si_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, si_c_n_def.search, si_c_n_def.search_coinc_type, create_new = True, description = si_c_n_def.description)
else:
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
#
# get coinc table, create one if needed
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
except ValueError:
self.coinctable = lsctables.New(lsctables.CoincTable)
xmldoc.childNodes[0].appendChild(self.coinctable)
self.coinctable.sync_next_id()
#
# get coinc_map table, create one if needed
#
try:
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
except ValueError:
self.coincmaptable = lsctables.New(lsctables.CoincMapTable)
xmldoc.childNodes[0].appendChild(self.coincmaptable)
#
# index the document
#
# index sngl_burst table
index = dict((row.event_id, row) for row in self.snglbursttable)
# find IDs of burst<-->burst coincs
self.coincs = dict((row.coinc_event_id, []) for row in self.coinctable if row.coinc_def_id == b_b_coinc_def_id)
# construct event list for each burst<-->burst coinc
for row in self.coincmaptable:
try:
self.coincs[row.coinc_event_id].append(index[row.event_id])
except KeyError:
pass
del index
# sort the event list for each coin by peak time and
# convert to tuples for speed
for coinc_event_id, events in self.coincs.items():
events.sort(key = lambda event: event.peak)
self.coincs[coinc_event_id] = tuple(events)
# convert dictionary to a list of (coinc_event_id, events)
# tuples and create a coinc_event_id to offset vector
# look-up table
self.coincs = self.coincs.items()
self.coincoffsets = dict((row.coinc_event_id, self.offsetvectors[row.time_slide_id]) for row in self.coinctable if row.coinc_def_id == b_b_coinc_def_id)
#
# represent the sngl_burst table as a dictionary indexed by
# instrument whose values are the event lists for those
# instruments sorted by peak time. sort the coincs list by
# the peak time of the first (earliest) event in each coinc
# (recall that the event tuple for each coinc has been
# time-ordered)
#
self.snglbursttable = dict((instrument, sorted((event for event in self.snglbursttable if event.ifo == instrument), key = lambda event: event.peak)) for instrument in set(self.snglbursttable.getColumnByName("ifo")))
self.coincs.sort(key = lambda coinc_id_events: coinc_id_events[1][0].peak)