def get_segments(connection,
xmldoc,
table_name,
live_time_program,
veto_segments_name=None,
data_segments_name="datasegments"):
segs = segments.segmentlistdict()
if table_name == dbtables.lsctables.CoincInspiralTable.tableName:
if live_time_program == "gstlal_inspiral":
segs = ligolw_segments.segmenttable_get_by_name(
xmldoc, data_segments_name).coalesce()
segs &= ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, live_time_program).coalesce()
elif live_time_program == "thinca":
segs = db_thinca_rings.get_thinca_zero_lag_segments(
connection, program_name=live_time_program).coalesce()
else:
raise ValueError(
"for burst tables livetime program must be one of gstlal_inspiral, thinca"
)
if veto_segments_name is not None:
veto_segs = db_thinca_rings.get_veto_segments(
connection, veto_segments_name)
segs -= veto_segs
return segs
elif table_name == dbtables.lsctables.CoincRingdownTable.tableName:
segs = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, live_time_program).coalesce()
if veto_segments_name is not None:
veto_segs = ligolw_segments.segmenttable_get_by_name(
xmldoc, veto_segments_name).coalesce()
segs -= veto_segs
return segs
elif table_name == dbtables.lsctables.MultiBurstTable.tableName:
if live_time_program == "omega_to_coinc":
segs = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, live_time_program).coalesce()
if veto_segments_name is not None:
veto_segs = ligolw_segments.segmenttable_get_by_name(
xmldoc, veto_segments_name).coalesce()
segs -= veto_segs
elif live_time_program == "waveburst":
segs = db_thinca_rings.get_thinca_zero_lag_segments(
connection, program_name=live_time_program).coalesce()
if veto_segments_name is not None:
veto_segs = db_thinca_rings.get_veto_segments(
connection, veto_segments_name)
segs -= veto_segs
else:
raise ValueError(
"for burst tables livetime program must be one of omega_to_coinc, waveburst"
)
return segs
else:
raise ValueError("table must be in " +
" ".join(allowed_analysis_table_names()))
def __init__(self, xmldoc, vetoes = None, program = u"lalapps_ring"):
snglcoinc.CoincTables.__init__(self, xmldoc)
#
# create a string uniquifier
#
self.uniquifier = {}
#
# find the coinc_ringdown table or create one if not found
#
try:
self.coinc_ringdown_table = lsctables.table.get_table(xmldoc, lsctables.CoincRingdownTable.tableName)
except ValueError:
self.coinc_ringdown_table = lsctables.New(lsctables.CoincRingdownTable)
xmldoc.childNodes[0].appendChild(self.coinc_ringdown_table)
#
# extract the coalesced out segment lists from lalapps_ring
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce()
if vetoes is not None:
self.seglists -= vetoes
def retrieve_ring_boundaries(xmldoc): # # grab the segment list for any instrument selected at random (they # are all the same) # rings = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = "thinca").popitem()[1] # # because the input often contains two thinca jobs the rings might # be duplicated; use set() to uniqueify them then sort them. # rings = segments.segmentlist(set(rings)) rings.sort() # # check that the (sorted) rings are non-intersecting # for i in range(len(rings) - 1): if rings[i].intersects(rings[i + 1]): raise ValueError, "non-disjoint thinca rings detected in search_summary table" # # cast to int to prevent explosions later # for i, ring in enumerate(rings): rings[i] = segments.segment(int(ring[0]), int(ring[1])) # # done # return rings
def __init__(self, xmldoc, vetoes = None, program = u"inspiral", likelihood_func = None, likelihood_params_func = None):
snglcoinc.CoincTables.__init__(self, xmldoc)
#
# configure the likelihood ratio evaluator
#
if likelihood_func is None and likelihood_params_func is not None or likelihood_func is not None and likelihood_params_func is None:
raise ValueError("must provide both a likelihood function and a parameter function or neither")
self.likelihood_func = likelihood_func
self.likelihood_params_func = likelihood_params_func
#
# create a string uniquifier
#
self.uniquifier = {}
#
# find the coinc_inspiral table or create one if not found
#
try:
self.coinc_inspiral_table = lsctables.CoincInspiralTable.get_table(xmldoc)
except ValueError:
self.coinc_inspiral_table = lsctables.New(lsctables.CoincInspiralTable)
xmldoc.childNodes[0].appendChild(self.coinc_inspiral_table)
#
# extract the coalesced out segment lists from the trigger generator
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce()
if vetoes is not None:
self.seglists -= vetoes
def retrieve_ring_boundaries(xmldoc): # # grab the segment list for any instrument selected at random (they # are all the same) # rings = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = "thinca").popitem()[1] # # because the input often contains two thinca jobs the rings might # be duplicated; use set() to uniqueify them then sort them. # rings = segments.segmentlist(set(rings)) rings.sort() # # check that the (sorted) rings are non-intersecting # for i in range(len(rings) - 1): if rings[i].intersects(rings[i + 1]): raise ValueError, "non-disjoint thinca rings detected in search_summary table" # # cast to int to prevent explosions later # for i, ring in enumerate(rings): rings[i] = segments.segment(int(ring[0]), int(ring[1])) # # done # return rings
def get_segments(connection, xmldoc, program_name): seglists = segments.segmentlistdict() if program_name == "thinca": seglists = db_thinca_rings.get_thinca_zero_lag_segments(connection, program_name) if program_name == "gstlal_inspiral" or program_name == "lalapps_ring": seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program_name).coalesce() return seglists
def __init__(self, xmldoc, vetoes=None, program=u"lalapps_ring"):
snglcoinc.CoincTables.__init__(self, xmldoc)
#
# create a string uniquifier
#
self.uniquifier = {}
#
# find the coinc_ringdown table or create one if not found
#
try:
self.coinc_ringdown_table = lsctables.CoincRingdownTable.table.get_table(
xmldoc)
except ValueError:
self.coinc_ringdown_table = lsctables.New(
lsctables.CoincRingdownTable)
xmldoc.childNodes[0].appendChild(self.coinc_ringdown_table)
#
# extract the coalesced out segment lists from lalapps_ring
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program=program).coalesce()
if vetoes is not None:
self.seglists -= vetoes
def __init__(self, xmldoc, vetoes = None, program = u"inspiral", likelihood_func = None, likelihood_params_func = None):
snglcoinc.CoincTables.__init__(self, xmldoc)
#
# configure the likelihood ratio evaluator
#
if likelihood_func is None and likelihood_params_func is not None or likelihood_func is not None and likelihood_params_func is None:
raise ValueError("must provide both a likelihood function and a parameter function or neither")
self.likelihood_func = likelihood_func
self.likelihood_params_func = likelihood_params_func
#
# create a string uniquifier
#
self.uniquifier = {}
#
# find the coinc_inspiral table or create one if not found
#
try:
self.coinc_inspiral_table = lsctables.table.get_table(xmldoc, lsctables.CoincInspiralTable.tableName)
except ValueError:
self.coinc_inspiral_table = lsctables.New(lsctables.CoincInspiralTable)
xmldoc.childNodes[0].appendChild(self.coinc_inspiral_table)
#
# extract the coalesced out segment lists from the trigger generator
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce()
if vetoes is not None:
self.seglists -= vetoes
def get_segments(connection, xmldoc, table_name, live_time_program, veto_segments_name = None, data_segments_name = "datasegments"):
from pylal import db_thinca_rings
segs = segments.segmentlistdict()
if table_name == dbtables.lsctables.CoincInspiralTable.tableName:
if live_time_program == "gstlal_inspiral":
segs = ligolw_segments.segmenttable_get_by_name(xmldoc, data_segments_name).coalesce()
segs &= ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, live_time_program).coalesce()
elif live_time_program == "thinca":
segs = db_thinca_rings.get_thinca_zero_lag_segments(connection, program_name = live_time_program).coalesce()
else:
raise ValueError("for burst tables livetime program must be one of gstlal_inspiral, thinca")
if veto_segments_name is not None:
veto_segs = db_thinca_rings.get_veto_segments(connection, veto_segments_name)
segs -= veto_segs
return segs
elif table_name == dbtables.lsctables.CoincRingdownTable.tableName:
segs = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, live_time_program).coalesce()
if veto_segments_name is not None:
veto_segs = ligolw_segments.segmenttable_get_by_name(xmldoc, veto_segments_name).coalesce()
segs -= veto_segs
return segs
elif table_name == dbtables.lsctables.MultiBurstTable.tableName:
if live_time_program == "omega_to_coinc":
segs = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, live_time_program).coalesce()
if veto_segments_name is not None:
veto_segs = ligolw_segments.segmenttable_get_by_name(xmldoc, veto_segments_name).coalesce()
segs -= veto_segs
elif live_time_program == "waveburst":
segs = db_thinca_rings.get_thinca_zero_lag_segments(connection, program_name = live_time_program).coalesce()
if veto_segments_name is not None:
veto_segs = db_thinca_rings.get_veto_segments(connection, veto_segments_name)
segs -= veto_segs
else:
raise ValueError("for burst tables livetime program must be one of omega_to_coinc, waveburst")
return segs
else:
raise ValueError("table must be in " + " ".join(allowed_analysis_table_names()))
def get_segs_from_doc(doc):
"""
Return the segments from a document
@param doc: document containing the desired segments
"""
# get segments
seg_dict = ligolw_search_summary.segmentlistdict_fromsearchsummary(doc)
segs = seg_dict.union(seg_dict.iterkeys()).coalesce()
# typecast to ints, which are better behaved and faster than LIGOTimeGPS
segs = segments.segmentlist([segments.segment(int(seg[0]), int(seg[1]))\
for seg in segs])
return segs
def get_segs_from_doc(doc):
"""
Return the segments from a document
@param doc: document containing the desired segments
"""
# get segments
seg_dict = ligolw_search_summary.segmentlistdict_fromsearchsummary(doc)
segs = seg_dict.union(seg_dict.iterkeys()).coalesce()
# typecast to ints, which are better behaved and faster than LIGOTimeGPS
segs = segments.segmentlist([segments.segment(int(seg[0]), int(seg[1]))\
for seg in segs])
return segs
def __init__(self, xmldoc, program=None):
#
# Find the out segments
#
self.outsegs = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program).coalesce()
#
# Find the sngl_burst table
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
#
# Get the list of process IDs we care about
#
self.process_ids = set(
self.snglbursttable.getColumnByName("process_id"))
if program is not None:
self.process_ids &= lsctables.ProcessTable.get_table(
xmldoc).get_ids_by_program(program)
#
# Find the sim_burst table, or make a fake one
#
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except:
self.simbursttable = []
#
# Find the coinc tables, or make fake ones
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
self.multibursttable = lsctables.MultiBurstTable.get_table(xmldoc)
except:
self.coinctable = []
self.coincmaptable = []
self.multibursttable = []
def get_rinca_zero_lag_segments(connection, program_name="rinca"):
"""
Return the rinca rings from the database at the given connection. The
rings are returned as a coalesced glue.segments.segmentlistdict indexed
by instrument.
Example:
>>> seglists = get_rinca_zero_lag_segments(connection)
>>> print seglists.keys()
['H1', 'L1']
This function is most useful if only zero-lag segments are desired
because it allows for more convenient manipulation of the segment lists
using the methods in glue.segments. If information about background
segments or the original ring boundaries is desired the data returned by
get_rinca_rings_by_available_instruments() is required.
"""
# extract the raw rings indexed by instrument
xmldoc = dbtables.get_xml(connection)
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program_name)
xmldoc.unlink()
# remove rings that are exact duplicates on the assumption that there are
# zero-lag and time-slide rinca jobs represented in the same document
seglists = segments.segmentlistdict((key, segments.segmentlist(set(value)))
for key, value in seglists.items())
# coalesce the remaining segments making sure we don't loose livetime in
# the process
durations_before = abs(seglists)
seglists.coalesce()
if abs(seglists) != durations_before:
raise ValueError, "detected overlapping rinca rings"
# done
return seglists
def __init__(self, xmldoc, program = None):
#
# Find the out segments
#
self.outsegs = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program).coalesce()
#
# Find the sngl_burst table
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
#
# Get the list of process IDs we care about
#
self.process_ids = set(self.snglbursttable.getColumnByName("process_id"))
if program is not None:
self.process_ids &= lsctables.ProcessTable.get_table(xmldoc).get_ids_by_program(program)
#
# Find the sim_burst table, or make a fake one
#
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except:
self.simbursttable = []
#
# Find the coinc tables, or make fake ones
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
self.multibursttable = lsctables.MultiBurstTable.get_table(xmldoc)
except:
self.coinctable = []
self.coincmaptable = []
self.multibursttable = []
def get_thinca_zero_lag_segments(connection, program_name = "thinca"):
"""
Return the thinca rings from the database at the given connection. The
rings are returned as a coalesced glue.segments.segmentlistdict indexed
by instrument.
Example:
>>> seglists = get_thinca_zero_lag_segments(connection)
>>> print seglists.keys()
['H1', 'L1']
This function is most useful if only zero-lag segments are desired
because it allows for more convenient manipulation of the segment lists
using the methods in glue.segments. If information about background
segments or the original ring boundaries is desired the data returned by
get_thinca_rings_by_available_instruments() is required.
"""
# extract the raw rings indexed by instrument
xmldoc = dbtables.get_xml(connection)
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program_name)
xmldoc.unlink()
# remove rings that are exact duplicates on the assumption that there are
# zero-lag and time-slide thinca jobs represented in the same document
seglists = segments.segmentlistdict((key, segments.segmentlist(set(value))) for key, value in seglists.items())
# coalesce the remaining segments making sure we don't loose livetime in
# the process
durations_before = abs(seglists)
seglists.coalesce()
if abs(seglists) != durations_before:
raise ValueError, "detected overlapping thinca rings"
# done
return seglists
def GetSegListFromSearchSummaries(fileList, verbose=False):
"""
Read segment lists from search summary tables
@param fileList: list of input files.
"""
required_tables = [lsctables.SearchSummaryTable, lsctables.ProcessTable]
segList = segments.segmentlistdict()
for thisFile in fileList:
doc = ReadTablesFromFiles([thisFile], required_tables, verbose)
try:
segs = ligolw_search_summary.segmentlistdict_fromsearchsummary(doc)
except:
raise ValueError, "Cannot extract segments from the SearchSummaryTable of %s" % thisFile
#Now add these segments to the existing list
segList.extend(segs)
for value in segList.values():
value.sort()
return segList
def GetSegListFromSearchSummaries(fileList, verbose=False):
"""
Read segment lists from search summary tables
@param fileList: list of input files.
"""
required_tables = [lsctables.SearchSummaryTable, lsctables.ProcessTable]
segList = segments.segmentlistdict()
for thisFile in fileList:
doc = ReadTablesFromFiles([thisFile], required_tables, verbose)
try:
segs = ligolw_search_summary.segmentlistdict_fromsearchsummary(doc)
except:
raise ValueError, "Cannot extract segments from the SearchSummaryTable of %s" % thisFile
#Now add these segments to the existing list
segList.extend(segs)
for value in segList.values():
value.sort()
return segList
#
# Load Data
#
# =============================================================================
#
summary = {}
seglists = segments.segmentlistdict()
for n, filename in enumerate(ligolw_utils.sort_files_by_size(filenames, options.verbose, reverse = True)):
if options.verbose:
print >>sys.stderr, "%d/%d:" % (n + 1, len(filenames)),
xmldoc = ligolw_utils.load_filename(filename, verbose = options.verbose, contenthandler = ligolw.LIGOLWContentHandler)
seglists |= ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, options.livetime_program).coalesce()
xmldoc.unlink()
#
# =============================================================================
#
# Plot
#
# =============================================================================
#
def new_plots(ifo, plots):
l = (
RateVsPeakFreq(ifo, segments.segment(options.frequency_range), 4),
def __init__(self, xmldoc, b_b_def, sb_b_def, si_b_def, sb_c_e_def, sb_c_n_def, si_c_e_def, si_c_n_def, process, livetime_program):
#
# store the process row
#
self.process = process
#
# locate the sngl_burst, time_slide and injection tables
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except ValueError:
self.simbursttable = None
try:
self.siminspiraltable = lsctables.SimInspiralTable.get_table(xmldoc)
except ValueError:
self.siminspiraltable = None
try:
timeslidetable = lsctables.TimeSlideTable.get_table(xmldoc)
except ValueError:
timeslidetable = None
if timeslidetable is not None:
self.offsetvectors = timeslidetable.as_dict()
else:
self.offsetvectors = {}
#
# store the longest duration of a burst event
#
if self.snglbursttable:
self.longestduration = max(self.snglbursttable.getColumnByName("duration"))
else:
self.longestduration = 0.0
#
# get a segmentlistdict indicating the times when the jobs
# that produced burst events could have produced output
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, livetime_program).coalesce()
#
# FIXME: in the future, the sim_inspiral table should
# indicate time slide at which the injection was done, like
# the sim_burst table does. for now, fake it by giving
# each one a time_slide_id attribute. this is the only
# reason why a place-holder class is required for the
# SimInspiral row type; that class can be deleted when
# this is no longer needed
#
if self.siminspiraltable is not None:
time_slide_id = ligolw_time_slide.get_time_slide_id(xmldoc, {}.fromkeys(self.seglists, 0.0), create_new = process, superset_ok = True, nonunique_ok = False)
for sim in self.siminspiraltable:
sim.time_slide_id = time_slide_id
#
# get coinc_definer rows for sim_* <--> sngl_burst coincs
# for whichever sim_* tables are present; this creates a
# coinc_definer table if the document doesn't have one
#
if self.simbursttable is not None:
self.sb_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, sb_b_def.search, sb_b_def.search_coinc_type, create_new = True, description = sb_b_def.description)
else:
self.sb_b_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, si_b_def.search, si_b_def.search_coinc_type, create_new = True, description = si_b_def.description)
else:
self.si_b_coinc_def_id = None
#
# get coinc_def_id's for sngl_burst <--> sngl_burst, and
# both kinds of sim_* <--> coinc_event coincs. set all to
# None if this document does not contain any sngl_burst
# <--> sngl_burst coincs.
#
try:
b_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, b_b_def.search, b_b_def.search_coinc_type, create_new = False)
except KeyError:
b_b_coinc_def_id = None
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
else:
if self.simbursttable is not None:
self.sb_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, sb_c_e_def.search, sb_c_e_def.search_coinc_type, create_new = True, description = sb_c_e_def.description)
self.sb_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, sb_c_n_def.search, sb_c_n_def.search_coinc_type, create_new = True, description = sb_c_n_def.description)
else:
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, si_c_e_def.search, si_c_e_def.search_coinc_type, create_new = True, description = si_c_e_def.description)
self.si_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(xmldoc, si_c_n_def.search, si_c_n_def.search_coinc_type, create_new = True, description = si_c_n_def.description)
else:
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
#
# get coinc table, create one if needed
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
except ValueError:
self.coinctable = lsctables.New(lsctables.CoincTable)
xmldoc.childNodes[0].appendChild(self.coinctable)
self.coinctable.sync_next_id()
#
# get coinc_map table, create one if needed
#
try:
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
except ValueError:
self.coincmaptable = lsctables.New(lsctables.CoincMapTable)
xmldoc.childNodes[0].appendChild(self.coincmaptable)
#
# index the document
#
# index sngl_burst table
index = dict((row.event_id, row) for row in self.snglbursttable)
# find IDs of burst<-->burst coincs
self.coincs = dict((row.coinc_event_id, []) for row in self.coinctable if row.coinc_def_id == b_b_coinc_def_id)
# construct event list for each burst<-->burst coinc
for row in self.coincmaptable:
try:
self.coincs[row.coinc_event_id].append(index[row.event_id])
except KeyError:
pass
del index
# sort the event list for each coin by peak time and
# convert to tuples for speed
for coinc_event_id, events in self.coincs.items():
events.sort(lambda a, b: cmp(a.peak_time, b.peak_time) or cmp(a.peak_time_ns, b.peak_time_ns))
self.coincs[coinc_event_id] = tuple(events)
# convert dictionary to a list of (coinc_event_id, events)
# tuples and create a coinc_event_id to offset vector
# look-up table
self.coincs = self.coincs.items()
self.coincoffsets = dict((row.coinc_event_id, self.offsetvectors[row.time_slide_id]) for row in self.coinctable if row.coinc_def_id == b_b_coinc_def_id)
#
# represent the sngl_burst table as a dictionary indexed by
# instrument whose values are the event lists for those
# instruments sorted by peak time. sort the coincs list by
# the peak time of the first (earliest) event in each coinc
# (recall that the event tuple for each coinc has been
# time-ordered)
#
self.snglbursttable = dict((instrument, sorted((event for event in self.snglbursttable if event.ifo == instrument), lambda a, b: cmp(a.peak_time, b.peak_time) or cmp(a.peak_time_ns, b.peak_time_ns))) for instrument in set(self.snglbursttable.getColumnByName("ifo")))
self.coincs.sort(lambda (id_a, events_a), (id_b, events_b): cmp(events_a[0].peak_time, events_b[0].peak_time) or cmp(events_a[0].peak_time_ns, events_b[0].peak_time_ns))
def bucluster( xmldoc, program, process, prefunc, postfunc, testfunc, clusterfunc, sortfunc = None, bailoutfunc = None, verbose = False ): """ Run the clustering algorithm on the list of burst candidates. The return value is the tuple (xmldoc, changed), where xmldoc is the input document, and changed is a boolean that is True if the contents of the sngl_burst table were altered, and False if the triggers were not modified by the clustering process. If the document does not contain a sngl_burst table, then the document is not modified (including no modifications to the process metadata tables). """ # # Extract live time segment and sngl_burst table # try: sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc) except ValueError: # no-op: document does not contain a sngl_burst table if verbose: print >>sys.stderr, "document does not contain a sngl_burst table, skipping ..." return xmldoc, False seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce() # # Preprocess candidates # if verbose: print >>sys.stderr, "pre-processing ..." preprocess_output = prefunc(sngl_burst_table) # # Cluster # table_changed = snglcluster.cluster_events(sngl_burst_table, testfunc, clusterfunc, sortfunc = sortfunc, bailoutfunc = bailoutfunc, verbose = verbose) # # Postprocess candidates # if verbose: print >>sys.stderr, "post-processing ..." postfunc(sngl_burst_table, preprocess_output) # # Update instrument list in process table and add search summary # information # process.instruments = seglists.keys() ligolw_search_summary.append_search_summary(xmldoc, process, inseg = seglists.extent_all(), outseg = seglists.extent_all(), nevents = len(sngl_burst_table)) # # Done # return xmldoc, table_changed
def bucluster(xmldoc,
program,
process,
prefunc,
postfunc,
testfunc,
clusterfunc,
sortfunc=None,
bailoutfunc=None,
verbose=False):
"""
Run the clustering algorithm on the list of burst candidates. The
return value is the tuple (xmldoc, changed), where xmldoc is the
input document, and changed is a boolean that is True if the
contents of the sngl_burst table were altered, and False if the
triggers were not modified by the clustering process.
If the document does not contain a sngl_burst table, then the
document is not modified (including no modifications to the process
metadata tables).
"""
#
# Extract live time segment and sngl_burst table
#
try:
sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc)
except ValueError:
# no-op: document does not contain a sngl_burst table
if verbose:
print >> sys.stderr, "document does not contain a sngl_burst table, skipping ..."
return xmldoc, False
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program=program).coalesce()
#
# Preprocess candidates
#
if verbose:
print >> sys.stderr, "pre-processing ..."
preprocess_output = prefunc(sngl_burst_table)
#
# Cluster
#
table_changed = snglcluster.cluster_events(sngl_burst_table,
testfunc,
clusterfunc,
sortfunc=sortfunc,
bailoutfunc=bailoutfunc,
verbose=verbose)
#
# Postprocess candidates
#
if verbose:
print >> sys.stderr, "post-processing ..."
postfunc(sngl_burst_table, preprocess_output)
#
# Update instrument list in process table and add search summary
# information
#
process.instruments = seglists.keys()
ligolw_search_summary.append_search_summary(xmldoc,
process,
inseg=seglists.extent_all(),
outseg=seglists.extent_all(),
nevents=len(sngl_burst_table))
#
# Done
#
return xmldoc, table_changed
#
summary = {}
seglists = segments.segmentlistdict()
for n, filename in enumerate(
ligolw_utils.sort_files_by_size(filenames,
options.verbose,
reverse=True)):
if options.verbose:
print >> sys.stderr, "%d/%d:" % (n + 1, len(filenames)),
xmldoc = ligolw_utils.load_filename(
filename,
verbose=options.verbose,
contenthandler=ligolw.LIGOLWContentHandler)
seglists |= ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, options.livetime_program).coalesce()
xmldoc.unlink()
#
# =============================================================================
#
# Plot
#
# =============================================================================
#
def new_plots(ifo, plots):
l = (RateVsPeakFreq(ifo, segments.segment(options.frequency_range),
4), Durations(ifo), Delays(ifo, 0.25, 20),
RateVsSNR(ifo), RateVsConfidence(ifo), ConfidenceVsTime(ifo),
def ligolw_bucluster(xmldoc,
program,
process,
prefunc,
postfunc,
testfunc,
clusterfunc,
sortfunc=None,
bailoutfunc=None,
verbose=False):
"""
Run the clustering algorithm on the list of burst candidates. The
return value is the tuple (xmldoc, changed), where xmldoc is the
input document, and changed is a boolean that is True if the
contents of the sngl_burst table were altered, and False if the
triggers were not modified by the clustering process.
If the document does not contain a sngl_burst table, then the
document is not modified (including no modifications to the process
metadata tables).
"""
#
# Extract live time segment and sngl_burst table
#
try:
sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc)
except ValueError:
# no-op: document does not contain a sngl_burst table
if verbose:
print >> sys.stderr, "document does not contain a sngl_burst table, skipping ..."
return xmldoc, False
seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, program=program).coalesce()
#
# Remove all H2 triggers intersecting the frequency band
# 1138.6 Hz -- 1216.0 Hz
#
# FIXME: put this into the excess power pipeline, correctly
#
#bad_band = segments.segment(1138.586956521739, 1216.0326086956522)
#for i in xrange(len(sngl_burst_table) - 1, -1, -1):
# a = sngl_burst_table[i]
# if a.ifo == "H2" and a.band.intersects(bad_band):
# del sngl_burst_table[i]
#
# Preprocess candidates
#
if verbose:
print >> sys.stderr, "pre-processing ..."
preprocess_output = prefunc(sngl_burst_table)
#
# Cluster
#
table_changed = snglcluster.cluster_events(sngl_burst_table,
testfunc,
clusterfunc,
sortfunc=sortfunc,
bailoutfunc=bailoutfunc,
verbose=verbose)
#
# Postprocess candidates
#
if verbose:
print >> sys.stderr, "post-processing ..."
postfunc(sngl_burst_table, preprocess_output)
#
# Update instrument list in process table and add search summary
# information
#
process.instruments = seglists.keys()
ligolw_search_summary.append_search_summary(xmldoc,
process,
inseg=seglists.extent_all(),
outseg=seglists.extent_all(),
nevents=len(sngl_burst_table))
#
# Done
#
return xmldoc, table_changed
def ligolw_bucluster( xmldoc, program, process, prefunc, postfunc, testfunc, clusterfunc, sortfunc = None, bailoutfunc = None, verbose = False ): """ Run the clustering algorithm on the list of burst candidates. The return value is the tuple (xmldoc, changed), where xmldoc is the input document, and changed is a boolean that is True if the contents of the sngl_burst table were altered, and False if the triggers were not modified by the clustering process. If the document does not contain a sngl_burst table, then the document is not modified (including no modifications to the process metadata tables). """ # # Extract live time segment and sngl_burst table # try: sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc) except ValueError: # no-op: document does not contain a sngl_burst table if verbose: print >>sys.stderr, "document does not contain a sngl_burst table, skipping ..." return xmldoc, False seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce() # # Remove all H2 triggers intersecting the frequency band # 1138.6 Hz -- 1216.0 Hz # # FIXME: put this into the excess power pipeline, correctly # #bad_band = segments.segment(1138.586956521739, 1216.0326086956522) #for i in xrange(len(sngl_burst_table) - 1, -1, -1): # a = sngl_burst_table[i] # if a.ifo == "H2" and a.band.intersects(bad_band): # del sngl_burst_table[i] # # Preprocess candidates # if verbose: print >>sys.stderr, "pre-processing ..." preprocess_output = prefunc(sngl_burst_table) # # Cluster # table_changed = snglcluster.cluster_events(sngl_burst_table, testfunc, clusterfunc, sortfunc = sortfunc, bailoutfunc = bailoutfunc, verbose = verbose) # # Postprocess candidates # if verbose: print >>sys.stderr, "post-processing ..." postfunc(sngl_burst_table, preprocess_output) # # Update instrument list in process table and add search summary # information # process.instruments = seglists.keys() ligolw_search_summary.append_search_summary(xmldoc, process, inseg = seglists.extent_all(), outseg = seglists.extent_all(), nevents = len(sngl_burst_table)) # # Done # return xmldoc, table_changed
def __init__(self, xmldoc, b_b_def, sb_b_def, si_b_def, sb_c_e_def,
sb_c_n_def, si_c_e_def, si_c_n_def, process,
livetime_program):
#
# store the process row
#
self.process = process
#
# locate the sngl_burst, time_slide and injection tables
#
self.snglbursttable = lsctables.SnglBurstTable.get_table(xmldoc)
try:
self.simbursttable = lsctables.SimBurstTable.get_table(xmldoc)
except ValueError:
self.simbursttable = None
try:
self.siminspiraltable = lsctables.SimInspiralTable.get_table(
xmldoc)
except ValueError:
self.siminspiraltable = None
try:
timeslidetable = lsctables.TimeSlideTable.get_table(xmldoc)
except ValueError:
timeslidetable = None
if timeslidetable is not None:
self.offsetvectors = timeslidetable.as_dict()
else:
self.offsetvectors = {}
#
# store the longest duration of a burst event
#
if self.snglbursttable:
self.longestduration = max(
self.snglbursttable.getColumnByName("duration"))
else:
self.longestduration = 0.0
#
# get a segmentlistdict indicating the times when the jobs
# that produced burst events could have produced output
#
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
xmldoc, livetime_program).coalesce()
#
# FIXME: in the future, the sim_inspiral table should
# indicate time slide at which the injection was done, like
# the sim_burst table does. for now, fake it by giving
# each one a time_slide_id attribute. this is the only
# reason why a place-holder class is required for the
# SimInspiral row type; that class can be deleted when
# this is no longer needed
#
if self.siminspiraltable is not None:
time_slide_id = ligolw_time_slide.get_time_slide_id(
xmldoc, {}.fromkeys(self.seglists, 0.0),
create_new=process,
superset_ok=True,
nonunique_ok=False)
for sim in self.siminspiraltable:
sim.time_slide_id = time_slide_id
#
# get coinc_definer rows for sim_* <--> sngl_burst coincs
# for whichever sim_* tables are present; this creates a
# coinc_definer table if the document doesn't have one
#
if self.simbursttable is not None:
self.sb_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
sb_b_def.search,
sb_b_def.search_coinc_type,
create_new=True,
description=sb_b_def.description)
else:
self.sb_b_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
si_b_def.search,
si_b_def.search_coinc_type,
create_new=True,
description=si_b_def.description)
else:
self.si_b_coinc_def_id = None
#
# get coinc_def_id's for sngl_burst <--> sngl_burst, and
# both kinds of sim_* <--> coinc_event coincs. set all to
# None if this document does not contain any sngl_burst
# <--> sngl_burst coincs.
#
try:
b_b_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
b_b_def.search,
b_b_def.search_coinc_type,
create_new=False)
except KeyError:
b_b_coinc_def_id = None
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
else:
if self.simbursttable is not None:
self.sb_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
sb_c_e_def.search,
sb_c_e_def.search_coinc_type,
create_new=True,
description=sb_c_e_def.description)
self.sb_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
sb_c_n_def.search,
sb_c_n_def.search_coinc_type,
create_new=True,
description=sb_c_n_def.description)
else:
self.sb_c_e_coinc_def_id = None
self.sb_c_n_coinc_def_id = None
if self.siminspiraltable is not None:
self.si_c_e_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
si_c_e_def.search,
si_c_e_def.search_coinc_type,
create_new=True,
description=si_c_e_def.description)
self.si_c_n_coinc_def_id = ligolw_coincs.get_coinc_def_id(
xmldoc,
si_c_n_def.search,
si_c_n_def.search_coinc_type,
create_new=True,
description=si_c_n_def.description)
else:
self.si_c_e_coinc_def_id = None
self.si_c_n_coinc_def_id = None
#
# get coinc table, create one if needed
#
try:
self.coinctable = lsctables.CoincTable.get_table(xmldoc)
except ValueError:
self.coinctable = lsctables.New(lsctables.CoincTable)
xmldoc.childNodes[0].appendChild(self.coinctable)
self.coinctable.sync_next_id()
#
# get coinc_map table, create one if needed
#
try:
self.coincmaptable = lsctables.CoincMapTable.get_table(xmldoc)
except ValueError:
self.coincmaptable = lsctables.New(lsctables.CoincMapTable)
xmldoc.childNodes[0].appendChild(self.coincmaptable)
#
# index the document
#
# index sngl_burst table
index = dict((row.event_id, row) for row in self.snglbursttable)
# find IDs of burst<-->burst coincs
self.coincs = dict((row.coinc_event_id, []) for row in self.coinctable
if row.coinc_def_id == b_b_coinc_def_id)
# construct event list for each burst<-->burst coinc
for row in self.coincmaptable:
try:
self.coincs[row.coinc_event_id].append(index[row.event_id])
except KeyError:
pass
del index
# sort the event list for each coin by peak time and
# convert to tuples for speed
for coinc_event_id, events in self.coincs.items():
events.sort(key=lambda event: event.peak)
self.coincs[coinc_event_id] = tuple(events)
# convert dictionary to a list of (coinc_event_id, events)
# tuples and create a coinc_event_id to offset vector
# look-up table
self.coincs = self.coincs.items()
self.coincoffsets = dict(
(row.coinc_event_id, self.offsetvectors[row.time_slide_id])
for row in self.coinctable if row.coinc_def_id == b_b_coinc_def_id)
#
# represent the sngl_burst table as a dictionary indexed by
# instrument whose values are the event lists for those
# instruments sorted by peak time. sort the coincs list by
# the peak time of the first (earliest) event in each coinc
# (recall that the event tuple for each coinc has been
# time-ordered)
#
self.snglbursttable = dict(
(instrument,
sorted((event for event in self.snglbursttable
if event.ifo == instrument),
key=lambda event: event.peak))
for instrument in set(self.snglbursttable.getColumnByName("ifo")))
self.coincs.sort(
key=lambda coinc_id_events: coinc_id_events[1][0].peak)
def __init__(self,
connection,
live_time_program,
search="excesspower",
veto_segments_name=None):
"""
Compute and record some summary information about the
database. Call this after all the data has been inserted,
and before you want any of this information.
"""
self.connection = connection
self.xmldoc = dbtables.get_xml(connection)
# find the tables
try:
self.sngl_burst_table = lsctables.SnglBurstTable.get_table(
self.xmldoc)
except ValueError:
self.sngl_burst_table = None
try:
self.sim_burst_table = lsctables.SimBurstTable.get_table(
self.xmldoc)
except ValueError:
self.sim_burst_table = None
try:
self.coinc_def_table = lsctables.CoincDefTable.get_table(
self.xmldoc)
self.coinc_table = lsctables.CoincTable.get_table(self.xmldoc)
self.time_slide_table = lsctables.TimeSlideTable.get_table(
self.xmldoc)
except ValueError:
self.coinc_def_table = None
self.coinc_table = None
self.time_slide_table = None
try:
self.multi_burst_table = lsctables.MultiBurstTable.get_table(
self.xmldoc)
except ValueError:
self.multi_burst_table = None
# get the segment lists
self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(
self.xmldoc, live_time_program).coalesce()
self.instruments = set(self.seglists.keys())
if veto_segments_name is not None:
self.vetoseglists = ligolw_segments.segmenttable_get_by_name(
self.xmldoc, veto_segments_name).coalesce()
else:
self.vetoseglists = ligolw_segments.segments.segmentlistdict()
# determine a few coinc_definer IDs
# FIXME: don't hard-code the numbers
if self.coinc_def_table is not None:
try:
self.bb_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 0, create_new=False)
except KeyError:
self.bb_definer_id = None
try:
self.sb_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 1, create_new=False)
except KeyError:
self.sb_definer_id = None
try:
self.sce_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 2, create_new=False)
except KeyError:
self.sce_definer_id = None
try:
self.scn_definer_id = self.coinc_def_table.get_coinc_def_id(
search, 3, create_new=False)
except KeyError:
self.scn_definer_id = None
else:
self.bb_definer_id = None
self.sb_definer_id = None
self.sce_definer_id = None
self.scn_definer_id = None
def __init__(self, connection, live_time_program, search = "excesspower", veto_segments_name = None): """ Compute and record some summary information about the database. Call this after all the data has been inserted, and before you want any of this information. """ self.connection = connection self.xmldoc = dbtables.get_xml(connection) # find the tables try: self.sngl_burst_table = lsctables.SnglBurstTable.get_table(self.xmldoc) except ValueError: self.sngl_burst_table = None try: self.sim_burst_table = lsctables.SimBurstTable.get_table(self.xmldoc) except ValueError: self.sim_burst_table = None try: self.coinc_def_table = lsctables.CoincDefTable.get_table(self.xmldoc) self.coinc_table = lsctables.CoincTable.get_table(self.xmldoc) self.time_slide_table = lsctables.TimeSlideTable.get_table(self.xmldoc) except ValueError: self.coinc_def_table = None self.coinc_table = None self.time_slide_table = None try: self.multi_burst_table = lsctables.MultiBurstTable.get_table(self.xmldoc) except ValueError: self.multi_burst_table = None # get the segment lists self.seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(self.xmldoc, live_time_program).coalesce() self.instruments = set(self.seglists.keys()) if veto_segments_name is not None: self.vetoseglists = ligolw_segments.segmenttable_get_by_name(self.xmldoc, veto_segments_name).coalesce() else: self.vetoseglists = ligolw_segments.segments.segmentlistdict() # determine a few coinc_definer IDs # FIXME: don't hard-code the numbers if self.coinc_def_table is not None: try: self.bb_definer_id = self.coinc_def_table.get_coinc_def_id(search, 0, create_new = False) except KeyError: self.bb_definer_id = None try: self.sb_definer_id = self.coinc_def_table.get_coinc_def_id(search, 1, create_new = False) except KeyError: self.sb_definer_id = None try: self.sce_definer_id = self.coinc_def_table.get_coinc_def_id(search, 2, create_new = False) except KeyError: self.sce_definer_id = None try: self.scn_definer_id = self.coinc_def_table.get_coinc_def_id(search, 3, create_new = False) except KeyError: self.scn_definer_id = None else: self.bb_definer_id = None self.sb_definer_id = None self.sce_definer_id = None self.scn_definer_id = None
