def do_process_table(xmldoc, sim_tree, liv_tree):
try:
process_table = lsctables.ProcessTable.get_table(xmldoc)
except ValueError:
process_table = lsctables.New(lsctables.ProcessTable,
["process_id", "ifos", "comment", "program", "start_time", "jobid",
"end_time"])
xmldoc.childNodes[0].appendChild(process_table)
runids = set()
for i in range(0, sim_tree.GetEntries()) :
sim_tree.GetEntry(i)
# Id for the run processed by WaveBurst -> process ID
if sim_tree.run in runids :
continue
row = process_table.RowType()
row.process_id = type(process_table.next_id)(sim_tree.run)
runids.add(sim_tree.run)
# Imstruments involved in the search
row.ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
row.comment = "waveburst"
row.program = "waveburst"
# Begin and end time of the segment
# TODO: This is a typical offset on either side of the job for artifacts
# It can, and probably will change in the future, and should not be hardcoded
setattr(row, "start_time", None)
setattr(row, "end_time", None)
setattr(row, "jobid", sim_tree.run)
process_table.append(row)
def write_sky_map(
filename,
prob,
nest=False,
objid=None,
url=None,
instruments=None,
gps_time=None,
gps_creation_time=None,
creator=None,
origin=None,
runtime=None,
):
"""Write a gravitational-wave sky map to a file, populating the header
with optional metadata."""
#
# Populate optional header fieds.
#
extra_metadata = []
if objid is not None:
extra_metadata.append(("OBJECT", objid, "Unique identifier for this event"))
if url is not None:
extra_metadata.append(("REFERENC", url, "URL of this event"))
if instruments is not None:
if not isinstance(instruments, basestring):
instruments = str(lsctables.ifos_from_instrument_set(instruments))
extra_metadata.append(("INSTRUME", instruments, "Instruments that triggered this event"))
if gps_time is not None:
extra_metadata.append(("DATE-OBS", gps_to_iso8601(gps_time), "UTC date of the observation"))
extra_metadata.append(("MJD-OBS", gps_to_mjd(gps_time), "modified Julian date of the observation"))
if gps_creation_time is None:
gps_creation_time = lal.GPSTimeNow()
extra_metadata.append(("DATE", gps_to_iso8601(gps_creation_time), "UTC date of file creation"))
if creator is not None:
extra_metadata.append(("CREATOR", creator, "Program that created this file"))
if origin is not None:
extra_metadata.append(("ORIGIN", origin, "Organization responsible for this FITS file"))
if runtime is not None:
extra_metadata.append(("RUNTIME", runtime, "Runtime in seconds of the CREATOR program"))
write_map(
filename,
prob,
nest=nest,
fits_IDL=True,
coord="C",
column_names=("PROB",),
unit="pix-1",
extra_metadata=extra_metadata,
)
def set_ifos(self, instruments): """ Serialize a sequence of instruments into the ifos attribute. The instrument names must not contain the "," character. """ self.ifos = lsctables.ifos_from_instrument_set(instruments)
def do_process_table_from_segment(self, xmldoc, sim_tree, segment, jobid=-1):
"""
Create the process_table for the cWB job(s) from a job segment.
"""
try:
process_table = lsctables.ProcessTable.get_table(xmldoc)
except ValueError:
process_table = lsctables.New(lsctables.ProcessTable,
["process_id", "ifos", "comment", "program", "start_time", "jobid",
"end_time",
# Unused, but required for column synchronicity
"version", "cvs_repository", "cvs_entry_time", "is_online", "node",
"username", "unix_procid", "domain"])
xmldoc.childNodes[0].appendChild(process_table)
# get run IDs
if(jobid < 0):
N = sim_tree.GetEntries()
if N == 0:
exit("There is no information available to parse from the ROOT file, and no external information was provided.")
runs = set()
for i in xrange(N):
sim_tree.GetEntry(i)
run = sim_tree.run
runs.add(int(run))
else:
run = jobid
seg = segment
# Imstruments involved in the search
ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
if( ifos == None or len(ifos) == 0 ):
if( self.instruments ):
ifos = self.instruments
else: # Not enough information to completely fill out the table
sys.exit("Found a job with no IFOs on, or not enough to determine IFOs. Try specifying instruments directly.")
for run in runs:
row = process_table.RowType()
row.process_id = type(process_table.next_id)(run)
row.ifos = ifos
row.comment = u"waveburst"
row.program = u"waveburst"
row.start_time = None
row.end_time = None
row.jobid = run
row.version = __version__
row.cvs_repository = u"lscsoft"
# TODO: Need a handle on where this comes from
row.cvs_entry_time = 0
row.is_online = 0
row.username = os.environ['USER']
row.node = platform.node()
row.unix_procid = os.getpid()
row.domain = u"pylal"
process_table.append(row)
def __init__(self, joblist=None, start=None, end=None, instruments=None,
#cwbtables=False,
waveoffset=0, verbose=False):
self.job_list = joblist
self.start = start
self.end = end
#self.cwbtable = cwbtables
self.verbose = verbose
self.instruments = lsctables.ifos_from_instrument_set( instruments.split(",") )
self.waveoffset = waveoffset
def write_sky_map(filename, prob, nest=False, objid=None, url=None, instruments=None,
gps_time=None, gps_creation_time=None, creator=None, origin=None,
runtime=None):
"""Write a gravitational-wave sky map to a file, populating the header
with optional metadata."""
#
# Populate optional header fieds.
#
extra_header = []
if objid is not None:
extra_header.append(('OBJECT', objid,
'Unique identifier for this event'))
if url is not None:
extra_header.append(('REFERENC', url,
'URL of this event'))
if instruments is not None:
if not isinstance(instruments, six.string_types):
instruments = str(lsctables.ifos_from_instrument_set(instruments))
extra_header.append(('INSTRUME', instruments,
'Instruments that triggered this event'))
if gps_time is not None:
extra_header.append(('DATE-OBS', gps_to_iso8601(gps_time),
'UTC date of the observation'))
extra_header.append(('MJD-OBS', gps_to_mjd(gps_time),
'modified Julian date of the observation'))
if gps_creation_time is None:
gps_creation_time = lal.GPSTimeNow()
extra_header.append(('DATE', gps_to_iso8601(gps_creation_time),
'UTC date of file creation'))
if creator is not None:
extra_header.append(('CREATOR', creator,
'Program that created this file'))
if origin is not None:
extra_header.append(('ORIGIN', origin,
'Organization responsible for this FITS file'))
if runtime is not None:
extra_header.append(('RUNTIME', runtime,
'Runtime in seconds of the CREATOR program'))
hp.write_map(filename, prob, nest=nest, fits_IDL=True, coord='C',
column_names=('PROB',), column_units=('pix-1',),
extra_header=extra_header)
def do_summary_table(xmldoc, sim_tree, liv_tree):
try:
search_summary = lsctables.SearchSummaryTable.get_table(xmldoc)
except ValueError:
search_summary = lsctables.New(lsctables.SearchSummaryTable,
["process_id", "nevents", "ifos", "comment", "in_start_time",
"in_start_time_ns", "out_start_time", "out_start_time_ns",
"in_end_time", "in_end_time_ns", "out_end_time", "out_end_time_ns"])
xmldoc.childNodes[0].appendChild(search_summary)
process_id_type = lsctables.ProcessID
runids = set()
for i in range(0, sim_tree.GetEntries()) :
sim_tree.GetEntry(i)
# Id for the run processed by WaveBurst -> process ID
if sim_tree.run in runids :
continue
row = search_summary.RowType()
row.process_id = process_id_type(sim_tree.run)
runids.add(sim_tree.run)
# Search Summary Table
# events found in the run -> nevents
setattr(row, "nevents", sim_tree.GetEntries())
# Imstruments involved in the search
row.ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
setattr(row, "comment", "waveburst")
# Begin and end time of the segment
# TODO: This is a typical offset on either side of the job for artifacts
# It can, and probably will change in the future, and should not be hardcoded
# TODO: Make this work properly. We need a gps end from the livetime
waveoffset = 8
livetime = 600
#live_entries = live_tree.GetEntries()
# This is WAAAAAAAAAAAAAY too slow
#for l in range(0, live_entries):
#liv_tree.GetEntry(l)
#livetime = max(livetime, liv_tree.live)
#if livetime < 0:
#sys.exit("Could not find livetime, cannot fill all of summary table.")
# in -- with waveoffset
# out -- without waveoffset
row.set_in(segments.segment(LIGOTimeGPS(sim_tree.gps - waveoffset), LIGOTimeGPS(sim_tree.gps + livetime + waveoffset)))
row.set_out(segments.segment(LIGOTimeGPS(sim_tree.gps), LIGOTimeGPS(sim_tree.gps + livetime)))
search_summary.append(row)
def do_summary_table_from_segment(self, xmldoc, segment, sim_tree, jobid=-1):
"""
Create the search_summary table for a cWB from a segment specified from the command line. The function will try to determine the proper job intervals from the waveoffset, if specified.
"""
try:
search_summary = lsctables.SearchSummaryTable.get_table(xmldoc)
except ValueError:
search_summary = lsctables.New(lsctables.SearchSummaryTable,
["process_id", "nevents", "ifos", "comment", "in_start_time",
"in_start_time_ns", "out_start_time", "out_start_time_ns",
"in_end_time", "in_end_time_ns", "out_end_time", "out_end_time_ns"])
xmldoc.childNodes[0].appendChild(search_summary)
process_id_type = type(lsctables.ProcessTable.get_table(xmldoc).next_id)
sim_tree.GetEntry(0)
if(jobid < 0):
run = sim_tree.run
else: run = jobid
seg = segment
# Search Summary Table
# events found in the run -> nevents
row = search_summary.RowType()
row.process_id = process_id_type(run)
row.nevents = sim_tree.GetEntries()
ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
# Imstruments involved in the search
if( ifos == None or len(ifos) == 0 ):
if( self.instruments ):
ifos = self.instruments
else: # Not enough information to completely fill out the table
sys.exit("Found a job with no IFOs on, or not enough to determine IFOs. Try specifying instruments directly.")
row.ifos = ifos
row.comment = "waveburst"
# Begin and end time of the segment
waveoffset = self.waveoffset
if waveoffset == None: waveoffset = 0
# in -- with waveoffset
row.set_in(seg)
# out -- without waveoffset
waveoffset = LIGOTimeGPS(waveoffset)
row.set_out(segments.segment(seg[0]+waveoffset, seg[1]-waveoffset))
search_summary.append(row)
def playground_nonplayground_livetime(seglists, playground_segs=None, verbose=False):
playground_livetime = {}
nonplayground_livetime = {}
instruments = frozenset(seglists.keys())
for on_inst, off_inst in detector_combos(list(instruments)):
on_inst = frozenset(on_inst)
off_inst = frozenset(off_inst)
key = lsctables.ifos_from_instrument_set(on_inst)
selected_segs = seglists.intersection(list(on_inst))-seglists.union(list(off_inst))
if playground_segs:
playground_livetime[on_inst] = float(abs(selected_segs & playground_segs))
nonplayground_livetime[on_inst] = float(abs(selected_segs - playground_segs))
else:
playground_livetime[on_inst] = 0
nonplayground_livetime[on_inst] = float(abs(selected_segs))
return playground_livetime, nonplayground_livetime
def printmissed(connection, simulation_table, recovery_table, map_label, livetime_program,
param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ',
limit = None, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):
from pylal import ligolw_sqlutils as sqlutils
from pylal import ligolw_cbc_compute_durations as compute_dur
from glue import segments
from glue.ligolw import dbtables
# Get simulation/recovery tables
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper( connection )
connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag )
#
# Create and prepare the CloseMissedTable to store summary information
#
# Get simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table )
column_names = simulation_table_columns + \
['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag']
# define needed tables
class CloseMissedTable(table.Table):
tableName = "close_missed_injections:table"
validcolumns = {}
for col_name in column_names:
if 'rank' in col_name:
validcolumns[col_name] = "int_4u"
elif 'instruments_on' == col_name:
validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments']
elif 'veto_def_name' == col_name:
validcolumns[col_name] = lsctables.ExperimentSummaryTable.validcolumns['veto_def_name']
elif 'decisive_distance' == col_name:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, 'eff_dist_h')
elif 'gps_time' == col_name or 'gps_time_ns' == col_name:
validcolumns[col_name] = "int_4s"
elif 'sim_tag' == col_name:
validcolumns[col_name] = "lstring"
else:
validcolumns[col_name] = sqlutils.get_col_type(simulation_table, col_name, default = 'lstring')
class CloseMissed(object):
__slots__ = CloseMissedTable.validcolumns.keys()
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
CloseMissedTable.RowType = CloseMissed
# create the table
cmtable = lsctables.New(CloseMissedTable)
# set up sim_rec_map table
sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, None)
#
# Set table filters
#
# we force the include/exclude filters to None; will check for excluded/included ifo time
# when cycling through the ifo times
filter = """
WHERE
simulation_id NOT IN (
SELECT
sim_id
FROM
sim_rec_map )"""
af = create_filter( connection, simulation_table, param_name = param_name, param_ranges = param_ranges,
exclude_coincs = None, include_only_coincs = None, sim_tag = sim_tag, verbose = verbose)
af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af)
if af != '':
filter = '\n'.join([ filter, """
AND""", af])
# get desired instrument times
if include_only_coincs is not None:
include_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( include_only_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
if exclude_coincs is not None:
exclude_times = [on_instruments for on_instruments, type in
sqlutils.parse_coinc_options( exclude_coincs, verbose = verbose ).get_coinc_types().items()
if 'ALL' in type]
# get the usertags of inspiral jobs in this database
sqlquery = """
SELECT value
FROM process_params
WHERE param == "-userTag"
GROUP BY value
"""
usertags = set(usertag[0] for usertag in connection.cursor().execute(sqlquery) )
# Get the single-ifo science segments after CAT-1 vetoes
try:
if "FULL_DATA" in usertags:
tag = "FULL_DATA"
else:
tag = list(usertags)[0]
except IndexError:
# This is hacky anyway, so let's just take a guess
tag = "FULL_DATA"
ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name = livetime_program, usertag = tag)
if ifo_segments == {}:
raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program
if verbose:
print >> sys.stderr, "Getting all veto category names from the experiment_summary table..."
xmldoc = dbtables.get_xml(connection)
# get veto_segments
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# make a dictionary of zerolag "shifts" needed for the get_coinc_segments function
zerolag_dict = {}
for ifo in ifo_segments:
zerolag_dict[ifo] = 0.0
# Cycle over available veto categories
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
# make a dictionary of coincident segments by exclusive on-ifos
coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs, zerolag_dict)
#
# Get all the on_instrument times and cycle over them
#
sqlquery = """
SELECT DISTINCT experiment.instruments
FROM experiment
JOIN experiment_summary ON (
experiment.experiment_id == experiment_summary.experiment_id )
WHERE experiment_summary.veto_def_name == :1
"""
for on_instruments in connection.cursor().execute(sqlquery, (veto_def_name,)).fetchall():
on_instruments = lsctables.instrument_set_from_ifos(on_instruments[0])
# check if this on_instruments is desired; if not, skip
if include_only_coincs is not None and frozenset(on_instruments) not in include_times:
continue
if exclude_coincs is not None and frozenset(on_instruments) in exclude_times:
continue
on_times = coinc_segs[','.join(sorted(on_instruments))]
def is_in_on_time(gps_time, gps_time_ns):
return gps_time + (1e-9 * gps_time_ns) in on_times
connection.create_function('is_in_on_time', 2, is_in_on_time)
# add the check for on time to the filter
in_this_filter = filter
# figure out if simulation_table has end_times or start_times
end_or_start = any('end_time' in c for c in simulation_table_columns) and '_end_time' or '_start_time'
for instrument in on_instruments:
inst_time = instrument.lower()[0] + end_or_start
inst_time_ns = inst_time + '_ns'
in_this_filter = ''.join([ in_this_filter,
'\n\tAND is_in_on_time(', inst_time, ',', inst_time_ns, ')' ])
#
# Set up decisive distance argument
#
def get_decisive_distance( *args ):
return sorted(args)[1]
connection.create_function('get_decisive_distance', len(on_instruments), get_decisive_distance)
decisive_distance = ''.join(['get_decisive_distance(', ','.join(['eff_dist_'+inst.lower()[0] for inst in on_instruments]), ')' ])
#
# Initialize ranking. Statistics for ranking are based on decisive distance
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(simulation_table, decisive_distance, 'ASC')
# add requirement that stats not be found in the sim_rec_table to in_this_filter
ranker.populate_stats_list(connection, limit = limit, filter = in_this_filter)
connection.create_function( 'rank', 1, ranker.get_rank )
#
# Get the Data
#
sqlquery = ''.join(["""
SELECT
*,
get_sim_tag(process_id),
""", decisive_distance, """,
rank(""", decisive_distance, """)
FROM
""", simulation_table, """
""", in_this_filter, """
%s""" % (limit is not None and ''.join(['AND rank(', decisive_distance, ') <= ', str(limit)]) or ''), """
ORDER BY
rank(""", decisive_distance, """) ASC
"""])
if verbose:
print >> sys.stderr, "Getting injections..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute( sqlquery ).fetchall():
cmrow = CloseMissed()
[ setattr(cmrow, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ]
cmrow.decisive_distance = values[-2]
cmrow.rank = values[-1]
cmrow.instruments_on = lsctables.ifos_from_instrument_set(on_instruments)
cmrow.veto_def_name = veto_def_name
cmrow.sim_tag = values[-3]
cmrow.mini_followup = None
cmrow.omega_scan = None
cmrow.gps_time = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start)
cmrow.gps_time_ns = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start + '_ns')
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time)) for ifo in on_instruments]
cmrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ])
# set daily_ihope page
injection_time_utc = format_end_time_in_utc( cmrow.gps_time )
daily_ihope_address = get_daily_ihope_page(cmrow.gps_time, pages_location = daily_ihope_pages_location)
cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, injection_time_utc )
# add the row
cmtable.append(cmrow)
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return cmtable
def _get_ifo_string(self): return lsctables.ifos_from_instrument_set(self.ifos)
def do_summary_table(self, xmldoc, sim_tree):
"""
Create the search_summary table for the cWB job(s). This function exists as a backup in case no job list exists. It will try and reconstruct the job segments from the event data, but this list will be incomplete in the case where no events were found during a job.
"""
try:
search_summary = lsctables.SearchSummaryTable.get_table(xmldoc)
except ValueError:
search_summary = lsctables.New(lsctables.SearchSummaryTable,
["process_id", "nevents", "ifos", "comment", "in_start_time",
"in_start_time_ns", "out_start_time", "out_start_time_ns",
"in_end_time", "in_end_time_ns", "out_end_time", "out_end_time_ns"])
xmldoc.childNodes[0].appendChild(search_summary)
process_id_type = type(lsctables.ProcessTable.get_table(xmldoc).next_id)
runids = set()
entries = sim_tree.GetEntries()
for i in range(0, entries) :
sim_tree.GetEntry(i)
if self.start != None :
if float(self.start) > sim_tree.start[0] : continue
if self.end != None :
if float(self.end) < sim_tree.stop[0] : continue
# Id for the run processed by WaveBurst -> process ID
run = sim_tree.run
if run in runids :
continue
row = search_summary.RowType()
row.process_id = process_id_type(run)
runids.add(run)
# Search Summary Table
# events found in the run -> nevents
# TODO: Destroy ROOT, because it hates me
#row.nevents = sim_tree.nevent
row.nevents = 0
# Imstruments involved in the search
ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
# Imstruments involved in the search
if( ifos == None or len(ifos) == 0 ):
if( self.instruments ):
ifos = self.instruments
else: # Not enough information to completely fill out the table
sys.exit("Found a job with no IFOs on, or not enough to determine IFOs. Try specifying instruments directly.")
row.ifos = ifos
row.comment = "waveburst"
# Begin and end time of the segment
if self.waveoffset != None :
waveoffset = self.waveoffset
else:
waveoffset = 0
livetime = -1
# WARNING: This will fail miserably if there are no events in analyzed
# see do_search_summary_from_joblist for a better solution
livetime = sim_tree.left[0] + sim_tree.duration[0] + sim_tree.right[0]
if livetime < 0:
print >>sys.stderr, "WARNING: Run %d will have zero livetime because no events are recorded in this run, and therefore livetime cannot be calculated." % run
# in -- with waveoffset
row.set_in(segments.segment(LIGOTimeGPS(0), LIGOTimeGPS(1)))
# out -- without waveoffset
row.set_out(segments.segment(LIGOTimeGPS(0), LIGOTimeGPS(1)))
else:
seg_start_with_offset = sim_tree.gps - waveoffset
seg_start_without_offset = sim_tree.gps
seg_end_with_offset = sim_tree.gps + waveoffset + livetime
seg_end_without_offset = sim_tree.gps + livetime
# in -- with waveoffset
row.set_in(segments.segment(LIGOTimeGPS(seg_start_with_offset), LIGOTimeGPS(seg_end_with_offset)))
# out -- without waveoffset
row.set_out(segments.segment(LIGOTimeGPS(seg_start_without_offset), LIGOTimeGPS(seg_end_without_offset)))
search_summary.append(row)
if 'PLAYGROUND' in tag:
datatypes = ['playground', 'slide']
else:
datatypes = ['all_data', 'playground', 'slide']
lc_nodes = []
for datatype in datatypes:
print("\t\tfor %s..." % datatype)
# set file naming type
type_prefix = tag
type = '_'.join([ type_prefix, 'LOUDEST', datatype.upper(), 'EVENTS_BY', cp.get('printlc', 'ranking-stat').upper()])
# cycle over all ifos times, creating different tables for each
for on_instruments in distinct_instrument_sets:
on_instruments = lsctables.ifos_from_instrument_set(on_instruments)
# set output and extracted xml file names
if cp.has_option('printlc', 'output-format'):
summ_file_type = cp.get('printlc', 'output-format')
else:
summ_file_type = 'xml'
summary_filename = '.'.join([
'-'.join([ ''.join(on_instruments.split(',')), type + '_SUMMARY', experiment_start, experiment_duration ]),
summ_file_type ])
full_filename = '-'.join([ ''.join(on_instruments.split(',')), type, experiment_start, experiment_duration ])
# set node options
printlc_node = inspiral.PrintLCNode( printlc_job )
printlc_node.set_category('printlc')
printlc_node.set_tmp_space( tmp_space )
printlc_node.set_input( result_db.path )
printlc_node.set_output( summary_filename )
def printmissed(
connection,
simulation_table,
recovery_table,
map_label,
livetime_program,
param_name=None,
param_ranges=None,
exclude_coincs=None,
include_only_coincs=None,
sim_tag='ALLINJ',
limit=None,
daily_ihope_pages_location='https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily',
verbose=False):
from pylal import ligolw_sqlutils as sqlutils
from pylal import ligolw_cbc_compute_durations as compute_dur
from glue import segments
from glue.ligolw import dbtables
# Get simulation/recovery tables
simulation_table = sqlutils.validate_option(simulation_table)
recovery_table = sqlutils.validate_option(recovery_table)
# create the get_sim_tag function
sim_map = sqlutils.sim_tag_proc_id_mapper(connection)
connection.create_function('get_sim_tag', 1, sim_map.get_sim_tag)
#
# Create and prepare the CloseMissedTable to store summary information
#
# Get simulation table column names from database
simulation_table_columns = sqlutils.get_column_names_from_table(
connection, simulation_table)
column_names = simulation_table_columns + \
['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag']
# define needed tables
class CloseMissedTable(table.Table):
tableName = "close_missed_injections:table"
validcolumns = {}
for col_name in column_names:
if 'rank' in col_name:
validcolumns[col_name] = "int_4u"
elif 'instruments_on' == col_name:
validcolumns[
col_name] = lsctables.ExperimentTable.validcolumns[
'instruments']
elif 'veto_def_name' == col_name:
validcolumns[
col_name] = lsctables.ExperimentSummaryTable.validcolumns[
'veto_def_name']
elif 'decisive_distance' == col_name:
validcolumns[col_name] = sqlutils.get_col_type(
simulation_table, 'eff_dist_h')
elif 'gps_time' == col_name or 'gps_time_ns' == col_name:
validcolumns[col_name] = "int_4s"
elif 'sim_tag' == col_name:
validcolumns[col_name] = "lstring"
else:
validcolumns[col_name] = sqlutils.get_col_type(
simulation_table, col_name, default='lstring')
class CloseMissed(object):
__slots__ = CloseMissedTable.validcolumns.keys()
def get_pyvalue(self):
return generic_get_pyvalue(self)
# connect the rows to the tables
CloseMissedTable.RowType = CloseMissed
# create the table
cmtable = lsctables.New(CloseMissedTable)
# set up sim_rec_map table
sqlutils.create_sim_rec_map_table(connection, simulation_table,
recovery_table, map_label, None)
#
# Set table filters
#
# we force the include/exclude filters to None; will check for excluded/included ifo time
# when cycling through the ifo times
filter = """
WHERE
simulation_id NOT IN (
SELECT
sim_id
FROM
sim_rec_map )"""
af = create_filter(connection,
simulation_table,
param_name=param_name,
param_ranges=param_ranges,
exclude_coincs=None,
include_only_coincs=None,
sim_tag=sim_tag,
verbose=verbose)
af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af)
if af != '':
filter = '\n'.join([filter, """
AND""", af])
# get desired instrument times
if include_only_coincs is not None:
include_times = [
on_instruments
for on_instruments, type in sqlutils.parse_coinc_options(
include_only_coincs,
verbose=verbose).get_coinc_types().items() if 'ALL' in type
]
if exclude_coincs is not None:
exclude_times = [
on_instruments
for on_instruments, type in sqlutils.parse_coinc_options(
exclude_coincs, verbose=verbose).get_coinc_types().items()
if 'ALL' in type
]
# get the usertags of inspiral jobs in this database
sqlquery = """
SELECT value
FROM process_params
WHERE param == "-userTag"
GROUP BY value
"""
usertags = set(usertag[0]
for usertag in connection.cursor().execute(sqlquery))
# Get the single-ifo science segments after CAT-1 vetoes
try:
if "FULL_DATA" in usertags:
tag = "FULL_DATA"
else:
tag = list(usertags)[0]
except IndexError:
# This is hacky anyway, so let's just take a guess
tag = "FULL_DATA"
ifo_segments = compute_dur.get_single_ifo_segments(
connection, program_name=livetime_program, usertag=tag)
if ifo_segments == {}:
raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program
if verbose:
print >> sys.stderr, "Getting all veto category names from the experiment_summary table..."
xmldoc = dbtables.get_xml(connection)
# get veto_segments
veto_segments = compute_dur.get_veto_segments(xmldoc, verbose)
# make a dictionary of zerolag "shifts" needed for the get_coinc_segments function
zerolag_dict = {}
for ifo in ifo_segments:
zerolag_dict[ifo] = 0.0
# Cycle over available veto categories
for veto_def_name, veto_seg_dict in veto_segments.items():
post_vetoes_ifosegs = ifo_segments - veto_seg_dict
# make a dictionary of coincident segments by exclusive on-ifos
coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs,
zerolag_dict)
#
# Get all the on_instrument times and cycle over them
#
sqlquery = """
SELECT DISTINCT experiment.instruments
FROM experiment
JOIN experiment_summary ON (
experiment.experiment_id == experiment_summary.experiment_id )
WHERE experiment_summary.veto_def_name == :1
"""
for on_instruments in connection.cursor().execute(
sqlquery, (veto_def_name, )).fetchall():
on_instruments = lsctables.instrument_set_from_ifos(
on_instruments[0])
# check if this on_instruments is desired; if not, skip
if include_only_coincs is not None and frozenset(
on_instruments) not in include_times:
continue
if exclude_coincs is not None and frozenset(
on_instruments) in exclude_times:
continue
on_times = coinc_segs[','.join(sorted(on_instruments))]
def is_in_on_time(gps_time, gps_time_ns):
return LIGOTimeGPS(gps_time, gps_time_ns) in on_times
connection.create_function('is_in_on_time', 2, is_in_on_time)
# add the check for on time to the filter
in_this_filter = filter
# figure out if simulation_table has end_times or start_times
end_or_start = any('end_time' in c
for c in simulation_table_columns
) and '_end_time' or '_start_time'
for instrument in on_instruments:
inst_time = instrument.lower()[0] + end_or_start
inst_time_ns = inst_time + '_ns'
in_this_filter = ''.join([
in_this_filter, '\n\tAND is_in_on_time(', inst_time, ',',
inst_time_ns, ')'
])
#
# Set up decisive distance argument
#
def get_decisive_distance(*args):
return sorted(args)[1]
connection.create_function('get_decisive_distance',
len(on_instruments),
get_decisive_distance)
decisive_distance = ''.join([
'get_decisive_distance(', ','.join([
'eff_dist_' + inst.lower()[0] for inst in on_instruments
]), ')'
])
#
# Initialize ranking. Statistics for ranking are based on decisive distance
#
if verbose:
print >> sys.stderr, "Getting statistics for ranking..."
ranker = sqlutils.rank_stats(simulation_table, decisive_distance,
'ASC')
# add requirement that stats not be found in the sim_rec_table to in_this_filter
ranker.populate_stats_list(connection,
limit=limit,
filter=in_this_filter)
connection.create_function('rank', 1, ranker.get_rank)
#
# Get the Data
#
sqlquery = ''.join([
"""
SELECT
*,
get_sim_tag(process_id),
""", decisive_distance, """,
rank(""", decisive_distance, """)
FROM
""", simulation_table, """
""", in_this_filter,
"""
%s""" % (limit is not None and ''.join(
['AND rank(', decisive_distance, ') <= ',
str(limit)]) or ''), """
ORDER BY
rank(""", decisive_distance, """) ASC
"""
])
if verbose:
print >> sys.stderr, "Getting injections..."
print >> sys.stderr, "SQLite query used is:"
print >> sys.stderr, sqlquery
for values in connection.cursor().execute(sqlquery).fetchall():
cmrow = CloseMissed()
[
setattr(cmrow, column, values[ii])
for ii, column in enumerate(simulation_table_columns)
]
cmrow.decisive_distance = values[-2]
cmrow.rank = values[-1]
cmrow.instruments_on = lsctables.ifos_from_instrument_set(
on_instruments)
cmrow.veto_def_name = veto_def_name
cmrow.sim_tag = values[-3]
cmrow.mini_followup = None
cmrow.omega_scan = None
cmrow.gps_time = getattr(
cmrow,
sorted(on_instruments)[0][0].lower() + end_or_start)
cmrow.gps_time_ns = getattr(
cmrow,
sorted(on_instruments)[0][0].lower() + end_or_start +
'_ns')
# set elog page
elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time))
for ifo in on_instruments]
cmrow.elogs = ','.join([
create_hyperlink(elog[1], elog[0])
for elog in sorted(elog_pages)
])
# set daily_ihope page
injection_time_utc = format_end_time_in_utc(cmrow.gps_time)
daily_ihope_address = get_daily_ihope_page(
cmrow.gps_time, pages_location=daily_ihope_pages_location)
cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink(
daily_ihope_address, injection_time_utc)
# add the row
cmtable.append(cmrow)
# drop the sim_rec_map table
connection.cursor().execute("DROP TABLE sim_rec_map")
return cmtable
def do_summary_table_from_joblist(self, xmldoc, sim_tree):
"""
Create the search_summary table for the cWB job(s) and a provided cWB joblist. The function will try to determine the proper job intervals from the waveoffset, if specified.
"""
try:
search_summary = lsctables.SearchSummaryTable.get_table(xmldoc)
except ValueError:
search_summary = lsctables.New(lsctables.SearchSummaryTable,
["process_id", "nevents", "ifos", "comment", "in_start_time",
"in_start_time_ns", "out_start_time", "out_start_time_ns",
"in_end_time", "in_end_time_ns", "out_end_time", "out_end_time_ns"])
xmldoc.childNodes[0].appendChild(search_summary)
process_id_type = type(lsctables.ProcessTable.get_table(xmldoc).next_id)
runid = dict()
itr = 0
for line in file(self.job_list) :
if line[0] == '#' :
continue # skip comments
# Determine the file type
# WARNING: Mixing the two types could cause overwrite behavior
line = line.split()
if len(line) == 2 : # start and stop
seg = segments.segment( map( LIGOTimeGPS, map(float, line[0:2]) ) )
runid[itr] = seg
itr += 1
elif len(line) == 3 : # index start and stop
seg = segments.segment( map( LIGOTimeGPS, map(float, line[1:3]) ) )
runid[int(line[0])] = seg
elif len(line) == 4 : # index start and stop and length
seg = segments.segment( map( LIGOTimeGPS, map(float, line[1:3]) ) )
runid[int(line[0])] = seg
else : # dunno!
sys.exit("Unable to understand job list segment format.")
for run, seg in runid.iteritems() :
if self.start != None :
if float(self.start) > seg[0] : continue
if self.end != None :
if float(self.end) < seg[1] : continue
row = search_summary.RowType()
row.process_id = process_id_type(run)
# Search Summary Table
# events found in the run -> nevents
row.nevents = 0
#entries = sim_tree.GetEntries()
# Imstruments involved in the search
sim_tree.GetEntry(0)
ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
# Imstruments involved in the search
if( ifos == None or len(ifos) == 0 ):
if( self.instruments ):
ifos = self.instruments
else: # Not enough information to completely fill out the table
sys.exit("Found a job with no IFOs on, or not enough to determine IFOs. Try specifying instruments directly.")
row.ifos = ifos
row.comment = "waveburst"
# Begin and end time of the segment
# TODO: This is a typical offset on either side of the job for artifacts
# It can, and probably will change in the future, and shouldn't be hardcoded
#waveoffset, livetime = 8, -1
waveoffset, livetime = self.waveoffset, -1
if waveoffset == None: waveoffset = 0
livetime = abs(seg)
if livetime < 0:
print >>sys.stderr, "WARNING: Run %d will have zero livetime because no events are recorded in this run, and therefore livetime cannot be calculated." % run
# in -- with waveoffset
row.set_in(segments.segment(LIGOTimeGPS(0), LIGOTimeGPS(1)))
# out -- without waveoffset
row.set_out(segments.segment(LIGOTimeGPS(0), LIGOTimeGPS(1)))
else:
# in -- with waveoffset
row.set_in(seg)
# out -- without waveoffset
row.set_out(segments.segment(seg[0]+waveoffset, seg[1]-waveoffset))
search_summary.append(row)
def do_process_table(self, xmldoc, sim_tree):
"""
Create the process_table for the cWB job(s)
"""
try:
process_table = lsctables.ProcessTable.get_table(xmldoc)
except ValueError:
process_table = lsctables.New(lsctables.ProcessTable,
["process_id", "ifos", "comment", "program", "start_time", "jobid",
"end_time",
# Unused, but required for column synchronicity
"version", "cvs_repository", "cvs_entry_time", "is_online", "node",
"username", "unix_procid", "domain"])
xmldoc.childNodes[0].appendChild(process_table)
runids = set()
runid = dict()
itr = 0
for line in file(self.job_list) :
if line[0] == '#' :
continue # skip comments
# Determine the file type
# WARNING: Mixing the two types could cause overwrite behavior
line = line.split()
if len(line) == 2 : # start and stop
seg = segments.segment( map( LIGOTimeGPS, map(float, line[0:2]) ) )
runid[itr] = seg
itr += 1
elif len(line) == 3 : # index start and stop
seg = segments.segment( map( LIGOTimeGPS, map(float, line[1:3]) ) )
runid[int(line[0])] = seg
elif len(line) == 4 : # index start and stop and length
seg = segments.segment( map( LIGOTimeGPS, map(float, line[1:3]) ) )
runid[int(line[0])] = seg
else : # dunno!
sys.exit("Unable to understand job list segment format. Why do you confused me so?")
sim_tree.GetEntry(0)
for run, seg in runid.iteritems() :
if self.start != None :
if float(self.start) > seg[0] : continue
if self.end != None :
if float(self.end) < seg[1] : continue
row = process_table.RowType()
row.process_id = type(process_table.next_id)(run)
#print "Run id %d was given process_id %d" % (sim_tree.run, row.process_id)
runids.add(sim_tree.run)
# Imstruments involved in the search
ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
if( ifos == None or len(ifos) == 0 ):
if( self.instruments ):
ifos = self.instruments
else: # Not enough information to completely fill out the table
sys.exit("Found a job with no IFOs on, or not enough to determine IFOs. Try specifying instruments directly.")
row.ifos = ifos
row.comment = u"waveburst"
row.program = u"waveburst"
row.start_time = None
row.end_time = None
row.jobid = run
row.version = __version__
row.cvs_repository = u"lscsoft"
# TODO: Need a handle on where this comes from
row.cvs_entry_time = 0
row.is_online = 0
row.username = os.environ['USER']
row.node = platform.node()
row.unix_procid = os.getpid()
row.domain = "pylal"
process_table.append(row)
# Add Coinc table entry.
coinc = lsctables.Coinc()
coinc.coinc_event_id = coinc_table.get_next_id()
coinc.process_id = process.process_id
coinc.coinc_def_id = coinc_def_id
coinc.time_slide_id = time_slide_id
coinc.set_instruments(opts.detector)
coinc.nevents = len(opts.detector)
coinc.likelihood = None
coinc_table.append(coinc)
# Add CoincInspiral table entry.
coinc_inspiral = lsctables.CoincInspiral()
coinc_inspiral.coinc_event_id = coinc.coinc_event_id
coinc_inspiral.ifos = lsctables.ifos_from_instrument_set(
sngl_inspiral.ifo for sngl_inspiral in sngl_inspirals)
coinc_inspiral.end = lal.LIGOTimeGPS(
sum(sngl_inspiral.end.ns() for sngl_inspiral in sngl_inspirals)
// len(sngl_inspirals) * 1e-9) # FIXME: should only be detected sngls
coinc_inspiral.mass = sim_inspiral.mass1 + sim_inspiral.mass2
coinc_inspiral.mchirp = sim_inspiral.mchirp
coinc_inspiral.combined_far = 0.0 # Not provided
coinc_inspiral.false_alarm_rate = 0.0 # Not provided
coinc_inspiral.minimum_duration = None # Not provided
coinc_inspiral.snr = net_snr
coinc_inspiral_table.append(coinc_inspiral)
# Record all sngl_inspiral records and associate them with coincidences.
for sngl_inspiral in sngl_inspirals:
# Give this sngl_inspiral record an id and add it to the table.
sngl_inspiral.event_id = sngl_inspiral_table.get_next_id()
def do_summary_table(xmldoc, sim_tree, liv_tree):
try:
search_summary = table.get_table(
xmldoc, lsctables.SearchSummaryTable.tableName)
except ValueError:
search_summary = lsctables.New(lsctables.SearchSummaryTable, [
"process_id", "nevents", "ifos", "comment", "in_start_time",
"in_start_time_ns", "out_start_time", "out_start_time_ns",
"in_end_time", "in_end_time_ns", "out_end_time", "out_end_time_ns"
])
xmldoc.childNodes[0].appendChild(search_summary)
process_id_type = type(
table.get_table(xmldoc, lsctables.ProcessTable.tableName).next_id)
runids = set()
for i in range(0, sim_tree.GetEntries()):
sim_tree.GetEntry(i)
# Id for the run processed by WaveBurst -> process ID
if sim_tree.run in runids:
continue
row = search_summary.RowType()
row.process_id = process_id_type(sim_tree.run)
runids.add(sim_tree.run)
# Search Summary Table
# events found in the run -> nevents
setattr(row, "nevents", sim_tree.GetEntries())
# Imstruments involved in the search
row.ifos = lsctables.ifos_from_instrument_set(
get_ifos_from_index(
branch_array_to_list(sim_tree.ifo, sim_tree.ndim)))
setattr(row, "comment", "waveburst")
# Begin and end time of the segment
# TODO: This is a typical offset on either side of the job for artifacts
# It can, and probably will change in the future, and should not be hardcoded
# TODO: Make this work properly. We need a gps end from the livetime
waveoffset = 8
livetime = 600
#live_entries = live_tree.GetEntries()
# This is WAAAAAAAAAAAAAY too slow
#for l in range(0, live_entries):
#liv_tree.GetEntry(l)
#livetime = max(livetime, liv_tree.live)
#if livetime < 0:
#sys.exit("Could not find livetime, cannot fill all of summary table.")
# in -- with waveoffset
# out -- without waveoffset
row.set_in(
segments.segment(LIGOTimeGPS(sim_tree.gps - waveoffset),
LIGOTimeGPS(sim_tree.gps + livetime +
waveoffset)))
row.set_out(
segments.segment(LIGOTimeGPS(sim_tree.gps),
LIGOTimeGPS(sim_tree.gps + livetime)))
search_summary.append(row)
if 'PLAYGROUND' in tag:
datatypes = ['playground', 'slide']
else:
datatypes = ['all_data', 'playground', 'slide']
lc_nodes = []
for datatype in datatypes:
print "\t\tfor %s..." % datatype
# set file naming type
type_prefix = tag
type = '_'.join([ type_prefix, 'LOUDEST', datatype.upper(), 'EVENTS_BY', cp.get('printlc', 'ranking-stat').upper()])
# cycle over all ifos times, creating different tables for each
for on_instruments in distinct_instrument_sets:
on_instruments = lsctables.ifos_from_instrument_set(on_instruments)
# set output and extracted xml file names
if cp.has_option('printlc', 'output-format'):
summ_file_type = cp.get('printlc', 'output-format')
else:
summ_file_type = 'xml'
summary_filename = '.'.join([
'-'.join([ ''.join(on_instruments.split(',')), type + '_SUMMARY', experiment_start, experiment_duration ]),
summ_file_type ])
full_filename = '-'.join([ ''.join(on_instruments.split(',')), type, experiment_start, experiment_duration ])
# set node options
printlc_node = inspiral.PrintLCNode( printlc_job )
printlc_node.set_category('printlc')
printlc_node.set_tmp_space( tmp_space )
printlc_node.set_input( result_db.path )
printlc_node.set_output( summary_filename )
opts, filenames = parse_command_line() if opts.veto_segments_name is not None: working_filename = dbtables.get_connection_filename(opts.full_data_file, verbose = opts.verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) veto_segments = db_thinca_rings.get_veto_segments(connection, opts.veto_segments_name) connection.close() dbtables.discard_connection_filename(opts.full_data_file, working_filename, verbose = opts.verbose) dbtables.DBTable_set_connection(None) else: veto_segments = segments.segmentlistdict() if not opts.burst_found and not opts.burst_missed: FAR, seglists = get_far_threshold_and_segments(opts.full_data_file, opts.live_time_program, instruments=lsctables.ifos_from_instrument_set(opts.instruments),verbose = opts.verbose) # times when only exactly the required instruments are on seglists -= veto_segments zero_lag_segments = seglists.intersection(opts.instruments) - seglists.union(set(seglists.keys()) - opts.instruments) live_time = float(abs(zero_lag_segments)) print(FAR, live_time) Found, Missed = get_injections(opts.injfnames, FAR, zero_lag_segments, verbose = opts.verbose) else: Found = get_burst_injections(opts.burst_found) Missed = get_burst_injections(opts.burst_missed)
def create_tables(self, xmldoc, rootfiles):
"""
Sets up table structures and calls populating methods.
"""
if os.path.splitext(rootfiles[0])[1] == ".root":
sim_tree = TChain("waveburst")
else: # If the file is (for example) text, use a proxy class
sim_tree = CWBTextConverter()
for rootfile in rootfiles:
sim_tree.Add(rootfile)
# Define tables
sngl_burst_table = lsctables.New(lsctables.SnglBurstTable,
["peak_time_ns", "start_time_ns", "stop_time_ns",
"process_id", "ifo", "peak_time", "start_time", "stop_time",
"duration", "time_lag", "peak_frequency", "search",
"flow", "fhigh", "bandwidth", "tfvolume", "hrss", "event_id", "snr"])
xmldoc.childNodes[0].appendChild(sngl_burst_table)
sngl_burst_table.sync_next_id()
coinc_event_table = lsctables.New(lsctables.CoincTable,
[ "process_id", "coinc_event_id", "nevents", "instruments", "time_slide_id",
"coinc_def_id", "likelihood"])
xmldoc.childNodes[0].appendChild(coinc_event_table)
coinc_event_table.sync_next_id()
# TODO: Reimplement this when the cwb_table module is included
#if self.cwbtable:
#cohwb_table = lsctables.New(cwb_table.CoherentWaveburstTable,
#["ellipticity", "correlated_energy", "eff_correlated_energy",
#"coherent_network_amp", "penalty", "network_correlation",
#"energy_disbalance", "ellip_energy_disbalance", "process_id",
#"coinc_event_id", "cwb_id"])
#xmldoc.childNodes[0].appendChild(cohwb_table)
#cohwb_table.sync_next_id()
multi_burst_table = lsctables.New(lsctables.MultiBurstTable,
["process_id", "peak_time", "peak_time_ns", "coinc_event_id", "snr", "ifos",
# NOTE: Added to the table definition
"false_alarm_rate", "ligo_axis_ra", "ligo_axis_dec"])
xmldoc.childNodes[0].appendChild(multi_burst_table)
coinc_event_map_table = lsctables.New(lsctables.CoincMapTable)
xmldoc.childNodes[0].appendChild(coinc_event_map_table)
jobsegment = None
if self.start and self.end :
jobsegment = segments.segment(LIGOTimeGPS(self.start), LIGOTimeGPS(self.end))
if self.verbose:
print "Creating Process Table...",
if self.job_list:
self.do_process_table(xmldoc, sim_tree)
else :
self.do_process_table_from_segment(xmldoc, sim_tree, jobsegment)
process_index = dict((int(row.process_id), row) for row in lsctables.ProcessTable.get_table(xmldoc))
if self.verbose:
print " done."
if self.verbose:
print "Creating Summary Table...",
# If we are provided a joblist, use it to generate the list
if self.job_list :
self.do_summary_table_from_joblist(xmldoc, sim_tree)
elif self.job_list == None and self.start and self.end :
self.do_summary_table_from_segment(xmldoc, jobsegment, sim_tree)
else :
self.do_summary_table(xmldoc, sim_tree)
if self.verbose:
print " done."
# create coinc_definer row
row = self.get_coinc_def_row(sim_tree)
coinc_def_id = llwapp.get_coinc_def_id(xmldoc, row.search, row.search_coinc_type, description = row.description)
entries = sim_tree.GetEntries()
for i in range(0, entries):
sim_tree.GetEntry(i)
if self.start != None :
if float(self.start) > sim_tree.start[0] : continue
if self.end != None :
if float(self.end) < sim_tree.stop[0] : continue
offset_vector = dict((get_ifos_from_index(instrument_index), offset) for instrument_index, offset in zip(sim_tree.ifo, sim_tree.lag))
coinc_event = coinc_event_table.RowType()
coinc_event.process_id = process_index[sim_tree.run].process_id
coinc_event.coinc_event_id = coinc_event_table.get_next_id()
coinc_event.coinc_def_id = coinc_def_id
coinc_event.nevents = sim_tree.ndim
coinc_event.instruments = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
coinc_event.time_slide_id = time_slide.get_time_slide_id(xmldoc, offset_vector, process_index[sim_tree.run])
coinc_event.likelihood = sim_tree.likelihood
coinc_event_table.append(coinc_event)
for d in range(0, sim_tree.ndim):
sngl_burst = self.get_sngl_burst_row(sngl_burst_table, sim_tree, d)
sngl_burst.process_id = coinc_event.process_id
sngl_burst.event_id = sngl_burst_table.get_next_id()
sngl_burst_table.append(sngl_burst)
coinc_event_map = coinc_event_map_table.RowType()
coinc_event_map.event_id = sngl_burst.event_id
coinc_event_map.table_name = sngl_burst.event_id.table_name
coinc_event_map.coinc_event_id = coinc_event.coinc_event_id
coinc_event_map_table.append(coinc_event_map)
# TODO: Reimplement when cwb_table module is included
#if self.cwbtable:
#cwb = get_cwb_row(cohwb_table, sim_tree)
#cwb.process_id = coinc_event.process_id
#cwb.coinc_event_id = coinc_event.coinc_event_id
#cwb.cwb_id = cohwb_table.get_next_id()
#cohwb_table.append(cwb)
multi_burst = self.get_multi_burst_row(multi_burst_table, sim_tree)
multi_burst.process_id = coinc_event.process_id
multi_burst.coinc_event_id = coinc_event.coinc_event_id
# NOTE: Until we have an embedded cwb table definition, this will be
# copied here so official tools have a ranking statistic to use
try:
multi_burst.snr = sim_tree.rho[1]
except TypeError: # difference in definition between ROOT and text
multi_burst.snr = sim_tree.rho
# NOTE: To be filled in later by farburst
multi_burst.false_alarm_rate = -1.0
# Reconstructed right ascension and declination
multi_burst.ligo_axis_ra = sim_tree.phi[2]
multi_burst.ligo_axis_dec = sim_tree.theta[2]
multi_burst.ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
multi_burst_table.append(multi_burst)
opts, filenames = parse_command_line() if opts.veto_segments_name is not None: working_filename = dbtables.get_connection_filename(opts.full_data_file, verbose = opts.verbose) connection = sqlite3.connect(working_filename) dbtables.DBTable_set_connection(connection) veto_segments = db_thinca_rings.get_veto_segments(connection, opts.veto_segments_name) connection.close() dbtables.discard_connection_filename(opts.full_data_file, working_filename, verbose = opts.verbose) dbtables.DBTable_set_connection(None) else: veto_segments = segments.segmentlistdict() if not opts.burst_found and not opts.burst_missed: FAR, seglists = get_far_threshold_and_segments(opts.full_data_file, opts.live_time_program, instruments=lsctables.ifos_from_instrument_set(opts.instruments),verbose = opts.verbose) # times when only exactly the required instruments are on seglists -= veto_segments zero_lag_segments = seglists.intersection(opts.instruments) - seglists.union(set(seglists.keys()) - opts.instruments) live_time = float(abs(zero_lag_segments)) print FAR, live_time Found, Missed = get_injections(opts.injfnames, FAR, zero_lag_segments, verbose = opts.verbose) else: Found = get_burst_injections(opts.burst_found) Missed = get_burst_injections(opts.burst_missed)
