def load_cache(xmldoc, cache, sieve_pattern, exact_match=False, verbose=False):
"""
Return a parsed and ligolw_added XML document from the files matched by
sieve_pattern in the given cache.
"""
subcache = cache.sieve(description=sieve_pattern, exact_match=exact_match)
found, missed = subcache.checkfilesexist()
if len(found) == 0:
print >>sys.stderr, "warning: no files found for pattern %s" \
% sieve_pattern
# turn on event_id mangling
old_id = lsctables.SnglInspiralTable.next_id
lsctables.SnglInspiralTable.next_id = SnglInspiralID_old(0)
# reduce memory footprint at the expense of speed
# table.TableStream.RowBuilder = table.InterningRowBuilder
urls = [c.url for c in found]
try:
xmldoc = ligolw_add.ligolw_add(ligolw.Document(),
urls,
verbose=verbose)
except ligolw.ElementError:
# FIXME: backwards compatibility for int_8s SnglInspiralTable event_ids
lsctables.SnglInspiralTable.validcolumns["event_id"] = "int_8s"
lsctables.SnglInspiralID = int
xmldoc = ligolw_add.ligolw_add(ligolw.Document(),
urls,
verbose=verbose)
# turn off event_id mangling
lsctables.SnglInspiralTable.next_id = old_id
return xmldoc
def gen_likelihood_control(coinc_params_distributions,
seglists,
name=u"ligolw_burca_tailor",
comment=u""):
xmldoc = ligolw.Document()
node = xmldoc.appendChild(ligolw.LIGO_LW())
process = ligolw_process.register_to_xmldoc(xmldoc,
program=process_program_name,
paramdict={},
version=__version__,
cvs_repository="lscsoft",
cvs_entry_time=__date__,
comment=comment)
coinc_params_distributions.process_id = process.process_id
ligolw_search_summary.append_search_summary(xmldoc,
process,
ifos=seglists.keys(),
inseg=seglists.extent_all(),
outseg=seglists.extent_all())
node.appendChild(coinc_params_distributions.to_xml(name))
ligolw_process.set_process_end_time(process)
return xmldoc
def create_xmldoc(tables):
"""
Creates an xmldoc from the list of given LIGOLW tables.
"""
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
for table in tables:
xmldoc.childNodes[0].appendChild(table)
return xmldoc
def load_external_triggers(filename):
doc = ligolw_add.ligolw_add(ligolw.Document(), [filename])
ext_trigger_tables = lsctables.table.getTablesByName(doc, lsctables.ExtTriggersTable.tableName)
if ext_trigger_tables is None:
print >>sys.stderr, "No tables named external_trigger:table found in " + filename
else:
assert len(ext_trigger_tables) == 1 # ligolw_add should merge them
ext_triggers = ext_trigger_tables[0]
return ext_triggers
def make_psd_xmldoc(psddict):
Attributes = ligolw.sax.xmlreader.AttributesImpl
xmldoc = ligolw.Document()
root_name = u"psd"
lw = xmldoc.appendChild(ligolw.LIGO_LW(Attributes({u"Name": root_name})))
for instrument, psd in psddict.items():
xmlseries = _build_series(psd, (u"Frequency,Real", u"Frequency"), None,
'deltaF', 's^-1')
fs = lw.appendChild(xmlseries)
fs.appendChild(ligolw_param.from_pyvalue(u"instrument", instrument))
return xmldoc
def run_segment_operation(outdoc, filenames, segments, use_segment_table, operation, result_name = 'RESULT', preserve = True):
"""
Performs an operation (intersect or union) across a set of segments.
That is, given a set of files each with segment definers DMT-FLAG1,
DMT-FLAG2 etc and a list of segments DMT-FLAG1,DMT-FLAG1 this returns
RESULT = (table 1's DMT-FLAG1 union table 2's DMT-FLAG1 union ...)
operation
(table 1's DMT-FLAG2 union table 2's DMT-FLAG2 union ...)
operation
etc
"""
proc_id = lsctables.ProcessTable.get_table(outdoc)[0].process_id
if preserve:
indoc = ligolw_add.ligolw_add(outdoc, filenames)
else:
indoc = ligolw_add.ligolw_add(ligolw.Document(), filenames)
# Start with a segment covering all of time, then
# intersect with each of the fields of interest
keys = segments.split(',')
if operation == INTERSECT:
sgmntlist = glue.segments.segmentlist([glue.segments.segment(-glue.segments.infinity(), glue.segments.infinity())])
for key in keys:
sgmntlist &= find_segments(indoc, key, use_segment_table)
elif operation == UNION:
sgmntlist = glue.segments.segmentlist([])
for key in keys:
sgmntlist |= find_segments(indoc, key, use_segment_table)
elif operation == DIFF:
sgmntlist = find_segments(indoc, keys[0], use_segment_table)
for key in keys[1:]:
sgmntlist -= find_segments(indoc, key, use_segment_table)
else:
raise NameError("%s is not a known operation (intersect, union or diff)" % operation)
# Add a segment definer and segments
seg_def_id = add_to_segment_definer(outdoc, proc_id, '', result_name, 1)
if use_segment_table:
add_to_segment(outdoc, proc_id, seg_def_id, sgmntlist)
else:
add_to_segment_summary(outdoc, proc_id, seg_def_id, sgmntlist)
return outdoc, abs(sgmntlist)
def new_doc(comment=None, **kwargs):
doc = ligolw.Document()
doc.appendChild(ligolw.LIGO_LW())
process = ligolw_process.register_to_xmldoc(doc,
program=u"ligolw_tisi",
paramdict=kwargs,
version=__version__,
cvs_repository=u"lscsoft",
cvs_entry_time=__date__,
comment=comment)
timeslidetable = lsctables.New(lsctables.TimeSlideTable)
doc.childNodes[0].appendChild(timeslidetable)
return doc, process
def write_to_xml(cells,
intr_prms,
pin_prms={},
fvals=None,
fname=None,
verbose=False):
"""
Write a set of cells, with dimensions corresponding to intr_prms to an XML file as sim_inspiral rows.
"""
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
procrow = process.append_process(xmldoc, program=sys.argv[0])
procid = procrow.process_id
process.append_process_params(
xmldoc, procrow, process.process_params_from_dict(opts.__dict__))
rows = ["simulation_id", "process_id", "numrel_data"]
# Override eff_lambda to with psi0, its shoehorn column
if "eff_lambda" in intr_prms:
intr_prms[intr_prms.index("eff_lambda")] = "psi0"
if "deff_lambda" in intr_prms:
intr_prms[intr_prms.index("deff_lambda")] = "psi3"
rows += list(intr_prms)
rows += list(pin_prms)
if fvals is not None:
rows.append("alpha1")
sim_insp_tbl = lsctables.New(lsctables.SimInspiralTable, rows)
for itr, intr_prm in enumerate(cells):
sim_insp = sim_insp_tbl.RowType()
# FIXME: Need better IDs
sim_insp.numrel_data = "INTR_SET_%d" % itr
sim_insp.simulation_id = ilwd.ilwdchar(
"sim_inspiral:sim_inspiral_id:%d" % itr)
sim_insp.process_id = procid
if fvals:
sim_insp.alpha1 = fvals[itr]
for p, v in zip(intr_prms, intr_prm._center):
setattr(sim_insp, p, v)
for p, v in pin_prms.iteritems():
setattr(sim_insp, p, v)
sim_insp_tbl.append(sim_insp)
xmldoc.childNodes[0].appendChild(sim_insp_tbl)
if fname is None:
channel_name = ["H=H", "L=L"]
ifos = "".join([o.split("=")[0][0] for o in channel_name])
#start = int(event_time)
start = 0
fname = "%s-MASS_POINTS-%d-1.xml.gz" % (ifos, start)
utils.write_filename(xmldoc, fname, gz=True, verbose=verbose)
def ReadTablesFromFiles(fileList, table_list, verbose=False):
"""
Return a parsed document containing only the tables enumerated in
table_list that appear in the files in fileList.
"""
_is_table = _table_filter_generator(table_list)
doc = ligolw.Document()
searchsummary_handler = ligolw.PartialLIGOLWContentHandler(doc, _is_table)
for thisFile in fileList:
if thisFile.endswith(".gz"):
fileobj = gzip.open(thisFile)
else:
fileobj = open(thisFile)
ligolw.make_parser(searchsummary_handler).parse(fileobj)
return doc
def write_rows(rows, table_type, filename):
"""
Create an empty LIGO_LW XML document, add a table of table_type,
insert the given rows, then write the document to a file.
"""
# prepare a new XML document
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
tbl = lsctables.New(table_type)
xmldoc.childNodes[-1].appendChild(tbl)
# insert our rows
tbl.extend(rows)
# write out the document
utils.write_filename(xmldoc, filename)
def setUp(self):
available_detectors = get_available_detectors()
available_detectors = [a[0] for a in available_detectors]
self.assertTrue('H1' in available_detectors)
self.assertTrue('L1' in available_detectors)
self.assertTrue('V1' in available_detectors)
self.detectors = [Detector(d) for d in ['H1', 'L1', 'V1']]
self.sample_rate = 4096.
self.earth_time = lal.REARTH_SI / lal.C_SI
# create a few random injections
self.injections = []
start_time = float(lal.GPSTimeNow())
taper_choices = ('TAPER_NONE', 'TAPER_START', 'TAPER_END',
'TAPER_STARTEND')
for i, taper in zip(xrange(20), itertools.cycle(taper_choices)):
inj = MyInjection()
inj.end_time = start_time + 40000 * i + \
numpy.random.normal(scale=3600)
random = numpy.random.uniform
inj.mass1 = random(low=1., high=20.)
inj.mass2 = random(low=1., high=20.)
inj.distance = random(low=0.9, high=1.1) * 1e6 * lal.PC_SI
inj.latitude = numpy.arccos(random(low=-1, high=1))
inj.longitude = random(low=0, high=2 * lal.PI)
inj.inclination = numpy.arccos(random(low=-1, high=1))
inj.polarization = random(low=0, high=2 * lal.PI)
inj.taper = taper
self.injections.append(inj)
# create LIGOLW document
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
# create sim inspiral table, link it to document and fill it
sim_table = lsctables.New(lsctables.SimInspiralTable)
xmldoc.childNodes[-1].appendChild(sim_table)
for i in xrange(len(self.injections)):
row = sim_table.RowType()
self.injections[i].fill_sim_inspiral_row(row)
row.process_id = 'process:process_id:0'
row.simulation_id = 'sim_inspiral:simulation_id:%d' % i
sim_table.append(row)
# write document to temp file
self.inj_file = tempfile.NamedTemporaryFile(suffix='.xml')
ligolw_utils.write_fileobj(xmldoc, self.inj_file)
def make_psd_xmldoc(psddict, xmldoc=None):
"""Add a set of PSDs to a LIGOLW XML document. If the document is not
given, a new one is created first.
"""
xmldoc = ligolw.Document() if xmldoc is None else xmldoc.childNodes[0]
# the PSDs must be children of a LIGO_LW with name "psd"
root_name = u"psd"
Attributes = ligolw.sax.xmlreader.AttributesImpl
lw = xmldoc.appendChild(ligolw.LIGO_LW(Attributes({u"Name": root_name})))
for instrument, psd in psddict.items():
xmlseries = _build_series(psd, (u"Frequency,Real", u"Frequency"), None,
'deltaF', 's^-1')
fs = lw.appendChild(xmlseries)
fs.appendChild(LIGOLWParam.from_pyvalue(u'instrument', instrument))
return xmldoc
def __getitem__(self, coinc_event_id):
"""
Construct and return an XML document containing the
sngl_inspiral<-->sngl_inspiral coinc carrying the given
coinc_event_id.
"""
newxmldoc = ligolw.Document()
ligolw_elem = newxmldoc.appendChild(ligolw.LIGO_LW())
# when making these, we can't use .copy() method of Table
# instances because we need to ensure we have a Table
# subclass, not a DBTable subclass
new_process_table = ligolw_elem.appendChild(lsctables.New(lsctables.ProcessTable, self.process_table.columnnames))
new_process_params_table = ligolw_elem.appendChild(lsctables.New(lsctables.ProcessParamsTable, self.process_params_table.columnnames))
new_search_summary_table = ligolw_elem.appendChild(lsctables.New(lsctables.SearchSummaryTable, self.search_summary_table.columnnames))
new_sngl_inspiral_table = ligolw_elem.appendChild(lsctables.New(lsctables.SnglInspiralTable, self.sngl_inspiral_table.columnnames))
new_coinc_def_table = ligolw_elem.appendChild(lsctables.New(lsctables.CoincDefTable, self.coinc_def_table.columnnames))
new_coinc_event_table = ligolw_elem.appendChild(lsctables.New(lsctables.CoincTable, self.coinc_event_table.columnnames))
new_coinc_inspiral_table = ligolw_elem.appendChild(lsctables.New(lsctables.CoincInspiralTable, self.coinc_inspiral_table.columnnames))
new_coinc_event_map_table = ligolw_elem.appendChild(lsctables.New(lsctables.CoincMapTable, self.coinc_event_map_table.columnnames))
new_time_slide_table = ligolw_elem.appendChild(lsctables.New(lsctables.TimeSlideTable, self.time_slide_table.columnnames))
new_coinc_def_table.append(self.coinc_def)
coinc_event = self.coinc_event_index[coinc_event_id]
new_coinc_event_table.append(coinc_event)
new_coinc_inspiral_table.append(self.coinc_inspiral_index[coinc_event_id])
map(new_coinc_event_map_table.append, self.coinc_event_map_index[coinc_event_id])
map(new_time_slide_table.append, self.time_slide_index[coinc_event.time_slide_id])
for row in new_coinc_event_map_table:
new_sngl_inspiral_table.append(self.sngl_inspiral_index[row.event_id])
for process_id in set(new_sngl_inspiral_table.getColumnByName("process_id")) | set(new_coinc_event_table.getColumnByName("process_id")) | set(new_time_slide_table.getColumnByName("process_id")):
# process row is required
new_process_table.append(self.process_index[process_id])
try:
map(new_process_params_table.append, self.process_params_index[process_id])
except KeyError:
# process_params rows are optional
pass
try:
new_search_summary_table.append(self.search_summary_index[process_id])
except KeyError:
# search_summary rows are optional
pass
return newxmldoc
def create_xml(ts_data,psd_segment_length,window_fraction,event_list,station,setname="MagneticFields"):
__program__ = 'pyburst_excesspower'
start_time = LIGOTimeGPS(int(ts_data.start_time))
end_time = LIGOTimeGPS(int(ts_data.end_time))
inseg = segment(start_time,end_time)
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
ifo = 'H1'#channel_name.split(":")[0]
straindict = psd.insert_psd_option_group.__dict__
proc_row = register_to_xmldoc(xmldoc, __program__,straindict, ifos=[ifo],version=git_version.id, cvs_repository=git_version.branch, cvs_entry_time=git_version.date)
outseg = determine_output_segment(inseg, psd_segment_length, ts_data.sample_rate, window_fraction)
ss = append_search_summary(xmldoc, proc_row, ifos=(station,), inseg=inseg, outseg=outseg)
for sb in event_list:
sb.process_id = proc_row.process_id
sb.search = proc_row.program
sb.ifo, sb.channel = station, setname
xmldoc.childNodes[0].appendChild(event_list)
fname = make_filename(station, inseg)
utils.write_filename(xmldoc, fname, gz=fname.endswith("gz"))
def multi_segments_to_file(seg_list, filename, names, ifos):
""" Save segments to an xml file
Parameters
----------
seg_list: glue.segments.segmentlist
List of segment lists to write to disk
filename : str
name of the output file
names :
name of each segment list
ifos :
list of ifos
Returns
-------
File : Return a pycbc.core.File reference to the file
"""
from pycbc.workflow.core import File
# create XML doc and add process table
outdoc = ligolw.Document()
outdoc.appendChild(ligolw.LIGO_LW())
process = ligolw_utils.process.register_to_xmldoc(outdoc, argv[0], {})
for segs, ifo, name in zip(seg_list, ifos, names):
fsegs = [(lal.LIGOTimeGPS(seg[0]), lal.LIGOTimeGPS(seg[1])) \
for seg in segs]
# add segments, segments summary, and segment definer tables using glue library
with ligolw_segments.LigolwSegments(outdoc, process) as xmlsegs:
xmlsegs.insert_from_segmentlistdict({ifo: fsegs}, name)
# write file
ligolw_utils.write_filename(outdoc, filename)
# return a File instance
url = urlparse.urlunparse(
['file', 'localhost', filename, None, None, None])
f = File(ifo, name, segs, file_url=url, tags=[name])
f.PFN(os.path.abspath(filename), site='local')
return f
def make_tseries_xmldoc(psddict, xmldoc=None):
"""
Construct an XML document tree representation of a dictionary of
frequency series objects containing complex data. See also
read_tseries_xmldoc() for a function to parse the resulting XML
documents.
If xmldoc is None (the default), then a new XML document is created
and the frequency series dictionary added to it. If xmldoc is not None
then the frequency series dictionary is appended to the children of that
element inside a new LIGO_LW element.
"""
if xmldoc is None:
xmldoc = ligolw.Document()
lw = xmldoc.appendChild(ligolw.LIGO_LW())
for instrument, psd in psddict.items():
fs = lw.appendChild(build_REAL8TimeSeries(psd))
if instrument is not None:
fs.appendChild(param.from_pyvalue(u"instrument", instrument))
return xmldoc
def make_psd_xmldoc(psddict, xmldoc = None, root_name = u"psd"):
"""
Construct an XML document tree representation of a dictionary of
frequency series objects containing PSDs. See also read_psd_xmldoc()
for a function to parse the resulting XML documents.
If xmldoc is None (the default), then a new XML document is created and
the PSD dictionary added to it inside a LIGO_LW element. If xmldoc is
not None then the PSD dictionary is appended to the children of that
element inside a new LIGO_LW element. In both cases, the LIGO_LW
element's Name attribute is set to root_name. This will be looked for
by read_psd_xmldoc() when parsing the PSD document.
"""
if xmldoc is None:
xmldoc = ligolw.Document()
lw = xmldoc.appendChild(ligolw.LIGO_LW(Attributes({u"Name": root_name})))
for instrument, psd in psddict.items():
fs = lw.appendChild(build_REAL8FrequencySeries(psd))
if instrument is not None:
fs.appendChild(ligolw_param.Param.from_pyvalue(u"instrument", instrument))
return xmldoc
def write(filename, samples, write_params=None, static_args=None):
"""Writes the injection samples to the given xml.
Parameters
----------
filename : str
The name of the file to write to.
samples : io.FieldArray
FieldArray of parameters.
write_params : list, optional
Only write the given parameter names. All given names must be keys
in ``samples``. Default is to write all parameters in ``samples``.
static_args : dict, optional
Dictionary mapping static parameter names to values. These are
written to the ``attrs``.
"""
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
simtable = lsctables.New(lsctables.SimInspiralTable)
xmldoc.childNodes[0].appendChild(simtable)
if static_args is None:
static_args = {}
if write_params is None:
write_params = samples.fieldnames
for ii in range(samples.size):
sim = lsctables.SimInspiral()
# initialize all elements to None
for col in sim.__slots__:
setattr(sim, col, None)
for field in write_params:
data = samples[ii][field]
set_sim_data(sim, field, data)
# set any static args
for (field, value) in static_args.items():
set_sim_data(sim, field, value)
simtable.append(sim)
ligolw_utils.write_filename(xmldoc,
filename,
gz=filename.endswith('gz'))
def create_empty_sim_inspiral_xml(geocent_end_time=0.0):
""" Creates a string of a LIGOLW XML file with an empty sim_inspiral table.
Parameters
----------
geocent_end_time: float
The geocentric end time to add to row in sim_inspiral table.
Retuns
----------
file_contents: str
A string that contains a LIGOLW XML file with an empty
sim_inspiral table.
"""
# create a new sim_inspiral table
cols = lsctables.SimInspiralTable.validcolumns.keys()
sim_table = lsctables.New(lsctables.SimInspiralTable, cols)
# create new LIGOLW XML document and add the new sim_inspiral table
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
xmldoc.childNodes[0].appendChild(sim_table)
# add a row with the geocentric end time column filled
sim = create_empty_sim_inspiral_row()
sim.geocent_end_time = int(geocent_end_time)
sim.geocent_end_time_ns = int(geocent_end_time % 1 * 1e9)
sim_table.append(sim)
# get XML content as a str
fp = StringIO.StringIO()
xmldoc.write(fp)
fp.seek(0)
file_contents = fp.read()
fp.close()
return file_contents
def make_random_document(num_procs, num_seg_defs, num_segs, num_seg_sums,
start_time, end_time, max_len):
"""Create a ligolw document with random segments and segment_summary"""
doc = ligolw.Document()
doc.appendChild(ligolw.LIGO_LW())
# Add some processes
proc_ids = []
seg_def_ids = {}
segment_map = {}
segment_sum_map = {}
for count in range(num_procs):
proc_id = append_process(doc, "Test program %d" % count).process_id
proc_ids.append(proc_id)
seg_def_ids[proc_id] = []
# Add some segment definers
for count in range(num_seg_defs):
proc_id = proc_ids[int(uniform(0, num_procs))]
seg_def_id = add_to_segment_definer(doc, proc_id, "H1",
"TEST_SEG_%d" % count, 1)
seg_def_ids[proc_id].append(seg_def_id)
# Add some segments
sgmntlst = random_segments(start_time, end_time, num_segs, max_len)
add_to_segment(doc, proc_id, seg_def_id, sgmntlst)
sgmntlst.coalesce()
segment_map["TEST_SEG_%d" % count] = sgmntlst
# Add some segment summaries
sgmntlst = random_segments(start_time, end_time, num_segs, max_len)
add_to_segment_summary(doc, proc_id, seg_def_id, sgmntlst)
sgmntlst.coalesce()
segment_sum_map["TEST_SEG_%d" % count] = sgmntlst
return doc, segment_map, segment_sum_map
def tosegmentxml(file, segs):
"""
Write the glue.segments.segmentlist object segs to file object file in xml
format with appropriate tables.
"""
# generate empty document
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
xmldoc.childNodes[-1].appendChild(lsctables.New(lsctables.ProcessTable))
xmldoc.childNodes[-1].appendChild(
lsctables.New(lsctables.ProcessParamsTable))
# append process to table
process = ligolw_process.append_process(xmldoc,\
program='pylal.dq.dqSegmentUtils',\
version=__version__,\
cvs_repository='lscsoft',\
cvs_entry_time=__date__)
gpssegs = segments.segmentlist()
for seg in segs:
gpssegs.append(
segments.segment(LIGOTimeGPS(seg[0]), LIGOTimeGPS(seg[1])))
# append segs and seg definer
segments_tables = ligolw_segments.LigolwSegments(xmldoc)
segments_tables.add(ligolw_segments.LigolwSegmentList(active=gpssegs))
# finalise
segments_tables.coalesce()
segments_tables.optimize()
segments_tables.finalize(process)
ligolw_process.set_process_end_time(process)
# write file
with utils.SignalsTrap():
utils.write_fileobj(xmldoc, file, gz=False)
def gettriggers(plotdesc):
doc = ligolw_add.ligolw_add(ligolw.Document(),
CacheURLs(
eventdisplay.cache[plotdesc.instrument],
plotdesc.segment),
verbose=False,
non_lsc_tables_ok=False)
try:
plotdesc.seglist = table.get_table(
doc,
lsctables.SearchSummaryTable.tableName).get_outlist().coalesce()
except:
plotdesc.seglist = segments.segmentlist()
try:
bursttable = table.get_table(doc, lsctables.SnglBurstTable.tableName)
except:
bursttable = lsctables.New(lsctables.SnglBurstTable)
try:
simtable = table.get_table(doc, lsctables.SimBurstTable.tableName)
except:
simtable = lsctables.New(lsctables.SnglBurstTable)
# cluster
if plotdesc.cluster:
ligolw_bucluster.ClusterSnglBurstTable(
bursttable, SnglBurstUtils.CompareSnglBurstByPeakTimeAndFreq,
ligolw_bucluster.SnglBurstCluster,
SnglBurstUtils.CompareSnglBurstByPeakTime)
# remove triggers and injections that lie outside the required segment
bursttable.filterRows(
lambda row: row.get_peak() in plotdesc.trig_segment())
simtable.filterRows(
lambda row: row.get_time_geocent() in plotdesc.trig_segment())
return bursttable, simtable
def write_round_xml(vetosegs, vetotrigs, winner, ifo, opts):
"""
Write out the products from this round of hveto: veto segments and the vetoed triggers.
"""
# Create a new document
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
# Append the process information
procrow = utils.process.append_process(xmldoc, program="laldetchar-hveto")
utils.process.append_process_params(
xmldoc, procrow, utils.process.process_params_from_dict(opts))
# Add the vetoed triggers
xmldoc.childNodes[0].childNodes.append(vetotrigs)
# Append the veto segments
segl = segmentlistdict()
segl[ifo] = vetosegs
lwsegs = ligolw_segments.LigolwSegments(xmldoc)
lwsegs.insert_from_segmentlistdict(segl, "hveto")
lwsegs.finalize(procrow)
return xmldoc
def to_coinc_xml_object(self, file_name):
outdoc = ligolw.Document()
outdoc.appendChild(ligolw.LIGO_LW())
ifos = list(self.sngl_files.keys())
proc_id = ligolw_process.register_to_xmldoc(
outdoc,
'pycbc', {},
ifos=ifos,
comment='',
version=pycbc_version.git_hash,
cvs_repository='pycbc/' + pycbc_version.git_branch,
cvs_entry_time=pycbc_version.date).process_id
search_summ_table = lsctables.New(lsctables.SearchSummaryTable)
coinc_h5file = self.coinc_file.h5file
try:
start_time = coinc_h5file['segments']['coinc']['start'][:].min()
end_time = coinc_h5file['segments']['coinc']['end'][:].max()
except KeyError:
start_times = []
end_times = []
for ifo_comb in coinc_h5file['segments']:
if ifo_comb == 'foreground_veto':
continue
seg_group = coinc_h5file['segments'][ifo_comb]
start_times.append(seg_group['start'][:].min())
end_times.append(seg_group['end'][:].max())
start_time = min(start_times)
end_time = max(end_times)
num_trigs = len(self.sort_arr)
search_summary = return_search_summary(start_time, end_time, num_trigs,
ifos)
search_summ_table.append(search_summary)
outdoc.childNodes[0].appendChild(search_summ_table)
sngl_inspiral_table = lsctables.New(lsctables.SnglInspiralTable)
coinc_def_table = lsctables.New(lsctables.CoincDefTable)
coinc_event_table = lsctables.New(lsctables.CoincTable)
coinc_inspiral_table = lsctables.New(lsctables.CoincInspiralTable)
coinc_event_map_table = lsctables.New(lsctables.CoincMapTable)
time_slide_table = lsctables.New(lsctables.TimeSlideTable)
# Set up time_slide table
time_slide_id = lsctables.TimeSlideID(0)
for ifo in ifos:
time_slide_row = lsctables.TimeSlide()
time_slide_row.instrument = ifo
time_slide_row.time_slide_id = time_slide_id
time_slide_row.offset = 0
time_slide_row.process_id = proc_id
time_slide_table.append(time_slide_row)
# Set up coinc_definer table
coinc_def_id = lsctables.CoincDefID(0)
coinc_def_row = lsctables.CoincDef()
coinc_def_row.search = "inspiral"
coinc_def_row.description = \
"sngl_inspiral<-->sngl_inspiral coincidences"
coinc_def_row.coinc_def_id = coinc_def_id
coinc_def_row.search_coinc_type = 0
coinc_def_table.append(coinc_def_row)
bank_col_names = ['mass1', 'mass2', 'spin1z', 'spin2z']
bank_col_vals = {}
for name in bank_col_names:
bank_col_vals[name] = self.get_bankfile_array(name)
coinc_event_names = ['ifar', 'time', 'fap', 'stat']
coinc_event_vals = {}
for name in coinc_event_names:
if name == 'time':
coinc_event_vals[name] = self.get_end_time()
else:
coinc_event_vals[name] = self.get_coincfile_array(name)
sngl_col_names = [
'snr', 'chisq', 'chisq_dof', 'bank_chisq', 'bank_chisq_dof',
'cont_chisq', 'cont_chisq_dof', 'end_time', 'template_duration',
'coa_phase', 'sigmasq'
]
sngl_col_vals = {}
for name in sngl_col_names:
sngl_col_vals[name] = self.get_snglfile_array_dict(name)
sngl_event_count = 0
for idx in range(len(self.sort_arr)):
# Set up IDs and mapping values
coinc_id = lsctables.CoincID(idx)
# Set up sngls
# FIXME: As two-ifo is hardcoded loop over all ifos
sngl_combined_mchirp = 0
sngl_combined_mtot = 0
net_snrsq = 0
for ifo in ifos:
# If this ifo is not participating in this coincidence then
# ignore it and move on.
if not sngl_col_vals['snr'][ifo][1][idx]:
continue
event_id = lsctables.SnglInspiralID(sngl_event_count)
sngl_event_count += 1
sngl = return_empty_sngl()
sngl.event_id = event_id
sngl.ifo = ifo
net_snrsq += sngl_col_vals['snr'][ifo][0][idx]**2
for name in sngl_col_names:
val = sngl_col_vals[name][ifo][0][idx]
if name == 'end_time':
sngl.set_end(LIGOTimeGPS(val))
else:
setattr(sngl, name, val)
for name in bank_col_names:
val = bank_col_vals[name][idx]
setattr(sngl, name, val)
sngl.mtotal, sngl.eta = pnutils.mass1_mass2_to_mtotal_eta(
sngl.mass1, sngl.mass2)
sngl.mchirp, _ = pnutils.mass1_mass2_to_mchirp_eta(
sngl.mass1, sngl.mass2)
sngl.eff_distance = (sngl.sigmasq)**0.5 / sngl.snr
sngl_combined_mchirp += sngl.mchirp
sngl_combined_mtot += sngl.mtotal
sngl_inspiral_table.append(sngl)
# Set up coinc_map entry
coinc_map_row = lsctables.CoincMap()
coinc_map_row.table_name = 'sngl_inspiral'
coinc_map_row.coinc_event_id = coinc_id
coinc_map_row.event_id = event_id
coinc_event_map_table.append(coinc_map_row)
sngl_combined_mchirp = sngl_combined_mchirp / len(ifos)
sngl_combined_mtot = sngl_combined_mtot / len(ifos)
# Set up coinc inspiral and coinc event tables
coinc_event_row = lsctables.Coinc()
coinc_inspiral_row = lsctables.CoincInspiral()
coinc_event_row.coinc_def_id = coinc_def_id
coinc_event_row.nevents = len(ifos)
coinc_event_row.instruments = ','.join(ifos)
coinc_inspiral_row.set_ifos(ifos)
coinc_event_row.time_slide_id = time_slide_id
coinc_event_row.process_id = proc_id
coinc_event_row.coinc_event_id = coinc_id
coinc_inspiral_row.coinc_event_id = coinc_id
coinc_inspiral_row.mchirp = sngl_combined_mchirp
coinc_inspiral_row.mass = sngl_combined_mtot
coinc_inspiral_row.set_end(
LIGOTimeGPS(coinc_event_vals['time'][idx]))
coinc_inspiral_row.snr = net_snrsq**0.5
coinc_inspiral_row.false_alarm_rate = coinc_event_vals['fap'][idx]
coinc_inspiral_row.combined_far = 1. / coinc_event_vals['ifar'][idx]
# Transform to Hz
coinc_inspiral_row.combined_far = \
coinc_inspiral_row.combined_far / YRJUL_SI
coinc_event_row.likelihood = coinc_event_vals['stat'][idx]
coinc_inspiral_row.minimum_duration = 0.
coinc_event_table.append(coinc_event_row)
coinc_inspiral_table.append(coinc_inspiral_row)
outdoc.childNodes[0].appendChild(coinc_def_table)
outdoc.childNodes[0].appendChild(coinc_event_table)
outdoc.childNodes[0].appendChild(coinc_event_map_table)
outdoc.childNodes[0].appendChild(time_slide_table)
outdoc.childNodes[0].appendChild(coinc_inspiral_table)
outdoc.childNodes[0].appendChild(sngl_inspiral_table)
ligolw_utils.write_filename(outdoc, file_name)
])
cart_grid2 /= lal.LAL_MSUN_SI
#np.savetxt("m1_m2_pts.txt", cart_grid2)
Njobs = int(np.ceil(len(cart_grid2) / float(pts_per_job)))
cart_grid3 = np.array_split(cart_grid2, Njobs)
for i in range(Njobs):
fname = "m1_m2_pts_%i.txt" % i
np.savetxt(fname, cart_grid3[i])
elapsed = elapsed_time() - elapsed
print("Time to distribute points, split and write to file:", elapsed)
#dag_utils.write_integrate_likelihood_extrinsic_sub('test')
#dag_utils.write_extrinsic_marginalization_dag(cart_grid2, 'test.sub')
xmldoc = ligolw.Document()
xmldoc.childNodes.append(ligolw.LIGO_LW())
#proc_id = process.register_to_xmldoc(xmldoc, sys.argv[0], opts.__dict__)
proc_id = process.register_to_xmldoc(xmldoc, sys.argv[0], {})
proc_id = proc_id.process_id
xmldoc.childNodes[0].appendChild(write_sngl_params(cart_grid3, proc_id))
utils.write_filename(xmldoc, "m1m2_grid.xml.gz", gz=True)
#
# N.B. Below here, the real code will divy up cart_grid into blocks of intrinsic
# parameters and compute the marginalized likelihood on each point
# in intrinsic parameter space
#
# For testing purposes, simply evaluate the overlap on cart_grid and plot
# to confirm it is placing points in the expected ellipsoid with the
# proper distribution, and the overlap behaves properly.
print "RUNNING DATAFIND"
datafinds, scienceSegs = _workflow.setup_datafind_workflow(
workflow, scienceSegs, dfDir, segsList)
# This is needed to know what times will be analysed by daily ahope
# Template bank stuff
banks = _workflow.setup_tmpltbank_workflow(workflow, scienceSegs, datafinds,
dfDir)
# Do matched-filtering
insps = _workflow.setup_matchedfltr_workflow(workflow, scienceSegs, datafinds,
banks, dfDir)
# Now construct the summary XML file
outdoc = ligolw.Document()
outdoc.appendChild(ligolw.LIGO_LW())
# FIXME: PROGRAM NAME and dictionary of opts should be variables defined up above
proc_id = ligolw_process.register_to_xmldoc(outdoc, 'dayhopetest',
vars(args)).process_id
for ifo in workflow.ifos:
# Lets get the segment lists we need
segIfoFiles = segsList.find_output_with_ifo(ifo)
# SCIENCE
sciSegFile = segIfoFiles.find_output_with_tag('SCIENCE')
assert (len(sciSegFile) == 1)
sciSegFile = sciSegFile[0]
sciSegs = sciSegFile.segmentList
# SCIENCE_OK
sciokSegFile = segIfoFiles.find_output_with_tag('SCIENCE_OK')
assert (len(sciokSegFile) == 1)
def make_exttrig_file(cp, ifos, sci_seg, out_dir):
'''
Make an ExtTrig xml file containing information on the external trigger
Parameters
----------
cp : pycbc.workflow.configuration.WorkflowConfigParser object
The parsed configuration options of a pycbc.workflow.core.Workflow.
ifos : str
String containing the analysis interferometer IDs.
sci_seg : glue.segments.segment
The science segment for the analysis run.
out_dir : str
The output directory, destination for xml file.
Returns
-------
xml_file : pycbc.workflow.File object
The xml file with external trigger information.
'''
# Initialise objects
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
tbl = lsctables.New(lsctables.ExtTriggersTable)
cols = tbl.validcolumns
xmldoc.childNodes[-1].appendChild(tbl)
row = tbl.appendRow()
# Add known attributes for this GRB
setattr(row, "event_ra", float(cp.get("workflow", "ra")))
setattr(row, "event_dec", float(cp.get("workflow", "dec")))
setattr(row, "start_time", int(cp.get("workflow", "trigger-time")))
setattr(row, "event_number_grb", str(cp.get("workflow", "trigger-name")))
# Fill in all empty rows
for entry in cols.keys():
if not hasattr(row, entry):
if cols[entry] in ['real_4', 'real_8']:
setattr(row, entry, 0.)
elif cols[entry] == 'int_4s':
setattr(row, entry, 0)
elif cols[entry] == 'lstring':
setattr(row, entry, '')
elif entry == 'process_id':
row.process_id = ilwd.ilwdchar("external_trigger:process_id:0")
elif entry == 'event_id':
row.event_id = ilwd.ilwdchar("external_trigger:event_id:0")
else:
print >> sys.stderr, "Column %s not recognized" % (entry)
raise ValueError
# Save file
xml_file_name = "triggerGRB%s.xml" % str(cp.get("workflow",
"trigger-name"))
xml_file_path = os.path.join(out_dir, xml_file_name)
utils.write_filename(xmldoc, xml_file_path)
xml_file_url = urlparse.urljoin("file:",
urllib.pathname2url(xml_file_path))
xml_file = File(ifos, xml_file_name, sci_seg, file_url=xml_file_url)
xml_file.PFN(xml_file_url, site="local")
return xml_file
def table_to_document(table):
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
xmldoc.childNodes[0].appendChild(table)
return xmldoc
def __init__(self, ifos, coinc_results, **kwargs):
"""Initialize a ligolw xml representation of a zerolag trigger
for upload from pycbc live to gracedb.
Parameters
----------
ifos: list of strs
A list of the ifos pariticipating in this trigger
coinc_results: dict of values
A dictionary of values. The format is defined in
pycbc/events/coinc.py and matches the on disk representation
in the hdf file for this time.
psds: dict of FrequencySeries
Dictionary providing PSD estimates for all involved detectors.
low_frequency_cutoff: float
Minimum valid frequency for the PSD estimates.
followup_data: dict of dicts, optional
Dictionary providing SNR time series for each detector,
to be used in sky localization with BAYESTAR. The format should
be `followup_data['H1']['snr_series']`. More detectors can be
present than given in `ifos`. If so, the extra detectors will only
be used for sky localization.
channel_names: dict of strings, optional
Strain channel names for each detector.
Will be recorded in the sngl_inspiral table.
"""
self.template_id = coinc_results['foreground/%s/template_id' % ifos[0]]
self.coinc_results = coinc_results
self.ifos = ifos
# remember if this should be marked as HWINJ
self.is_hardware_injection = ('HWINJ' in coinc_results
and coinc_results['HWINJ'])
if 'followup_data' in kwargs:
fud = kwargs['followup_data']
assert len({fud[ifo]['snr_series'].delta_t for ifo in fud}) == 1, \
"delta_t for all ifos do not match"
self.snr_series = {ifo: fud[ifo]['snr_series'] for ifo in fud}
usable_ifos = fud.keys()
followup_ifos = list(set(usable_ifos) - set(ifos))
else:
self.snr_series = None
usable_ifos = ifos
followup_ifos = []
# Set up the bare structure of the xml document
outdoc = ligolw.Document()
outdoc.appendChild(ligolw.LIGO_LW())
proc_id = ligolw_process.register_to_xmldoc(
outdoc, 'pycbc', {}, ifos=usable_ifos, comment='',
version=pycbc_version.git_hash,
cvs_repository='pycbc/'+pycbc_version.git_branch,
cvs_entry_time=pycbc_version.date).process_id
# Set up coinc_definer table
coinc_def_table = lsctables.New(lsctables.CoincDefTable)
coinc_def_id = lsctables.CoincDefID(0)
coinc_def_row = lsctables.CoincDef()
coinc_def_row.search = "inspiral"
coinc_def_row.description = "sngl_inspiral<-->sngl_inspiral coincs"
coinc_def_row.coinc_def_id = coinc_def_id
coinc_def_row.search_coinc_type = 0
coinc_def_table.append(coinc_def_row)
outdoc.childNodes[0].appendChild(coinc_def_table)
# Set up coinc inspiral and coinc event tables
coinc_id = lsctables.CoincID(0)
coinc_event_table = lsctables.New(lsctables.CoincTable)
coinc_event_row = lsctables.Coinc()
coinc_event_row.coinc_def_id = coinc_def_id
coinc_event_row.nevents = len(usable_ifos)
coinc_event_row.instruments = ','.join(usable_ifos)
coinc_event_row.time_slide_id = lsctables.TimeSlideID(0)
coinc_event_row.process_id = proc_id
coinc_event_row.coinc_event_id = coinc_id
coinc_event_row.likelihood = 0.
coinc_event_table.append(coinc_event_row)
outdoc.childNodes[0].appendChild(coinc_event_table)
# Set up sngls
sngl_inspiral_table = lsctables.New(lsctables.SnglInspiralTable)
coinc_event_map_table = lsctables.New(lsctables.CoincMapTable)
sngl_populated = None
network_snrsq = 0
for sngl_id, ifo in enumerate(usable_ifos):
sngl = return_empty_sngl(nones=True)
sngl.event_id = lsctables.SnglInspiralID(sngl_id)
sngl.process_id = proc_id
sngl.ifo = ifo
names = [n.split('/')[-1] for n in coinc_results
if 'foreground/%s' % ifo in n]
for name in names:
val = coinc_results['foreground/%s/%s' % (ifo, name)]
if name == 'end_time':
sngl.set_end(lal.LIGOTimeGPS(val))
else:
try:
setattr(sngl, name, val)
except AttributeError:
pass
if sngl.mass1 and sngl.mass2:
sngl.mtotal, sngl.eta = pnutils.mass1_mass2_to_mtotal_eta(
sngl.mass1, sngl.mass2)
sngl.mchirp, _ = pnutils.mass1_mass2_to_mchirp_eta(
sngl.mass1, sngl.mass2)
sngl_populated = sngl
if sngl.snr:
sngl.eff_distance = (sngl.sigmasq)**0.5 / sngl.snr
network_snrsq += sngl.snr ** 2.0
if 'channel_names' in kwargs and ifo in kwargs['channel_names']:
sngl.channel = kwargs['channel_names'][ifo]
sngl_inspiral_table.append(sngl)
# Set up coinc_map entry
coinc_map_row = lsctables.CoincMap()
coinc_map_row.table_name = 'sngl_inspiral'
coinc_map_row.coinc_event_id = coinc_id
coinc_map_row.event_id = sngl.event_id
coinc_event_map_table.append(coinc_map_row)
if self.snr_series is not None:
snr_series_to_xml(self.snr_series[ifo], outdoc, sngl.event_id)
# for subthreshold detectors, respect BAYESTAR's assumptions and checks
bayestar_check_fields = ('mass1 mass2 mtotal mchirp eta spin1x '
'spin1y spin1z spin2x spin2y spin2z').split()
subthreshold_sngl_time = numpy.mean(
[coinc_results['foreground/{}/end_time'.format(ifo)]
for ifo in ifos])
for sngl in sngl_inspiral_table:
if sngl.ifo in followup_ifos:
for bcf in bayestar_check_fields:
setattr(sngl, bcf, getattr(sngl_populated, bcf))
sngl.set_end(lal.LIGOTimeGPS(subthreshold_sngl_time))
outdoc.childNodes[0].appendChild(coinc_event_map_table)
outdoc.childNodes[0].appendChild(sngl_inspiral_table)
# Set up the coinc inspiral table
coinc_inspiral_table = lsctables.New(lsctables.CoincInspiralTable)
coinc_inspiral_row = lsctables.CoincInspiral()
# This seems to be used as FAP, which should not be in gracedb
coinc_inspiral_row.false_alarm_rate = 0
coinc_inspiral_row.minimum_duration = 0.
coinc_inspiral_row.set_ifos(usable_ifos)
coinc_inspiral_row.coinc_event_id = coinc_id
coinc_inspiral_row.mchirp = sngl_populated.mchirp
coinc_inspiral_row.mass = sngl_populated.mtotal
coinc_inspiral_row.end_time = sngl_populated.end_time
coinc_inspiral_row.end_time_ns = sngl_populated.end_time_ns
coinc_inspiral_row.snr = network_snrsq ** 0.5
far = 1.0 / (lal.YRJUL_SI * coinc_results['foreground/ifar'])
coinc_inspiral_row.combined_far = far
coinc_inspiral_table.append(coinc_inspiral_row)
outdoc.childNodes[0].appendChild(coinc_inspiral_table)
# append the PSDs
self.psds = kwargs['psds']
psds_lal = {}
for ifo in self.psds:
psd = self.psds[ifo]
kmin = int(kwargs['low_frequency_cutoff'] / psd.delta_f)
fseries = lal.CreateREAL8FrequencySeries(
"psd", psd.epoch, kwargs['low_frequency_cutoff'], psd.delta_f,
lal.StrainUnit**2 / lal.HertzUnit, len(psd) - kmin)
fseries.data.data = psd.numpy()[kmin:] / pycbc.DYN_RANGE_FAC ** 2.0
psds_lal[ifo] = fseries
make_psd_xmldoc(psds_lal, outdoc)
self.outdoc = outdoc
self.time = sngl_populated.get_end()
def run_file_operation(outdoc,
filenames,
use_segment_table,
operation,
preserve=True):
"""
Performs an operation (intersect or union) across a set of files.
That is, given a set of files each with segment definers DMT-FLAG1,
DMT-FLAG2 etc the result is a file where
DMT-FLAG1 = (file 1's DMT-FLAG1 operation file 2's DMT-FLAG1 operation ...)
DMT-FLAG2 = (file 1's DMT-FLAG2 operation file 2's DMT-FLAG2 operation ...)
etc
"""
proc_id = table.get_table(outdoc,
lsctables.ProcessTable.tableName)[0].process_id
# load up the files into individual documents
xmldocs = [
ligolw_add.ligolw_add(ligolw.Document(), [fname])
for fname in filenames
]
# Get the list of dinstinct segment_definers across all docs
segment_definers = {}
def register_definer(seg_def):
key = (seg_def.ifos, seg_def.name, seg_def.version)
segment_definers[key] = True
return key
for xmldoc in xmldocs:
seg_def_table = table.get_table(xmldoc,
lsctables.SegmentDefTable.tableName)
map(register_definer, seg_def_table)
# For each unique segment definer, find the intersection
for ifo, name, version in segment_definers:
if operation == INTERSECT:
# If I were feeling especially functional-ist I'd write this
# with reduce()
result = glue.segments.segmentlist([
glue.segments.segment(-glue.segments.infinity(),
glue.segments.infinity())
])
for xmldoc in xmldocs:
result &= find_segments(xmldoc,
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
elif operation == UNION:
result = glue.segments.segmentlist([])
for xmldoc in xmldocs:
result |= find_segments(xmldoc,
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
elif operation == DIFF:
result = find_segments(xmldocs[0],
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
for xmldoc in xmldocs[1:]:
result -= find_segments(xmldoc,
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
else:
raise NameError(
"%s is not a known operation (intersect, union or diff)" %
operation)
# Add a segment definer for the result
seg_def_id = add_to_segment_definer(outdoc, proc_id, ifo, name,
version)
# Add the segments
if use_segment_table:
add_to_segment(outdoc, proc_id, seg_def_id, result)
else:
add_to_segment_summary(outdoc, proc_id, seg_def_id, result)
# If we're preserving, also load up everything into the output document.
if preserve:
# Add them to the output document
map(lambda x: outdoc.appendChild(x.childNodes[0]), xmldocs)
# Merge the ligolw elements and tables
ligolw_add.merge_ligolws(outdoc)
ligolw_add.merge_compatible_tables(outdoc)
return outdoc, abs(result)