def load_likelihood_data(filenames, verbose=False):
coinc_params = None
seglists = None
for n, filename in enumerate(filenames, 1):
if verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(
filename,
verbose=verbose,
contenthandler=StringCoincParamsDistributions.LIGOLWContentHandler)
this_coinc_params = StringCoincParamsDistributions.from_xml(
xmldoc, u"string_cusp_likelihood")
this_seglists = lsctables.SearchSummaryTable.get_table(
xmldoc).get_out_segmentlistdict(
lsctables.ProcessTable.get_table(xmldoc).get_ids_by_program(
u"lalapps_string_meas_likelihood")).coalesce()
xmldoc.unlink()
if coinc_params is None:
coinc_params = this_coinc_params
else:
coinc_params += this_coinc_params
if seglists is None:
seglists = this_seglists
else:
seglists |= this_seglists
return coinc_params, seglists
def from_filenames(cls, filenames, name, verbose = False):
"""
Convenience function to deserialize
CoincParamsDistributions objects from a collection of XML
files and return their sum. The return value is a
two-element tuple. The first element is the deserialized
and summed CoincParamsDistributions object, the second is a
segmentlistdict indicating the interval of time spanned by
the out segments in the search_summary rows matching the
process IDs that were attached to the
CoincParamsDistributions objects in the XML.
"""
self = None
for n, filename in enumerate(filenames, 1):
if verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(filename, verbose = verbose, contenthandler = cls.contenthandler)
if self is None:
self = cls.from_xml(xmldoc, name)
seglists = lsctables.SearchSummaryTable.get_table(xmldoc).get_out_segmentlistdict(set([self.process_id])).coalesce()
else:
other = cls.from_xml(xmldoc, name)
self += other
seglists |= lsctables.SearchSummaryTable.get_table(xmldoc).get_out_segmentlistdict(set([other.process_id])).coalesce()
del other
xmldoc.unlink()
return self, seglists
def load_xml_file(filename):
"""Wrapper to ligolw's utils.load_filename"""
xml_doc = utils.load_filename(filename, gz=filename.endswith("gz"),
contenthandler=LIGOLWContentHandler)
return xml_doc
def coinc_without_inj(coinc, tmpdir):
"""Produce a coinc.xml file with the found coincs stripped out."""
filename = str(tmpdir / 'coinc_without_inj.xml')
xmldoc = ligolw_utils.load_filename(coinc, contenthandler=ContentHandler)
# Prune coinc_def table
coinc_def_table = lsctables.CoincDefTable.get_table(xmldoc)
included = [row for row in coinc_def_table
if row.search_coinc_type == InspiralCoincDef.search_coinc_type
and row.search == InspiralCoincDef.search]
included_coinc_def_ids = {row.coinc_def_id for row in included}
coinc_def_table[:] = included
# Prune coinc table
coinc_table = lsctables.CoincTable.get_table(xmldoc)
included = [row for row in coinc_table
if row.coinc_def_id in included_coinc_def_ids]
included_coinc_ids = {row.coinc_event_id for row in included}
coinc_table[:] = included
# Prune coinc_map table
coinc_map_table = lsctables.CoincMapTable.get_table(xmldoc)
coinc_map_table[:] = [row for row in coinc_map_table
if row.coinc_event_id in included_coinc_ids]
ligolw_utils.write_filename(xmldoc, filename)
return filename
def from_filenames(cls, filenames, name, verbose=False):
"""
Convenience function to deserialize
CoincParamsDistributions objects from a collection of XML
files and return their sum. The return value is a
two-element tuple. The first element is the deserialized
and summed CoincParamsDistributions object, the second is a
segmentlistdict indicating the interval of time spanned by
the out segments in the search_summary rows matching the
process IDs that were attached to the
CoincParamsDistributions objects in the XML.
"""
self = None
for n, filename in enumerate(filenames, 1):
if verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(
filename, verbose=verbose, contenthandler=cls.contenthandler)
if self is None:
self = cls.from_xml(xmldoc, name)
seglists = lsctables.SearchSummaryTable.get_table(
xmldoc).get_out_segmentlistdict(set([self.process_id
])).coalesce()
else:
other = cls.from_xml(xmldoc, name)
self += other
seglists |= lsctables.SearchSummaryTable.get_table(
xmldoc).get_out_segmentlistdict(set([other.process_id
])).coalesce()
del other
xmldoc.unlink()
return self, seglists
def create_inj_WFs(filename,
PSD,
sample_rate,
instrument,
N_inj=None,
plot_dir=None):
#https://git.ligo.org/RatesAndPopulations/lvc-rates-and-pop/-/blob/master/bin/lvc_rates_injections
class LIGOLWContentHandler(ligolw.LIGOLWContentHandler):
pass
lsctables.use_in(LIGOLWContentHandler)
xmldoc = utils.load_filename(filename,
verbose=True,
contenthandler=LIGOLWContentHandler)
sim_inspiral_table = lsctables.SimInspiralTable.get_table(xmldoc)
WFs = [] #a list of tuples
if N_inj is None: N_inj = len(sim_inspiral_table)
for row in tqdm(sim_inspiral_table):
WFs.append(get_WF_and_snr(row, PSD, sample_rate, instrument, plot_dir))
if len(WFs) >= N_inj: break
return WFs
def load_xml_table(file_name, table_name):
"""Load xml table from file."""
xml_doc = utils.load_filename(
file_name,
gz=file_name.endswith("gz"),
contenthandler=glsctables.use_in(LIGOLWContentHandler))
return get_table(xml_doc, table_name)
def read_multiinspiral_timeslides_from_files(file_list):
"""
Read time-slid multiInspiral tables from a list of files
"""
multis = None
time_slides = []
contenthandler = glsctables.use_in(LIGOLWContentHandler)
for this_file in file_list:
doc = utils.load_filename(this_file,
gz=this_file.endswith("gz"),
contenthandler=contenthandler)
# Extract the time slide table
time_slide_table = get_table(doc, lsctables.TimeSlideTable.tableName)
slide_mapping = {}
curr_slides = {}
for slide in time_slide_table:
curr_id = int(slide.time_slide_id)
if curr_id not in curr_slides:
curr_slides[curr_id] = {}
curr_slides[curr_id][slide.instrument] = slide.offset
elif slide.instrument not in curr_slides[curr_id]:
curr_slides[curr_id][slide.instrument] = slide.offset
for slide_id, offset_dict in curr_slides.items():
try:
# Is the slide already in the list and where?
offset_index = time_slides.index(offset_dict)
slide_mapping[slide_id] = offset_index
except ValueError:
# If not then add it
time_slides.append(offset_dict)
slide_mapping[slide_id] = len(time_slides) - 1
# Extract the multi inspiral table
try:
multi_inspiral_table = get_table(doc, 'multi_inspiral')
# Remap the time slide IDs
for multi in multi_inspiral_table:
new_id = slide_mapping[int(multi.time_slide_id)]
multi.time_slide_id = gilwdchar(
f"time_slide:time_slide_id:{new_id}")
if multis:
multis.extend(multi_inspiral_table)
else:
multis = multi_inspiral_table
except Exception as exc:
err_msg = "Unable to read a time-slid multiInspiral table "
err_msg += f"from {this_file}."
raise RuntimeError(err_msg) from exc
return multis, time_slides
def marginalize_rankingstatpdf(filenames, verbose = False):
rankingstatpdf = None
for n, filename in enumerate(filenames, 1):
if verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(filename, verbose = verbose, contenthandler = RankingStat.LIGOLWContentHandler)
if rankingstatpdf is None:
rankingstatpdf = RankingStatPDF.from_xml(xmldoc)
else:
rankingstatpdf += RankingStatPDF.from_xml(xmldoc)
xmldoc.unlink()
return rankingstatpdf
def get_segment_summary_times(scienceFile, segmentName):
"""
This function will find the times for which the segment_summary is set
for the flag given by segmentName.
Parameters
-----------
scienceFile : SegFile
The segment file that we want to use to determine this.
segmentName : string
The DQ flag to search for times in the segment_summary table.
Returns
---------
summSegList : ligo.segments.segmentlist
The times that are covered in the segment summary table.
"""
# Parse the segmentName
segmentName = segmentName.split(':')
if not len(segmentName) in [2,3]:
raise ValueError("Invalid channel name %s." %(segmentName))
ifo = segmentName[0]
channel = segmentName[1]
version = ''
if len(segmentName) == 3:
version = int(segmentName[2])
# Load the filename
xmldoc = utils.load_filename(scienceFile.cache_entry.path,
gz=scienceFile.cache_entry.path.endswith("gz"),
contenthandler=LIGOLWContentHandler)
# Get the segment_def_id for the segmentName
segmentDefTable = table.Table.get_table(xmldoc, "segment_definer")
for entry in segmentDefTable:
if (entry.ifos == ifo) and (entry.name == channel):
if len(segmentName) == 2 or (entry.version==version):
segDefID = entry.segment_def_id
break
else:
raise ValueError("Cannot find channel %s in segment_definer table."\
%(segmentName))
# Get the segmentlist corresponding to this segmentName in segment_summary
segmentSummTable = table.Table.get_table(xmldoc, "segment_summary")
summSegList = segments.segmentlist([])
for entry in segmentSummTable:
if entry.segment_def_id == segDefID:
segment = segments.segment(entry.start_time, entry.end_time)
summSegList.append(segment)
summSegList.coalesce()
return summSegList
def parse_segments_xml(path):
llwsegments = ligolw_segments.LigolwSegments(
ligolw_utils.load_filename(path, contenthandler=ligolwcontenthandler))
seglistdicts = { #'frame gate': llwsegments.get_by_name(u'framesegments')}
'h(t) gate': llwsegments.get_by_name(u'whitehtsegments'),
'state vector': llwsegments.get_by_name(u'statevectorsegments'),
'trigger buffers': llwsegments.get_by_name(u'triggersegments')
}
# FIXME This needs to be generalized to more than two IFOs
seglistdicts['joint segments'] = {
'H1L1': seglistdicts['state vector'].intersection(['H1', 'L1'])
}
return seglistdicts
def _read_xml(f, fallbackpath=None):
if f is None:
doc = filename = None
elif isinstance(f, Element):
doc = f
filename = ''
elif isinstance(f, str):
try:
doc = load_filename(f, contenthandler=ContentHandler)
except IOError as e:
if e.errno == errno.ENOENT and fallbackpath and \
not os.path.isabs(f):
f = os.path.join(fallbackpath, f)
doc = load_filename(f, contenthandler=ContentHandler)
else:
raise
filename = f
else:
doc = load_fileobj(f, contenthandler=ContentHandler)
try:
filename = f.name
except AttributeError:
filename = ''
return doc, filename
def select_segments_by_definer(segment_file, segment_name=None, ifo=None):
""" Return the list of segments that match the segment name
Parameters
----------
segment_file: str
path to segment xml file
segment_name: str
Name of segment
ifo: str, optional
Returns
-------
seg: list of segments
"""
from pycbc.io.ligolw import LIGOLWContentHandler as h
indoc = ligolw_utils.load_filename(segment_file, False, contenthandler=h)
segment_table = table.Table.get_table(indoc, 'segment')
seg_def_table = table.Table.get_table(indoc, 'segment_definer')
def_ifos = seg_def_table.getColumnByName('ifos')
def_names = seg_def_table.getColumnByName('name')
def_ids = seg_def_table.getColumnByName('segment_def_id')
valid_id = []
for def_ifo, def_name, def_id in zip(def_ifos, def_names, def_ids):
if ifo and ifo != def_ifo:
continue
if segment_name and segment_name != def_name:
continue
valid_id += [def_id]
start = numpy.array(segment_table.getColumnByName('start_time'))
start_ns = numpy.array(segment_table.getColumnByName('start_time_ns'))
end = numpy.array(segment_table.getColumnByName('end_time'))
end_ns = numpy.array(segment_table.getColumnByName('end_time_ns'))
start, end = start + 1e-9 * start_ns, end + 1e-9 * end_ns
did = segment_table.getColumnByName('segment_def_id')
keep = numpy.array([d in valid_id for d in did])
if sum(keep) > 0:
return start_end_to_segments(start[keep], end[keep])
else:
return segmentlist([])
def load_likelihood_data(filenames, verbose = False):
coinc_params = None
seglists = None
for n, filename in enumerate(filenames, 1):
if verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(filename, verbose = verbose, contenthandler = StringCoincParamsDistributions.LIGOLWContentHandler)
this_coinc_params = StringCoincParamsDistributions.from_xml(xmldoc, u"string_cusp_likelihood")
this_seglists = lsctables.SearchSummaryTable.get_table(xmldoc).get_out_segmentlistdict(lsctables.ProcessTable.get_table(xmldoc).get_ids_by_program(u"lalapps_string_meas_likelihood")).coalesce()
xmldoc.unlink()
if coinc_params is None:
coinc_params = this_coinc_params
else:
coinc_params += this_coinc_params
if seglists is None:
seglists = this_seglists
else:
seglists |= this_seglists
return coinc_params, seglists
def open_xmldoc(fobj, **kwargs):
"""Try and open an existing LIGO_LW-format file, or create a new Document
Parameters
----------
fobj : `str`, `file`
file path or open file object to read
**kwargs
other keyword arguments to pass to
:func:`~ligo.lw.utils.load_filename`, or
:func:`~ligo.lw.utils.load_fileobj` as appropriate
Returns
--------
xmldoc : :class:`~ligo.lw.ligolw.Document`
either the `Document` as parsed from an existing file, or a new, empty
`Document`
"""
from ligo.lw.ligolw import (Document, LIGOLWContentHandler)
from ligo.lw.lsctables import use_in
from ligo.lw.utils import (load_filename, load_fileobj)
use_in(kwargs.setdefault('contenthandler', LIGOLWContentHandler))
try: # try and load existing file
if isinstance(fobj, str):
return load_filename(fobj, **kwargs)
if isinstance(fobj, FILE_LIKE):
return load_fileobj(fobj, **kwargs)[0]
except (OSError, IOError): # or just create a new Document
return Document()
except LigolwElementError as exc:
if LIGO_LW_COMPAT_ERROR.search(str(exc)):
try:
return open_xmldoc(fobj, ilwdchar_compat=True, **kwargs)
except Exception: # for any reason, raise original
pass
raise
def open_xmldoc(fobj, **kwargs):
"""Try and open an existing LIGO_LW-format file, or create a new Document
Parameters
----------
fobj : `str`, `file`
file path or open file object to read
**kwargs
other keyword arguments to pass to
:func:`~ligo.lw.utils.load_filename`, or
:func:`~ligo.lw.utils.load_fileobj` as appropriate
Returns
--------
xmldoc : :class:`~ligo.lw.ligolw.Document`
either the `Document` as parsed from an existing file, or a new, empty
`Document`
"""
from ligo.lw.ligolw import (Document, LIGOLWContentHandler)
from ligo.lw.lsctables import use_in
from ligo.lw.utils import (load_filename, load_fileobj)
use_in(kwargs.setdefault('contenthandler', LIGOLWContentHandler))
try: # try and load existing file
if isinstance(fobj, string_types):
return load_filename(fobj, **kwargs)
if isinstance(fobj, FILE_LIKE):
return load_fileobj(fobj, **kwargs)[0]
except (OSError, IOError): # or just create a new Document
return Document()
except LigolwElementError as exc:
if LIGO_LW_COMPAT_ERROR.search(str(exc)):
try:
return open_xmldoc(fobj, ilwdchar_compat=True, **kwargs)
except Exception: # for any reason, raise original
pass
raise
def start_end_from_segments(segment_file):
"""
Return the start and end time arrays from a segment file.
Parameters
----------
segment_file: xml segment file
Returns
-------
start: numpy.ndarray
end: numpy.ndarray
"""
from pycbc.io.ligolw import LIGOLWContentHandler as h
indoc = ligolw_utils.load_filename(segment_file, False, contenthandler=h)
segment_table = lsctables.SegmentTable.get_table(indoc)
start = numpy.array(segment_table.getColumnByName('start_time'))
start_ns = numpy.array(segment_table.getColumnByName('start_time_ns'))
end = numpy.array(segment_table.getColumnByName('end_time'))
end_ns = numpy.array(segment_table.getColumnByName('end_time_ns'))
return start + start_ns * 1e-9, end + end_ns * 1e-9
def plotbank(in_filename,
out_filename=None,
column1='mchirp',
column2='mtotal'):
"""Plot template bank parameters from a file generated by lalapps_tmpltbank."""
from ligo.lw import ligolw, utils, lsctables
class LIGOLWContentHandler(ligolw.LIGOLWContentHandler):
pass
lsctables.use_in(LIGOLWContentHandler)
table = lsctables.SnglInspiralTable.get_table(
utils.load_filename(in_filename, contenthandler=LIGOLWContentHandler))
pylab.figure()
pylab.title('%s: placement of %d templates' % (in_filename, len(table)))
pylab.plot(table.get_column(column1), table.get_column(column2), ',')
pylab.xlabel(labels[column1])
pylab.ylabel(labels[column2])
pylab.grid()
if out_filename is None:
pylab.show()
else:
pylab.savefig(out_filename)
pylab.close()
# get chunk lengths from the values in the ini file
#
short_segment_duration = config_parser.getint('lalapps_StringSearch', 'short-segment-duration')
pad = config_parser.getint('lalapps_StringSearch', 'pad')
min_segment_length = config_parser.getint('pipeline', 'segment-length') # not including pad at each end
trig_overlap = config_parser.getint('pipeline', 'trig_overlap')
overlap = short_segment_duration / 2 + 2 * pad # FIXME: correct?
#
# get the instruments and raw segments
#
instruments = lsctables.instrumentsproperty.get(config_parser.get('pipeline','ifos'))
segments_cache = set([CacheEntry(None, "SEG", None, "file://localhost" + os.path.abspath(options.segments_file))])
seglists = ligolw_segments.segmenttable_get_by_name(ligolw_utils.load_filename(options.segments_file, contenthandler = ligolw_segments.LIGOLWContentHandler, verbose = options.verbose), options.segments_name).coalesce()
# remove extra instruments
for instrument in set(seglists) - instruments:
if options.verbose:
print("warning: ignoring segments for '%s' found in '%s'" % (instrument, options.segments_file), file=sys.stderr)
del seglists[instrument]
# check for missing instruments
if not instruments.issubset(set(seglists)):
raise ValueError("segment lists retrieved from '%s' missing segments for instruments %s" % (options.segments_file, ", ".join(instruments - set(seglists))))
# now rely on seglists' keys to provide the instruments
del instruments
#
# Using time slide information, construct segment lists describing times
# requiring trigger construction.
#
def test_ligolw_document():
"""Test reading events from LIGO-LW XML document."""
xmldoc = load_filename(os.path.join(DATA_PATH, '2016_subset.xml.gz'),
contenthandler=ContentHandler)
source = events.open(xmldoc, fallbackpath=DATA_PATH)
ligolw_assertions(source)
def __init__(self, sim_file, **kwds):
self.indoc = ligolw_utils.load_filename(
sim_file, False, contenthandler=LIGOLWContentHandler)
self.table = lsctables.SimInspiralTable.get_table(self.indoc)
self.extra_args = kwds
def load_veto_segments(filename, verbose = False, contenthandler = None): return ligolw_segments.segmenttable_get_by_name(ligolw_utils.load_filename(filename, verbose = verbose, contenthandler = contenthandler), "sngl_burst_veto").coalesce()
# =============================================================================
#
options, cachenames = parse_command_line()
cache = []
for filename in cachenames:
cache.extend(cafe.load_cache(filename, options.verbose))
@lsctables.use_in
class LIGOLWContentHandler(lsctables.ligolw.LIGOLWContentHandler):
pass
seglists, outputcaches = cafe.ligolw_cafe(
cache,
lsctables.TimeSlideTable.get_table(
ligolw_utils.load_filename(
options.time_slides,
verbose=options.verbose,
contenthandler=LIGOLWContentHandler)).as_dict().values(),
options.verbose)
instruments = set(seglists.keys())
if options.single_instrument:
cafe.write_single_instrument_caches(options.base, outputcaches,
instruments, options.verbose)
else:
cafe.write_caches(options.base, outputcaches, instruments, options.verbose)
ntuple_comparefunc = None
#
# Iterate over files.
#
for n, filename in enumerate(filenames, start = 1):
#
# Load the file.
#
if options.verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(filename, verbose = options.verbose, contenthandler = ligolw.LIGOLWContentHandler)
#
# Have we already processed it?
#
if ligolw_process.doc_includes_process(xmldoc, process_program_name):
if options.verbose:
print("warning: %s already processed," % (filename or "stdin"), end=' ', file=sys.stderr)
if not options.force:
if options.verbose:
print("skipping", file=sys.stderr)
continue
if options.verbose:
print("continuing by --force", file=sys.stderr)
keep_this_sngl_burst = add_test(keep_this_sngl_burst, lambda burst: burst.stop <= options.max_stop_time)
return keep_this_sngl_burst
keep_this_sngl_burst = make_keep_this_sngl_burst(options)
#
# Get veto segment information.
#
if options.veto_file:
veto_segments = load_veto_segments(options.veto_file, verbose = options.verbose, contenthandler = ContentHandler)
else:
veto_segments = segments.segmentlistdict()
#
# Do the work.
#
for filename in filenames:
xmldoc = ligolw_utils.load_filename(filename, verbose = options.verbose, contenthandler = ContentHandler)
xmldoc = ligolw_bucut(xmldoc, options, keep_this_sngl_burst, veto_segments = veto_segments, del_non_coincs = options.coinc_only, del_skipped_injections = options.inj_made_only, program = options.program, verbose = options.verbose)
ligolw_utils.write_filename(xmldoc, filename, verbose = options.verbose, gz = (filename or "stdout").endswith(".gz"))
xmldoc.unlink()
"inspiral": (InspiralSnglCompare, NearCoincCompare)
}[options.match_algorithm]
#
# loop over files
#
for n, filename in enumerate(filenames, start=1):
#
# load the document
#
if options.verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(
filename,
contenthandler=inspinjfind.LIGOLWContentHandler,
verbose=options.verbose)
#
# process
#
if options.revert:
inspinjfind.revert(xmldoc,
process_program_name,
verbose=options.verbose)
else:
#
# have we already processed it?
#
def do_test(self, n_ifos, n_ifos_followup):
# choose a random selection of interferometers
# n_ifos will be used to generate the simulated trigger
# n_ifos_followup will be used as followup-only
all_ifos = random.sample(self.possible_ifos, n_ifos + n_ifos_followup)
trig_ifos = all_ifos[0:n_ifos]
results = {'foreground/stat': np.random.uniform(4, 20),
'foreground/ifar': np.random.uniform(0.01, 1000)}
followup_data = {}
for ifo in all_ifos:
offset = 10000 + np.random.uniform(-0.02, 0.02)
amplitude = np.random.uniform(4, 20)
# generate a mock SNR time series with a peak
n = 201
dt = 1. / 2048.
t = np.arange(n) * dt
t_peak = dt * n / 2
snr = np.exp(-(t - t_peak) ** 2 * 3e-3 ** -2) * amplitude
snr_series = TimeSeries((snr + 1j * 0).astype(np.complex64),
delta_t=dt, epoch=offset)
# generate a mock PSD
psd_samples = np.random.exponential(size=1024)
psd = FrequencySeries(psd_samples, delta_f=1.)
# fill in the various fields
if ifo in trig_ifos:
base = 'foreground/' + ifo + '/'
results[base + 'end_time'] = t_peak + offset
results[base + 'snr'] = amplitude
results[base + 'sigmasq'] = np.random.uniform(1e6, 2e6)
followup_data[ifo] = {'snr_series': snr_series,
'psd': psd}
for ifo, k in itertools.product(trig_ifos, self.template):
results['foreground/' + ifo + '/' + k] = self.template[k]
channel_names = {ifo: 'TEST' for ifo in all_ifos}
kwargs = {'psds': {ifo: followup_data[ifo]['psd'] for ifo in all_ifos},
'low_frequency_cutoff': 20.,
'followup_data': followup_data,
'channel_names': channel_names}
coinc = SingleCoincForGraceDB(trig_ifos, results, **kwargs)
tempdir = tempfile.mkdtemp()
coinc_file_name = os.path.join(tempdir, 'coinc.xml.gz')
if GraceDb is not None:
# pretend to upload the event to GraceDB.
# The upload will fail, but it should not raise an exception
# and it should still leave the event file around
coinc.upload(coinc_file_name, gracedb_server='localhost',
testing=True)
else:
# no GraceDb module, so just save the coinc file
coinc.save(coinc_file_name)
# read back and check the coinc document
read_coinc = ligolw_utils.load_filename(
coinc_file_name, verbose=False,
contenthandler=LIGOLWContentHandler)
single_table = lsctables.SnglInspiralTable.get_table(read_coinc)
self.assertEqual(len(single_table), len(all_ifos))
coinc_table = lsctables.CoincInspiralTable.get_table(read_coinc)
self.assertEqual(len(coinc_table), 1)
# make sure lalseries can read the PSDs
psd_doc = ligolw_utils.load_filename(
coinc_file_name, verbose=False,
contenthandler=lalseries.PSDContentHandler)
psd_dict = lalseries.read_psd_xmldoc(psd_doc)
self.assertEqual(set(psd_dict.keys()), set(all_ifos))
shutil.rmtree(tempdir)
def __init__(self, options):
"""!
Initialize a DataSourceInfo class instance from command line options specified by append_options()
"""
## A list of possible, valid data sources ("frames", "framexmit", "lvshm", "white", "silence")
self.data_sources = set(
("framexmit", "lvshm", "frames", "white", "silence", "white_live"))
self.live_sources = set(("framexmit", "lvshm", "white_live"))
assert self.live_sources <= self.data_sources
# Sanity check the options
if options.data_source not in self.data_sources:
raise ValueError("--data-source must be one of %s" %
", ".join(self.data_sources))
if options.data_source == "frames" and options.frame_cache is None:
raise ValueError(
"--frame-cache must be specified when using --data-source=frames"
)
if options.frame_segments_file is not None and options.data_source != "frames":
raise ValueError(
"can only give --frame-segments-file if --data-source=frames")
if options.frame_segments_name is not None and options.frame_segments_file is None:
raise ValueError(
"can only specify --frame-segments-name if --frame-segments-file is given"
)
if not (options.channel_list or options.channel_name):
raise ValueError(
"must specify a channel list in the form --channel-list=/path/to/file or --channel-name=H1:AUX-CHANNEL-NAME:RATE --channel-name=H1:SOMETHING-ELSE:RATE"
)
if (options.channel_list and options.channel_name):
raise ValueError(
"must specify a channel list in the form --channel-list=/path/to/file or --channel-name=H1:AUX-CHANNEL-NAME:RATE --channel-name=H1:SOMETHING-ELSE:RATE"
)
## Generate a dictionary of requested channels from channel INI file
# known/permissible values of safety and fidelity flags
self.known_safety = set(
("safe", "unsafe", "unsafeabove2kHz", "unknown"))
self.known_fidelity = set(("clean", "flat", "glitchy", "unknown"))
# ensure safety and fidelity options are valid
options.safety_include = set(options.safety_include)
options.fidelity_exclude = set(options.fidelity_exclude)
for safety in options.safety_include:
assert safety in self.known_safety, '--safety-include=%s is not understood. Must be one of %s' % (
safety, ", ".join(self.known_safety))
for fidelity in options.fidelity_exclude:
assert fidelity in self.known_fidelity, '--fidelity-exclude=%s is not understood. Must be one of %s' % (
fidelity, ", ".join(self.known_fidelity))
# dictionary of the requested channels, e.g., {"H1:LDAS-STRAIN": 16384, "H1:ODC-LARM": 2048}
if options.channel_list:
name, self.extension = options.channel_list.rsplit('.', 1)
if self.extension == 'ini':
self.channel_dict = channel_dict_from_channel_ini(options)
else:
self.channel_dict = channel_dict_from_channel_file(
options.channel_list)
elif options.channel_name:
self.extension = 'none'
self.channel_dict = channel_dict_from_channel_list(
options.channel_name)
# set instrument; it is assumed all channels from a given channel list are from the same instrument
self.instrument = self.channel_dict[next(iter(
self.channel_dict))]['ifo']
# set the maximum number of streams to be run by a single pipeline.
self.max_streams = options.max_streams
# set the frequency ranges considered by channels with splitting into multiple frequency bands.
# If channel sampling rate doesn't fall within this range, it will not be split into multiple bands.
self.max_sample_rate = options.max_sample_rate
self.min_sample_rate = options.min_sample_rate
# split up channels requested into partitions for serial processing
if options.equal_subsets:
self.channel_subsets = partition_channels_to_equal_subsets(
self.channel_dict, self.max_streams, self.min_sample_rate,
self.max_sample_rate)
else:
self.channel_subsets = partition_channels_to_subsets(
self.channel_dict, self.max_streams, self.min_sample_rate,
self.max_sample_rate)
## A dictionary for shared memory partition, e.g., {"H1": "LHO_Data", "H2": "LHO_Data", "L1": "LLO_Data", "V1": "VIRGO_Data"}
self.shm_part_dict = {
"H1": "LHO_Data",
"H2": "LHO_Data",
"L1": "LLO_Data",
"V1": "VIRGO_Data"
}
if options.shared_memory_partition is not None:
self.shm_part_dict.update(
datasource.channel_dict_from_channel_list(
options.shared_memory_partition))
## options for shared memory
self.shm_assumed_duration = options.shared_memory_assumed_duration
self.shm_block_size = options.shared_memory_block_size # NOTE: should this be incorporated into options.block_size? currently only used for offline data sources
## A dictionary of framexmit addresses
self.framexmit_addr = framexmit_ports["CIT"]
if options.framexmit_addr is not None:
self.framexmit_addr.update(
datasource.framexmit_dict_from_framexmit_list(
options.framexmit_addr))
self.framexmit_iface = options.framexmit_iface
## Analysis segment. Default is None
self.seg = None
## Set latency output
self.latency_output = options.latency_output
if options.gps_start_time is not None:
if options.gps_end_time is None:
raise ValueError(
"must provide both --gps-start-time and --gps-end-time")
try:
start = LIGOTimeGPS(options.gps_start_time)
except ValueError:
raise ValueError("invalid --gps-start-time '%s'" %
options.gps_start_time)
try:
end = LIGOTimeGPS(options.gps_end_time)
except ValueError:
raise ValueError("invalid --gps-end-time '%s'" %
options.gps_end_time)
if start >= end:
raise ValueError(
"--gps-start-time must be < --gps-end-time: %s < %s" %
(options.gps_start_time, options.gps_end_time))
## Segment from gps start and stop time if given
self.seg = segments.segment(LIGOTimeGPS(options.gps_start_time),
LIGOTimeGPS(options.gps_end_time))
elif options.gps_end_time is not None:
raise ValueError(
"must provide both --gps-start-time and --gps-end-time")
elif options.data_source not in self.live_sources:
raise ValueError(
"--gps-start-time and --gps-end-time must be specified when --data-source not one of %s"
% ", ".join(sorted(self.live_sources)))
if options.frame_segments_file is not None:
## Frame segments from a user defined file
self.frame_segments = ligolw_segments.segmenttable_get_by_name(
ligolw_utils.load_filename(
options.frame_segments_file,
contenthandler=ligolw_segments.LIGOLWContentHandler),
options.frame_segments_name).coalesce()
if self.seg is not None:
# Clip frame segments to seek segment if it
# exists (not required, just saves some
# memory and I/O overhead)
self.frame_segments = segments.segmentlistdict(
(instrument, seglist & segments.segmentlist([self.seg]))
for instrument, seglist in self.frame_segments.items())
else:
## if no frame segments provided, set them to an empty segment list dictionary
self.frame_segments = segments.segmentlistdict(
{self.instrument: None})
## frame cache file
self.frame_cache = options.frame_cache
## block size in bytes to read data from disk
self.block_size = options.block_size
## Data source, one of python.datasource.DataSourceInfo.data_sources
self.data_source = options.data_source
# FIXME: this is ugly, but we have to protect against busted shared memory partitions
if self.data_source == "lvshm":
import subprocess
subprocess.call([
"smrepair", "--bufmode", "5",
self.shm_part_dict[self.instrument]
])
parser.add_option('--injXML', action='store', type='string', dest='injxml',
help='sim_inspiral XML file for injections')
parser.add_option('--outdir', action='store', type='string',
help='output directory')
parser.add_option('--postsamples', action='store', type='string',
default='posterior_samples.dat',
help='filename for posterior samples files')
parser.add_option('--par', action='append', default=[], type='string',
help='parameter names for the p-p plot')
parser.add_option('--skyPPfolder', action='store',dest='skypp',type='string',default=None,help='Path to folder containing png/pdf with 2D skyarea PP plots')
(options, args) = parser.parse_args()
injs = lsctables.SimInspiralTable.get_table(utils.load_filename(options.injxml,contenthandler=lsctables.use_in(ligolw.LIGOLWContentHandler)))
if options.par == []:
parameters = ['m1', 'm2', 'mc', 'eta', 'q', 'theta_jn', 'a1', 'a2', 'tilt1', 'tilt2', 'phi12', 'phi_jl', 'ra', 'dec', 'distance', 'time', 'phi_orb', 'psi']
else:
parameters = options.par
try:
os.mkdir(options.outdir)
except:
pass
pvalues = { }
posfiles=args
Ninj=0
for index,posfile in enumerate(posfiles):
options, filenames = parse_command_line()
#
# Load initial time slides.
#
@lsctables.use_in
class LIGOLWContentHandler(ligolw.LIGOLWContentHandler):
pass
time_slides = {}
for filename in options.add_to:
time_slide_table = lsctables.TimeSlideTable.get_table(ligolw_utils.load_filename(filename, verbose = options.verbose, contenthandler = LIGOLWContentHandler))
extra_time_slides = time_slide_table.as_dict().values()
if options.verbose:
print("Loaded %d time slides." % len(extra_time_slides), file=sys.stderr)
for offsetvect in extra_time_slides:
time_slides[lsctables.TimeSlideTable.get_next_id()] = offsetvect
#
# Make new time slides.
#
if options.verbose:
print("Computing new time slides ...", file=sys.stderr)
def process_options(options, gw_data_source_opts, pipeline, mainloop):
# Locate and load the initialization file
if not options.infile:
print >> sys.stderr, "Initialization file required."
elif not os.path.exists(options.infile):
print >> sys.stderr, "Initialization file path is invalid."
sys.exit(-1)
cfg = SafeConfigParser()
cfg.read(options.infile)
#
# This supplants the ligo_data_find step and is mostly convenience
#
# TODO: Move to a utility library
if gw_data_source_opts.data_source == "frames" and gw_data_source_opts.frame_cache is None:
if gw_data_source_opts.seg is None:
sys.exit(
"No frame cache present, and no GPS times set. Cannot query for data without an interval to query in."
)
# Shamelessly stolen from gw_data_find
print "Querying LDR server for data location."
try:
server, port = os.environ["LIGO_DATAFIND_SERVER"].split(":")
except ValueError:
sys.exit("Invalid LIGO_DATAFIND_SERVER environment variable set")
print "Server is %s:%s" % (server, port)
try:
frame_type = cfg.get("instrument", "frame_type")
except ConfigParser.NoOptionError:
sys.exit(
"Invalid cache location, and no frame type set, so I can't query LDR for the file locations."
)
if frame_type == "":
sys.exit("No frame type set, aborting.")
print "Frame type is %s" % frame_type
connection = datafind.GWDataFindHTTPConnection(host=server, port=port)
print "Equivalent command line is "
# FIXME: Multiple instruments?
inst = gw_data_source_opts.channel_dict.keys()[0]
print "gw_data_find -o %s -s %d -e %d -u file -t %s" % (
inst[0], gw_data_source_opts.seg[0], gw_data_source_opts.seg[1],
frame_type)
cache = connection.find_frame_urls(inst[0],
frame_type,
gw_data_source_opts.seg[0],
gw_data_source_opts.seg[1],
urltype="file",
on_gaps="error")
tmpfile, tmpname = tempfile.mkstemp()
print "Writing cache of %d files to %s" % (len(cache), tmpname)
with open(tmpname, "w") as tmpfile:
cache.tofile(tmpfile)
connection.close()
gw_data_source_opts.frame_cache = tmpname
handler = EPHandler(mainloop, pipeline)
# Enable the periodic output of trigger statistics
if options.channel_monitoring:
handler.channel_monitoring = True
# If a sample rate other than the native rate is requested, we'll need to
# keep track of it
if options.sample_rate is not None:
handler.rate = options.sample_rate
# Does the user want a cache file to track the trigger files we spit out?
# And if so, if you give us a name, we'll update it every time we output,
# else only at the end of the run
if options.file_cache_name is not None:
handler.output_cache_name = options.file_cache_name
# Clustering on/off
handler.clustering = options.clustering
# Be verbose?
handler.verbose = options.verbose
# Instruments and channels
# FIXME: Multiple instruments
if len(gw_data_source_opts.channel_dict.keys()) == 1:
handler.inst = gw_data_source_opts.channel_dict.keys()[0]
else:
sys.exit("Unable to determine instrument.")
# FIXME: Multiple instruments
if gw_data_source_opts.channel_dict[handler.inst] is not None:
handler.channel = gw_data_source_opts.channel_dict[handler.inst]
else:
# TODO: In the future, we may request multiple channels for the same
# instrument -- e.g. from a single raw frame
sys.exit("Unable to determine channel.")
print "Channel name(s): " + handler.channel
# FFT and time-frequency parameters
# Low frequency cut off -- filter bank begins here
handler.flow = cfg.getfloat("tf_parameters", "min-frequency")
# High frequency cut off -- filter bank ends here
handler.fhigh = cfg.getfloat("tf_parameters", "max-frequency")
# Frequency resolution of the finest filters
handler.base_band = cfg.getfloat("tf_parameters", "min-bandwidth")
# Tile duration should not exceed this value
handler.max_duration = cfg.getfloat("tf_parameters", "max-duration")
# Number of resolutions levels. Can't be less than 1, and can't be greater
# than log_2((fhigh-flow)/base_band)
handler.max_bandwidth = cfg.getfloat("tf_parameters", "max-bandwidth")
handler.max_level = int(
math.floor(math.log(handler.max_bandwidth / handler.base_band, 2))) + 1
# Frequency band overlap -- in our case, handler uses 1 - frequency overlap
if options.frequency_overlap > 1 or options.frequency_overlap < 0:
sys.exit("Frequency overlap must be between 0 and 1.")
handler.frequency_overlap = options.frequency_overlap
# DOF options -- this affects which tile types will be calculated
if cfg.has_option("tf_parameters", "max-dof"):
handler.max_dof = cfg.getint("tf_parameters", "max-dof")
if cfg.has_option("tf_parameters", "fix-dof"):
handler.fix_dof = cfg.getint("tf_parameters", "fix-dof")
if cfg.has_option("tf_parameters", "fft-length"):
handler.fft_length = cfg.getfloat("tf_parameters", "fft-length")
if cfg.has_option("cache", "cache-psd-every"):
handler.cache_psd = cfg.getint("cache", "cache-psd-every")
print "PSD caching enabled. PSD will be recorded every %d seconds" % handler.cache_psd
else:
handler.cache_psd = None
if cfg.has_option("cache", "cache-psd-dir"):
handler.cache_psd_dir = cfg.get("cache", "cache-psd-dir")
print "Caching PSD to %s" % handler.cache_psd_dir
# Used to keep track if we need to lock the PSD into the whitener
psdfile = None
if cfg.has_option("cache", "reference-psd"):
psdfile = cfg.get("cache", "reference-psd")
try:
handler.psd = lal.series.read_psd_xmldoc(
ligolw_utils.load_filename(
psdfile,
contenthandler=lal.series.PSDContentHandler))[handler.inst]
print "Reference PSD for instrument %s from file %s loaded" % (
handler.inst, psdfile)
# Reference PSD disables caching (since we already have it)
handler.cache_psd = None
handler.psd_mode = 1
except KeyError: # Make sure we have a PSD for this instrument
sys.exit(
"PSD for instrument %s requested, but not found in file %s. Available instruments are %s"
% (handler.inst, psdfile, str(handler.psd.keys())))
# Triggering options
if cfg.has_option("triggering", "output-file-stride"):
handler.dump_frequency = cfg.getint("triggering", "output-file-stride")
if cfg.has_option("triggering", "output-directory"):
handler.outdir = cfg.get("triggering", "output-directory")
if cfg.has_option("triggering", "output-dir-format"):
handler.outdirfmt = cfg.get("triggering", "output-dir-format")
handler.output = not options.disable_triggers
# FAP thresh overrides SNR thresh, because multiple resolutions will have
# different SNR thresholds, nominally.
if cfg.has_option("triggering", "snr-thresh"):
handler.snr_thresh = cfg.getfloat("triggering", "snr-thresh")
if cfg.has_option("triggering", "fap-thresh"):
handler.fap = cfg.getfloat("triggering", "fap-thresh")
if handler.fap is not None:
print "False alarm probability threshold (in Gaussian noise) is %g" % handler.fap
if handler.snr_thresh is not None:
print "Trigger SNR threshold sqrt(E/ndof-1) is %f" % handler.snr_thresh
# Maximum number of events (+/- a few in the buffer) before which we drop an
# output file
if cfg.has_option("triggering", "events_per_file"):
handler.max_events = cfg.get_int("triggering", "events_per_file")
return handler
#
# Load initial time slides.
#
@lsctables.use_in
class LIGOLWContentHandler(ligolw.LIGOLWContentHandler):
pass
time_slides = {}
for filename in options.add_to:
time_slide_table = lsctables.TimeSlideTable.get_table(
ligolw_utils.load_filename(filename,
verbose=options.verbose,
contenthandler=LIGOLWContentHandler))
extra_time_slides = time_slide_table.as_dict().values()
if options.verbose:
print("Loaded %d time slides." % len(extra_time_slides),
file=sys.stderr)
for offsetvect in extra_time_slides:
time_slides[lsctables.TimeSlideTable.get_next_id()] = offsetvect
#
# Make new time slides.
#
if options.verbose:
print("Computing new time slides ...", file=sys.stderr)
def read_psd(filename, verbose=False):
return psd_instrument_dict(
utils.load_filename(filename,
verbose=verbose,
contenthandler=lalseries.PSDContentHandler), )
#
# =============================================================================
#
options, filenames = parse_command_line()
if options.verbose:
print("time-frequency tiles have %g degrees of freedom" % (2 * options.delta_t * options.delta_f), file=sys.stderr)
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
process = ligolw_process.register_to_xmldoc(xmldoc, u"lalapps_binj_pic", options.options_dict, version = git_version.version)
time_slide_table = xmldoc.childNodes[-1].appendChild(lsctables.TimeSlideTable.get_table(ligolw_utils.load_filename(options.time_slide_xml, verbose = options.verbose, contenthandler = LIGOLWContentHandler)))
time_slide_id = time_slide_table[0].time_slide_id
if options.verbose:
print("associating injections with time slide ID \"%s\": %s" % (time_slide_id, time_slide_table.as_dict()[time_slide_id]), file=sys.stderr)
for row in time_slide_table:
row.process_id = process.process_id
sim_burst_tbl = xmldoc.childNodes[-1].appendChild(lsctables.New(lsctables.SimBurstTable, ["process_id", "simulation_id", "time_slide_id", "waveform", "waveform_number", "ra", "dec", "psi", "time_geocent_gps", "time_geocent_gps_ns", "duration", "frequency", "bandwidth", "egw_over_rsquared", "pol_ellipse_angle", "pol_ellipse_e"]))
for filename in filenames:
if options.verbose:
print("loading %s ..." % filename, file=sys.stderr)
img = Image.open(filename)
width, height = img.size
width, height = int(round(width / float(height) * options.height)), options.height
# Grab all relevant Omicron trigger files
omicron_times = []
omicron_snr = []
omicron_freq = []
for era in eras:
# Generate list of all Omicron SnglBurst xml trigger files
file_list = glob.glob(args.omicron_dir +
'/%s/%s_Omicron/%s/%s-%s_Omicron-*.xml.gz' %
(args.ifo, args.omicron_channel, era, args.ifo,
args.omicron_channel.replace('-', '_')))
# Parse trigger files into SNR, time, and frequency for Omicron triggers
for file_name in file_list:
omicron_xml = utils.load_filename(file_name,
contenthandler=LIGOLWContentHandler)
snglburst_table = lsctables.SnglBurstTable.get_table(omicron_xml)
for row in snglburst_table:
if (row.snr > args.omicron_snr_thresh
and omicron_start_time < row.peak_time < omicron_end_time):
omicron_times.append(row.peak_time +
row.peak_time_ns * 10**(-9))
omicron_snr.append(row.snr)
omicron_freq.append(row.peak_frequency)
# Generate inspiral waveform and calculate f(t) to plot on top of Omicron triggers
hp, hc = get_td_waveform(approximant='SEOBNRv2',
mass1=m1,
mass2=m2,
spin1x=0,
trig_overlap = config_parser.getint('pipeline', 'trig_overlap')
overlap = short_segment_duration / 2 + 2 * pad # FIXME: correct?
#
# get the instruments and raw segments
#
instruments = lsctables.instrumentsproperty.get(
config_parser.get('pipeline', 'ifos'))
segments_cache = set([
CacheEntry(None, "SEG", None,
"file://localhost" + os.path.abspath(options.segments_file))
])
seglists = ligolw_segments.segmenttable_get_by_name(
ligolw_utils.load_filename(
options.segments_file,
contenthandler=ligolw_segments.LIGOLWContentHandler,
verbose=options.verbose), options.segments_name).coalesce()
# remove extra instruments
for instrument in set(seglists) - instruments:
if options.verbose:
print("warning: ignoring segments for '%s' found in '%s'" %
(instrument, options.segments_file),
file=sys.stderr)
del seglists[instrument]
# check for missing instruments
if not instruments.issubset(set(seglists)):
raise ValueError(
"segment lists retrieved from '%s' missing segments for instruments %s"
% (options.segments_file, ", ".join(instruments - set(seglists))))
# now rely on seglists' keys to provide the instruments
del instruments
def load_veto_segments(filename, verbose = False, contenthandler = None): return ligolw_segments.segmenttable_get_by_name(ligolw_utils.load_filename(filename, verbose = verbose, contenthandler = contenthandler), "sngl_burst_veto").coalesce()
default=[],
type='string',
help='parameter names for the p-p plot')
parser.add_option(
'--skyPPfolder',
action='store',
dest='skypp',
type='string',
default=None,
help='Path to folder containing png/pdf with 2D skyarea PP plots')
(options, args) = parser.parse_args()
injs = lsctables.SimBurstTable.get_table(
utils.load_filename(
options.injxml,
contenthandler=LIGOLWContentHandlerExtractSimBurstTable))
if options.par == []:
parameters = [
'frequency', 'quality', 'hrss', 'ra', 'dec', 'psi', 'time',
'alpha', 'polar_eccentricity'
]
else:
parameters = options.par
try:
os.mkdir(options.outdir)
except:
pass
}[options.match_algorithm]
#
# loop over files
#
for n, filename in enumerate(filenames, start = 1):
#
# load the document
#
if options.verbose:
print("%d/%d:" % (n, len(filenames)), end=' ', file=sys.stderr)
xmldoc = ligolw_utils.load_filename(filename, contenthandler = inspinjfind.LIGOLWContentHandler, verbose = options.verbose)
#
# process
#
if options.revert:
inspinjfind.revert(xmldoc, process_program_name, verbose = options.verbose)
else:
#
# have we already processed it?
#
if ligolw_process.doc_includes_process(xmldoc, process_program_name):
if options.verbose:
print("warning: %s already processed," % (filename or "stdin"), end=' ', file=sys.stderr)