def downtime(vetosegs, segments, channel): myflag = DataQualityFlag() myflag.active = vetosegs myflag.known = segments plot = myflag.plot() plot.set_title(r'Active and vetoed segments for %s' %channel) plot.savefig(r'%s_downtime.png'%channel)
def evaluate_flag(flag, triggers=None, metrics=['deadtime'], injections=None,
minduration=0, vetotag='', channel=None, etg=None):
"""Evaluate the performance of a set of a `~gwpy.segments.DataQualityFlag`
Parameters
----------
flag : `~gwpy.segments.DataQualityFlag`
the data-quality flag to be tested
triggers : `~glue.ligolw.table.Table`, optional
the set of analysis event triggers against which to test
metrics : `list`, optional
the list of `Metrics <~gwvet.Metric>`
injections : `~glue.ligolw.table.Table`, `~gwpy.segments.SegmentList`, optional
a list of injections, or injection segments, against which to test
flag safety
minduration : `float`, optional
the minimum duration of post-veto segments, if applicable, default: 0
Returns
-------
results, after : `OrderedDict`, `~glue.ligolw.table.Table`
the results of each metric test, and the triggers after vetoes have
been applied (or `None` if not given)
"""
# format as flag
if not isinstance(flag, DataQualityFlag):
flag = DataQualityFlag(active=flag)
else:
flag = flag.copy()
# get inverse of veto segments
if minduration:
post = type(flag.known)([s for s in (flag.known - flag.active)
if float(abs(s)) >= minduration])
flag.active = flag.known - post
# apply vetoes to triggers
triggers.etg = etg
if triggers is not None:
after = veto(triggers, flag, tag=vetotag, channel=channel, etg=etg)
else:
after = None
# test each metric
out = OrderedDict()
for metric in metrics:
if isinstance(metric, Metric):
_metric = metric
else:
_metric = get_metric(metric)
if _metric.name.lower() == 'safety':
out[metric] = _metric(flag, injections)
elif _metric.needs_triggers:
out[metric] = _metric(flag, triggers, after=after)
else:
out[metric] = _metric(flag)
return out, after
def test_deprecated(self):
with pytest.warns(DeprecationWarning):
flag = DataQualityFlag(FLAG1, active=ACTIVE, valid=KNOWN)
with pytest.warns(DeprecationWarning):
flag.valid
with pytest.warns(DeprecationWarning):
flag.valid = flag.known
with pytest.warns(DeprecationWarning):
del flag.valid
def test_coalesce(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
self.assertFalse(flag.regular,
'flag.regular test failed (should be False)')
flag.coalesce()
self.assertTrue(flag.known == KNOWN, 'flag.known changed by coalesce')
self.assertTrue(flag.active == KNOWNACTIVE,
'flag.active misset by coalesce')
self.assertTrue(flag.regular,
'flag.regular test failed (should be True)')
def test_pad(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
# test without arguments (and no padding)
padded = flag.pad()
self.assertListEqual(padded.known, flag.known)
self.assertListEqual(padded.active, flag.active)
# test without arguments (and no padding)
flag.padding = PADDING
padded = flag.pad()
self.assertListEqual(padded.known, KNOWNPAD)
self.assertListEqual(padded.active, ACTIVEPAD)
# test with arguments
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
padded = flag.pad(*PADDING)
self.assertListEqual(padded.known, KNOWNPAD)
self.assertListEqual(padded.active, ACTIVEPAD)
# test coalesce
padded.coalesce()
self.assertListEqual(padded.active, ACTIVEPADC)
# test in-place
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
padded = flag.pad(*PADDING)
self.assertIsNot(flag, padded)
padded = flag.pad(*PADDING, inplace=True)
self.assertIs(flag, padded)
# test other kwargs fail
self.assertRaises(TypeError, flag.pad, *PADDING, kwarg='test')
def test_write_hdf5(self, delete=True):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
hdfout = self.tmpfile % 'hdf'
try:
flag.write(hdfout)
except ImportError as e:
self.skipTest(str(e))
else:
if delete:
os.remove(hdfout)
return hdfout
def summary_stats(statistics, bbh_trigs, omicron_trigs, channel, vetosegs, segments):
# Metrics being considered
eff = get_metric('efficiency')
dt = get_metric('deadtime')
eff_over_dt = get_metric('efficiency/deadtime')
usep = get_metric('my use percentage')
loudbysnr = get_metric('loudest event by snr')
get_percentile= get_metric('percentile')
myflag = DataQualityFlag()
myflag.active = vetosegs
myflag.known = segments
statistics[i] = (channel, eff(myflag, bbh_trigs).value, dt(myflag).value,\
eff_over_dt(myflag, bbh_trigs).value, usep(myflag, omic_trigs).value,\
loudbysnr(myflag, bbh_trigs).value)
def check_flag(flag, gpstime, duration, pad):
"""Check that a state flag is active during an entire analysis segment
Parameters
----------
flag : `str`
state flag to check
gpstime : `float`
GPS time of required data
duration : `float`
duration (in seconds) of required data
pad : `float`
amount of extra data to read in at the start and end for filtering
Returns
-------
check : `bool`
Boolean switch to pass (`True`) or fail (`False`) depending on whether
the given flag is active
"""
# set GPS start and end time
start = gpstime - duration/2. - pad
end = gpstime + duration/2. + pad
seg = Segment(start, end)
# query for state segments
active = DataQualityFlag.query(flag, start, end,
url=DEFAULT_SEGMENT_SERVER).active
# check that state flag is active during the entire analysis
if (not active.intersects_segment(seg)) or (abs(active[0]) < abs(seg)):
return False
return True
def test_read_ligolw(self):
flag = DataQualityFlag.read(SEGXML, FLAG1, coalesce=False)
self.assertTrue(flag.active == ACTIVE,
'DataQualityFlag.read(ligol) mismatch:\n\n%s\n\n%s'
% (ACTIVE, flag.active))
self.assertTrue(flag.known == KNOWN,
'DataQualityFlag.read(ligol) mismatch:\n\n%s\n\n%s'
% (KNOWN, flag.known))
def query_state_segments(flag, start, end, url=DEFAULT_SEGMENT_SERVER,
pad=(0, 0)):
"""Query a segment database for active segments associated with a flag
"""
# NOTE: DQF.pad pads forward in time at end
return DataQualityFlag.query(
flag, start-pad[0], end+pad[1], url=url,
).coalesce().pad(pad[0], -pad[1]).active
def test_query_dqsegdb(self):
try:
flag = DataQualityFlag.query_dqsegdb(
QUERY_FLAG, QUERY_START, QUERY_END, url=QUERY_URL)
except (ImportError, URLError) as e:
self.skipTest(str(e))
else:
self.assertEqual(flag.known, QUERY_KNOWN)
self.assertEqual(flag.active, QUERY_ACTIVE)
def test_query_segdb(self):
flag = QUERY_FLAGS[0]
try:
result = DataQualityFlag.query_segdb(flag, QUERY_START, QUERY_END,
url=QUERY_URL_SEGDB)
except (SystemExit, LDBDClientException) as e:
self.skipTest(str(e))
self.assertEqual(result.known, QUERY_RESULT[flag].known)
self.assertEqual(result.active, QUERY_RESULT[flag].active)
def test_query_dqsegdb_multi(self):
querymid = int(QUERY_START + (QUERY_END - QUERY_START) /2.)
segs = SegmentList([Segment(QUERY_START, querymid),
Segment(querymid, QUERY_END)])
try:
flag = DataQualityFlag.query_dqsegdb(
QUERY_FLAG, segs, url=QUERY_URL)
except (ImportError, URLError) as e:
self.skipTest(str(e))
else:
self.assertEqual(flag.known, QUERY_KNOWN)
self.assertEqual(flag.active, QUERY_ACTIVE)
def _read_segments(self, filename):
segs = DataQualityFlag.read(filename, self.definition)
# XXX HACK around malformed segment files with no segment_summary table
if segs.active and not segs.known:
segs.known = type(segs.active)(segs.active)
if self.known:
self.known = self.known & segs.known
self.active = self.known & segs.active
else:
self.known = segs.known
self.active = segs.active
return self
def test_read_hdf5(self):
try:
hdfout = self.test_write_hdf5(delete=False)
except ImportError as e:
self.skipTest(str(e))
else:
flag = DataQualityFlag.read(hdfout)
os.remove(hdfout)
self.assertTrue(flag.active == ACTIVE,
'DataQualityFlag.read(hdf5) mismatch:\n\n%s\n\n%s'
% (ACTIVE, flag.active))
self.assertTrue(flag.known == KNOWN,
'DataQualityFlag.read(hdf5) mismatch:\n\n%s\n\n%s'
% (KNOWN, flag.known))
def process(self, *args, **kwargs):
# read the segment files
if os.path.isfile(self.segmentfile):
segs = DataQualityFlag.read(self.segmentfile, coalesce=False)
self.states[0].known = segs.known
self.states[0].active = segs.active
self.states[0].ready = True
else:
warn('Segment file %s not found.' % self.segmentfile)
return
if len(self.states[0].active) == 0:
warn('No segments analysed by daily ahope.')
return
# read the cache files
if os.path.isfile(self.inspiralcachefile):
with open(self.inspiralcachefile, 'r') as fobj:
try:
self.inspiralcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError as e:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.inspiralcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.inspiralcachefile)
return
if os.path.isfile(self.tmpltbankcachefile):
with open(self.tmpltbankcachefile, 'r') as fobj:
try:
self.tmpltbankcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.tmpltbankcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.tmpltbankcachefile)
self.tmpltbankcache = Cache()
# only process if the cachfile was found
super(DailyAhopeTab, self).process(*args, **kwargs)
def test_pad(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
# test without arguments (and no padding)
padded = flag.pad()
self.assertListEqual(padded.known, flag.known)
self.assertListEqual(padded.active, flag.active)
# test without arguments (and no padding)
flag.padding = PADDING
padded = flag.pad()
self.assertListEqual(padded.known, KNOWNPAD)
self.assertListEqual(padded.active, ACTIVEPAD)
# test with arguments
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
padded = flag.pad(*PADDING)
self.assertListEqual(padded.known, KNOWNPAD)
self.assertListEqual(padded.active, ACTIVEPAD)
# test coalesce
padded.coalesce()
self.assertListEqual(padded.active, ACTIVEPADC)
def test_pad(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
# test without arguments (and no padding)
padded = flag.pad()
self.assertListEqual(padded.known, flag.known)
self.assertListEqual(padded.active, flag.active)
# test without arguments (and no padding)
flag.padding = PADDING
padded = flag.pad()
self.assertListEqual(padded.known, KNOWNPAD)
self.assertListEqual(padded.active, ACTIVEPAD)
# test with arguments
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
padded = flag.pad(*PADDING)
self.assertListEqual(padded.known, KNOWNPAD)
self.assertListEqual(padded.active, ACTIVEPAD)
# test coalesce
padded.coalesce()
self.assertListEqual(padded.active, ACTIVEPADC)
# test in-place
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
padded = flag.pad(*PADDING)
self.assertIsNot(flag, padded)
padded = flag.pad(*PADDING, inplace=True)
self.assertIs(flag, padded)
# test other kwargs fail
self.assertRaises(TypeError, flag.pad, *PADDING, kwarg='test')
def query(flag, start, end, url=DEFAULT_SEGMENT_SERVER):
"""Query a segment database for active segments associated with a flag
"""
return DataQualityFlag.query(flag, start, end, url=url)
def flag():
known = SegmentList([Segment(0, 3), Segment(6, 7)])
active = SegmentList([Segment(1, 2), Segment(3, 4), Segment(5, 7)])
return DataQualityFlag(name='Test segments',
known=known,
active=active)
import shutil
from unittest.mock import patch
from gwpy.segments import DataQualityFlag
from matplotlib import (use, rcParams)
use('agg')
from .. import plot
__author__ = 'Alex Urban <*****@*****.**>'
# global test objects
FLAG = DataQualityFlag(known=[(0, 66)],
active=[(16, 42)],
name='X1:TEST-FLAG:1')
# -- test utilities -----------------------------------------------------------
def test_texify():
name = 'X1:TEST-CHANNEL_NAME'
# test with LaTeX
with patch.dict(rcParams, {'text.usetex': True}):
assert plot.texify(name) == name.replace('_', r'\_')
# test without LaTeX
with patch.dict(rcParams, {'text.usetex': False}):
assert plot.texify(name) == name
def test_repr_str(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
repr(flag)
str(flag)
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
FIXME
Returns
-------
FIXME
"""
if isinstance(flag, (unicode, str)):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag, isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEAFULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(operator.and_, (globalv.SEGMENTS.get(
f, DataQualityFlag(f)).known
for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= segs.known
out[compound].active &= segs.known
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, basestring):
return out[flag]
else:
return out
FLAG_HTML_WITH_PLOTS = FLAG_CONTENT.format(
content='<pre># seg\tstart\tstop\tduration\n0\t0\t66\t66.0\n</pre>',
plots='\n<div class="row scaffold">\n<div class="col-sm-12">\n<a '
'href="plots/X1-TEST_FLAG-0-66.png" id="a_X1-TEST_FLAG_66" '
'title="Known (small) and active (large) analysis segments for '
'X1:TEST_FLAG" class="fancybox" target="_blank" data-caption='
'"Known (small) and active (large) analysis segments for '
'X1:TEST_FLAG" data-fancybox="gallery" data-fancybox-group="images"'
'>\n<img id="img_X1-TEST_FLAG_66" alt="X1-TEST_FLAG-0-66.png" '
'class="img-fluid w-100" src="plots/X1-TEST_FLAG-0-66.png" />\n</a>'
'\n</div>\n</div>')
FLAG_HTML_NO_SEGMENTS = FLAG_CONTENT.format(
content='<p>No segments were found.</p>', plots='')
FLAG = DataQualityFlag(known=[(0, 66)], active=[(0, 66)], name='X1:TEST_FLAG')
OMEGA_SCAFFOLD = """<div class="card card-x1">
<div class="card-header pb-0">
<h5 class="card-title"><a class="cis-link" href="https://cis.ligo.org/channel/byname/X1:STRAIN" title="CIS entry for X1:STRAIN" target="_blank">X1:STRAIN</a></h5>
</div>
<div class="card-body">
<ul class="list-group">
<li class="list-group-item">
<div class="container">
<div class="row">
<h6>
<a href="./1126259462" class="text-dark">1126259462</a>
</h6>
</div>
<div class="row scaffold">
def test_read_segwizard(self):
flag = DataQualityFlag.read(SEGWIZ, FLAG1, coalesce=False)
self.assertTrue(flag.active == ACTIVE,
'DataQualityFlag.read(segwizard) mismatch:\n\n%s\n\n%s'
% (ACTIVE, flag.active))
self.assertTrue(flag.known == flag.active)
def test_plot(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
plot = flag.plot()
self.assertIsInstance(plot, SegmentPlot)
self.assertIsInstance(plot.gca(), SegmentAxes)
self.assertEqual(plot.gca().get_epoch(), flag.known[0][0])
args = parser.parse_args()
freq = args.freq
year = args.year
month = args.month
day = args.day
if len(month) < 2:
month = "0" + month
if len(day) < 2:
day = "0" + day
#=============Get locked segments=============
locked = DataQualityFlag.read("/home/detchar/Segments/K1-DET_FOR_GRB200415A/" +
year + "/K1-DET_FOR_GRB200415A_UTC_" + year +
"-" + month + "-" + day + ".xml")
# Remove segments shorter than 94 sec
act = SegmentList()
for seg in locked.active:
duration = seg[1] - seg[0]
if duration >= 94:
act.append(seg)
# Remove last 30 sec and margin 2 sec
act = act.contract(17)
act = act.shift(-15)
locked.active = act
def test_protract(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
flag.protract(.1)
self.assertListEqual(flag.active, ACTIVE_PROTRACTED)
def validate_segments(ifos, start, end, cp, trigger_time=None):
"""determine analysis ready segments during requested analysis time
Parameters
----------
ifos : `str`
list of ifos used in X-Pipeline analysis
start : `float`, :class:`~gwpy.time.LIGOTimeGPS`
end : `float`, `~gwpy.time.LIGOTimeGPS`
cp : `object` ConfigParser object
trigger_time
Returns
-------
DataQualityDict : ~gwpy.segments.DataQualityDict`
"""
analysis_seg_files = []
# If simulating noise skip all segment, veto
# and network validation checks. The on source and off source is
# simulated and therefore has no DQ issues.
if cp.has_option('parameters', 'makeSimulatedNoise'):
for ifo in ifos:
print('Making simulated noise, creating temp segment file')
f = open('segments_{0}.txt'.format(ifo), 'w')
f.write('0 {0} {1} {2}'.format(start, end, end - start))
f.close()
analysis_seg_files.append('segments_{0}.txt'.format(ifo))
else:
for ifo in ifos:
if cp.has_option(ifo, 'segment-list'):
if not os.path.isfile(cp.get(ifo, 'segment-list')):
raise ValueError('Please uncomment the '
'the segment file in ini file '
'as it does not exist. '
'If you want to use a '
'supplied file please provide one.')
else:
analysis_seg_files.append(cp.get(ifo, 'segment-list'))
else:
# Query for veto definer file
vdf = query_veto_definer_file(ifo, start, end, cp)
# Filter for cat1 vetos
segs = filter_for_cat_type(vdf, ifo, [1])
# ---- Write out cat1 veto to text file.
filename_cat1 = "input/" + ifo + "-veto-cat1.txt"
segs.write(filename_cat1)
# Compute analysis ready segments in order to
# subtract out cat1 vetos
analysis_ready = DataQualityFlag.query(
'{0}:DMT-ANALYSIS_READY:1'.format(ifo), start, end)
# Subtract cat 1 veto from analysis_ready
analysis_ready_minus_cat1 = analysis_ready.active - segs
# Save new segment list to file
filename_analysis_ready_minus_cat1 = "input/" + ifo + "_science_cat1.txt"
analysis_ready_minus_cat1.write(
filename_analysis_ready_minus_cat1)
analysis_seg_files.append(filename_analysis_ready_minus_cat1)
return analysis_seg_files
#! /usr/bin/env python
import sys
from gwpy.segments import DataQualityFlag
def add_job(the_file, job_type, job_number, **kwargs):
job_id = "%s%.6u" % (job_type, job_number)
the_file.write("JOB %s %s.sub\n" % (job_id, job_type))
vars_line = " ".join(
['%s="%s"' % (arg, str(val)) for arg, val in kwargs.iteritems()])
the_file.write("VARS %s %s\n" % (job_id, vars_line))
the_file.write("\n")
if __name__ == "__main__":
segment_file = sys.argv[1]
sci_segs = DataQualityFlag.read(segment_file)
ifo = sci_segs.ifo
segs = sci_segs.active
jobtypes = ['target', 'aux']
fdag = open("my.dag", 'w')
for idx, seg in enumerate(segs):
for jobtype in jobtypes:
add_job(fdag, jobtype, idx, ifo=ifo, st=seg.start, et=seg.end)
def test_plot(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
plot = flag.plot()
self.assertIsInstance(plot, SegmentPlot)
self.assertIsInstance(plot.gca(), SegmentAxes)
self.assertEqual(plot.gca().get_epoch(), flag.known[0][0])
__author__ = 'Duncan Macleod <*****@*****.**>'
# .. currentmodule:: gwpy.segments
#
# Getting the segments
# --------------------
#
# First, we need to fetch the Open Data timeline segments from LOSC.
# To do that we can call the :meth:`DataQualityFlag.fetch_open_data` method
# using ``'H1_DATA'`` as the flag (for an explanation of what this means,
# read up on `The S6 Data Release <https://losc.ligo.org/S6/>`__).
from gwpy.segments import DataQualityFlag
h1segs = DataQualityFlag.fetch_open_data('H1_DATA', 'Sep 16 2010',
'Sep 17 2010')
# For sanity, lets plot these segments:
splot = h1segs.plot(figsize=[12, 3])
splot.show()
splot.close() # hide
# We see that the LIGO Hanford Observatory detector was operating for the
# majority of the day, with a few outages of ~30 minutes or so.
# We can use the :func:`abs` function to display the total amount of time
# spent taking data:
print(abs(h1segs.active))
def test_protract(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
flag.protract(.1)
self.assertListEqual(flag.active, ACTIVE_PROTRACTED)
def main(args=None):
"""Run the online Guardian node visualization tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# parse command line options
ifo = args.ifo
if not args.ifo:
parser.error('--ifo must be given if not obvious from the host')
start = getattr(args, 'gpsstart')
end = getattr(args, 'gpsend')
duration = int(ceil(end) - floor(start))
categories = args.categories.split(',')
for i, c in enumerate(categories):
try:
categories[i] = int(c)
except (TypeError, ValueError):
pass
vetofile = getattr(args, 'veto-definer-file')
vetofile = (urlparse(vetofile).netloc or os.path.abspath(vetofile))
args.metric = args.metric or DEFAULT_METRICS
# -- setup --------------------------------------
tag = '%d-%d' % (start.seconds, end.seconds)
outdir = os.path.abspath(os.path.join(args.output_directory, tag))
mkdir(outdir)
os.chdir(outdir)
mkdir('etc', 'segments', 'condor')
# -- segment handling ---------------------------
os.chdir('segments')
ALLSEGMENTS = DataQualityDict()
# -- get analysis segments ----------------------
aflags = args.analysis_segments
asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo)
for i, flag in enumerate(aflags):
# use union of segments from a file
if os.path.isfile(flag):
asegments += DataQualityFlag.read(flag)
# or intersection of segments from multiple flags
else:
new = DataQualityFlag.query(flag, start, end, url=args.segdb)
if i:
asegments.known &= new.known
asegments.active &= new.active
else:
asegments.known = new.known
asegments.active = new.active
ALLSEGMENTS[asegments.name] = asegments
if os.path.isfile(aflags[0]):
asegments.filename = aflags
# -- read veto definer and process --------------
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
with open(vetofile, 'w') as f:
f.write(tmp.read())
LOGGER.info('Downloaded veto definer file')
vdf = DataQualityDict.from_veto_definer_file(vetofile,
format='ligolw',
start=start,
end=end,
ifo=ifo)
LOGGER.info('Read %d flags from veto definer' % len(vdf.keys()))
# populate veto definer file from database
vdf.populate(source=args.segdb, on_error=args.on_segdb_error)
ALLSEGMENTS += vdf
# organise flags into categories
flags = dict((c, DataQualityDict()) for c in categories)
for name, flag in vdf.items():
try:
flags[flag.category][name] = flag
except KeyError:
pass
# find the states and segments for each category
states, after, oldtitle = (dict(), None, '')
for i, category in enumerate(categories):
title = isinstance(category, int) and 'Cat %d' % category or category
tag = re_cchar.sub('_', str(title).upper())
states[category] = SummaryState(
'After %s' % oldtitle,
key=tag,
known=after.known,
active=after.active,
definition=after.name,
) if i else SummaryState(
args.analysis_name,
key=args.analysis_name,
definition=asegments.name,
)
try:
segs = flags[category].union()
except TypeError: # no flags
segs = DataQualityFlag()
segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag)
ALLSEGMENTS[segs.name] = segs
after = (after - segs) if i else (asegments - segs)
after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag)
ALLSEGMENTS[after.name] = after
oldtitle = title
# write all segments to disk
segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' %
(ifo, start.seconds, duration))
ALLSEGMENTS.write(segfile)
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("All segments accessed and written to\n%s" % segfile)
# -- job preparation ----------------------------
os.chdir('etc')
configs = []
for category in categories:
title = (isinstance(category, int) and 'Category %d' % category
or category)
tab = 'tab-%s' % title
config = ConfigParser()
# add segment-database configuration
add_config_section(config, 'segment-database', url=args.segdb)
# add plot configurations
pconfig = ConfigParser()
pconfig.read(args.config_file)
for section in pconfig.sections():
if section.startswith('plot-'):
config._sections[section] = pconfig._sections[section].copy()
try:
plots = pconfig.items('plots-%s' % category, raw=True)
except NoSectionError:
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
# add state
if args.independent:
state = states[categories[0]]
else:
state = states[category]
sname = 'state-%s' % state.key
add_config_section(config,
sname,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
# add plugin
add_config_section(config, 'plugins', **{'gwvet.tabs': ''})
# define metrics
if category == 1:
metrics = ['Deadtime']
else:
metrics = args.metric
# define summary tab
if category == 1:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{'veto-name': title})
else:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{
'veto-name': title,
'event-channel': args.event_channel,
'event-generator':
args.event_generator,
})
if len(categories) == 1:
config.set(tab, 'index',
'%(gps-start-time)s-%(gps-end-time)s/index.html')
for key, value in plots:
if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605
config.set(tab, key, '%%(union)s,%s' % value)
if '%s-labels' % key not in plots:
config.set(tab, '%s-labels' % key, 'Union,%(flags)s')
else:
config.set(tab, key, value)
# now a tab for each flag
for flag in flags[category]:
if category == 1:
tab = configure_veto_tab(config, flag, title, state, [flag],
segfile, metrics)
else:
tab = configure_veto_tab(
config, flag, title, state, [flag], segfile, metrics, **{
'event-channel': args.event_channel,
'event-generator': args.event_generator
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
if len(categories) > 1 and category != categories[-1]:
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
# configure summary job
if len(categories) > 1:
state = states[categories[0]]
add_config_section(config,
'state-%s' % state.key,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
flags = [f for c in categories for f in flags[c].keys()]
tab = configure_veto_tab(
config,
'Impact of full veto definer file',
None,
state,
flags,
segfile,
args.metric,
shortname='Summary',
index='%(gps-start-time)s-%(gps-end-time)s/index.html',
**{
'event-channel': args.event_channel,
'event-generator': args.event_generator,
'veto-name': 'All vetoes'
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("Generated configuration files for each category")
# -- condor preparation -------------------------
os.chdir(os.pardir)
# get condor variables
if getuser() == 'detchar':
accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy'
else:
accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy'
gwsumm_args = [
'--gps-start-time',
str(start.seconds),
'--gps-end-time',
str(end.seconds),
'--ifo',
ifo,
'--file-tag',
'gwpy-vet',
'--condor-command',
'accounting_group=%s' % accgroup,
'--condor-command',
'accounting_group_user=%s' % getuser(),
'--on-segdb-error',
args.on_segdb_error,
'--output-dir',
args.output_directory,
]
for cf in args.global_config:
gwsumm_args.extend(('--global-config', cf))
for cf in configs:
gwsumm_args.extend(('--config-file', cf))
if args.verbose:
gwsumm_args.append('--verbose')
if args.verbose:
LOGGER.debug('Generating summary DAG via:\n')
LOGGER.debug(' '.join([batch.PROG] + gwsumm_args))
# execute gwsumm in batch mode
batch.main(args=gwsumm_args)
#NOTE TO SELF: If there were no active segments, so we still want to produce the DQ Flag? Or should we tell it to quit out?
known_start = [knownsegments[i, 0] for i in range(len(knownsegments))]
known_end = [knownsegments[i, 1] for i in range(len(knownsegments))]
# reading in trigger files
data = numpy.loadtxt('total_' + args.type_dq_flag + '_trigs.txt')
# get an array for the start_time and end_time of each segment
start_time = [data[i, 0] for i in range(len(data))]
end_time = [data[i, 1] for i in range(len(data))]
# create a data quality flag object
#zip will truncate the start and end time. is this OK?
flag = DataQualityFlag(flag_name,
active=zip(start_time, end_time),
known=zip(known_start, known_end))
# write flag
flag.write(name)
print "Created DQ Flag: " + flag_name + " in .xml form as: " + name
###################################################
##############CREATING VET .INI FILE###############
###################################################
config = ConfigParser.RawConfigParser()
config.add_section('plugins')
config.set('plugins', 'gwvet.tabs', ' ')
params = parse_command_line()
st_dir = int(str(params.st)[0:5])
et_dir = int(str(params.et)[0:5])
dirs = np.arange(st_dir, et_dir + 1)
pipeline_dict = coh_io.read_pipeline_ini(params.ini)
env_params, run_params = coh_io.check_ini_params(pipeline_dict)
segs = []
for directory in dirs:
seg_files = sorted(
glob.glob('%s/SEGMENTS/%d' %
(env_params['base_directory'], directory)))
for f in seg_files:
temps = DataQualityFlag.read(f)
segs.append(temps)
plots = sorted(glob.glob('%s/*.png' % params.directory))
notches_page = 'webpage/notches.html'
params_page = 'webpage/params.html'
diagnostic_plots_page = 'webpage/diagnostic_plots.html'
index_page = 'webpage/index.html'
os.system('mkdir -p webpage')
os.system('touch %s' % notches_page)
os.system('touch %s' % params_page)
os.system('touch %s' % diagnostic_plots_page)
os.system('touch %s' % index_page)
os.system('cp main.css webpage/')
make_index_page(index_page, params)
make_diagnostic_page(diagnostic_plots_page, params)
# set up channel name for grabbing data
channel = ['%s,m-trend' % args.channel]
# Threshold used to generate this flag
thresh = 190 #nm/s
# set up segment file name
chan = channel[0].replace(':', '_')
segment_file = 'L1_EARTHQUAKE_GT_%s_%s-%s.xml' % (thresh, args.start, abs(args.end-args.start))
# name the DQ flag
optic = channel[0].split('_')[2]
flag_name = 'L1:DCH-EQ_%s_GT_%s:1' % (optic, thresh)
# grab all observing (or whatever is defined) time
active = DataQualityFlag.query_dqsegdb(args.science, args.start, args.end).active
# grab only data for the STS channel in observing time
data = get_timeseries_dict(channel, active, frametype='L1_M')
# find times above threshold
time = [j.times[j > thresh] for j in data[channel[0]]]
times = numpy.concatenate(time)
# put all times above threshold in to segments
segs = segments.segmentlist()
segs.extend([segments.segment(int(t.value), int(t.value)+args.stride) for t in times])
segs = segs.coalesce()
# set up the xml file by making a list of the start and end times of the flag
start_time = []
def process(self, config=GWSummConfigParser(), **kwargs):
# set params
self.rounds = None
if not os.path.isdir(self.directory):
self.rounds = None
return
# get some basic info
ifo = config.get('DEFAULT', 'ifo')
# read the configuration
d = os.path.realpath(self.directory).rstrip('/')
self.conf = dict()
confs = glob(os.path.join(d, '%s-HVETO_CONF-*-*.txt' % ifo))
if len(confs) != 1:
self.rounds = 'FAIL'
return
conffile = confs[0]
try:
with open(conffile) as f:
self.conf = dict()
lines = f.readlines()[3:]
for line in lines:
try:
key, val = line.split(': ', 1)
self.conf[key.strip()] = eval(val)
except (ValueError, SyntaxError, NameError):
pass
except IOError:
self.rounds = 'FAIL'
return
else:
etg = self.conf.pop('AUXtype', None)
if 'DEfnm' in self.conf:
name = re_quote.sub('', self.conf['DEfnm'])
self.primary = '%s:%s' % (ifo, name)
if 'DEtype' in self.conf:
hetg = re_quote.sub('', self.conf['DEtype'])
if re.search('_%s\Z' % hetg, self.primary, re.I):
self.primary = self.primary[:-len(hetg) - 1]
else:
self.primary = None
# find the segments
try:
ce = CacheEntry.from_T050017(conffile)
except (ValueError):
start = int(self.span[0])
duration = int(abs(self.span))
span = self.span
else:
start = int(ce.segment[0])
duration = int(abs(ce.segment))
span = ce.segment
try:
statefile = self.conf['dqfnm']
except KeyError:
statefile = '%s-HVETO_DQ_SEGS-%d-%d.txt' % (ifo, start, duration)
if not os.path.isfile(os.path.join(self.directory, statefile)):
self.rounds = 'NOSEGMENTS'
return
# find the results table
resultsfile = os.path.join(self.directory, 'summary_stats.txt')
if not os.path.isfile(resultsfile):
self.rounds = 'FAIL'
return
# determine the Hveto state
cache = Cache(
[CacheEntry.from_T050017(os.path.join(self.directory, statefile))])
segments = SegmentList.read(cache)
globalv.SEGMENTS[self.states[0].definition] = DataQualityFlag(
self.states[0].definition, known=[span], active=segments)
self.finalize_states(config=config, query=False)
# read results file
self.rounds = []
with open(resultsfile, 'r') as f:
for line in f.readlines():
self.rounds.append(
dict(zip(self.summaryrows,
line.split(' ')[1:])))
# fix channel name
c = '%s:%s' % (ifo, self.rounds[-1]['Winning channel'])
if etg and re.search('_%s\Z' % etg, c, re.I):
c = c.rsplit('_', 1)[0]
self.rounds[-1]['Winning channel'] = c
# read starting triggers
rawfile = ('%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' %
(ifo, start, duration))
cache = Cache(
[CacheEntry.from_T050017(os.path.join(self.directory, rawfile))])
get_triggers('%s:hveto_start' % ifo,
'hveto', [self.span],
config=config,
cache=cache,
tablename='sngl_burst',
return_=False)
get_triggers('%s:hveto_vetoed_all' % ifo,
'hveto', [self.span],
config=config,
cache=Cache(),
tablename='sngl_burst')
for r in range(1, len(self.rounds) + 1):
# read round veto triggers
rawfile = ('%s-HVETO_VETOED_TRIGS_ROUND_%d-%d-%d.txt' %
(ifo, r, start, duration))
cache = Cache([
CacheEntry.from_T050017(os.path.join(self.directory, rawfile))
])
trigs = get_triggers('%s:hveto_vetoed_round %d' % (ifo, r),
'hveto', [self.span],
config=config,
cache=cache,
tablename='sngl_burst')
globalv.TRIGGERS['%s:hveto_vetoed_all,hveto' % ifo].extend(trigs)
# read round veto segments
segfile = ('%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' %
(ifo, r, start, duration))
cache = Cache([
CacheEntry.from_T050017(os.path.join(self.directory, segfile))
])
get_segments('%s:hveto_veto_segs_round_%d' % (ifo, r), [self.span],
config=config,
cache=cache,
return_=False)
for plot in self.plots:
if isinstance(plot, HvetoSegmentSummaryPlot):
plot.find_flags()
kwargs['trigcache'] = Cache()
kwargs['segmentcache'] = Cache()
super(HvetoTab, self).process(config=config, **kwargs)
# find some plots
for plot in ['OVERAL_HISTOGRAM', 'OVERAL_EFF_DT'][::-1]:
filename = ('%s-HVETO_%s-%d-%d.png' % (ifo, plot, start, duration))
plotfile = os.path.join(self.directory, filename)
if os.path.isfile(plotfile):
p = SummaryPlot(os.path.join(self.url, filename), new=False)
p.state = self.states[0]
self.plots.insert(0, p)
# delete data from archive
del globalv.SEGMENTS[self.states[0].definition]
for row in range(1, len(self.rounds) + 1):
del globalv.SEGMENTS['%s:hveto_veto_segs_round_%s' % (ifo, row)]
from gwpy.timeseries import (TimeSeries, TimeSeriesDict)
from gwpy.segments import (Segment, DataQualityFlag)
from .. import datafind
__author__ = 'Alex Urban <*****@*****.**>'
# global test objects
HOFT = TimeSeries(numpy.random.normal(loc=1, scale=.5, size=16384 * 66),
sample_rate=16384,
epoch=0,
name='X1:TEST-STRAIN')
FLAG = DataQualityFlag(known=[(-33, 33)],
active=[(-33, 33)],
name='X1:TEST-FLAG:1')
# -- make sure data can be read -----------------------------------------------
@mock.patch('gwpy.segments.DataQualityFlag.query', return_value=FLAG)
def test_check_flag(segserver):
# attempt to query segments database for an analysis flag
flag = 'X1:TEST-FLAG:1'
assert datafind.check_flag(flag, gpstime=0, duration=64, pad=1) is True
assert datafind.check_flag(flag, gpstime=800, duration=64, pad=1) is False
@mock.patch('gwpy.io.gwf.iter_channel_names', return_value=['X1:TEST-STRAIN'])
def test_remove_missing_channels(io_gwf):
def read_data_archive(sourcefile):
"""Read archived data from an HDF5 archive source
This method reads all found data into the data containers defined by
the `gwsumm.globalv` module, then returns nothing.
Parameters
----------
sourcefile : `str`
path to source HDF5 file
"""
from h5py import File
with File(sourcefile, 'r') as h5file:
# -- channels ---------------------------
try:
ctable = Table.read(h5file['channels'])
except KeyError: # no channels table written
pass
else:
for row in ctable:
chan = get_channel(row['name'])
for p in ctable.colnames[1:]:
if row[p]:
setattr(chan, p, row[p])
# -- timeseries -------------------------
for dataset in h5file.get('timeseries', {}).values():
ts = TimeSeries.read(dataset, format='hdf5')
if (re.search(r'\.(rms|min|mean|max|n)\Z', ts.channel.name) and
ts.sample_rate.value == 1.0):
ts.channel.type = 's-trend'
elif re.search(r'\.(rms|min|mean|max|n)\Z', ts.channel.name):
ts.channel.type = 'm-trend'
ts.channel = get_channel(ts.channel)
try:
add_timeseries(ts, key=ts.channel.ndsname)
except ValueError:
if mode.get_mode() != mode.Mode.day:
raise
warnings.warn('Caught ValueError in combining daily archives')
# get end time
globalv.DATA[ts.channel.ndsname].pop(-1)
t = globalv.DATA[ts.channel.ndsname][-1].span[-1]
add_timeseries(ts.crop(start=t), key=ts.channel.ndsname)
# -- statevector -- ---------------------
for dataset in h5file.get('statevector', {}).values():
sv = StateVector.read(dataset, format='hdf5')
sv.channel = get_channel(sv.channel)
add_timeseries(sv, key=sv.channel.ndsname)
# -- spectrogram ------------------------
for tag, add_ in zip(
['spectrogram', 'coherence-components'],
[add_spectrogram, add_coherence_component_spectrogram]):
for key, dataset in h5file.get(tag, {}).items():
key = key.rsplit(',', 1)[0]
spec = Spectrogram.read(dataset, format='hdf5')
spec.channel = get_channel(spec.channel)
add_(spec, key=key)
# -- segments ---------------------------
for name, dataset in h5file.get('segments', {}).items():
dqflag = DataQualityFlag.read(h5file, path=dataset.name,
format='hdf5')
globalv.SEGMENTS += {name: dqflag}
# -- triggers ---------------------------
for dataset in h5file.get('triggers', {}).values():
load_table(dataset)
def read_data_archive(sourcefile):
"""Read archived data from an HDF5 archive source.
Parameters
----------
sourcefile : `str`
path to source HDF5 file
"""
from h5py import File
with File(sourcefile, 'r') as h5file:
# read all time-series data
try:
group = h5file['timeseries']
except KeyError:
group = dict()
for dataset in group.itervalues():
ts = TimeSeries.read(dataset, format='hdf')
if (re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name) and
ts.sample_rate.value == 1.0):
ts.channel.type = 's-trend'
elif re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name):
ts.channel.type = 'm-trend'
ts.channel = get_channel(ts.channel)
try:
add_timeseries(ts, key=ts.channel.ndsname)
except ValueError:
if mode.get_mode() == mode.SUMMARY_MODE_DAY:
raise
warnings.warn('Caught ValueError in combining daily archives')
# get end time
globalv.DATA[ts.channel.ndsname].pop(-1)
t = globalv.DATA[ts.channel.ndsname][-1].span[-1]
add_timeseries(ts.crop(start=t), key=ts.channel.ndsname)
# read all state-vector data
try:
group = h5file['statevector']
except KeyError:
group = dict()
for dataset in group.itervalues():
sv = StateVector.read(dataset, format='hdf')
sv.channel = get_channel(sv.channel)
add_timeseries(sv, key=sv.channel.ndsname)
# read all spectrogram data
try:
group = h5file['spectrogram']
except KeyError:
group = dict()
for key, dataset in group.iteritems():
key = key.rsplit(',', 1)[0]
spec = Spectrogram.read(dataset, format='hdf')
spec.channel = get_channel(spec.channel)
add_spectrogram(spec, key=key)
try:
group = h5file['segments']
except KeyError:
group = dict()
for name, dataset in group.iteritems():
dqflag = DataQualityFlag.read(dataset, format='hdf')
globalv.SEGMENTS += {name: dqflag}
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
flag : `str`, `list`
either the name of one flag, or a list of names
validity : `~gwpy.segments.SegmentList`
the segments over which to search for other segments
query : `bool`, optional, default: `True`
actually execute a read/query operation (if needed), otherwise
just retrieve segments that have already been cached
config : `~configparser.ConfigParser`, optional
the configuration for your analysis, if you have one. If
present the ``[segment-database]`` section will be queried
for the following options
- ``gps-start-time``, and ``gps-end-time``, if ``validity`` is
not given
- ``url`` (the remote hostname for the segment database) if
the ``url`` keyword is not given
cache : :class:`glue.lal.Cache`, optional
a cache of files from which to read segments, otherwise segments
will be downloaded from the segment database
coalesce : `bool`, optional, default: `True`
coalesce all segmentlists before returning, otherwise just return
segments as they were downloaded/read
padding : `tuple`, or `dict` of `tuples`, optional
`(start, end)` padding with which to pad segments that are
downloaded/read
segdb_error : `str`, optional, default: ``'raise'``
how to handle errors returned from the segment database, one of
- ``'raise'`` (default) : raise the exception as normal
- ``'warn'`` : print the exception as a warning, but return no
segments
- ``'ignore'`` : silently ignore the error and return no segments
url : `str`, optional
the remote hostname for the target segment database
return_ : `bool`, optional, default: `True`
internal flag to enable (True) or disable (False) actually returning
anything. This is useful if you want to download/read segments now
but not use them until later (e.g. plotting)
Returns
-------
flag : `~gwpy.segments.DataQualityFlag`
the flag object representing segments for the given single flag, OR
flagdict : `~gwpy.segments.DataQualityDict`
the dict of `~gwpy.segments.DataQualityFlag` objects for multiple
flags, if ``flag`` is given as a `list`, OR
None
if ``return_=False``
"""
if isinstance(flag, str):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag,
isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEFAULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(
operator.and_,
(globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, str):
return out[flag]
else:
return out
def setUp(self):
self.segments = SegmentList([Segment(0, 3), Segment(6, 7)])
active = SegmentList([Segment(1, 2), Segment(3, 4), Segment(5, 7)])
self.flag = DataQualityFlag(name='Test segments',
known=self.segments,
active=active)
def test_fail_write_segwizard(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN)
self.assertRaises(ValueError, flag.write, StringIO, format='segwizard')
os.path.split(__file__)[0], 'data', 'X1-GWPY_TEST_SEGMENTS-0-10.xml.gz')
SEGWIZ = os.path.join(
os.path.split(__file__)[0], 'data', 'X1-GWPY_TEST_SEGMENTS-0-10.txt')
FLAG1 = 'X1:GWPY-TEST_SEGMENTS:1'
FLAG2 = 'X1:GWPY-TEST_SEGMENTS:2'
QUERY_START = 968630415
QUERY_END = 968716815
QUERY_FLAGS = ['H1:DMT-SCIENCE:4', 'L1:DMT-SCIENCE:4']
QUERY_RESULT = DataQualityDict()
QUERY_RESULT['H1:DMT-SCIENCE:4'] = DataQualityFlag('H1:DMT-SCIENCE:4',
known=[(968630415,
968716815)],
active=[
(968632249, 968641010),
(968642871, 968644430),
(968646220, 968681205),
(968686357, 968686575),
(968688760, 968690950),
(968692881, 968714403)
])
QUERY_RESULT['L1:DMT-SCIENCE:4'] = DataQualityFlag('L1:DMT-SCIENCE:4',
known=[(968630415,
968716815)],
active=[
(968630415, 968634911),
(968638548, 968644632),
(968646025, 968675387),
(968676835, 968679443),
(968680215, 968686803),
(968688905, 968691838),
def query(flag, start, end, url='https://segments.ligo.org'):
"""Query a segment database for active segments associated with a flag
"""
return DataQualityFlag.query(flag, start, end, url=url)
def main(args=None):
"""Run the hveto command-line interface
"""
# declare global variables
# this is needed for multiprocessing utilities
global acache, analysis, areadkw, atrigfindkw, auxiliary, auxetg
global auxfreq, counter, livetime, minsnr, naux, pchannel, primary
global rnd, snrs, windows
# parse command-line
parser = create_parser()
args = parser.parse_args(args=args)
ifo = args.ifo
start = int(args.gpsstart)
end = int(args.gpsend)
duration = end - start
# log startup
LOGGER.info("-- Welcome to Hveto --")
LOGGER.info("GPS start time: %d" % start)
LOGGER.info("GPS end time: %d" % end)
LOGGER.info("Interferometer: %s" % ifo)
# -- initialisation -------------------------
# read configuration
cp = config.HvetoConfigParser(ifo=ifo)
cp.read(args.config_file)
LOGGER.info("Parsed configuration file(s)")
# format output directory
outdir = _abs_path(args.output_directory)
if not os.path.isdir(outdir):
os.makedirs(outdir)
os.chdir(outdir)
LOGGER.info("Working directory: %s" % outdir)
segdir = 'segments'
plotdir = 'plots'
trigdir = 'triggers'
omegadir = 'scans'
for d in [segdir, plotdir, trigdir, omegadir]:
if not os.path.isdir(d):
os.makedirs(d)
# prepare html variables
htmlv = {
'title': '%s Hveto | %d-%d' % (ifo, start, end),
'config': None,
'context': ifo.lower(),
}
# get segments
aflag = cp.get('segments', 'analysis-flag')
url = cp.get('segments', 'url')
padding = tuple(cp.getfloats('segments', 'padding'))
if args.analysis_segments:
segs_ = DataQualityDict.read(args.analysis_segments, gpstype=float)
analysis = segs_[aflag]
span = SegmentList([Segment(start, end)])
analysis.active &= span
analysis.known &= span
analysis.coalesce()
LOGGER.debug("Segments read from disk")
else:
analysis = DataQualityFlag.query(aflag, start, end, url=url)
LOGGER.debug("Segments recovered from %s" % url)
if padding != (0, 0):
mindur = padding[0] - padding[1]
analysis.active = type(
analysis.active)([s for s in analysis.active if abs(s) >= mindur])
analysis.pad(*padding, inplace=True)
LOGGER.debug("Padding %s applied" % str(padding))
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("Retrieved %d segments for %s with %ss (%.2f%%) livetime" %
(len(analysis.active), aflag, livetime, livetimepc))
# apply vetoes from veto-definer file
try:
vetofile = cp.get('segments', 'veto-definer-file')
except configparser.NoOptionError:
vetofile = None
else:
try:
categories = cp.getfloats('segments', 'veto-definer-categories')
except configparser.NoOptionError:
categories = None
# read file
vdf = read_veto_definer_file(vetofile, start=start, end=end, ifo=ifo)
LOGGER.debug("Read veto-definer file from %s" % vetofile)
# get vetoes from segdb
vdf.populate(source=url, segments=analysis.active, on_error='warn')
# coalesce flags from chosen categories
vetoes = DataQualityFlag('%s:VDF-VETOES:1' % ifo)
nflags = 0
for flag in vdf:
if not categories or vdf[flag].category in categories:
vetoes += vdf[flag]
nflags += 1
try:
deadtime = int(abs(vetoes.active)) / int(abs(vetoes.known)) * 100
except ZeroDivisionError:
deadtime = 0
LOGGER.debug("Coalesced %ss (%.2f%%) of deadtime from %d veto flags" %
(abs(vetoes.active), deadtime, nflags))
# apply to analysis segments
analysis -= vetoes
LOGGER.debug("Applied vetoes from veto-definer file")
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("%ss (%.2f%%) livetime remaining after vetoes" %
(livetime, livetimepc))
snrs = cp.getfloats('hveto', 'snr-thresholds')
minsnr = min(snrs)
windows = cp.getfloats('hveto', 'time-windows')
# record all segments
segments = DataQualityDict()
segments[analysis.name] = analysis
# -- load channels --------------------------
# get primary channel name
pchannel = cp.get('primary', 'channel')
# read auxiliary cache
if args.auxiliary_cache is not None:
acache = read_cache(args.auxiliary_cache)
else:
acache = None
# load auxiliary channels
auxetg = cp.get('auxiliary', 'trigger-generator')
auxfreq = cp.getfloats('auxiliary', 'frequency-range')
try:
auxchannels = cp.get('auxiliary', 'channels').strip('\n').split('\n')
except config.configparser.NoOptionError:
auxchannels = find_auxiliary_channels(auxetg, (start, end),
ifo=ifo,
cache=acache)
cp.set('auxiliary', 'channels', '\n'.join(auxchannels))
LOGGER.debug("Auto-discovered %d "
"auxiliary channels" % len(auxchannels))
else:
auxchannels = sorted(set(auxchannels))
LOGGER.debug("Read list of %d auxiliary channels" % len(auxchannels))
# load unsafe channels list
_unsafe = cp.get('safety', 'unsafe-channels')
if os.path.isfile(_unsafe): # from file
unsafe = set()
with open(_unsafe, 'rb') as f:
for c in f.read().rstrip('\n').split('\n'):
if c.startswith('%(IFO)s'):
unsafe.add(c.replace('%(IFO)s', ifo))
elif not c.startswith('%s:' % ifo):
unsafe.add('%s:%s' % (ifo, c))
else:
unsafe.add(c)
else: # or from line-seprated list
unsafe = set(_unsafe.strip('\n').split('\n'))
unsafe.add(pchannel)
cp.set('safety', 'unsafe-channels', '\n'.join(sorted(unsafe)))
LOGGER.debug("Read list of %d unsafe channels" % len(unsafe))
# remove unsafe channels
nunsafe = 0
for i in range(len(auxchannels) - 1, -1, -1):
if auxchannels[i] in unsafe:
LOGGER.warning("Auxiliary channel %r identified as unsafe and has "
"been removed" % auxchannels[i])
auxchannels.pop(i)
nunsafe += 1
LOGGER.debug("%d auxiliary channels identified as unsafe" % nunsafe)
naux = len(auxchannels)
LOGGER.info("Identified %d auxiliary channels to process" % naux)
# record INI file in output HTML directory
inifile = '%s-HVETO_CONFIGURATION-%d-%d.ini' % (ifo, start, duration)
if os.path.isfile(inifile) and any(
os.path.samefile(inifile, x) for x in args.config_file):
LOGGER.debug("Cannot write INI file to %s, file was given as input")
else:
with open(inifile, 'w') as f:
cp.write(f)
LOGGER.info("Configuration recorded as %s" % inifile)
htmlv['config'] = inifile
# -- load primary triggers ------------------
# read primary cache
if args.primary_cache is not None:
pcache = read_cache(args.primary_cache)
else:
pcache = None
# load primary triggers
petg = cp.get('primary', 'trigger-generator')
psnr = cp.getfloat('primary', 'snr-threshold')
pfreq = cp.getfloats('primary', 'frequency-range')
preadkw = cp.getparams('primary', 'read-')
if pcache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the primary trigger file format')
preadkw['format'] = None
preadkw['path'] = 'triggers'
ptrigfindkw = cp.getparams('primary', 'trigfind-')
primary = get_triggers(pchannel,
petg,
analysis.active,
snr=psnr,
frange=pfreq,
cache=pcache,
nproc=args.nproc,
trigfind_kwargs=ptrigfindkw,
**preadkw)
fcol, scol = primary.dtype.names[1:3]
if len(primary):
LOGGER.info("Read %d events for %s" % (len(primary), pchannel))
else:
message = "No events found for %r in %d seconds of livetime" % (
pchannel, livetime)
LOGGER.critical(message)
# cluster primary triggers
clusterkwargs = cp.getparams('primary', 'cluster-')
if clusterkwargs:
primary = primary.cluster(**clusterkwargs)
LOGGER.info("%d primary events remain after clustering over %s" %
(len(primary), clusterkwargs['rank']))
# -- bail out early -------------------------
# the bail out is done here so that we can at least generate the eventual
# configuration file, mainly for HTML purposes
# no segments
if livetime == 0:
message = ("No active segments found for analysis flag %r in interval "
"[%d, %d)" % (aflag, start, end))
LOGGER.critical(message)
htmlv['context'] = 'info'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# no primary triggers
if len(primary) == 0:
htmlv['context'] = 'danger'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# otherwise write all primary triggers to ASCII
trigfile = os.path.join(
trigdir,
'%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration),
)
primary.write(trigfile, format='ascii', overwrite=True)
# -- load auxiliary triggers ----------------
LOGGER.info("Reading triggers for aux channels...")
counter = multiprocessing.Value('i', 0)
areadkw = cp.getparams('auxiliary', 'read-')
if acache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the auxiliary trigger file format')
areadkw['format'] = None
areadkw['path'] = 'triggers'
atrigfindkw = cp.getparams('auxiliary', 'trigfind-')
# map with multiprocessing
if args.nproc > 1:
pool = multiprocessing.Pool(processes=args.nproc)
results = pool.map(_get_aux_triggers, auxchannels)
pool.close()
# map without multiprocessing
else:
results = map(_get_aux_triggers, auxchannels)
LOGGER.info("All aux events loaded")
auxiliary = dict(x for x in results if x is not None)
auxchannels = sorted(auxiliary.keys())
chanfile = '%s-HVETO_CHANNEL_LIST-%d-%d.txt' % (ifo, start, duration)
with open(chanfile, 'w') as f:
for chan in auxchannels:
print(chan, file=f)
LOGGER.info("Recorded list of valid auxiliary channels in %s" % chanfile)
# -- execute hveto analysis -----------------
minsig = cp.getfloat('hveto', 'minimum-significance')
pevents = [primary]
pvetoed = []
auxfcol, auxscol = auxiliary[auxchannels[0]].dtype.names[1:3]
slabel = plot.get_column_label(scol)
flabel = plot.get_column_label(fcol)
auxslabel = plot.get_column_label(auxscol)
auxflabel = plot.get_column_label(auxfcol)
rounds = []
rnd = core.HvetoRound(1, pchannel, rank=scol)
rnd.segments = analysis.active
while True:
LOGGER.info("-- Processing round %d --" % rnd.n)
# write segments for this round
segfile = os.path.join(
segdir, '%s-HVETO_ANALYSIS_SEGS_ROUND_%d-%d-%d.txt' %
(ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.segments)
# calculate significances for this round
if args.nproc > 1: # multiprocessing
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_find_max_significance, chunks)
pool.close()
winners, sigsets = zip(*results)
# find winner of chunk winners
winner = sorted(winners, key=lambda w: w.significance)[-1]
# flatten sets of significances into one list
newsignificances = sigsets[0]
for subdict in sigsets[1:]:
newsignificances.update(subdict)
else: # single process
winner, newsignificances = core.find_max_significance(
primary, auxiliary, pchannel, snrs, windows, rnd.livetime)
LOGGER.info("Round %d winner: %s" % (rnd.n, winner.name))
# plot significance drop here for the last round
# only now do we actually have the new data to
# calculate significance drop
if rnd.n > 1:
svg = (pngname % 'SIG_DROP').replace('.png', '.svg') # noqa: F821
plot.significance_drop(
svg,
oldsignificances,
newsignificances, # noqa: F821
title=' | '.join([title, subtitle]), # noqa: F821
bbox_inches='tight')
LOGGER.debug("Figure written to %s" % svg)
svg = FancyPlot(svg, caption=plot.ROUND_CAPTION['SIG_DROP'])
rounds[-1].plots.append(svg)
oldsignificances = newsignificances # noqa: F841
# break out of the loop if the significance is below stopping point
if winner.significance < minsig:
LOGGER.info("Maximum signifiance below stopping point")
LOGGER.debug(" (%.2f < %.2f)" % (winner.significance, minsig))
LOGGER.info("-- Rounds complete! --")
break
# work out the vetoes for this round
allaux = auxiliary[winner.name][
auxiliary[winner.name][auxscol] >= winner.snr]
winner.events = allaux
coincs = allaux[core.find_coincidences(allaux['time'],
primary['time'],
dt=winner.window)]
rnd.vetoes = winner.get_segments(allaux['time'])
flag = DataQualityFlag('%s:HVT-ROUND_%d:1' % (ifo, rnd.n),
active=rnd.vetoes,
known=rnd.segments,
description="winner=%s, window=%s, snr=%s" %
(winner.name, winner.window, winner.snr))
segments[flag.name] = flag
LOGGER.debug("Generated veto segments for round %d" % rnd.n)
# link events before veto for plotting
before = primary
beforeaux = auxiliary[winner.name]
# apply vetoes to primary
primary, vetoed = core.veto(primary, rnd.vetoes)
pevents.append(primary)
pvetoed.append(vetoed)
LOGGER.debug("Applied vetoes to primary")
# record results
rnd.winner = winner
rnd.efficiency = (len(vetoed), len(primary) + len(vetoed))
rnd.use_percentage = (len(coincs), len(winner.events))
if rnd.n > 1:
rnd.cum_efficiency = (len(vetoed) + rounds[-1].cum_efficiency[0],
rounds[0].efficiency[1])
rnd.cum_deadtime = (rnd.deadtime[0] + rounds[-1].cum_deadtime[0],
livetime)
else:
rnd.cum_efficiency = rnd.efficiency
rnd.cum_deadtime = rnd.deadtime
# apply vetoes to auxiliary
if args.nproc > 1: # multiprocess
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_veto, chunks)
pool.close()
auxiliary = results[0]
for subdict in results[1:]:
auxiliary.update(subdict)
else: # single process
auxiliary = core.veto_all(auxiliary, rnd.vetoes)
LOGGER.debug("Applied vetoes to auxiliary channels")
# log results
LOGGER.info("""Results for round %d:\n\n
winner : %s
significance : %s
mu : %s
snr : %s
dt : %s
use_percentage : %s
efficiency : %s
deadtime : %s
cum. efficiency : %s
cum. deadtime : %s\n\n""" %
(rnd.n, rnd.winner.name, rnd.winner.significance,
rnd.winner.mu, rnd.winner.snr, rnd.winner.window,
rnd.use_percentage, rnd.efficiency, rnd.deadtime,
rnd.cum_efficiency, rnd.cum_deadtime))
# write segments
segfile = os.path.join(
segdir, '%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' %
(ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.vetoes)
LOGGER.debug("Round %d vetoes written to %s" % (rnd.n, segfile))
rnd.files['VETO_SEGS'] = (segfile, )
# write triggers
trigfile = os.path.join(
trigdir, '%s-HVETO_%%s_TRIGS_ROUND_%d-%d-%d.txt' %
(ifo, rnd.n, start, duration))
for tag, arr in zip(['WINNER', 'VETOED', 'RAW'],
[winner.events, vetoed, primary]):
f = trigfile % tag
arr.write(f, format='ascii', overwrite=True)
LOGGER.debug("Round %d %s events written to %s" %
(rnd.n, tag.lower(), f))
rnd.files[tag] = f
# record times to omega scan
if args.omega_scans:
N = len(vetoed)
ind = random.sample(range(0, N), min(args.omega_scans, N))
rnd.scans = vetoed[ind]
LOGGER.debug("Collected %d events to omega scan:\n\n%s\n\n" %
(len(rnd.scans), rnd.scans))
# -- make some plots --
pngname = os.path.join(
plotdir,
'%s-HVETO_%%s_ROUND_%d-%d-%d.png' % (ifo, rnd.n, start, duration))
wname = texify(rnd.winner.name)
beforel = 'Before\n[%d]' % len(before)
afterl = 'After\n[%d]' % len(primary)
vetoedl = 'Vetoed\n(primary)\n[%d]' % len(vetoed)
beforeauxl = 'All\n[%d]' % len(beforeaux)
usedl = 'Used\n(aux)\n[%d]' % len(winner.events)
coincl = 'Coinc.\n[%d]' % len(coincs)
title = '%s Hveto round %d' % (ifo, rnd.n)
ptitle = '%s: primary impact' % title
atitle = '%s: auxiliary use' % title
subtitle = 'winner: %s [%d-%d]' % (wname, start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
plot.before_after_histogram(png,
before[scol],
primary[scol],
label1=beforel,
label2=afterl,
xlabel=slabel,
title=ptitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['HISTOGRAM'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(png,
before,
vetoed,
x='time',
y=scol,
label1=beforel,
label2=vetoedl,
epoch=start,
xlim=[start, end],
ylabel=slabel,
title=ptitle,
subtitle=subtitle,
legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'SNR_%s' % fcol.upper()
plot.veto_scatter(png,
before,
vetoed,
x=fcol,
y=scol,
label1=beforel,
label2=vetoedl,
xlabel=flabel,
ylabel=slabel,
xlim=pfreq,
title=ptitle,
subtitle=subtitle,
legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % '%s_TIME' % fcol.upper()
plot.veto_scatter(png,
before,
vetoed,
x='time',
y=fcol,
color=scol,
label1=None,
label2=None,
ylabel=flabel,
clabel=slabel,
clim=[3, 100],
cmap='YlGnBu',
epoch=start,
xlim=[start, end],
ylim=pfreq,
title=ptitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['TIME'])
rnd.plots.append(png)
# aux used versus frequency
png = pngname % 'USED_SNR_TIME'
plot.veto_scatter(png,
winner.events,
vetoed,
x='time',
y=[auxscol, scol],
label1=usedl,
label2=vetoedl,
ylabel=slabel,
epoch=start,
xlim=[start, end],
title=atitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['USED_SNR_TIME'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'AUX_SNR_TIME'
plot.veto_scatter(png,
beforeaux, (winner.events, coincs),
x='time',
y=auxscol,
label1=beforeauxl,
label2=(usedl, coincl),
epoch=start,
xlim=[start, end],
ylabel=auxslabel,
title=atitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'AUX_SNR_FREQUENCY'
plot.veto_scatter(png,
beforeaux, (winner.events, coincs),
x=auxfcol,
y=auxscol,
label1=beforeauxl,
label2=(usedl, coincl),
xlabel=auxflabel,
ylabel=auxslabel,
title=atitle,
subtitle=subtitle,
legend_title="Aux:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_FREQUENCY'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % 'AUX_FREQUENCY_TIME'
plot.veto_scatter(png,
beforeaux, (winner.events, coincs),
x='time',
y=auxfcol,
color=auxscol,
label1=None,
label2=[None, None],
ylabel=auxflabel,
clabel=auxslabel,
clim=[3, 100],
cmap='YlGnBu',
epoch=start,
xlim=[start, end],
title=atitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_FREQUENCY_TIME'])
rnd.plots.append(png)
# move to the next round
rounds.append(rnd)
rnd = core.HvetoRound(rnd.n + 1,
pchannel,
rank=scol,
segments=rnd.segments - rnd.vetoes)
# write file with all segments
segfile = os.path.join(
segdir, '%s-HVETO_SEGMENTS-%d-%d.h5' % (ifo, start, duration))
segments.write(segfile, overwrite=True)
LOGGER.debug("Segment summary written to %s" % segfile)
LOGGER.debug("Making summary figures...")
# -- exit early if no rounds above threshold
if not rounds:
message = ("No rounds completed above threshold. Analysis stopped "
"with %s achieving significance of %.2f" %
(winner.name, winner.significance))
LOGGER.critical(message)
message = message.replace(winner.name,
cis_link(winner.name, class_='alert-link'))
message += '<br>[T<sub>win</sub>: %ss, SNR: %s]' % (winner.window,
winner.snr)
htmlv['context'] = 'warning'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# -- plot all rounds impact
pngname = os.path.join(
plotdir, '%s-HVETO_%%s_ALL_ROUNDS-%d-%d.png' % (ifo, start, duration))
plots = []
title = '%s Hveto all rounds' % args.ifo
subtitle = '%d rounds | %d-%d' % (len(rounds), start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
beforel = 'Before analysis [%d events]' % len(pevents[0])
afterl = 'After %d rounds [%d]' % (len(pevents) - 1, len(pevents[-1]))
plot.before_after_histogram(png,
pevents[0][scol],
pevents[-1][scol],
label1=beforel,
label2=afterl,
xlabel=slabel,
title=title,
subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['HISTOGRAM'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# efficiency/deadtime curve
png = pngname % 'ROC'
plot.hveto_roc(png, rounds, title=title, subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['ROC'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# frequency versus time
png = pngname % '%s_TIME' % fcol.upper()
labels = [str(r.n) for r in rounds]
legtitle = 'Vetoed at\nround'
plot.veto_scatter(png,
pevents[0],
pvetoed,
label1='',
label2=labels,
title=title,
subtitle=subtitle,
ylabel=flabel,
x='time',
y=fcol,
epoch=start,
xlim=[start, end],
legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(png,
pevents[0],
pvetoed,
label1='',
label2=labels,
title=title,
subtitle=subtitle,
ylabel=slabel,
x='time',
y=scol,
epoch=start,
xlim=[start, end],
legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['SNR_TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# -- write summary states to ASCII table and JSON
json_ = {
'user': getuser(),
'host': getfqdn(),
'date': str(datetime.datetime.now()),
'configuration': inifile,
'ifo': ifo,
'gpsstart': start,
'gpsend': end,
'call': ' '.join(sys.argv),
'rounds': [],
}
with open('summary-stats.txt', 'w') as f:
# print header
print(
'#N winner window SNR significance nveto use-percentage '
'efficiency deadtime cumulative-efficiency cumulative-deadtime',
file=f)
for r in rounds:
# extract relevant statistics
results = [
('round', r.n),
('name', r.winner.name),
('window', r.winner.window),
('snr', r.winner.snr),
('significance', r.winner.significance),
('nveto', r.efficiency[0]),
('use-percentage',
r.use_percentage[0] / r.use_percentage[1] * 100.),
('efficiency', r.efficiency[0] / r.efficiency[1] * 100.),
('deadtime', r.deadtime[0] / r.deadtime[1] * 100.),
('cumulative-efficiency',
r.cum_efficiency[0] / r.cum_efficiency[1] * 100.),
('cumulative-deadtime',
r.cum_deadtime[0] / r.cum_deadtime[1] * 100.),
]
# write to ASCII
print(' '.join(map(str, list(zip(*results))[1])), file=f)
# write to JSON
results.append(('files', r.files))
json_['rounds'].append(dict(results))
LOGGER.debug("Summary table written to %s" % f.name)
with open('summary-stats.json', 'w') as f:
json.dump(json_, f, sort_keys=True)
LOGGER.debug("Summary JSON written to %s" % f.name)
# -- generate workflow for omega scans
if args.omega_scans:
omegatimes = list(
map(
str,
sorted(
numpy.unique([t['time'] for r in rounds
for t in r.scans]))))
LOGGER.debug("Collected %d times to omega scan" % len(omegatimes))
newtimes = [
t for t in omegatimes
if not os.path.exists(os.path.join(omegadir, str(t)))
]
LOGGER.debug("%d scans already complete or in progress, %d remaining" %
(len(omegatimes) - len(newtimes), len(newtimes)))
if len(newtimes) > 0:
LOGGER.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(ifo=ifo,
ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(timeout=4, gps=start)
batch.generate_dag(newtimes,
flags=flags,
submit=True,
outdir=omegadir,
condor_commands=condorcmds)
LOGGER.info('Launched {} omega scans to condor'.format(
len(newtimes)))
else:
LOGGER.debug('Skipping omega scans')
# -- write HTML and finish
index = html.write_hveto_page(ifo,
start,
end,
rounds,
plots,
winners=[r.winner.name for r in rounds],
**htmlv)
LOGGER.debug("HTML written to %s" % index)
LOGGER.debug("Analysis completed in %d seconds" % (time.time() - JOBSTART))
LOGGER.info("-- Hveto complete --")
else:
print filename + " does not exist. Looking for the segment file in next time increment."
break
f.close()
#construct flag and filename
flag_name = 'H1:UPVh-RND:1' #NEEDS TO BE CHANGED
name = 'segments_UPVh_RND.xml' #NEEDS TO BE CHANGED
try: knownsegments = numpy.loadtxt('total_UPVh_segs.txt')
except:
print 'No total_UPVh_segs.txt file in current working directory. It should have been produced from last loop.'
#knownsegments = numpy.loadtxt(total_UPVh_segs.txt'
known_start = [knownsegments[i,0] for i in range(len(knownsegments))]
known_end = [knownsegments[i,1] for i in range(len(knownsegments))]
# read the data
data = numpy.loadtxt('total_UPVh_trigs.txt', dtype=float)
# get an array for the start_time and end_time of each segment
start_time = [data[i,0] for i in range(len(data))]
end_time = [data[i,1] for i in range(len(data))]
# create a data quality flag object
#zip will truncate the start and end time. is this OK?
flag = DataQualityFlag(flag_name, active=zip(start_time, end_time), known=zip(known_start, known_end))
# write flag
flag.write(name)
#xarm = mylib.GetDQFlag(gpsstart,gpsend,config="xarm",kamioka=kamioka)
#LSC = mylib.GetDQFlag(gpsstart,gpsend,config="LSC",kamioka=kamioka)
#FPMI = mylib.GetDQFlag(gpsstart,gpsend,config="FPMI",kamioka=kamioka)
Observation = mylib.GetDQFlag(gpsstart,
gpsend,
config="Observation",
kamioka=kamioka)
IFO = mylib.GetDQFlag(gpsstart, gpsend, config="IFO", kamioka=kamioka)
IMC = mylib.GetDQFlag(gpsstart, gpsend, config="IMC", kamioka=kamioka)
PMC = mylib.GetDQFlag(gpsstart, gpsend, config="PMC", kamioka=kamioka)
FSS = mylib.GetDQFlag(gpsstart, gpsend, config="FSS", kamioka=kamioka)
ISS = mylib.GetDQFlag(gpsstart, gpsend, config="ISS", kamioka=kamioka)
#DAQ = mylib.GetDQFlag(gpsstart,gpsend,config="DAQ",kamioka=kamioka)
#MICH = mylib.GetDQFlag(gpsstart,gpsend,config="MICH",kamioka=kamioka)
trigger = DataQualityFlag(name="Trigger",
known=IFO.known,
active=[(triggertime, triggertime + duration)])
untriggered = ~trigger
plot = untriggered.plot(figsize=(12, 8))
ax = plot.gca()
ax.set_title("Interferometer state summary")
#ax.plot(LSC)
#ax.plot(FPMI)
ax.plot(Observation)
ax.plot(IFO)
#ax.plot(xarm)
#ax.plot(MICH)
ax.plot(ISS)
ax.plot(IMC)
ax.plot(PMC)
def get_triggers():
# Obtain segments that are analysis ready
analysis_ready = DataQualityFlag.query('{0}:DMT-ANALYSIS_READY:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
# Display segments for which this flag is true
print "Segments for which the ANALYSIS READY Flag is active: {0}".format(analysis_ready.active)
if opts.applyallDQ:
print("We are finding all previously created DQ cuts")
# Obtain segments of all DQ cuts if requested DQ list can be found
# https://code.pycbc.phy.syr.edu/detchar/veto-definitions/blob/master/burst/O1/H1L1-HOFT_C02_O1_BURST.xml
# First obtain those flags that are for both H1 and L1
O1_MISSING_HOFT_C02 = DataQualityFlag.query('{0}:DCS-MISSING_{0}_HOFT_C02:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_ETMY_ESD_DAC_OVERFLOW = DataQualityFlag.query('{0}:DMT-ETMY_ESD_DAC_OVERFLOW:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_OMC_DCPD_A_SATURATION = DataQualityFlag.query('{0}:DCH-OMC_DCPD_A_SATURATION:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_OMC_DCPD_ADC_OVERFLOW = DataQualityFlag.query('{0}:DMT-OMC_DCPD_ADC_OVERFLOW:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_ETMY_SATURATION_SNR200 = DataQualityFlag.query('{0}:DCH-ETMY_SATURATION_SNR200:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_CW_INJECTION_TRANSITION = DataQualityFlag.query('{0}:DCH-CW_INJECTION_TRANSITION:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_BAD_KAPPA_BASIC_CUT_HOFT_C02 = DataQualityFlag.query('{0}:DCS-BAD_KAPPA_BASIC_CUT_{0}_HOFT_C02:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_PARTIAL_FRAME_LOSS_HOFT_C02 = DataQualityFlag.query('{0}:DCS-PARTIAL_FRAME_LOSS_{0}_HOFT_C02:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
# Obtain detector specific flags
if opts.detector == "H1":
O1_RF45_AM_STABILIZATION = DataQualityFlag.query('{0}:DCH-RF45_AM_STABILIZATION:4'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_ETMY_SATURATION = DataQualityFlag.query('{0}:DCH-ETMY_SATURATION:2'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_BAD_DATA_BEFORE_LOCKLOSS = DataQualityFlag.query('{0}:DCH-BAD_DATA_BEFORE_LOCKLOSS:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_ETMY_VIOLIN_MODE_2NDHARMONIC_RINGING = DataQualityFlag.query('{0}:DCH-ETMY_VIOLIN_MODE_2NDHARMONIC_RINGING:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_RF45_SEVERE_GLITCHING = DataQualityFlag.query('{0}:DCH-RF45_SEVERE_GLITCHING:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_EY_BECKHOFF_CHASSIS_PROBLEM = DataQualityFlag.query('{0}:DCH-EY_BECKHOFF_CHASSIS_PROBLEM:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_ASC_AS_B_RF36_GLITCHING = DataQualityFlag.query('{0}:DCH-ASC_AS_B_RF36_GLITCHING:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_BAD_STRAIN_HOFT_C02 = DataQualityFlag.query('{0}:DCS-BAD_STRAIN_{0}_HOFT_C02:2'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_TOGGLING_BAD_KAPPA_HOFT_C02= DataQualityFlag.query('{0}:DCS-TOGGLING_BAD_KAPPA_{0}_HOFT_C02:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
else:
O1_ETMY_SATURATION = DataQualityFlag.query('{0}:DCH-ETMY_SATURATION:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_BAD_DATA_BEFORE_LOCKLOSS = DataQualityFlag.query('{0}:DCH-BAD_DATA_BEFORE_LOCKLOSS:2'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_PCAL_GLITCHES_GT_20P = DataQualityFlag.query('{0}:DCH-PCAL_GLITCHES_GT_20P:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_SUDDEN_PSD_CHANGE = DataQualityFlag.query('{0}:DCH-SUDDEN_PSD_CHANGE:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_BAD_VCO_OFFSET = DataQualityFlag.query('{0}:DCH-BAD_VCO_OFFSET:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
O1_SEVERE_60_200_HZ_NOISE = DataQualityFlag.query('{0}:DCH-SEVERE_60_200_HZ_NOISE:1'.format(opts.detector),opts.gpsStart,opts.gpsEnd)
# Fetch raw omicron triggers and apply filter which is defined in a function above.
omicrontriggers = SnglBurstTable.fetch(detchannelname,'Omicron',\
opts.gpsStart,opts.gpsEnd,filt=threshold)
print "List of available metadata information for a given glitch provided by omicron: {0}".format(omicrontriggers.columnnames)
print "Number of triggers after SNR and Freq cuts but before ANALYSIS READY flag filtering: {0}".format(len(omicrontriggers))
# Filter the raw omicron triggers against the ANALYSIS READY flag.
omicrontriggers = omicrontriggers.vetoed(analysis_ready.active)
# If requested filter out DQ flags
if opts.applyallDQ:
print("We are applying all previously created DQ cuts")
# Obtain segments of all DQ cuts if requested DQ list can be found
# https://code.pycbc.phy.syr.edu/detchar/veto-definitions/blob/master/burst/O1/H1L1-HOFT_C02_O1_BURST.xml
# First obtain those flags that are for both H1 and L1
omicrontriggers = omicrontriggers.veto(O1_MISSING_HOFT_C02.active)
omicrontriggers = omicrontriggers.veto(O1_ETMY_ESD_DAC_OVERFLOW.active)
omicrontriggers = omicrontriggers.veto(O1_OMC_DCPD_A_SATURATION.active)
omicrontriggers = omicrontriggers.veto(O1_OMC_DCPD_ADC_OVERFLOW.active)
omicrontriggers = omicrontriggers.veto(O1_ETMY_SATURATION_SNR200.active)
omicrontriggers = omicrontriggers.veto(O1_CW_INJECTION_TRANSITION.active)
omicrontriggers = omicrontriggers.veto(O1_BAD_KAPPA_BASIC_CUT_HOFT_C02.active)
omicrontriggers = omicrontriggers.veto(O1_PARTIAL_FRAME_LOSS_HOFT_C02.active)
# Obtain detector specific flags
if opts.detector == "H1":
omicrontriggers = omicrontriggers.veto(O1_RF45_AM_STABILIZATION.active)
omicrontriggers = omicrontriggers.veto(O1_ETMY_SATURATION.active)
omicrontriggers = omicrontriggers.veto(O1_BAD_DATA_BEFORE_LOCKLOSS.active)
omicrontriggers = omicrontriggers.veto(O1_ETMY_VIOLIN_MODE_2NDHARMONIC_RINGING.active)
omicrontriggers = omicrontriggers.veto(O1_RF45_SEVERE_GLITCHING.active)
omicrontriggers = omicrontriggers.veto(O1_EY_BECKHOFF_CHASSIS_PROBLEM.active)
omicrontriggers = omicrontriggers.veto(O1_ASC_AS_B_RF36_GLITCHING.active)
omicrontriggers = omicrontriggers.veto(O1_BAD_STRAIN_HOFT_C02.active)
omicrontriggers = omicrontriggers.veto(O1_TOGGLING_BAD_KAPPA_HOFT_C02.active)
else:
omicrontriggers = omicrontriggers.veto(O1_ETMY_SATURATION.active)
omicrontriggers = omicrontriggers.veto(O1_BAD_DATA_BEFORE_LOCKLOSS.active)
omicrontriggers = omicrontriggers.veto(O1_PCAL_GLITCHES_GT_20P.active)
omicrontriggers = omicrontriggers.veto(O1_SUDDEN_PSD_CHANGE.active)
omicrontriggers = omicrontriggers.veto(O1_BAD_VCO_OFFSET.active)
omicrontriggers = omicrontriggers.veto(O1_SEVERE_60_200_HZ_NOISE.active)
print "Final trigger length: {0}".format(len(omicrontriggers))
return omicrontriggers
import pytest
from gwpy.segments import (Segment, SegmentList, DataQualityFlag,
DataQualityDict)
from .. import segments
TEST_SEGMENTS = SegmentList([
Segment(0.1, 1.234567),
Segment(5.64321, 6.234567890),
])
TEST_SEGMENTS_2 = SegmentList(
[Segment(round(a, 6), round(b, 6)) for a, b in TEST_SEGMENTS])
TEST_FLAG = DataQualityFlag(known=SegmentList([Segment(0, 7)]),
active=TEST_SEGMENTS,
name='X1:TEST-FLAG')
TEST_DICT = DataQualityDict({TEST_FLAG.name: TEST_FLAG})
# -- unit tests ---------------------------------------------------------------
@mock.patch('gwpy.segments.DataQualityFlag.query', return_value=TEST_FLAG)
def test_query(dqflag):
flag = segments.query('X1:TEST-FLAG', 0, 7)
assert flag.known == TEST_FLAG.known
assert flag.active == SegmentList(
[Segment((int(seg[0]), int(seg[1]))) for seg in TEST_FLAG.active])
@pytest.mark.parametrize('ncol', (2, 4))
K1:AUX-INVALID_DATA,17.0,103.0,69.0,103.5999984741211,14.4,5.0,1.445966601520126,-55
""" # noqa: E501
# -- test data
GPS = 17
SIGNAL = TimeSeries(
gausspulse(numpy.arange(-1, 1, 1./4096), bw=100),
sample_rate=4096,
epoch=GPS - 1,
)
TEST_FLAG = DataQualityFlag(
name="K1:DCH-TEST_FLAG:1",
active=SegmentList(),
known=SegmentList([Segment(0, 34)]),
)
K1_DATA = TimeSeriesDict({
"K1:GW-PRIMARY_CHANNEL": TimeSeries(
numpy.random.normal(loc=1, scale=.5, size=4096 * GPS * 2),
sample_rate=4096,
epoch=0,
).zpk([], [0], 1).inject(SIGNAL),
"K1:AUX-HIGH_SIGNIFICANCE": TimeSeries(
numpy.random.normal(loc=1, scale=.5, size=4096 * GPS * 2),
sample_rate=4096,
epoch=0,
).zpk([], [0], 1).inject(SIGNAL),
"K1:AUX-LOW_SIGNIFICANCE": TimeSeries(
def test_round(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE_CONTRACTED, known=KNOWN)
flag2 = flag.round()
self.assertListEqual(flag2.active, ACTIVE & KNOWN)
"""
__author__ = 'Duncan Macleod <*****@*****.**>'
# .. currentmodule:: gwpy.segments
#
# Getting the segments
# --------------------
#
# First, we need to fetch the Open Data timeline segments from LOSC.
# To do that we can call the :meth:`DataQualityFlag.fetch_open_data` method
# using ``'H1_DATA'`` as the flag (for an explanation of what this means,
# read up on `The S6 Data Release <https://losc.ligo.org/S6/>`__).
from gwpy.segments import DataQualityFlag
h1segs = DataQualityFlag.fetch_open_data('H1_DATA',
'Sep 16 2010', 'Sep 17 2010')
# For sanity, lets plot these segments:
splot = h1segs.plot(figsize=[12, 3])
splot.show()
splot.close() # hide
# We see that the LIGO Hanford Observatory detector was operating for the
# majority of the day, with a few outages of ~30 minutes or so.
# We can use the :func:`abs` function to display the total amount of time
# spent taking data:
print(abs(h1segs.active))
import fir
from utils import chunk_segments
parser = argparse.ArgumentParser(description='Generate target features for machine learning on LIGO data.')
parser.add_argument('--ifo',type=str,required=True)
parser.add_argument('-f','--segment-file',type=str)
parser.add_argument('-s','--start-time',type=int)
parser.add_argument('-e','--end-time',type=int)
parser.add_argument('-p','--path', help='path to output directory', required=False)
args = parser.parse_args()
ifo=args.ifo
if args.segment_file:
sci_segs=DataQualityFlag.read(args.segment_file, path='%s:DMT-ANALYSIS_READY:1' % ifo)
assert sci_segs.ifo == ifo
segs=sci_segs.active
elif args.start_time and args.end_time:
segs=[Segment(args.start_time, args.end_time)]
else:
print "Either --segment-file, or both start and end time must be provided."
exit(2)
st=segs[0].start
et=segs[-1].end
chunk=4096
pad=256
target_chan=ifo+':GDS-CALIB_STRAIN'
`Observing Run 1 (O1) <https://www.gw-openscience.org/O1/>`__
have been released by |GWOSC|_.
This example demonstrates how to download segment information into a
:class:`~gwpy.segments.DataQualityFlag`, and then plot them.
"""
__author__ = 'Duncan Macleod <*****@*****.**>'
__currentmodule__ = 'gwpy.segments'
# All we need to do is import the `DataQualityFlag` object, and then call
# the :meth:`DataQualityFlag.fetch_open_data` method to query for, and download
# the segments for all of O1:
from gwpy.segments import DataQualityFlag
h1segs = DataQualityFlag.fetch_open_data('H1_DATA', 'Sep 12 2015',
'Jan 19 2016')
# We can then generate a plot of the times when LIGO-Hanford was operating:
plot = h1segs.plot(color='gwpy:ligo-hanford')
plot.show()
# That's a lot of segments. We can pare-down the list a little to display
# only the segments from the first month of the run:
h1month1 = DataQualityFlag.fetch_open_data('H1_DATA', 'Sep 12 2015',
'Oct 12 2015')
# We can also download the LIGO-Livingston segments from the same period
# and display them alongside, as well as those segments during which both
# interferometers were operating at the same time
def read_data_archive(sourcefile):
"""Read archived data from an HDF5 archive source.
Parameters
----------
sourcefile : `str`
path to source HDF5 file
"""
from h5py import File
with File(sourcefile, 'r') as h5file:
# read all time-series data
try:
group = h5file['timeseries']
except KeyError:
group = dict()
for dataset in group.itervalues():
ts = TimeSeries.read(dataset, format='hdf5')
if (re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name)
and ts.sample_rate.value == 1.0):
ts.channel.type = 's-trend'
elif re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name):
ts.channel.type = 'm-trend'
ts.channel = get_channel(ts.channel)
try:
add_timeseries(ts, key=ts.channel.ndsname)
except ValueError:
if mode.get_mode() != mode.Mode.day:
raise
warnings.warn('Caught ValueError in combining daily archives')
# get end time
globalv.DATA[ts.channel.ndsname].pop(-1)
t = globalv.DATA[ts.channel.ndsname][-1].span[-1]
add_timeseries(ts.crop(start=t), key=ts.channel.ndsname)
# read all state-vector data
try:
group = h5file['statevector']
except KeyError:
group = dict()
for dataset in group.itervalues():
sv = StateVector.read(dataset, format='hdf5')
sv.channel = get_channel(sv.channel)
add_timeseries(sv, key=sv.channel.ndsname)
# read all spectrogram data
for tag in ['spectrogram', 'coherence-components']:
if tag == 'coherence-components':
add_ = add_coherence_component_spectrogram
else:
add_ = add_spectrogram
try:
group = h5file[tag]
except KeyError:
group = dict()
for key, dataset in group.iteritems():
key = key.rsplit(',', 1)[0]
spec = Spectrogram.read(dataset, format='hdf5')
spec.channel = get_channel(spec.channel)
add_(spec, key=key)
# read all segments
try:
group = h5file['segments']
except KeyError:
group = dict()
for name in group:
dqflag = DataQualityFlag.read(group, path=name, format='hdf5')
globalv.SEGMENTS += {name: dqflag}
# read all triggers
try:
group = h5file['triggers']
except KeyError:
group = dict()
for key in group:
load_table(group[key])
def test_round(self):
flag = DataQualityFlag(FLAG1, active=ACTIVE_CONTRACTED, known=KNOWN)
flag2 = flag.round()
self.assertListEqual(flag2.active, ACTIVE & KNOWN)
# determine times when f2 above some threshold
idxhighscat = argwhere(scatf2>=thresh)
highscatf2 = scatf2[idxhighscat]
highscattimes = times[idxhighscat]
highscattimesgps = highscattimes+start_time
# save text file with values above threshold [GPS f2 index]
outdata = hstack((highscattimesgps,highscatf2,idxhighscat))
savetxt('%s-ALL-TIMES-SCATTER-GT%dHZ-%d-%d.txt' % (ifo,thresh,start_time,dur),outdata,fmt='%f %f %i')
# save segments XML file with segments (based off code from Duncan Macleod)
from math import (floor, ceil)
from gwpy.segments import (Segment, DataQualityFlag)
flag = '%s:DCH-SCATTERED_LIGHT_GT%dHZ:1' % (ifo,thresh)
flag = DataQualityFlag(flag)
segs = []
append = segs.append
for gps in highscattimesgps:
if len(segs) and gps in segs[-1]:
continue
seg = Segment(floor(gps), ceil(gps))
append(seg)
flag.active = segs
flag.known = [Segment(start_time, end_time)]
flag.coalesce()
flag.write('%s-%s_%d-%d-%d.xml.gz' % (flag.ifo, flag.tag.replace('-', '_'), flag.version,start_time, dur))
#EOF
def main(args=None):
"""Run the zero-crossing counter tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
span = Segment(args.gpsstart, args.gpsend)
LOGGER.info('-- Processing channel %s over span %d - %d' %
(args.channel, args.gpsstart, args.gpsend))
if args.state_flag:
state = DataQualityFlag.query(
args.state_flag,
int(args.gpsstart),
int(args.gpsend),
url=const.DEFAULT_SEGMENT_SERVER,
)
statea = state.active
else:
statea = SegmentList([span])
duration = abs(span)
# initialize output files for each threshold and store them in a dict
outfiles = {}
for thresh in args.threshold:
outfiles[str(thresh)] = (os.path.join(
args.output_path, '%s_%s_DAC-%d-%d.h5' %
(args.channel.replace('-', '_').replace(':', '-'), str(
int(thresh)).replace('-', 'n'), int(args.gpsstart), duration)))
# get frame cache
cache = gwdatafind.find_urls(args.ifo[0], args.frametype,
int(args.gpsstart), int(args.gpsend))
cachesegs = statea & cache_segments(cache)
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
# initialize a ligolw table for each threshold and store them in a dict
names = ("time", "frequency", "snr")
dtypes = ("f8", ) * len(names)
tables = {}
for thresh in args.threshold:
tables[str(thresh)] = EventTable(
names=names,
dtype=dtypes,
meta={"channel": args.channel},
)
# for each science segment, read in the data from frames, check for
# threshold crossings, and if the rate of crossings is less than
# rate_thresh, write to a sngl_burst table
for seg in cachesegs:
LOGGER.debug("Processing {}:".format(seg))
c = sieve_cache(cache, segment=seg)
if not c:
LOGGER.warning(" No {} data files for this segment, "
"skipping".format(args.frametype))
continue
data = get_data(args.channel,
seg[0],
seg[1],
nproc=args.nproc,
source=c,
verbose="Reading data:".rjust(30))
for thresh in args.threshold:
times = find_crossings(data, thresh)
rate = float(times.size) / abs(seg) if times.size else 0
LOGGER.info(" Found {0} crossings of {1}, rate: {2} Hz".format(
times.size,
thresh,
rate,
))
if times.size and rate < args.rate_thresh:
existing = tables[str(thresh)]
tables[str(thresh)] = vstack_tables(
(
existing,
table_from_times(times,
snr=10.,
frequency=100.,
names=existing.colnames),
),
join_type="exact",
)
n = max(map(len, tables.values()))
for thresh, outfile in outfiles.items():
tables[thresh].write(
outfile,
path="triggers",
format="hdf5",
overwrite=True,
)
LOGGER.info("{0} events written to {1}".format(
str(len(tables[thresh])).rjust(len(str(n))),
outfile,
))
