def test_populate(self):
# read veto definer
start, end = VETO_DEFINER_TEST_SEGMENTS[0]
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
# test query that should fail with 404
try:
vdf.populate(url='https://segments.ligo.org')
except (ImportError, UnboundLocalError) as e:
self.skipTest(str(e))
except URLError as e:
if e.code == 401: # 401 is uninformative
self.skipTest(str(e))
elif e.code == 404:
pass
else:
raise
else:
raise AssertionError("URLError not raised")
# check reduction to warning
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
with pytest.warns(UserWarning):
vdf.populate(url='https://segments.ligo.org', on_error='warn')
# check results
self.assertEqual(
len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
# check use of specific segments
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1')
vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
def test_populate(self):
# read veto definer
start, end = VETO_DEFINER_TEST_SEGMENTS[0]
vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
ifo='H1',
start=start,
end=end)
# test query that should fail with 404
try:
vdf.populate(url='https://segments.ligo.org')
except (ImportError, UnboundLocalError) as e:
self.skipTest(str(e))
except URLError as e:
if e.code == 401: # 401 is uninformative
self.skipTest(str(e))
elif e.code == 404:
pass
else:
raise
else:
raise AssertionError("URLError not raised")
# check reduction to warning
vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
ifo='H1',
start=start,
end=end)
with pytest.warns(UserWarning):
vdf.populate(url='https://segments.ligo.org', on_error='warn')
# check results
self.assertEqual(len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
# check use of specific segments
vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
ifo='H1')
vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
def test_read_ligolw(self):
flags = DataQualityDict.read(SEGXML)
self.assertEquals(len(flags.keys()), 2)
self.assertIn(FLAG1, flags)
self.assertIn(FLAG2, flags)
flags = DataQualityDict.read(SEGXML, [FLAG2])
self.assertEquals(len(flags.keys()), 1)
self.assertEquals(flags[FLAG2].known, KNOWN2)
self.assertEquals(flags[FLAG2].active, ACTIVE2)
def test_read_ligolw(self):
flags = DataQualityDict.read(SEGXML)
self.assertEquals(len(flags.keys()), 2)
self.assertIn(FLAG1, flags)
self.assertIn(FLAG2, flags)
flags = DataQualityDict.read(SEGXML, [FLAG2])
self.assertEquals(len(flags.keys()), 1)
self.assertEquals(flags[FLAG2].known, KNOWN2)
self.assertEquals(flags[FLAG2].active, ACTIVE2)
def test_from_veto_definer_file(self):
vdf = DataQualityDict.from_veto_definer_file(self.VETO_DEFINER)
self.assertNotEqual(len(vdf.keys()), 0)
# test missing h(t) flag
self.assertIn('H1:DCH-MISSING_H1_HOFT_C00:1', vdf)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][0],
1073779216)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][-1],
PosInfinity)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].category, 1)
# test injections padding
self.assertEquals(vdf['H1:ODC-INJECTION_CBC:1'].padding, Segment(-8, 8))
# test download URL
vdf2 = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE)
self.assertEqual(len(vdf.keys()), len(vdf2.keys()))
def test_populate(self):
start, end = VETO_DEFINER_TEST_SEGMENTS[0]
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
vdf.pop("H1:FAKE_CAT5:1", None)
try:
vdf.populate(url='https://segments.ligo.org')
except ImportError as e:
self.skipTest(str(e))
else:
self.assertEqual(
len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1')
vdf.pop("H1:FAKE_CAT5:1", None)
vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS)
def test_table_from_segments():
segs = DataQualityDict()
segs["test1"] = DataQualityFlag(active=[(0, 1), (3, 4)], )
segs["test2"] = DataQualityFlag(active=[(5, 6)], )
assert_table_equal(
utils.table_from_segments(segs),
EventTable(
rows=[(0., 100., 0., 1., 10., "test1"),
(3., 100., 3., 4., 10., "test1"),
(5., 100., 5., 6., 10., "test2")],
names=("time", "frequency", "start_time", "end_time", "snr",
"channel"),
),
)
assert_table_equal(
utils.table_from_segments(segs,
frequency=1234.,
snr=-4.,
sngl_burst=True),
EventTable(
rows=[(0., 1234., -4., "test1"), (3., 1234., -4., "test1"),
(5., 1234., -4., "test2")],
names=("peak", "peak_frequency", "snr", "channel"),
),
)
def test_table_from_segments_empty():
segs = DataQualityDict()
segs['test'] = DataQualityFlag(active=[])
assert_table_equal(
utils.table_from_segments(segs),
EventTable(names=("time", "frequency", "start_time", "end_time", "snr",
"channel")))
def test_from_veto_definer_file(self):
vdf = DataQualityDict.from_veto_definer_file(self.VETO_DEFINER)
self.assertNotEqual(len(vdf.keys()), 0)
# test missing h(t) flag
self.assertIn('H1:DCH-MISSING_H1_HOFT_C00:1', vdf)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][0],
1073779216)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][-1],
PosInfinity)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].category, 1)
# test injections padding
self.assertEquals(vdf['H1:ODC-INJECTION_CBC:1'].padding,
Segment(-8, 8))
# test download URL
vdf2 = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE)
self.assertEqual(len(vdf.keys()), len(vdf2.keys()))
def create_activity_list(station, data_order):
"""
Create consecutive list of available data segment.
Parameters
----------
station : string
Name of the station
data_order : dictionary
List of all the HDF5 data file for each segment
Return
------
full_seglist : dictionary
Ordered list of segment
"""
# Generate an ASCII representation of the GPS timestamped segments of time covered by the input data
seglist = segmentlist(data_order.keys())
# Sort the segment list
seglist.sort()
# Initialise dictionary for segment information
full_seglist = DataQualityDict()
# Save time span for each segment in ASCII file
with open("segments.txt", "w") as fout:
for seg in seglist:
print >> fout, "%10.9f %10.9f" % seg
# FIXME: Active should be masked from the sanity channel
full_seglist[station] = DataQualityFlag(station,
active=seglist.coalesce(),
known=seglist.coalesce())
return full_seglist
def create(self):
flgd = DataQualityDict()
flgd['flag1'] = DataQualityFlag(name='flag1',
active=ACTIVE,
known=KNOWN)
flgd['flag2'] = DataQualityFlag(name='flag2',
active=ACTIVE2,
known=KNOWN2)
return flgd
def process(self,
nds=None,
nproc=1,
config=GWSummConfigParser(),
datacache=None,
datafind_error='raise',
**kwargs):
"""Process time accounting data
"""
for p in self.plots:
if p.outputfile in globalv.WRITTEN_PLOTS:
p.new = False
# get archived GPS time
tag = self.segmenttag % list(self.modes)[0]
try:
lastgps = globalv.SEGMENTS[tag].known[-1][-1]
except (IndexError, KeyError):
lastgps = self.span[0]
# get data
new = SegmentList([type(self.span)(lastgps, self.span[1])])
data = get_timeseries(self.channel,
new,
config=config,
nds=nds,
dtype='int16',
datafind_error=datafind_error,
nproc=nproc,
cache=datacache)
vprint(" All time-series data loaded\n")
# find segments
for ts in data:
modesegments = DataQualityDict()
for idx, name in self.modes.items():
# get segments for state
tag = self.segmenttag % idx
instate = ts == idx * ts.unit
modesegments[tag] = instate.to_dqflag(name=name.strip('*'))
# append segments for group
group = int(idx // 10. * 10)
gtag = self.segmenttag % group
try:
modesegments[gtag] += modesegments[tag]
except KeyError:
modesegments[gtag] = modesegments[tag]
globalv.SEGMENTS += modesegments
kwargs['segdb_error'] = 'ignore'
super(AccountingTab, self).process(config=config,
nds=nds,
nproc=nproc,
segmentcache=Cache(),
datacache=datacache,
datafind_error=datafind_error,
**kwargs)
def test_query_segdb(self):
try:
result = DataQualityDict.query_segdb(
QUERY_FLAGS, QUERY_START, QUERY_END, url=QUERY_URL_SEGDB)
except (SystemExit, LDBDClientException) as e:
self.skipTest(str(e))
self.assertListEqual(result.keys(), QUERY_FLAGS)
for flag in result:
self.assertEqual(result[flag].known, QUERY_RESULT[flag].known)
self.assertEqual(result[flag].active, QUERY_RESULT[flag].active)
def test_write_ligolw(self):
tmpfile = self.tmpfile % 'xml.gz'
try:
flags = DataQualityDict.read(SEGXML)
except Exception as e:
self.skipTest(str(e))
try:
flags.write(tmpfile)
finally:
os.remove(tmpfile)
def test_write_ligolw(self):
tmpfile = self.tmpfile % 'xml.gz'
try:
flags = DataQualityDict.read(SEGXML)
except Exception as e:
self.skipTest(str(e))
try:
flags.write(tmpfile)
finally:
os.remove(tmpfile)
def read_veto_definer_file(vetofile, start=None, end=None, ifo=None):
"""Read a veto definer file, downloading it if necessary
"""
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
with open(vetofile, 'w') as f:
f.write(str(tmp.read()))
return DataQualityDict.from_veto_definer_file(
vetofile, format='ligolw', start=start, end=end, ifo=ifo)
def test_query_segdb(self):
try:
result = DataQualityDict.query_segdb(QUERY_FLAGS,
QUERY_START,
QUERY_END,
url=QUERY_URL_SEGDB)
except (SystemExit, LDBDClientException) as e:
self.skipTest(str(e))
self.assertListEqual(result.keys(), QUERY_FLAGS)
for flag in result:
self.assertEqual(result[flag].known, QUERY_RESULT[flag].known)
self.assertEqual(result[flag].active, QUERY_RESULT[flag].active)
def query_veto_definer_file(ifo, start, end, cp):
# Obtain segments that are analysis ready
analysis_ready = DataQualityFlag.query(
'{0}:DMT-ANALYSIS_READY:1'.format(ifo), start, end)
# Define the start and stop time of analysis ready
analysis_ready_start = analysis_ready.active.extent()[0]
analysis_ready_end = analysis_ready.active.extent()[1]
# Query for vetos found during this time.
vdf = DataQualityDict.from_veto_definer_file(
cp.get('segfind', 'veto-file'), analysis_ready_start,
analysis_ready_end)
# Populate only for analysis ready segments.
vdf.populate(segments=analysis_ready.active)
return vdf
def segments_from_sngl_burst(table, padding, known=None):
"""Create a `DataQualityDict` of segments from the given `SnglBurstTable`
This method creates a `~gwpy.segments.DataQualityFlag` for each unique
channel found in the given table by padding the peak time by the given
amount on each side
"""
out = DataQualityDict()
for row in table:
t = row.get_peak()
seg = Segment(t-padding, t+padding)
try:
out[row.channel].active.append(seg)
except KeyError:
out[row.channel] = DataQualityFlag(row.channel, known=known,
active=[seg])
return out
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
FIXME
Returns
-------
FIXME
"""
if isinstance(flag, (unicode, str)):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag, isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEAFULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(operator.and_, (globalv.SEGMENTS.get(
f, DataQualityFlag(f)).known
for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= segs.known
out[compound].active &= segs.known
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, basestring):
return out[flag]
else:
return out
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None, **read_kw):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
flag : `str`, `list`
either the name of one flag, or a list of names
validity : `~gwpy.segments.SegmentList`
the segments over which to search for other segments
query : `bool`, optional, default: `True`
actually execute a read/query operation (if needed), otherwise
just retrieve segments that have already been cached
config : `~configparser.ConfigParser`, optional
the configuration for your analysis, if you have one. If
present the ``[segment-database]`` section will be queried
for the following options
- ``gps-start-time``, and ``gps-end-time``, if ``validity`` is
not given
- ``url`` (the remote hostname for the segment database) if
the ``url`` keyword is not given
cache : :class:`glue.lal.Cache`, optional
a cache of files from which to read segments, otherwise segments
will be downloaded from the segment database
coalesce : `bool`, optional, default: `True`
coalesce all segmentlists before returning, otherwise just return
segments as they were downloaded/read
padding : `tuple`, or `dict` of `tuples`, optional
`(start, end)` padding with which to pad segments that are
downloaded/read
segdb_error : `str`, optional, default: ``'raise'``
how to handle errors returned from the segment database, one of
- ``'raise'`` (default) : raise the exception as normal
- ``'warn'`` : print the exception as a warning, but return no
segments
- ``'ignore'`` : silently ignore the error and return no segments
url : `str`, optional
the remote hostname for the target segment database
return_ : `bool`, optional, default: `True`
internal flag to enable (True) or disable (False) actually returning
anything. This is useful if you want to download/read segments now
but not use them until later (e.g. plotting)
**read_kw : `dict`, optional
additional keyword arguments to `~gwpy.segments.DataQualityDict.read`
or `~gwpy.segments.DataQualityFlag.read`
Returns
-------
flag : `~gwpy.segments.DataQualityFlag`
the flag object representing segments for the given single flag, OR
flagdict : `~gwpy.segments.DataQualityDict`
the dict of `~gwpy.segments.DataQualityFlag` objects for multiple
flags, if ``flag`` is given as a `list`, OR
None
if ``return_=False``
"""
if isinstance(flag, str):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag,
isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEFAULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(
operator.and_,
(globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
if isinstance(cache, str) and cache.endswith(
(".h5", ".hdf", ".hdf5")) and (
'path' not in read_kw):
read_kw['path'] = config.get(
'DEFAULT', 'segments-hdf5-path', fallback='segments')
try:
new = DataQualityDict.read(cache, list(allflags), **read_kw)
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(
cache, f, coalesce=False, **read_kw)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, str):
return out[flag]
else:
return out
(-.5, 4), (5.5, 8))
# padded version of above 'active' segments
ACTIVEPAD = _as_segmentlist(
(.5, 3), (2.5, 5), (4.5, 8))
# padded, coalesed version of above 'active' segments
ACTIVEPADC = _as_segmentlist(
(.5, 4), (5.5, 8))
# -- query helpers ------------------------------------------------------------
QUERY_FLAGS = ['X1:TEST-FLAG:1', 'Y1:TEST-FLAG2:4']
QUERY_RESULT = DataQualityDict()
QUERY_RESULT['X1:TEST-FLAG:1'] = DataQualityFlag(
'X1:TEST-FLAG:1',
known=[(0, 10)],
active=[(0, 1), (1, 2), (3, 4), (6, 9)])
QUERY_RESULT['Y1:TEST-FLAG2:4'] = DataQualityFlag(
'Y1:TEST-FLAG2:4',
known=[(0, 5), (9, 10)],
active=[])
QUERY_RESULTC = type(QUERY_RESULT)({x: y.copy().coalesce() for
x, y in QUERY_RESULT.items()})
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
flag : `str`, `list`
either the name of one flag, or a list of names
validity : `~gwpy.segments.SegmentList`
the segments over which to search for other segments
query : `bool`, optional, default: `True`
actually execute a read/query operation (if needed), otherwise
just retrieve segments that have already been cached
config : `~configparser.ConfigParser`, optional
the configuration for your analysis, if you have one. If
present the ``[segment-database]`` section will be queried
for the following options
- ``gps-start-time``, and ``gps-end-time``, if ``validity`` is
not given
- ``url`` (the remote hostname for the segment database) if
the ``url`` keyword is not given
cache : :class:`glue.lal.Cache`, optional
a cache of files from which to read segments, otherwise segments
will be downloaded from the segment database
coalesce : `bool`, optional, default: `True`
coalesce all segmentlists before returning, otherwise just return
segments as they were downloaded/read
padding : `tuple`, or `dict` of `tuples`, optional
`(start, end)` padding with which to pad segments that are
downloaded/read
segdb_error : `str`, optional, default: ``'raise'``
how to handle errors returned from the segment database, one of
- ``'raise'`` (default) : raise the exception as normal
- ``'warn'`` : print the exception as a warning, but return no
segments
- ``'ignore'`` : silently ignore the error and return no segments
url : `str`, optional
the remote hostname for the target segment database
return_ : `bool`, optional, default: `True`
internal flag to enable (True) or disable (False) actually returning
anything. This is useful if you want to download/read segments now
but not use them until later (e.g. plotting)
Returns
-------
flag : `~gwpy.segments.DataQualityFlag`
the flag object representing segments for the given single flag, OR
flagdict : `~gwpy.segments.DataQualityDict`
the dict of `~gwpy.segments.DataQualityFlag` objects for multiple
flags, if ``flag`` is given as a `list`, OR
None
if ``return_=False``
"""
if isinstance(flag, str):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag,
isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEFAULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(
operator.and_,
(globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, str):
return out[flag]
else:
return out
def main(args=None):
"""Run the hveto command-line interface
"""
# declare global variables
# this is needed for multiprocessing utilities
global acache, analysis, areadkw, atrigfindkw, auxiliary, auxetg
global auxfreq, counter, livetime, minsnr, naux, pchannel, primary
global rnd, snrs, windows
# parse command-line
parser = create_parser()
args = parser.parse_args(args=args)
ifo = args.ifo
start = int(args.gpsstart)
end = int(args.gpsend)
duration = end - start
# log startup
LOGGER.info("-- Welcome to Hveto --")
LOGGER.info("GPS start time: %d" % start)
LOGGER.info("GPS end time: %d" % end)
LOGGER.info("Interferometer: %s" % ifo)
# -- initialisation -------------------------
# read configuration
cp = config.HvetoConfigParser(ifo=ifo)
cp.read(args.config_file)
LOGGER.info("Parsed configuration file(s)")
# format output directory
outdir = _abs_path(args.output_directory)
if not os.path.isdir(outdir):
os.makedirs(outdir)
os.chdir(outdir)
LOGGER.info("Working directory: %s" % outdir)
segdir = 'segments'
plotdir = 'plots'
trigdir = 'triggers'
omegadir = 'scans'
for d in [segdir, plotdir, trigdir, omegadir]:
if not os.path.isdir(d):
os.makedirs(d)
# prepare html variables
htmlv = {
'title': '%s Hveto | %d-%d' % (ifo, start, end),
'config': None,
'prog': PROG,
'context': ifo.lower(),
}
# get segments
aflag = cp.get('segments', 'analysis-flag')
url = cp.get('segments', 'url')
padding = tuple(cp.getfloats('segments', 'padding'))
if args.analysis_segments:
segs_ = DataQualityDict.read(args.analysis_segments, gpstype=float)
analysis = segs_[aflag]
span = SegmentList([Segment(start, end)])
analysis.active &= span
analysis.known &= span
analysis.coalesce()
LOGGER.debug("Segments read from disk")
else:
analysis = DataQualityFlag.query(aflag, start, end, url=url)
LOGGER.debug("Segments recovered from %s" % url)
if padding != (0, 0):
mindur = padding[0] - padding[1]
analysis.active = type(analysis.active)([s for s in analysis.active if
abs(s) >= mindur])
analysis.pad(*padding, inplace=True)
LOGGER.debug("Padding %s applied" % str(padding))
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("Retrieved %d segments for %s with %ss (%.2f%%) livetime"
% (len(analysis.active), aflag, livetime, livetimepc))
# apply vetoes from veto-definer file
try:
vetofile = cp.get('segments', 'veto-definer-file')
except configparser.NoOptionError:
vetofile = None
else:
try:
categories = cp.getfloats('segments', 'veto-definer-categories')
except configparser.NoOptionError:
categories = None
# read file
vdf = read_veto_definer_file(vetofile, start=start, end=end, ifo=ifo)
LOGGER.debug("Read veto-definer file from %s" % vetofile)
# get vetoes from segdb
vdf.populate(source=url, segments=analysis.active, on_error='warn')
# coalesce flags from chosen categories
vetoes = DataQualityFlag('%s:VDF-VETOES:1' % ifo)
nflags = 0
for flag in vdf:
if not categories or vdf[flag].category in categories:
vetoes += vdf[flag]
nflags += 1
try:
deadtime = int(abs(vetoes.active)) / int(abs(vetoes.known)) * 100
except ZeroDivisionError:
deadtime = 0
LOGGER.debug("Coalesced %ss (%.2f%%) of deadtime from %d veto flags"
% (abs(vetoes.active), deadtime, nflags))
# apply to analysis segments
analysis -= vetoes
LOGGER.debug("Applied vetoes from veto-definer file")
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("%ss (%.2f%%) livetime remaining after vetoes"
% (livetime, livetimepc))
snrs = cp.getfloats('hveto', 'snr-thresholds')
minsnr = min(snrs)
windows = cp.getfloats('hveto', 'time-windows')
# record all segments
segments = DataQualityDict()
segments[analysis.name] = analysis
# -- load channels --------------------------
# get primary channel name
pchannel = cp.get('primary', 'channel')
# read auxiliary cache
if args.auxiliary_cache is not None:
acache = read_cache(args.auxiliary_cache)
else:
acache = None
# load auxiliary channels
auxetg = cp.get('auxiliary', 'trigger-generator')
auxfreq = cp.getfloats('auxiliary', 'frequency-range')
try:
auxchannels = cp.get('auxiliary', 'channels').strip('\n').split('\n')
except config.configparser.NoOptionError:
auxchannels = find_auxiliary_channels(auxetg, (start, end), ifo=ifo,
cache=acache)
cp.set('auxiliary', 'channels', '\n'.join(auxchannels))
LOGGER.debug("Auto-discovered %d "
"auxiliary channels" % len(auxchannels))
else:
auxchannels = sorted(set(auxchannels))
LOGGER.debug("Read list of %d auxiliary channels" % len(auxchannels))
# load unsafe channels list
_unsafe = cp.get('safety', 'unsafe-channels')
if os.path.isfile(_unsafe): # from file
unsafe = set()
with open(_unsafe, 'rb') as f:
for c in f.read().rstrip('\n').split('\n'):
if c.startswith('%(IFO)s'):
unsafe.add(c.replace('%(IFO)s', ifo))
elif not c.startswith('%s:' % ifo):
unsafe.add('%s:%s' % (ifo, c))
else:
unsafe.add(c)
else: # or from line-seprated list
unsafe = set(_unsafe.strip('\n').split('\n'))
unsafe.add(pchannel)
cp.set('safety', 'unsafe-channels', '\n'.join(sorted(unsafe)))
LOGGER.debug("Read list of %d unsafe channels" % len(unsafe))
# remove unsafe channels
nunsafe = 0
for i in range(len(auxchannels) - 1, -1, -1):
if auxchannels[i] in unsafe:
LOGGER.warning("Auxiliary channel %r identified as unsafe and has "
"been removed" % auxchannels[i])
auxchannels.pop(i)
nunsafe += 1
LOGGER.debug("%d auxiliary channels identified as unsafe" % nunsafe)
naux = len(auxchannels)
LOGGER.info("Identified %d auxiliary channels to process" % naux)
# record INI file in output HTML directory
inifile = '%s-HVETO_CONFIGURATION-%d-%d.ini' % (ifo, start, duration)
if os.path.isfile(inifile) and any(
os.path.samefile(inifile, x) for x in args.config_file):
LOGGER.debug("Cannot write INI file to %s, file was given as input")
else:
with open(inifile, 'w') as f:
cp.write(f)
LOGGER.info("Configuration recorded as %s" % inifile)
htmlv['config'] = inifile
# -- load primary triggers ------------------
# read primary cache
if args.primary_cache is not None:
pcache = read_cache(args.primary_cache)
else:
pcache = None
# load primary triggers
petg = cp.get('primary', 'trigger-generator')
psnr = cp.getfloat('primary', 'snr-threshold')
pfreq = cp.getfloats('primary', 'frequency-range')
preadkw = cp.getparams('primary', 'read-')
if pcache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the primary trigger file format')
preadkw['format'] = None
preadkw['path'] = 'triggers'
ptrigfindkw = cp.getparams('primary', 'trigfind-')
primary = get_triggers(pchannel, petg, analysis.active, snr=psnr,
frange=pfreq, cache=pcache, nproc=args.nproc,
trigfind_kwargs=ptrigfindkw, **preadkw)
fcol, scol = primary.dtype.names[1:3]
if len(primary):
LOGGER.info("Read %d events for %s" % (len(primary), pchannel))
else:
message = "No events found for %r in %d seconds of livetime" % (
pchannel, livetime)
LOGGER.critical(message)
# cluster primary triggers
clusterkwargs = cp.getparams('primary', 'cluster-')
if clusterkwargs:
primary = primary.cluster(**clusterkwargs)
LOGGER.info("%d primary events remain after clustering over %s" %
(len(primary), clusterkwargs['rank']))
# -- bail out early -------------------------
# the bail out is done here so that we can at least generate the eventual
# configuration file, mainly for HTML purposes
# no segments
if livetime == 0:
message = ("No active segments found for analysis flag %r in interval "
"[%d, %d)" % (aflag, start, end))
LOGGER.critical(message)
htmlv['context'] = 'info'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# no primary triggers
if len(primary) == 0:
htmlv['context'] = 'danger'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# otherwise write all primary triggers to ASCII
trigfile = os.path.join(
trigdir,
'%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration),
)
primary.write(trigfile, format='ascii', overwrite=True)
# -- load auxiliary triggers ----------------
LOGGER.info("Reading triggers for aux channels...")
counter = multiprocessing.Value('i', 0)
areadkw = cp.getparams('auxiliary', 'read-')
if acache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the auxiliary trigger file format')
areadkw['format'] = None
areadkw['path'] = 'triggers'
atrigfindkw = cp.getparams('auxiliary', 'trigfind-')
# map with multiprocessing
if args.nproc > 1:
pool = multiprocessing.Pool(processes=args.nproc)
results = pool.map(_get_aux_triggers, auxchannels)
pool.close()
# map without multiprocessing
else:
results = map(_get_aux_triggers, auxchannels)
LOGGER.info("All aux events loaded")
auxiliary = dict(x for x in results if x is not None)
auxchannels = sorted(auxiliary.keys())
chanfile = '%s-HVETO_CHANNEL_LIST-%d-%d.txt' % (ifo, start, duration)
with open(chanfile, 'w') as f:
for chan in auxchannels:
print(chan, file=f)
LOGGER.info("Recorded list of valid auxiliary channels in %s" % chanfile)
# -- execute hveto analysis -----------------
minsig = cp.getfloat('hveto', 'minimum-significance')
pevents = [primary]
pvetoed = []
auxfcol, auxscol = auxiliary[auxchannels[0]].dtype.names[1:3]
slabel = plot.get_column_label(scol)
flabel = plot.get_column_label(fcol)
auxslabel = plot.get_column_label(auxscol)
auxflabel = plot.get_column_label(auxfcol)
rounds = []
rnd = core.HvetoRound(1, pchannel, rank=scol)
rnd.segments = analysis.active
while True:
LOGGER.info("-- Processing round %d --" % rnd.n)
# write segments for this round
segfile = os.path.join(
segdir, '%s-HVETO_ANALYSIS_SEGS_ROUND_%d-%d-%d.txt'
% (ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.segments)
# calculate significances for this round
if args.nproc > 1: # multiprocessing
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_find_max_significance, chunks)
pool.close()
winners, sigsets = zip(*results)
# find winner of chunk winners
winner = sorted(winners, key=lambda w: w.significance)[-1]
# flatten sets of significances into one list
newsignificances = sigsets[0]
for subdict in sigsets[1:]:
newsignificances.update(subdict)
else: # single process
winner, newsignificances = core.find_max_significance(
primary, auxiliary, pchannel, snrs, windows, rnd.livetime)
LOGGER.info("Round %d winner: %s" % (rnd.n, winner.name))
# plot significance drop here for the last round
# only now do we actually have the new data to
# calculate significance drop
if rnd.n > 1:
svg = (pngname % 'SIG_DROP').replace('.png', '.svg') # noqa: F821
plot.significance_drop(
svg, oldsignificances, newsignificances, # noqa: F821
title=' | '.join([title, subtitle]), # noqa: F821
bbox_inches='tight')
LOGGER.debug("Figure written to %s" % svg)
svg = FancyPlot(svg, caption=plot.ROUND_CAPTION['SIG_DROP'])
rounds[-1].plots.append(svg)
oldsignificances = newsignificances # noqa: F841
# break out of the loop if the significance is below stopping point
if winner.significance < minsig:
LOGGER.info("Maximum signifiance below stopping point")
LOGGER.debug(" (%.2f < %.2f)" % (winner.significance, minsig))
LOGGER.info("-- Rounds complete! --")
break
# work out the vetoes for this round
allaux = auxiliary[winner.name][
auxiliary[winner.name][auxscol] >= winner.snr]
winner.events = allaux
coincs = allaux[core.find_coincidences(allaux['time'], primary['time'],
dt=winner.window)]
rnd.vetoes = winner.get_segments(allaux['time'])
flag = DataQualityFlag(
'%s:HVT-ROUND_%d:1' % (ifo, rnd.n), active=rnd.vetoes,
known=rnd.segments,
description="winner=%s, window=%s, snr=%s" % (
winner.name, winner.window, winner.snr))
segments[flag.name] = flag
LOGGER.debug("Generated veto segments for round %d" % rnd.n)
# link events before veto for plotting
before = primary
beforeaux = auxiliary[winner.name]
# apply vetoes to primary
primary, vetoed = core.veto(primary, rnd.vetoes)
pevents.append(primary)
pvetoed.append(vetoed)
LOGGER.debug("Applied vetoes to primary")
# record results
rnd.winner = winner
rnd.efficiency = (len(vetoed), len(primary) + len(vetoed))
rnd.use_percentage = (len(coincs), len(winner.events))
if rnd.n > 1:
rnd.cum_efficiency = (
len(vetoed) + rounds[-1].cum_efficiency[0],
rounds[0].efficiency[1])
rnd.cum_deadtime = (
rnd.deadtime[0] + rounds[-1].cum_deadtime[0],
livetime)
else:
rnd.cum_efficiency = rnd.efficiency
rnd.cum_deadtime = rnd.deadtime
# apply vetoes to auxiliary
if args.nproc > 1: # multiprocess
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_veto, chunks)
pool.close()
auxiliary = results[0]
for subdict in results[1:]:
auxiliary.update(subdict)
else: # single process
auxiliary = core.veto_all(auxiliary, rnd.vetoes)
LOGGER.debug("Applied vetoes to auxiliary channels")
# log results
LOGGER.info("""Results for round %d:\n\n
winner : %s
significance : %s
mu : %s
snr : %s
dt : %s
use_percentage : %s
efficiency : %s
deadtime : %s
cum. efficiency : %s
cum. deadtime : %s\n\n""" % (
rnd.n, rnd.winner.name, rnd.winner.significance,
rnd.winner.mu, rnd.winner.snr, rnd.winner.window,
rnd.use_percentage, rnd.efficiency, rnd.deadtime,
rnd.cum_efficiency, rnd.cum_deadtime))
# write segments
segfile = os.path.join(
segdir,
'%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % (
ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.vetoes)
LOGGER.debug("Round %d vetoes written to %s" % (rnd.n, segfile))
rnd.files['VETO_SEGS'] = (segfile,)
# write triggers
trigfile = os.path.join(
trigdir,
'%s-HVETO_%%s_TRIGS_ROUND_%d-%d-%d.txt' % (
ifo, rnd.n, start, duration))
for tag, arr in zip(
['WINNER', 'VETOED', 'RAW'],
[winner.events, vetoed, primary]):
f = trigfile % tag
arr.write(f, format='ascii', overwrite=True)
LOGGER.debug("Round %d %s events written to %s"
% (rnd.n, tag.lower(), f))
rnd.files[tag] = f
# record times to omega scan
if args.omega_scans:
N = len(vetoed)
ind = random.sample(range(0, N), min(args.omega_scans, N))
rnd.scans = vetoed[ind]
LOGGER.debug("Collected %d events to omega scan:\n\n%s\n\n"
% (len(rnd.scans), rnd.scans))
# -- make some plots --
pngname = os.path.join(plotdir, '%s-HVETO_%%s_ROUND_%d-%d-%d.png' % (
ifo, rnd.n, start, duration))
wname = texify(rnd.winner.name)
beforel = 'Before\n[%d]' % len(before)
afterl = 'After\n[%d]' % len(primary)
vetoedl = 'Vetoed\n(primary)\n[%d]' % len(vetoed)
beforeauxl = 'All\n[%d]' % len(beforeaux)
usedl = 'Used\n(aux)\n[%d]' % len(winner.events)
coincl = 'Coinc.\n[%d]' % len(coincs)
title = '%s Hveto round %d' % (ifo, rnd.n)
ptitle = '%s: primary impact' % title
atitle = '%s: auxiliary use' % title
subtitle = 'winner: %s [%d-%d]' % (wname, start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
plot.before_after_histogram(
png, before[scol], primary[scol],
label1=beforel, label2=afterl, xlabel=slabel,
title=ptitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['HISTOGRAM'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(
png, before, vetoed, x='time', y=scol, label1=beforel,
label2=vetoedl, epoch=start, xlim=[start, end], ylabel=slabel,
title=ptitle, subtitle=subtitle, legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'SNR_%s' % fcol.upper()
plot.veto_scatter(
png, before, vetoed, x=fcol, y=scol, label1=beforel,
label2=vetoedl, xlabel=flabel, ylabel=slabel, xlim=pfreq,
title=ptitle, subtitle=subtitle, legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % '%s_TIME' % fcol.upper()
plot.veto_scatter(
png, before, vetoed, x='time', y=fcol, color=scol,
label1=None, label2=None, ylabel=flabel,
clabel=slabel, clim=[3, 100], cmap='YlGnBu',
epoch=start, xlim=[start, end], ylim=pfreq,
title=ptitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['TIME'])
rnd.plots.append(png)
# aux used versus frequency
png = pngname % 'USED_SNR_TIME'
plot.veto_scatter(
png, winner.events, vetoed, x='time', y=[auxscol, scol],
label1=usedl, label2=vetoedl, ylabel=slabel, epoch=start,
xlim=[start, end], title=atitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['USED_SNR_TIME'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'AUX_SNR_TIME'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x='time', y=auxscol,
label1=beforeauxl, label2=(usedl, coincl), epoch=start,
xlim=[start, end], ylabel=auxslabel, title=atitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'AUX_SNR_FREQUENCY'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x=auxfcol, y=auxscol,
label1=beforeauxl, label2=(usedl, coincl), xlabel=auxflabel,
ylabel=auxslabel, title=atitle, subtitle=subtitle,
legend_title="Aux:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_FREQUENCY'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % 'AUX_FREQUENCY_TIME'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x='time', y=auxfcol,
color=auxscol, label1=None, label2=[None, None], ylabel=auxflabel,
clabel=auxslabel, clim=[3, 100], cmap='YlGnBu', epoch=start,
xlim=[start, end], title=atitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_FREQUENCY_TIME'])
rnd.plots.append(png)
# move to the next round
rounds.append(rnd)
rnd = core.HvetoRound(rnd.n + 1, pchannel, rank=scol,
segments=rnd.segments-rnd.vetoes)
# write file with all segments
segfile = os.path.join(
segdir, '%s-HVETO_SEGMENTS-%d-%d.h5' % (ifo, start, duration))
segments.write(segfile, overwrite=True)
LOGGER.debug("Segment summary written to %s" % segfile)
LOGGER.debug("Making summary figures...")
# -- exit early if no rounds above threshold
if not rounds:
message = ("No rounds completed above threshold. Analysis stopped "
"with %s achieving significance of %.2f"
% (winner.name, winner.significance))
LOGGER.critical(message)
message = message.replace(
winner.name, cis_link(winner.name, class_='alert-link'))
message += '<br>[T<sub>win</sub>: %ss, SNR: %s]' % (
winner.window, winner.snr)
htmlv['context'] = 'warning'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# -- plot all rounds impact
pngname = os.path.join(plotdir, '%s-HVETO_%%s_ALL_ROUNDS-%d-%d.png' % (
ifo, start, duration))
plots = []
title = '%s Hveto all rounds' % args.ifo
subtitle = '%d rounds | %d-%d' % (len(rounds), start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
beforel = 'Before analysis [%d events]' % len(pevents[0])
afterl = 'After %d rounds [%d]' % (len(pevents) - 1, len(pevents[-1]))
plot.before_after_histogram(
png, pevents[0][scol], pevents[-1][scol],
label1=beforel, label2=afterl, xlabel=slabel,
title=title, subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['HISTOGRAM'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# efficiency/deadtime curve
png = pngname % 'ROC'
plot.hveto_roc(png, rounds, title=title, subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['ROC'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# frequency versus time
png = pngname % '%s_TIME' % fcol.upper()
labels = [str(r.n) for r in rounds]
legtitle = 'Vetoed at\nround'
plot.veto_scatter(
png, pevents[0], pvetoed,
label1='', label2=labels, title=title,
subtitle=subtitle, ylabel=flabel, x='time', y=fcol,
epoch=start, xlim=[start, end], legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(
png, pevents[0], pvetoed, label1='', label2=labels, title=title,
subtitle=subtitle, ylabel=slabel, x='time', y=scol,
epoch=start, xlim=[start, end], legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['SNR_TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# -- write summary states to ASCII table and JSON
json_ = {
'user': getuser(),
'host': getfqdn(),
'date': str(datetime.datetime.now()),
'configuration': inifile,
'ifo': ifo,
'gpsstart': start,
'gpsend': end,
'call': ' '.join(sys.argv),
'rounds': [],
}
with open('summary-stats.txt', 'w') as f:
# print header
print('#N winner window SNR significance nveto use-percentage '
'efficiency deadtime cumulative-efficiency cumulative-deadtime',
file=f)
for r in rounds:
# extract relevant statistics
results = [
('round', r.n),
('name', r.winner.name),
('window', r.winner.window),
('snr', r.winner.snr),
('significance', r.winner.significance),
('nveto', r.efficiency[0]),
('use-percentage',
r.use_percentage[0] / r.use_percentage[1] * 100.),
('efficiency', r.efficiency[0] / r.efficiency[1] * 100.),
('deadtime', r.deadtime[0] / r.deadtime[1] * 100.),
('cumulative-efficiency',
r.cum_efficiency[0] / r.cum_efficiency[1] * 100.),
('cumulative-deadtime',
r.cum_deadtime[0] / r.cum_deadtime[1] * 100.),
]
# write to ASCII
print(' '.join(map(str, list(zip(*results))[1])), file=f)
# write to JSON
results.append(('files', r.files))
json_['rounds'].append(dict(results))
LOGGER.debug("Summary table written to %s" % f.name)
with open('summary-stats.json', 'w') as f:
json.dump(json_, f, sort_keys=True)
LOGGER.debug("Summary JSON written to %s" % f.name)
# -- generate workflow for omega scans
if args.omega_scans:
omegatimes = list(map(str, sorted(numpy.unique(
[t['time'] for r in rounds for t in r.scans]))))
LOGGER.debug("Collected %d times to omega scan" % len(omegatimes))
newtimes = [t for t in omegatimes if not
os.path.exists(os.path.join(omegadir, str(t)))]
LOGGER.debug("%d scans already complete or in progress, %d remaining"
% (len(omegatimes) - len(newtimes), len(newtimes)))
if len(newtimes) > 0:
LOGGER.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(
ifo=ifo,
ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(
timeout=4,
extra_commands=["request_disk='1G'"],
gps=start)
batch.generate_dag(
newtimes,
flags=flags,
submit=True,
outdir=omegadir,
condor_commands=condorcmds)
LOGGER.info('Launched {} omega scans to condor'.format(
len(newtimes)))
else:
LOGGER.debug('Skipping omega scans')
# -- write HTML and finish
index = html.write_hveto_page(
ifo, start, end, rounds, plots,
winners=[r.winner.name for r in rounds], **htmlv)
LOGGER.debug("HTML written to %s" % index)
LOGGER.debug("Analysis completed in %d seconds" % (time.time() - JOBSTART))
LOGGER.info("-- Hveto complete --")
PADDING = (-0.5, 1)
# padded version of above 'known' segments
KNOWNPAD = _as_segmentlist((-.5, 4), (5.5, 8))
# padded version of above 'active' segments
ACTIVEPAD = _as_segmentlist((.5, 3), (2.5, 5), (4.5, 8))
# padded, coalesed version of above 'active' segments
ACTIVEPADC = _as_segmentlist((.5, 4), (5.5, 8))
# -- query helpers ------------------------------------------------------------
QUERY_FLAGS = ['X1:TEST-FLAG:1', 'Y1:TEST-FLAG2:4']
QUERY_RESULT = DataQualityDict()
QUERY_RESULT['X1:TEST-FLAG:1'] = DataQualityFlag('X1:TEST-FLAG:1',
known=[(0, 10)],
active=[(0, 1), (1, 2),
(3, 4), (6, 9)])
QUERY_RESULT['Y1:TEST-FLAG2:4'] = DataQualityFlag('Y1:TEST-FLAG2:4',
known=[(0, 5), (9, 10)],
active=[])
QUERY_RESULTC = type(QUERY_RESULT)(
{x: y.copy().coalesce()
for x, y in QUERY_RESULT.items()})
def main(args=None):
"""Run the software saturation command-line interface
"""
parser = create_parser()
args = parser.parse_args(args=args)
# get IFO
ifo = args.ifo.upper()
site = ifo[0]
frametype = args.frametype or '%s_R' % ifo
# let's go
LOGGER.info('{} Software saturations {}-{}'.format(
args.ifo, int(args.gpsstart), int(args.gpsend)))
# get segments
span = Segment(args.gpsstart, args.gpsend)
if args.state_flag:
state = DataQualityFlag.query(args.state_flag, int(args.gpsstart),
int(args.gpsend),
url=const.DEFAULT_SEGMENT_SERVER)
for i, seg in enumerate(state.active):
state.active[i] = type(seg)(seg[0], seg[1]-args.pad_state_end)
segs = state.active.coalesce()
LOGGER.debug("Recovered %d seconds of time for %s"
% (abs(segs), args.state_flag))
else:
segs = SegmentList([Segment(args.gpsstart, args.gpsend)])
# find frames
cache = gwdatafind.find_urls(
site, frametype, int(args.gpsstart), int(args.gpsend))
# find channels
if not os.getenv('LIGO_DATAFIND_SERVER'):
raise RuntimeError("No LIGO_DATAFIND_SERVER variable set, don't know "
"how to discover channels")
else:
LOGGER.debug("Identifying channels in frame files")
if len(cache) == 0:
raise RuntimeError(
"No frames recovered for %s in interval [%s, %s)" %
(frametype, int(args.gpsstart),
int(args.gpsend)))
allchannels = get_channel_names(cache[0])
LOGGER.debug(" Found %d channels" % len(allchannels))
sys.stdout.flush()
channels = core.find_limit_channels(allchannels, skip=args.skip)
LOGGER.info(
" Parsed %d channels with '_LIMIT' and '_LIMEN' or '_SWSTAT'"
% sum(map(len, channels)))
# -- read channels and check limits -------------
saturations = DataQualityDict()
bad = set()
# TODO: use multiprocessing to separate channel list into discrete chunks
# should give a factor of X for X processes
# check limens
for suffix, clist in zip(['LIMEN', 'SWSTAT'], channels):
nchans = len(clist)
# group channels in sets for batch processing
# min of <number of channels>, user group size (sensible number),
# and 512 Mb of RAM for single-precision EPICS
try:
dur = max([float(abs(s)) for s in segs])
except ValueError:
ngroup = args.group_size
else:
ngroup = int(
min(nchans, args.group_size, 2 * 1024**3 / 4. / 16. / dur))
LOGGER.info('Processing %s channels in groups of %d' % (
suffix, ngroup))
sys.stdout.flush()
sets = core.grouper(clist, ngroup)
for i, cset in enumerate(sets):
# remove empty entries use to pad the list to 8 elements
cset = list(cset)
while cset[-1] is None:
cset.pop(-1)
for seg in segs:
cache2 = sieve_cache(cache, segment=seg)
if not len(cache2):
continue
saturated = core.is_saturated(
cset, cache2, seg[0], seg[1], indicator=suffix,
nproc=args.nproc)
for new in saturated:
try:
saturations[new.name] += new
except KeyError:
saturations[new.name] = new
for j, c in enumerate(cset):
try:
sat = saturations[c]
except KeyError:
LOGGER.debug('%40s: SKIP [%d/%d]'
% (c, i*ngroup + j + 1, nchans))
else:
if abs(sat.active):
LOGGER.debug('%40s: ---- FAIL ---- [%d/%d]'
% (c, i*ngroup + j + 1, nchans))
for seg in sat.active:
LOGGER.debug(" " * 42 + str(seg))
bad.add(c)
else:
LOGGER.debug('%40s: PASS [%d/%d]'
% (c, i*ngroup + j + 1, nchans))
sys.stdout.flush()
# -- log results and exit -----------------------
if len(bad):
LOGGER.info("Saturations were found for all of the following:\n\n")
for c in bad:
print(c)
print('\n\n')
else:
LOGGER.info("No software saturations were found in any channels")
# write segments to file
outfile = ('%s-SOFTWARE_SATURATIONS-%d-%d.h5'
% (ifo, int(args.gpsstart),
int(args.gpsend) - int(args.gpsstart)))
LOGGER.info("Writing saturation segments to %s" % outfile)
saturations.write(outfile, path="segments", overwrite=True)
if args.html:
# get base path
base = os.path.dirname(args.html)
os.chdir(base)
if args.plot:
args.plot = os.path.curdir
segfile = os.path.relpath(outfile, os.path.dirname(args.html))
if os.path.basename(args.html) == 'index.html':
links = [
'%d-%d' % (int(args.gpsstart), int(args.gpsend)),
('Parameters', '#parameters'),
('Segments', [('Software saturations',
'#software-saturations')]),
('Results', '#results'),
]
if args.state_flag:
links[2][1].insert(0, ('State flag', '#state-flag'))
(brand, class_) = htmlio.get_brand(ifo, 'Saturations',
args.gpsstart)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(
navbar=navbar, title='%s Saturations | %d-%d' % (
ifo, int(args.gpsstart), int(args.gpsend)))
else:
page = markup.page()
page.div(class_='container')
# -- header
page.div(class_='pb-2 mt-3 mb-2 border-bottom')
page.h1('%s Software Saturations: %d-%d'
% (ifo, int(args.gpsstart), int(args.gpsend)))
page.div.close()
# -- paramters
content = [
('State end padding', args.pad_state_end),
('Skip', ', '.join(map(repr, args.skip)))]
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(
content, start=args.gpsstart, end=args.gpsend,
flag=args.state_flag))
page.div.close() # col-md-9 col-sm-12
page.div(class_='col-md-3 col-sm-12')
page.add(htmlio.download_btn(
[('Segments (HDF)', segfile)],
btnclass='btn btn-%s dropdown-toggle' % ifo.lower(),
))
page.div.close() # col-md-9 col-sm-12
page.div.close() # row
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
# -- segments
page.h2('Segments', class_='mt-4', id_='segments')
msg = ("This analysis searched {0} filter bank readback channels for "
"time periods during which their OUTPUT value matched or "
"exceeded the LIMIT value set in software. Signals that "
"achieve saturation are shown below, and saturation segments "
"are available by expanding a given panel.").format(
sum(map(len, channels)))
page.add(htmlio.alert(msg, context=ifo.lower()))
# record state segments
if args.state_flag:
page.h3('State flag', class_='mt-3', id_='state-flag')
page.div(id_='accordion1')
page.add(htmlio.write_flag_html(
state, span, 'state', parent='accordion1', context='success',
plotdir=args.plot, facecolor=(0.2, 0.8, 0.2),
edgecolor='darkgreen', known={
'facecolor': 'red',
'edgecolor': 'darkred',
'height': 0.4},
))
page.div.close()
# record saturation segments
if len(bad):
page.h3('Software saturations', class_='mt-3',
id_='software-saturations')
page.div(id_='accordion2')
for i, (c, flag) in enumerate(saturations.items()):
if abs(flag.active) > 0:
title = '%s [%d]' % (flag.name, len(flag.active))
page.add(htmlio.write_flag_html(
flag, span=span, id=i, parent='accordion2',
title=title, plotdir=args.plot))
page.div.close()
else:
page.add(htmlio.alert('No software saturations were found in this '
'analysis', context=ifo.lower(),
dismiss=False))
# -- results table
page.h2('Results summary', class_='mt-4', id_='results')
page.add(htmlio.alert('All channels for which the LIMIT setting was '
'active are shown below.', context=ifo.lower()))
page.table(class_='table table-striped table-hover')
# write table header
page.thead()
page.tr()
for header in ['Channel', 'Result', 'Num. saturations']:
page.th(header)
page.thead.close()
# write body
page.tbody()
for c, seglist in saturations.items():
passed = abs(seglist.active) == 0
if passed:
page.tr()
else:
page.tr(class_='table-warning')
page.td(c)
page.td(passed and 'Pass' or 'Fail')
page.td(len(seglist.active))
page.tr.close()
page.tbody.close()
page.table.close()
# close and write
htmlio.close_page(page, args.html)
"""Set of global memory variables for GWSumm package
"""
import time
from gwpy.time import tconvert
from gwpy.segments import DataQualityDict
from gwpy.detector import ChannelList
CHANNELS = ChannelList()
STATES = {}
DATA = {}
SPECTROGRAMS = {}
SPECTRUM = {}
SEGMENTS = DataQualityDict()
TRIGGERS = {}
VERBOSE = False
PROFILE = False
START = time.time()
# run time variables
MODE = 4
WRITTEN_PLOTS = []
NOW = tconvert('now').seconds
HTMLONLY = False
# comments
IFO = None
HTML_COMMENTS_NAME = None
def main(args=None):
"""Run the cache_events tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
ifo = args.ifo
start = int(args.gpsstart)
end = int(args.gpsend)
duration = end - start
LOGGER.info("-- Welcome to Hveto --")
LOGGER.info("GPS start time: %d" % start)
LOGGER.info("GPS end time: %d" % end)
LOGGER.info("Interferometer: %s" % ifo)
# -- initialisation -------------------------------
# read configuration
cp = config.HvetoConfigParser(ifo=args.ifo)
cp.read(map(str, args.config_file))
LOGGER.info("Parsed configuration file(s)")
# format output directory
outdir = args.output_directory
outdir.mkdir(parents=True, exist_ok=True)
LOGGER.info("Working directory: {}".format(outdir))
trigdir = outdir / 'triggers'
trigdir.mkdir(parents=True, exist_ok=True)
# get segments
aflag = cp.get('segments', 'analysis-flag')
url = cp.get('segments', 'url')
padding = cp.getfloats('segments', 'padding')
if args.analysis_segments:
segs_ = DataQualityDict.read(args.analysis_segments, gpstype=float)
analysis = segs_[aflag]
span = SegmentList([Segment(start, end)])
analysis.active &= span
analysis.known &= span
analysis.coalesce()
LOGGER.debug("Segments read from disk")
else:
analysis = DataQualityFlag.query(aflag, start, end, url=url)
LOGGER.debug("Segments recovered from %s" % url)
analysis.pad(*padding)
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("Retrieved %d segments for %s with %ss (%.2f%%) livetime" %
(len(analysis.active), aflag, livetime, livetimepc))
snrs = cp.getfloats('hveto', 'snr-thresholds')
minsnr = min(snrs)
# -- utility methods ------------------------------
def create_path(channel):
ifo, name = channel.split(':', 1)
name = name.replace('-', '_')
return trigdir / "{}-{}-{}-{}.h5".format(ifo, name, start, duration)
def read_and_cache_events(channel,
etg,
cache=None,
trigfind_kw={},
**read_kw):
cfile = create_path(channel)
# read existing cached triggers and work out new segments to query
if args.append and cfile.is_file():
previous = DataQualityFlag.read(
str(cfile),
path='segments',
format='hdf5',
).coalesce()
new = analysis - previous
else:
new = analysis.copy()
# get cache of files
if cache is None:
cache = find_trigger_files(channel, etg, new.active, **trigfind_kw)
else:
cache = list(
filter(
lambda e: new.active.intersects_segment(file_segment(e)),
cache,
))
# restrict 'active' segments to when we have data
try:
new.active &= cache_segments(cache)
except IndexError:
new.active = type(new.active)()
# find new triggers
try:
trigs = get_triggers(channel,
etg,
new.active,
cache=cache,
raw=True,
**read_kw)
# catch error and continue
except ValueError as e:
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
else:
path = write_events(channel, trigs, new)
try:
return path, len(trigs)
except TypeError: # None
return
def write_events(channel, tab, segments):
"""Write events to file with a given filename
"""
# get filename
path = create_path(channel)
h5f = h5py.File(str(path), 'a')
# read existing table from file
try:
old = tab.read(h5f["triggers"], format="hdf5")
except KeyError:
pass
else:
tab = vstack(old, tab)
# append event table
tab.write(h5f, path="triggers", append=True, overwrite=True)
# write segments
try:
oldsegs = DataQualityFlag.read(h5f, path="segments", format="hdf5")
except KeyError:
pass
else:
segments = oldsegs + segments
segments.write(h5f, path="segments", append=True, overwrite=True)
# write file to disk
h5f.close()
return path
# -- load channels --------------------------------
# get primary channel name
pchannel = cp.get('primary', 'channel')
# read auxiliary cache
if args.auxiliary_cache is not None:
acache = [e for c in args.auxiliary_cache for e in read_cache(str(c))]
else:
acache = None
# load auxiliary channels
auxetg = cp.get('auxiliary', 'trigger-generator')
auxfreq = cp.getfloats('auxiliary', 'frequency-range')
try:
auxchannels = cp.get('auxiliary', 'channels').strip('\n').split('\n')
except config.configparser.NoOptionError:
auxchannels = find_auxiliary_channels(auxetg,
start,
ifo=args.ifo,
cache=acache)
# load unsafe channels list
_unsafe = cp.get('safety', 'unsafe-channels')
if os.path.isfile(_unsafe): # from file
unsafe = set()
with open(_unsafe, 'rb') as f:
for c in f.read().rstrip('\n').split('\n'):
if c.startswith('%(IFO)s'):
unsafe.add(c.replace('%(IFO)s', ifo))
elif not c.startswith('%s:' % ifo):
unsafe.add('%s:%s' % (ifo, c))
else:
unsafe.add(c)
else: # or from line-seprated list
unsafe = set(_unsafe.strip('\n').split('\n'))
unsafe.add(pchannel)
cp.set('safety', 'unsafe-channels', '\n'.join(sorted(unsafe)))
LOGGER.debug("Read list of %d unsafe channels" % len(unsafe))
# remove duplicates
auxchannels = sorted(set(auxchannels))
LOGGER.debug("Read list of %d auxiliary channels" % len(auxchannels))
# remove unsafe channels
nunsafe = 0
for i in range(len(auxchannels) - 1, -1, -1):
if auxchannels[i] in unsafe:
LOGGER.warning("Auxiliary channel %r identified as unsafe and has "
"been removed" % auxchannels[i])
auxchannels.pop(i)
nunsafe += 1
LOGGER.debug("%d auxiliary channels identified as unsafe" % nunsafe)
naux = len(auxchannels)
LOGGER.info("Identified %d auxiliary channels to process" % naux)
# -- load primary triggers -------------------------
LOGGER.info("Reading events for primary channel...")
# read primary cache
if args.primary_cache is not None:
pcache = [e for c in args.primary_cache for e in read_cache(str(c))]
else:
pcache = None
# get primary params
petg = cp.get('primary', 'trigger-generator')
psnr = cp.getfloat('primary', 'snr-threshold')
pfreq = cp.getfloats('primary', 'frequency-range')
preadkw = cp.getparams('primary', 'read-')
ptrigfindkw = cp.getparams('primary', 'trigfind-')
# load primary triggers
out = read_and_cache_events(pchannel,
petg,
snr=psnr,
frange=pfreq,
cache=pcache,
trigfind_kw=ptrigfindkw,
**preadkw)
try:
e, n = out
except TypeError:
e = None
n = 0
if n:
LOGGER.info("Cached %d new events for %s" % (n, pchannel))
elif args.append and e.is_file():
LOGGER.info("Cached 0 new events for %s" % pchannel)
else:
message = "No events found for %r in %d seconds of livetime" % (
pchannel, livetime)
LOGGER.critical(message)
# write primary to local cache
pname = trigdir / '{}-HVETO_PRIMARY_CACHE-{}-{}.lcf'.format(
ifo,
start,
duration,
)
write_lal_cache(str(pname), [e])
LOGGER.info('Primary cache written to {}'.format(pname))
# -- load auxiliary triggers -----------------------
LOGGER.info("Reading triggers for aux channels...")
counter = multiprocessing.Value('i', 0)
areadkw = cp.getparams('auxiliary', 'read-')
atrigfindkw = cp.getparams('auxiliary', 'trigfind-')
def read_and_write_aux_triggers(channel):
if acache is None:
auxcache = None
else:
ifo, name = channel.split(':')
match = "{}-{}".format(ifo, name.replace('-', '_'))
auxcache = [e for e in acache if Path(e).name.startswith(match)]
out = read_and_cache_events(channel,
auxetg,
cache=auxcache,
snr=minsnr,
frange=auxfreq,
trigfind_kw=atrigfindkw,
**areadkw)
try:
e, n = out
except TypeError:
e = None
n = 0
# log result of load
with counter.get_lock():
counter.value += 1
tag = '[%d/%d]' % (counter.value, naux)
if e is None: # something went wrong
LOGGER.critical(" %s Failed to read events for %s" %
(tag, channel))
else: # either read events or nothing new
LOGGER.debug(" %s Cached %d new events for %s" %
(tag, n, channel))
return e
# map with multiprocessing
if args.nproc > 1:
pool = multiprocessing.Pool(processes=args.nproc)
results = pool.map(read_and_write_aux_triggers, auxchannels)
pool.close()
# map without multiprocessing
else:
results = map(read_and_write_aux_triggers, auxchannels)
acache = [x for x in results if x is not None]
aname = trigdir / '{}-HVETO_AUXILIARY_CACHE-{}-{}.lcf'.format(
ifo,
start,
duration,
)
write_lal_cache(str(aname), acache)
LOGGER.info('Auxiliary cache written to {}'.format(aname))
# -- finish ----------------------------------------
LOGGER.info('Done, you can use these cache files in an hveto analysis by '
'passing the following arguments:\n\n--primary-cache {} '
'--auxiliary-cache {}\n'.format(pname, aname))
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
FIXME
Returns
-------
FIXME
"""
if isinstance(flag, (unicode, str)):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag, isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEAFULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(operator.and_, (globalv.SEGMENTS.get(
f, DataQualityFlag(f)).known
for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except Exception as e:
if type(e) is not Exception:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= segs.known
out[compound].active &= segs.known
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, basestring):
return out[flag]
else:
return out
ifo = args.ifo
vetofile = getattr(args, 'veto-definer-file')
metrics = [
'efficiency', 'efficiency/deadtime', 'Efficiency/Deadtime | SNR>=8',
'Efficiency/Deadtime | SNR>=20', 'Efficiency/Deadtime | SNR>=100'
]
trigPath = args.trigFile
analysis = args.analysis
#create directories to hold stuff
tag = '%d-%d-%s' % (start.seconds, end.seconds, analysis)
outdir = os.path.abspath(os.path.join(args.output_directory, tag))
mkdir(outdir)
os.chdir(outdir)
ALLSEGMENTS = DataQualityDict()
# -- get analysis flags ----------------------
allFlags = get_known_flags(start, end, url, ifo=ifo, badonly=None)
for flag in allFlags[:]:
if 'ANALYSIS_READY' not in flag:
new = DataQualityFlag.query(flag, start,
end) #try get_segments gwvet.segments
ALLSEGMENTS[new.name] = new
else:
allFlags.remove(flag)
#Download VDFs
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
def main(args=None):
"""Run the online Guardian node visualization tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# parse command line options
ifo = args.ifo
if not args.ifo:
parser.error('--ifo must be given if not obvious from the host')
start = getattr(args, 'gpsstart')
end = getattr(args, 'gpsend')
duration = int(ceil(end) - floor(start))
categories = args.categories.split(',')
for i, c in enumerate(categories):
try:
categories[i] = int(c)
except (TypeError, ValueError):
pass
vetofile = getattr(args, 'veto-definer-file')
vetofile = (urlparse(vetofile).netloc or os.path.abspath(vetofile))
args.metric = args.metric or DEFAULT_METRICS
# -- setup --------------------------------------
tag = '%d-%d' % (start.seconds, end.seconds)
outdir = os.path.abspath(os.path.join(args.output_directory, tag))
mkdir(outdir)
os.chdir(outdir)
mkdir('etc', 'segments', 'condor')
# -- segment handling ---------------------------
os.chdir('segments')
ALLSEGMENTS = DataQualityDict()
# -- get analysis segments ----------------------
aflags = args.analysis_segments
asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo)
for i, flag in enumerate(aflags):
# use union of segments from a file
if os.path.isfile(flag):
asegments += DataQualityFlag.read(flag)
# or intersection of segments from multiple flags
else:
new = DataQualityFlag.query(flag, start, end, url=args.segdb)
if i:
asegments.known &= new.known
asegments.active &= new.active
else:
asegments.known = new.known
asegments.active = new.active
ALLSEGMENTS[asegments.name] = asegments
if os.path.isfile(aflags[0]):
asegments.filename = aflags
# -- read veto definer and process --------------
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
with open(vetofile, 'w') as f:
f.write(tmp.read())
LOGGER.info('Downloaded veto definer file')
vdf = DataQualityDict.from_veto_definer_file(vetofile,
format='ligolw',
start=start,
end=end,
ifo=ifo)
LOGGER.info('Read %d flags from veto definer' % len(vdf.keys()))
# populate veto definer file from database
vdf.populate(source=args.segdb, on_error=args.on_segdb_error)
ALLSEGMENTS += vdf
# organise flags into categories
flags = dict((c, DataQualityDict()) for c in categories)
for name, flag in vdf.items():
try:
flags[flag.category][name] = flag
except KeyError:
pass
# find the states and segments for each category
states, after, oldtitle = (dict(), None, '')
for i, category in enumerate(categories):
title = isinstance(category, int) and 'Cat %d' % category or category
tag = re_cchar.sub('_', str(title).upper())
states[category] = SummaryState(
'After %s' % oldtitle,
key=tag,
known=after.known,
active=after.active,
definition=after.name,
) if i else SummaryState(
args.analysis_name,
key=args.analysis_name,
definition=asegments.name,
)
try:
segs = flags[category].union()
except TypeError: # no flags
segs = DataQualityFlag()
segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag)
ALLSEGMENTS[segs.name] = segs
after = (after - segs) if i else (asegments - segs)
after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag)
ALLSEGMENTS[after.name] = after
oldtitle = title
# write all segments to disk
segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' %
(ifo, start.seconds, duration))
ALLSEGMENTS.write(segfile)
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("All segments accessed and written to\n%s" % segfile)
# -- job preparation ----------------------------
os.chdir('etc')
configs = []
for category in categories:
title = (isinstance(category, int) and 'Category %d' % category
or category)
tab = 'tab-%s' % title
config = ConfigParser()
# add segment-database configuration
add_config_section(config, 'segment-database', url=args.segdb)
# add plot configurations
pconfig = ConfigParser()
pconfig.read(args.config_file)
for section in pconfig.sections():
if section.startswith('plot-'):
config._sections[section] = pconfig._sections[section].copy()
try:
plots = pconfig.items('plots-%s' % category, raw=True)
except NoSectionError:
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
# add state
if args.independent:
state = states[categories[0]]
else:
state = states[category]
sname = 'state-%s' % state.key
add_config_section(config,
sname,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
# add plugin
add_config_section(config, 'plugins', **{'gwvet.tabs': ''})
# define metrics
if category == 1:
metrics = ['Deadtime']
else:
metrics = args.metric
# define summary tab
if category == 1:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{'veto-name': title})
else:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{
'veto-name': title,
'event-channel': args.event_channel,
'event-generator':
args.event_generator,
})
if len(categories) == 1:
config.set(tab, 'index',
'%(gps-start-time)s-%(gps-end-time)s/index.html')
for key, value in plots:
if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605
config.set(tab, key, '%%(union)s,%s' % value)
if '%s-labels' % key not in plots:
config.set(tab, '%s-labels' % key, 'Union,%(flags)s')
else:
config.set(tab, key, value)
# now a tab for each flag
for flag in flags[category]:
if category == 1:
tab = configure_veto_tab(config, flag, title, state, [flag],
segfile, metrics)
else:
tab = configure_veto_tab(
config, flag, title, state, [flag], segfile, metrics, **{
'event-channel': args.event_channel,
'event-generator': args.event_generator
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
if len(categories) > 1 and category != categories[-1]:
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
# configure summary job
if len(categories) > 1:
state = states[categories[0]]
add_config_section(config,
'state-%s' % state.key,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
flags = [f for c in categories for f in flags[c].keys()]
tab = configure_veto_tab(
config,
'Impact of full veto definer file',
None,
state,
flags,
segfile,
args.metric,
shortname='Summary',
index='%(gps-start-time)s-%(gps-end-time)s/index.html',
**{
'event-channel': args.event_channel,
'event-generator': args.event_generator,
'veto-name': 'All vetoes'
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("Generated configuration files for each category")
# -- condor preparation -------------------------
os.chdir(os.pardir)
# get condor variables
if getuser() == 'detchar':
accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy'
else:
accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy'
gwsumm_args = [
'--gps-start-time',
str(start.seconds),
'--gps-end-time',
str(end.seconds),
'--ifo',
ifo,
'--file-tag',
'gwpy-vet',
'--condor-command',
'accounting_group=%s' % accgroup,
'--condor-command',
'accounting_group_user=%s' % getuser(),
'--on-segdb-error',
args.on_segdb_error,
'--output-dir',
args.output_directory,
]
for cf in args.global_config:
gwsumm_args.extend(('--global-config', cf))
for cf in configs:
gwsumm_args.extend(('--config-file', cf))
if args.verbose:
gwsumm_args.append('--verbose')
if args.verbose:
LOGGER.debug('Generating summary DAG via:\n')
LOGGER.debug(' '.join([batch.PROG] + gwsumm_args))
# execute gwsumm in batch mode
batch.main(args=gwsumm_args)
def main(args=None):
"""Run the primary scattering command-line tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# set up logger
logger = cli.logger(
name=PROG.split('python -m ').pop(),
level='DEBUG' if args.verbose else 'INFO',
)
# useful variables
fthresh = (
int(args.frequency_threshold) if args.frequency_threshold.is_integer()
else args.frequency_threshold)
multiplier = args.multiplier_for_threshold
tstr = str(fthresh).replace('.', '_')
gpsstr = '%s-%s' % (int(args.gpsstart), int(args.gpsend - args.gpsstart))
args.optic = args.optic or list(OPTIC_MOTION_CHANNELS.keys())
# go to working directory
indir = os.getcwd()
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
os.chdir(args.output_dir)
# set up output files
summfile = '{}-SCATTERING_SUMMARY-{}.csv'.format(
args.ifo, gpsstr)
segfile = '{}-SCATTERING_SEGMENTS_{}_HZ-{}.h5'.format(
args.ifo, tstr, gpsstr)
# log start of process
logger.info('{} Scattering {}-{}'.format(
args.ifo, int(args.gpsstart), int(args.gpsend)))
# -- get state segments -----------
span = Segment(args.gpsstart, args.gpsend)
# get segments
if args.state_flag is not None:
state = DataQualityFlag.query(
args.state_flag, int(args.gpsstart), int(args.gpsend),
url=DEFAULT_SEGMENT_SERVER,
).coalesce()
statea = []
padding = args.segment_start_pad + args.segment_end_pad
for i, seg in enumerate(state.active):
if abs(seg) > padding:
statea.append(Segment(
seg[0] + args.segment_start_pad,
seg[1] - args.segment_end_pad,
))
else:
logger.debug(
"Segment length {} shorter than padding length {}, "
"skipping segment {}-{}".format(abs(seg), padding, *seg),
)
statea = SegmentList(statea)
logger.debug("Downloaded %d segments for %s"
% (len(statea), args.state_flag))
else:
statea = SegmentList([span])
livetime = float(abs(statea))
logger.debug("Processing %.2f s of livetime" % livetime)
# -- load h(t) --------------------
args.main_channel = args.main_channel.format(IFO=args.ifo)
logger.debug("Loading Omicron triggers for %s" % args.main_channel)
if args.gpsstart >= 1230336018: # Jan 1 2019
ext = "h5"
names = ["time", "frequency", "snr"]
read_kw = {
"columns": names,
"selection": [
"{0} < frequency < {1}".format(
args.fmin, multiplier * fthresh),
("time", in_segmentlist, statea),
],
"format": "hdf5",
"path": "triggers",
}
else:
ext = "xml.gz"
names = ['peak', 'peak_frequency', 'snr']
read_kw = {
"columns": names,
"selection": [
"{0} < peak_frequency < {1}".format(
args.fmin, multiplier * fthresh),
('peak', in_segmentlist, statea),
],
"format": 'ligolw',
"tablename": "sngl_burst",
}
fullcache = []
for seg in statea:
cache = gwtrigfind.find_trigger_files(
args.main_channel, 'omicron', seg[0], seg[1], ext=ext,
)
if len(cache) == 0:
warnings.warn(
"No Omicron triggers found for %s in segment [%d .. %d)"
% (args.main_channel, seg[0], seg[1]),
)
continue
fullcache.extend(cache)
# read triggers
if fullcache:
trigs = EventTable.read(fullcache, nproc=args.nproc, **read_kw)
else: # no files (no livetime?)
trigs = EventTable(names=names)
highsnrtrigs = trigs[trigs['snr'] >= 8]
logger.debug("%d read" % len(trigs))
# -- prepare HTML -----------------
links = [
'%d-%d' % (int(args.gpsstart), int(args.gpsend)),
('Parameters', '#parameters'),
('Segments', (
('State flag', '#state-flag'),
('Optical sensors', '#osems'),
('Transmons', '#transmons'),
)),
]
if args.omega_scans:
links.append(('Scans', '#omega-scans'))
(brand, class_) = htmlio.get_brand(args.ifo, 'Scattering', args.gpsstart)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(
title='%s Scattering | %d-%d' % (
args.ifo, int(args.gpsstart), int(args.gpsend)),
navbar=navbar)
page.div(class_='pb-2 mt-3 mb-2 border-bottom')
page.h1('%s Scattering: %d-%d'
% (args.ifo, int(args.gpsstart), int(args.gpsend)))
page.div.close() # pb-2 mt-3 mb-2 border-bottom
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(
start=int(args.gpsstart), end=int(args.gpsend), flag=args.state_flag))
page.div.close() # col-md-9 col-sm-12
# link to summary files
page.div(class_='col-md-3 col-sm-12')
page.add(htmlio.download_btn(
[('Segments (HDF)', segfile),
('Triggers (CSV)', summfile)],
btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(),
))
page.div.close() # col-md-3 col-sm-12
page.div.close() # row
# command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
# section header
page.h2('Segments', class_='mt-4', id_='segments')
if statea: # contextual information
paper = markup.oneliner.a(
'Accadia et al. (2010)', target='_blank', class_='alert-link',
href='http://iopscience.iop.org/article/10.1088/0264-9381/27'
'/19/194011')
msg = (
"Segments marked \"optical sensors\" below show evidence of beam "
"scattering between {0} and {1} Hz based on the velocity of optic "
"motion, with fringe frequencies projected using equation (3) of "
"{2}. Segments marked \"transmons\" are based on whitened, "
"band-limited RMS trends of transmon sensors. In both cases, "
"yellow panels denote weak evidence for scattering, while red "
"panels denote strong evidence."
).format(args.fmin, multiplier * fthresh, str(paper))
page.add(htmlio.alert(msg, context=args.ifo.lower()))
else: # null segments
page.add(htmlio.alert('No active analysis segments were found',
context='warning', dismiss=False))
# record state segments
if args.state_flag is not None:
page.h3('State flag', class_='mt-3', id_='state-flag')
page.div(id_='accordion1')
page.add(htmlio.write_flag_html(
state, span, 'state', parent='accordion1', context='success',
plotdir='', facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen',
known={'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4}))
page.div.close()
# -- find scattering evidence -----
# read data for OSEMs and transmons
osems = ['%s:%s' % (args.ifo, c) for optic in args.optic for
c in OPTIC_MOTION_CHANNELS[optic]]
transmons = ['%s:%s' % (args.ifo, c) for c in TRANSMON_CHANNELS]
allchannels = osems + transmons
logger.info("Reading all timeseries data")
alldata = []
n = len(statea)
for i, seg in enumerate(statea):
msg = "{0}/{1} {2}:".rjust(30).format(
str(i + 1).rjust(len(str(n))),
n,
str(seg),
) if args.verbose else False
alldata.append(
get_data(allchannels, seg[0], seg[1],
frametype=args.frametype.format(IFO=args.ifo),
verbose=msg, nproc=args.nproc).resample(128))
try: # ensure that only available channels are analyzed
osems = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(osems))
transmons = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(transmons))
except IndexError:
osems = []
transmons = []
# initialize scattering segments
scatter_segments = DataQualityDict()
actives = SegmentList()
# scattering based on OSEM velocity
if statea:
page.h3('Optical sensors (OSEMs)', class_='mt-3', id_='osems')
page.div(id_='osems-group')
logger.info('Searching for scatter based on OSEM velocity')
for i, channel in enumerate(sorted(osems)):
logger.info("-- Processing %s --" % channel)
chanstr = re.sub('[:-]', '_', channel).replace('_', '-', 1)
optic = channel.split('-')[1].split('_')[0]
flag = '%s:DCH-%s_SCATTERING_GE_%s_HZ:1' % (args.ifo, optic, tstr)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering above {0} Hz from {1} in "
"{2}".format(fthresh, optic, channel),
)
# set up plot(s)
plot = Plot(figsize=[12, 12])
axes = {}
axes['position'] = plot.add_subplot(
411, xscale='auto-gps', xlabel='')
axes['fringef'] = plot.add_subplot(
412, sharex=axes['position'], xlabel='')
axes['triggers'] = plot.add_subplot(
413, sharex=axes['position'], xlabel='')
axes['segments'] = plot.add_subplot(
414, projection='segments', sharex=axes['position'])
plot.subplots_adjust(bottom=.07, top=.95)
fringecolors = [None] * len(FREQUENCY_MULTIPLIERS)
histdata = dict((x, numpy.ndarray((0,))) for
x in FREQUENCY_MULTIPLIERS)
linecolor = None
# loop over state segments and find scattering fringes
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
ts = alldata[j][channel]
# get raw data and plot
line = axes['position'].plot(ts, color=linecolor)[0]
linecolor = line.get_color()
# get fringe frequency and plot
fringef = get_fringe_frequency(ts, multiplier=1)
for k, m in list(enumerate(FREQUENCY_MULTIPLIERS))[::-1]:
fm = fringef * m
line = axes['fringef'].plot(
fm, color=fringecolors[k],
label=(j == 0 and r'$f\times%d$' % m or None))[0]
fringecolors[k] = line.get_color()
histdata[m] = numpy.resize(
histdata[m], (histdata[m].size + fm.size,))
histdata[m][-fm.size:] = fm.value
# get segments and plot
scatter = get_segments(
fringef * multiplier,
fthresh,
name=flag,
pad=args.segment_padding
)
axes['segments'].plot(
scatter, facecolor='red', edgecolor='darkred',
known={'alpha': 0.6, 'facecolor': 'lightgray',
'edgecolor': 'gray', 'height': 0.4},
height=0.8, y=0, label=' ',
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# finalize plot
logger.debug("Plotting")
name = texify(channel)
axes['position'].set_title("Scattering evidence in %s" % name)
axes['position'].set_xlabel('')
axes['position'].set_ylabel(r'Position [$\mu$m]')
axes['position'].text(
0.01, 0.95, 'Optic position',
transform=axes['position'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['fringef'].plot(
span, [fthresh, fthresh], 'k--')
axes['fringef'].set_xlabel('')
axes['fringef'].set_ylabel(r'Frequency [Hz]')
axes['fringef'].yaxis.tick_right()
axes['fringef'].yaxis.set_label_position("right")
axes['fringef'].set_ylim(0, multiplier * fthresh)
axes['fringef'].text(
0.01, 0.95, 'Calculated fringe frequency',
transform=axes['fringef'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
handles, labels = axes['fringef'].get_legend_handles_labels()
axes['fringef'].legend(handles[::-1], labels[::-1], loc='upper right',
borderaxespad=0, bbox_to_anchor=(-0.01, 1.),
handlelength=1)
axes['triggers'].scatter(
trigs[names[0]],
trigs[names[1]],
c=trigs[names[2]],
edgecolor='none',
)
name = texify(args.main_channel)
axes['triggers'].text(
0.01, 0.95,
'%s event triggers (Omicron)' % name,
transform=axes['triggers'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['triggers'].set_ylabel('Frequency [Hz]')
axes['triggers'].set_ylim(args.fmin, multiplier * fthresh)
axes['triggers'].colorbar(cmap='YlGnBu', clim=(3, 100), norm='log',
label='Signal-to-noise ratio')
axes['segments'].set_ylim(-.55, .55)
axes['segments'].text(
0.01, 0.95,
r'Time segments with $f\times%d > %.2f$ Hz' % (
multiplier, fthresh),
transform=axes['segments'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
for ax in axes.values():
ax.set_epoch(int(args.gpsstart))
ax.set_xlim(*span)
png = '%s_SCATTERING_%s_HZ-%s.png' % (chanstr, tstr, gpsstr)
try:
plot.save(png)
except OverflowError as e:
warnings.warn(str(e))
plot.axes[1].set_ylim(0, multiplier * fthresh)
plot.refresh()
plot.save(png)
plot.close()
logger.debug("%s written." % png)
# make histogram
histogram = Plot(figsize=[12, 6])
ax = histogram.gca()
hrange = (0, multiplier * fthresh)
for m, color in list(zip(histdata, fringecolors))[::-1]:
if histdata[m].size:
ax.hist(
histdata[m], facecolor=color, alpha=.6, range=hrange,
bins=50, histtype='stepfilled', label=r'$f\times%d$' % m,
cumulative=-1, weights=ts.dx.value, bottom=1e-100,
log=True)
else:
ax.plot(histdata[m], color=color, label=r'$f\times%d$' % m)
ax.set_yscale('log')
ax.set_ylim(.01, float(livetime))
ax.set_ylabel('Time with fringe above frequency [s]')
ax.set_xlim(*hrange)
ax.set_xlabel('Frequency [Hz]')
ax.set_title(axes['position'].get_title())
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles[::-1], labels[::-1], loc='upper right')
hpng = '%s_SCATTERING_HISTOGRAM-%s.png' % (chanstr, gpsstr)
histogram.save(hpng)
histogram.close()
logger.debug("%s written." % hpng)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif ((deadtime != 0 and effdt < 2) or
(histdata[multiplier].size and
histdata[multiplier].max() >=
fthresh/2.)):
context = 'warning'
else:
continue
page.div(class_='card border-%s mb-1 shadow-sm' % context)
page.div(class_='card-header text-white bg-%s' % context)
page.a(channel, class_='collapsed card-link cis-link',
href='#osem%s' % i, **{'data-toggle': 'collapse'})
page.div.close() # card-header
page.div(id_='osem%s' % i, class_='collapse',
**{'data-parent': '#osems-group'})
page.div(class_='card-body')
page.div(class_='row')
img = htmlio.FancyPlot(
png, caption=SCATTER_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(img))
page.div.close() # col-md-10 offset-md-1
himg = htmlio.FancyPlot(
hpng, caption=HIST_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(himg))
page.div.close() # col-md-10 offset-md-1
page.div.close() # row
segs = StringIO()
if deadtime:
page.p("%d segments were found predicting a scattering fringe "
"above %.2f Hz." % (
len(scatter_segments[channel].active),
fthresh))
page.table(class_='table table-sm table-hover')
page.tbody()
page.tr()
page.th('Deadtime')
page.td('%.2f/%d seconds' % (deadtime, livetime))
page.td('%.2f%%' % deadtimepc)
page.tr.close()
page.tr()
page.th('Efficiency<br><small>(SNR≥8 and '
'%.2f Hz</sub><f<sub>peak</sub><%.2f Hz)</small>'
% (args.fmin, multiplier * fthresh))
page.td('%d/%d events' % (efficiency, len(highsnrtrigs)))
page.td('%.2f%%' % efficiencypc)
page.tr.close()
page.tr()
page.th('Efficiency/Deadtime')
page.td()
page.td('%.2f' % effdt)
page.tr.close()
page.tbody.close()
page.table.close()
scatter_segments[channel].active.write(segs, format='segwizard',
coltype=float)
page.pre(segs.getvalue())
else:
page.p("No segments were found with scattering above %.2f Hz."
% fthresh)
page.div.close() # card-body
page.div.close() # collapse
page.div.close() # card
if statea: # close accordion
page.div.close() # osems-group
# scattering based on transmon BLRMS
if statea:
page.h3('Transmons', class_='mt-3', id_='transmons')
page.div(id_='transmons-group')
logger.info('Searching for scatter based on band-limited RMS of transmons')
for i, channel in enumerate(sorted(transmons)):
logger.info("-- Processing %s --" % channel)
optic = channel.split('-')[1][:6]
flag = '%s:DCH-%s_SCATTERING_BLRMS:1' % (args.ifo, optic)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering from whitened, band-limited "
"RMS trends of {0}".format(channel),
)
# loop over state segments and compute BLRMS
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
wblrms = get_blrms(
alldata[j][channel],
flow=args.bandpass_flow,
fhigh=args.bandpass_fhigh,
)
scatter = get_segments(
wblrms,
numpy.mean(wblrms) + args.sigma * numpy.std(wblrms),
name=flag,
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
highsnrtrigs = trigs[trigs['snr'] <= 200]
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif deadtime != 0 and effdt < 2:
context = 'warning'
else:
continue
page.add(htmlio.write_flag_html(
scatter_segments[channel], span, i, parent='transmons-group',
title=channel, context=context, plotdir=''))
if statea: # close accordion
page.div.close() # transmons-group
actives = actives.coalesce() # merge contiguous segments
if statea and not actives:
page.add(htmlio.alert(
'No evidence of scattering found in the channels analyzed',
context=args.ifo.lower(), dismiss=False))
# identify triggers during active segments
logger.debug('Writing a summary CSV record')
ind = [i for i, trigtime in enumerate(highsnrtrigs[names[0]])
if trigtime in actives]
gps = highsnrtrigs[names[0]][ind]
freq = highsnrtrigs[names[1]][ind]
snr = highsnrtrigs[names[2]][ind]
segs = [y for x in gps for y in actives if x in y]
table = EventTable(
[gps, freq, snr, [seg[0] for seg in segs], [seg[1] for seg in segs]],
names=('trigger_time', 'trigger_frequency', 'trigger_snr',
'segment_start', 'segment_end'))
logger.info('The following {} triggers fell within active scattering '
'segments:\n\n'.format(len(table)))
print(table)
print('\n\n')
table.write(summfile, overwrite=True)
# -- launch omega scans -----------
nscans = min(args.omega_scans, len(table))
if nscans > 0:
# launch scans
scandir = 'scans'
ind = random.sample(range(0, len(table)), nscans)
omegatimes = [str(t) for t in table['trigger_time'][ind]]
logger.debug('Collected {} event times to omega scan: {}'.format(
nscans, ', '.join(omegatimes)))
logger.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(
ifo=args.ifo, ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(timeout=4, gps=args.gpsstart)
batch.generate_dag(omegatimes, flags=flags, submit=True,
outdir=scandir, condor_commands=condorcmds)
logger.info('Launched {} omega scans to condor'.format(nscans))
# render HTML
page.h2('Omega scans', class_='mt-4', id_='omega-scans')
msg = (
'The following event times correspond to significant Omicron '
'triggers that occur during the scattering segments found above. '
'To compare these against fringe frequency projections, please '
'use the "simple scattering" module:',
markup.oneliner.pre(
'$ python -m gwdetchar.scattering.simple --help',
),
)
page.add(htmlio.alert(msg, context=args.ifo.lower()))
page.add(htmlio.scaffold_omega_scans(
omegatimes, args.main_channel, scandir=scandir))
elif args.omega_scans:
logger.info('No events found during active scattering segments')
# -- finalize ---------------------
# write segments
scatter_segments.write(segfile, path="segments", overwrite=True)
logger.debug("%s written" % segfile)
# write HTML
htmlio.close_page(page, 'index.html')
logger.info("-- index.html written, all done --")
# return to original directory
os.chdir(indir)
def get_data(station,start_time,end_time,rep='/GNOMEDrive/gnome/serverdata/',
resample=None,activity=False,unit='V',output='all',segtxt=False,
channel='MagneticFields'):
"""
Glob all files withing user-defined period and extract data.
Parameters
----------
station : str
Name of the station to be analysed
start_time : int
GPS timestamp of the first required magnetic field data
end_time : int
GPS timestamp of the last required magnetic field data
rep : str
Data repository. Default is the GNOME server repository.
resample : int
New sampling rate
activity : bool
Output the activity of data
unit : str
Output unit format (V for voltage, pT for magnetic field)
output : str
Output data to be extracted. If output is equal to 'ts',
only the time series will be given.
Returns
-------
ts_data : pycbc.types.TimeSeries
Time series data for selected time period.
ts_list : dictionary
List of time series.
activity : gwpy.segments.DataQualityDict
List all the segment of data retrieved
t0 : astropy.time.Time
First timestamp
t1 : astropy.time.Time
Last timestamp
"""
if start_time==None or end_time==None:
print "ERROR: No start or end date given..."
quit()
# Define data attribute to be extracted from HDF5 files
setname = channel
dstr = ['%Y','%m','%d','%H','%M','%S','%f']
dsplit = '-'.join(dstr[:start_time.count('-')+1])
start = datetime.strptime(start_time,dsplit)
dsplit = '-'.join(dstr[:end_time.count('-')+1])
end = datetime.strptime(end_time,dsplit)
dataset = []
for date in numpy.arange(start,end,timedelta(minutes=1)):
date = date.astype(datetime)
path1 = rep+station+'/'+date.strftime("%Y/%m/%d/")
path2 = station+'_'+date.strftime("%Y%m%d_%H%M*.hdf5")
fullpath = os.path.join(path1,path2)
dataset += glob.glob(fullpath)
if len(dataset)==0:
print "ERROR: No data files were found..."
quit()
file_order,data_order = {},{}
for fname in dataset:
hfile = h5py.File(fname, "r")
# Extract all atributes from the data
attrs = hfile[setname].attrs
# Define each attribute
dstr, t0, t1 = attrs["Date"], attrs["t0"], attrs["t1"]
# Construct GPS starting time from data
start_utc = construct_utc_from_metadata(dstr, t0)
# Construct GPS ending time from data
end_utc = construct_utc_from_metadata(dstr, t1)
# Represent the range of times in the semi-open interval
segfile = segment(start_utc,end_utc)
file_order[segfile] = fname
data_order[segfile] = hfile
# Create list of time series from every segment
ts_list = TimeSeriesList()
for seg in sorted(file_order):
hfile = h5py.File(file_order[seg], "r")
dset = hfile[setname]
sample_rate = dset.attrs["SamplingRate(Hz)"]
gps_epoch = construct_utc_from_metadata(dset.attrs["Date"], dset.attrs["t0"])
data = dset[:]
if unit=='pT':
data = eval(dset.attrs['MagFieldEq'].replace('MagneticFields','data').replace('[pT]',''))
ts_data = TimeSeries(data, sample_rate=sample_rate, epoch=gps_epoch)
ts_list.append(ts_data)
hfile.close()
# Generate an ASCII representation of the GPS timestamped segments of time covered by the input data
seglist = segmentlist(data_order.keys())
# Sort the segment list
seglist.sort()
# Initialise dictionary for segment information
activity = DataQualityDict()
if segtxt:
# Save time span for each segment in ASCII file
with open("segments.txt", "w") as fout:
for seg in seglist:
print >>fout, "%10.9f %10.9f" % seg
# FIXME: Active should be masked from the sanity channel
activity[station] = DataQualityFlag(station,active=seglist.coalesce(),known=seglist.coalesce())
# Generate an ASCII representation of the GPS timestamped segments of time covered by the input data
seglist = segmentlist(data_order.keys())
# Sort the segment list
seglist.sort()
# Retrieve channel data for all the segments
if unit=='V':
full_data = numpy.hstack([data_order[seg][setname][:] for seg in seglist])
if unit=='pT':
full_data = []
for seg in seglist:
dset = data_order[seg][setname]
data = dset[:]
data = eval(dset.attrs['MagFieldEq'].replace('MagneticFields','data').replace('[pT]',''))
full_data = numpy.hstack((full_data,data))
for v in data_order.values():
v.close()
new_sample_rate = float(sample_rate) if resample==None else float(resample)
new_data_length = len(full_data)*new_sample_rate/float(sample_rate)
full_data = scipy.signal.resample(full_data,int(new_data_length))
# Models a time series consisting of uniformly sampled scalar values
ts_data = types.TimeSeries(full_data,delta_t=1./new_sample_rate,epoch=seglist[0][0])
if output=='ts':
return ts_data
t0,t1 = time_convert(start_time,end_time)
return ts_data,ts_list,activity,t0,t1
def main(args=None):
"""Run the online Guardian node visualization tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
fec_map = args.fec_map
simulink = args.simulink
daqsvn = args.daqsvn or ('https://daqsvn.ligo-la.caltech.edu/websvn/'
'listing.php?repname=daq_maps')
if args.ifo == 'H1':
if not fec_map:
fec_map = 'https://lhocds.ligo-wa.caltech.edu/exports/detchar/fec/'
if not simulink:
simulink = 'https://lhocds.ligo-wa.caltech.edu/daq/simulink/'
if args.ifo == 'L1':
if not fec_map:
fec_map = 'https://llocds.ligo-la.caltech.edu/exports/detchar/fec/'
if not simulink:
simulink = 'https://llocds.ligo-la.caltech.edu/daq/simulink/'
span = Segment(args.gpsstart, args.gpsend)
# let's go
LOGGER.info('{} Overflows {}-{}'.format(args.ifo, int(args.gpsstart),
int(args.gpsend)))
# get segments
if args.state_flag:
state = DataQualityFlag.query(args.state_flag,
int(args.gpsstart),
int(args.gpsend),
url=const.DEFAULT_SEGMENT_SERVER)
tmp = type(state.active)()
for i, seg in enumerate(state.active):
if abs(seg) < args.segment_end_pad:
continue
tmp.append(type(seg)(seg[0], seg[1] - args.segment_end_pad))
state.active = tmp.coalesce()
statea = state.active
else:
statea = SegmentList([span])
if not args.output_file:
duration = abs(span)
args.output_file = ('%s-OVERFLOWS-%d-%d.h5' %
(args.ifo, int(args.gpsstart), duration))
LOGGER.debug("Set default output file as %s" % args.output_file)
# set up container
overflows = DataQualityDict()
# prepare data access
if args.nds:
from gwpy.io import nds2 as io_nds2
host, port = args.nds.rsplit(':', 1)
ndsconnection = io_nds2.connect(host, port=int(port))
if ndsconnection.get_protocol() == 1:
cachesegs = SegmentList(
[Segment(int(args.gpsstart), int(args.gpsend))])
else:
cachesegs = io_nds2.get_availability(
['{0}:FEC-1_DAC_OVERFLOW_ACC_0_0'.format(args.ifo)],
int(args.gpsstart),
int(args.gpsend),
)
else: # get frame cache
cache = gwdatafind.find_urls(args.ifo[0], args.frametype,
int(args.gpsstart), int(args.gpsend))
cachesegs = statea & cache_segments(cache)
flag_desc = "ADC/DAC Overflow indicated by {0}"
# get channel and find overflows
for dcuid in args.dcuid:
LOGGER.info("Processing DCUID %d" % dcuid)
channel = daq.ligo_accum_overflow_channel(dcuid, args.ifo)
overflows[channel] = DataQualityFlag(channel, known=cachesegs)
if args.deep:
LOGGER.debug(" -- Getting list of overflow channels")
try:
channels = daq.ligo_model_overflow_channels(dcuid,
args.ifo,
args.frametype,
gpstime=span[0],
nds=args.nds)
except IndexError: # no frame found for GPS start, try GPS end
channels = daq.ligo_model_overflow_channels(dcuid,
args.ifo,
args.frametype,
gpstime=span[-1])
for chan in channels: # set up flags early
overflows[chan] = DataQualityFlag(
chan,
known=cachesegs,
description=flag_desc.format(chan),
isgood=False,
)
LOGGER.debug(" -- %d channels found" % len(channel))
for seg in cachesegs:
LOGGER.debug(" -- Processing {}-{}".format(*seg))
if args.nds:
read_kw = dict(connection=ndsconnection)
else:
read_kw = dict(source=cache, nproc=args.nproc)
msg = "Reading ACCUM_OVERFLOW data:".rjust(30)
data = get_data(channel,
seg[0],
seg[1],
pad=0.,
verbose=msg,
**read_kw)
new = daq.find_overflow_segments(
data,
cumulative=True,
)
overflows[channel] += new
LOGGER.info(" -- {} overflows found".format(len(new.active)))
if not new.active:
continue
# go deep!
for s, e in tqdm.tqdm(new.active.protract(2),
unit='ovfl',
desc='Going deep'.rjust(30)):
data = get_data(channels, s, e, **read_kw)
for ch in channels:
try:
overflows[ch] += daq.find_overflow_segments(
data[ch],
cumulative=True,
)
except KeyError:
warnings.warn("Skipping {}".format(ch), UserWarning)
continue
LOGGER.debug(" -- Search complete")
# write output
LOGGER.info("Writing segments to %s" % args.output_file)
table = table_from_segments(
overflows,
sngl_burst=args.output_file.endswith((".xml", ".xml.gz")),
)
if args.integer_segments:
for key in overflows:
overflows[key] = overflows[key].round()
if args.output_file.endswith((".h5", "hdf", ".hdf5")):
with h5py.File(args.output_file, "w") as h5f:
table.write(h5f, path="triggers")
overflows.write(h5f, path="segments")
else:
table.write(args.output_file, overwrite=True)
overflows.write(args.output_file, overwrite=True, append=True)
# write HTML
if args.html:
# get base path
base = os.path.dirname(args.html)
os.chdir(base)
if args.plot:
args.plot = os.path.curdir
if args.output_file:
args.output_file = os.path.relpath(args.output_file,
os.path.dirname(args.html))
if os.path.basename(args.html) == 'index.html':
links = [
'%d-%d' % (int(args.gpsstart), int(args.gpsend)),
('Parameters', '#parameters'),
('Segments', [('Overflows', '#overflows')]),
('Results', '#results'),
]
if args.state_flag:
links[2][1].insert(0, ('State flag', '#state-flag'))
(brand, class_) = htmlio.get_brand(args.ifo, 'Overflows',
args.gpsstart)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(
title='%s Overflows | %d-%d' %
(args.ifo, int(args.gpsstart), int(args.gpsend)),
navbar=navbar)
else:
page = htmlio.markup.page()
page.div(class_='container')
# -- header
page.div(class_='pb-2 mt-3 mb-2 border-bottom')
page.h1('%s ADC/DAC Overflows: %d-%d' %
(args.ifo, int(args.gpsstart), int(args.gpsend)))
page.div.close()
# -- paramters
content = [('DCUIDs', ' '.join(map(str, args.dcuid)))]
if daqsvn:
content.append(('FEC configuration', (
'<a href="{0}" target="_blank" title="{1} FEC configuration">'
'{0}</a>').format(daqsvn, args.ifo)))
if fec_map:
content.append(
('FEC map', '<a href="{0}" target="_blank" title="{1} FEC '
'map">{0}</a>'.format(fec_map, args.ifo)))
if simulink:
content.append(
('Simulink models', '<a href="{0}" target="_blank" title="{1} '
'Simulink models">{0}</a>'.format(simulink, args.ifo)))
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(
htmlio.parameter_table(content,
start=args.gpsstart,
end=args.gpsend,
flag=args.state_flag))
page.div.close() # col-md-9 col-sm-12
# link to summary file
if args.output_file:
ext = ('HDF' if args.output_file.endswith(
(".h5", "hdf", ".hdf5")) else 'XML')
page.div(class_='col-md-3 col-sm-12')
page.add(
htmlio.download_btn(
[('Segments ({})'.format(ext), args.output_file)],
btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(),
))
page.div.close() # col-md-3 col-sm-12
page.div.close() # row
# -- command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
# -- segments
page.h2('Segments', class_='mt-4', id_='segments')
# give contextual information
msg = ("This analysis searched for digital-to-analogue (DAC) or "
"analogue-to-digital (ADC) conversion overflows in the {0} "
"real-time controls system. ").format(
SITE_MAP.get(args.ifo, 'LIGO'))
if args.deep:
msg += (
"A hierarchichal search was performed, with one cumulative "
"overflow counter checked per front-end controller (FEC). "
"For those models that indicated an overflow, the card- and "
"slot-specific channels were then checked. ")
msg += (
"Consant overflow is shown as yellow, while transient overflow "
"is shown as red. If a data-quality flag was loaded for this "
"analysis, it will be displayed in green.")
page.add(htmlio.alert(msg, context=args.ifo.lower()))
# record state segments
if args.state_flag:
page.h3('State flag', class_='mt-3', id_='state-flag')
page.div(id_='accordion1')
page.add(
htmlio.write_flag_html(state,
span,
'state',
parent='accordion1',
context='success',
plotdir=args.plot,
facecolor=(0.2, 0.8, 0.2),
edgecolor='darkgreen',
known={
'facecolor': 'red',
'edgecolor': 'darkred',
'height': 0.4,
}))
page.div.close()
# record overflow segments
if sum(abs(s.active) for s in overflows.values()):
page.h3('Overflows', class_='mt-3', id_='overflows')
page.div(id_='accordion2')
for i, (c, flag) in enumerate(list(overflows.items())):
if abs(flag.active) == 0:
continue
if abs(flag.active) == abs(cachesegs):
context = 'warning'
else:
context = 'danger'
try:
channel = cds.get_real_channel(flag.name)
except Exception:
title = '%s [%d]' % (flag.name, len(flag.active))
else:
title = '%s (%s) [%d]' % (flag.name, channel,
len(flag.active))
page.add(
htmlio.write_flag_html(flag,
span,
i,
parent='accordion2',
title=title,
context=context,
plotdir=args.plot))
page.div.close()
else:
page.add(
htmlio.alert('No overflows were found in this analysis',
context=args.ifo.lower(),
dismiss=False))
# -- results table
page.h2('Results summary', class_='mt-4', id_='results')
page.table(class_='table table-striped table-hover')
# write table header
page.thead()
page.tr()
for header in ['Channel', 'Connected signal', 'Num. overflows']:
page.th(header)
page.thead.close()
# write body
page.tbody()
for c, seglist in overflows.items():
t = abs(seglist.active)
if t == 0:
page.tr()
elif t == abs(cachesegs):
page.tr(class_='table-warning')
else:
page.tr(class_='table-danger')
page.td(c)
try:
page.td(cds.get_real_channel(str(c)))
except Exception:
page.td()
page.td(len(seglist.active))
page.tr.close()
page.tbody.close()
page.table.close()
# -- close and write
htmlio.close_page(page, args.html)
LOGGER.info("HTML written to %s" % args.html)
def process(self,
nds='guess',
multiprocess=True,
config=GWSummConfigParser(),
datacache=None,
segmentcache=Cache(),
datafind_error='raise',
**kwargs):
"""Process data for the given state.
"""
ifo = self.ifo
for p in self.plots:
if p.outputfile in globalv.WRITTEN_PLOTS:
p.new = False
# --------------------------------------------------------------------
# work out which channels are needed
prefix = '%s:GRD-%s_%%s' % (self.ifo, self.node)
state = sorted(self.states, key=lambda s: abs(s.active))[0]
try:
version = get_timeseries(
prefix % 'VERSION',
state,
config=config,
nds=nds,
multiprocess=multiprocess,
cache=datacache,
datafind_error=datafind_error,
).join(gap='ignore').min().value
except ValueError:
version = 1201
prefices = ['STATE_N', 'REQUEST_N', 'NOMINAL_N', 'OK', 'MODE']
if version >= 1200:
prefices.append('OP')
alldata = get_timeseries_dict([prefix % x for x in prefices],
state,
config=config,
nds=nds,
multiprocess=multiprocess,
cache=datacache,
datafind_error=datafind_error,
dtype='int16').values()
vprint(" All time-series data loaded\n")
# --------------------------------------------------------------------
# find segments and transitions
self.transitions = dict((v, []) for v in self.grdstates)
for sdata, rdata, ndata, okdata in zip(*alldata[:4]):
ssegs = DataQualityDict()
rsegs = DataQualityDict()
nsegs = DataQualityDict()
oksegs = (okdata == 1).to_dqflag(name='Node OK')
for v, name in self.grdstates.iteritems():
# get segments for state
tag = self.segmenttag % name
instate = sdata == v
ssegs[tag] = instate.to_dqflag(name=name)
transin = (numpy.diff(instate.astype(int))
== 1).nonzero()[0] + 1
transout = (numpy.diff(instate.astype(int))
== -1).nonzero()[0] + 1
for i, j in zip(transin, transout):
t = sdata.times[i].value
from_ = sdata[i - 1].value
to_ = sdata[j].value
self.transitions[v].append((t, from_, to_))
# get segments for request
tag = self.segmenttag % name + REQUESTSTUB
instate = rdata == v
rsegs[tag] = instate.to_dqflag(name=name)
# get segments for nominal
tag = self.segmenttag % name + NOMINALSTUB
nom = ndata == v
nsegs[tag] = nom.to_dqflag(name=name)
globalv.SEGMENTS += ssegs
globalv.SEGMENTS += rsegs
globalv.SEGMENTS += nsegs
globalv.SEGMENTS += {self.segmenttag % 'OK': oksegs}
super(GuardianTab, self).process(config=config,
nds=nds,
multiprocess=multiprocess,
datacache=datacache,
segmentcache=segmentcache,
**kwargs)
from gwpy.segments import (Segment, SegmentList, DataQualityFlag,
DataQualityDict)
from .. import segments
TEST_SEGMENTS = SegmentList([
Segment(0.1, 1.234567),
Segment(5.64321, 6.234567890),
])
TEST_SEGMENTS_2 = SegmentList(
[Segment(round(a, 6), round(b, 6)) for a, b in TEST_SEGMENTS])
TEST_FLAG = DataQualityFlag(known=SegmentList([Segment(0, 7)]),
active=TEST_SEGMENTS,
name='X1:TEST-FLAG')
TEST_DICT = DataQualityDict({TEST_FLAG.name: TEST_FLAG})
# -- unit tests ---------------------------------------------------------------
@mock.patch('gwpy.segments.DataQualityFlag.query', return_value=TEST_FLAG)
def test_query(dqflag):
flag = segments.query('X1:TEST-FLAG', 0, 7)
assert flag.known == TEST_FLAG.known
assert flag.active == SegmentList(
[Segment((int(seg[0]), int(seg[1]))) for seg in TEST_FLAG.active])
@pytest.mark.parametrize('ncol', (2, 4))
def test_write_segments_ascii(ncol):
with NamedTemporaryFile(suffix='.txt', delete=False) as tmp:
ACTIVEPADC = SegmentList([
Segment(.5, 4),
Segment(5.5, 8),
])
SEGXML = os.path.join(
os.path.split(__file__)[0], 'data', 'X1-GWPY_TEST_SEGMENTS-0-10.xml.gz')
SEGWIZ = os.path.join(
os.path.split(__file__)[0], 'data', 'X1-GWPY_TEST_SEGMENTS-0-10.txt')
FLAG1 = 'X1:GWPY-TEST_SEGMENTS:1'
FLAG2 = 'X1:GWPY-TEST_SEGMENTS:2'
QUERY_START = 968630415
QUERY_END = 968716815
QUERY_FLAGS = ['H1:DMT-SCIENCE:4', 'L1:DMT-SCIENCE:4']
QUERY_RESULT = DataQualityDict()
QUERY_RESULT['H1:DMT-SCIENCE:4'] = DataQualityFlag('H1:DMT-SCIENCE:4',
known=[(968630415,
968716815)],
active=[
(968632249, 968641010),
(968642871, 968644430),
(968646220, 968681205),
(968686357, 968686575),
(968688760, 968690950),
(968692881, 968714403)
])
QUERY_RESULT['L1:DMT-SCIENCE:4'] = DataQualityFlag('L1:DMT-SCIENCE:4',
known=[(968630415,
968716815)],
active=[
def get_segments(flags,
segments,
cache=None,
url='https://segdb-er.ligo.caltech.edu',
**kwargs):
"""Fetch some segments from the segment database
Parameters
----------
flags : `str`, `list`
one of more flags for which to query
segments : `~gwpy.segments.DataQualityFlag`, `~gwpy.segments.SegmentList`
span over which to query for flag segments
cache : `~glue.lal.Cache`, optional
cache of files to use as data source
url : `str`
URL of segment database, if ``cache`` is not given
**kwargs
other keyword arguments to pass to either
`~gwpy.segments.DataQualityFlag.read` (if ``cache`` is given) or
`~gwpy.segments.DataQualityFlag.query` (otherwise)
Returns
-------
segments : `~gwpy.segments.DataQualityFlag`,
`~gwpy.segments.DataQualityDict`
a single `~gwpy.segments.DataQualityFlag` (if ``flags`` is given
as a `str`), or a `~gwpy.segments.DataQualityDict` (if ``flags``
is given as a `list`)
"""
# format segments
if isinstance(segments, DataQualityFlag):
segments = segments.active
elif isinstance(segments, tuple):
segments = [Segment(to_gps(segments[0]), to_gps(segments[1]))]
segments = SegmentList(segments)
# get format for files
if cache is not None and not isinstance(cache, Cache):
kwargs.setdefault(
'format',
_get_valid_format('read', DataQualityFlag, None, None,
(cache[0], ), {}))
# populate an existing set of flags
if isinstance(flags, (DataQualityFlag, DataQualityDict)):
return flags.populate(source=cache or url, segments=segments, **kwargs)
# query one flag
elif cache is None and isinstance(flags, str):
return DataQualityFlag.query(flags, segments, url=url, **kwargs)
# query lots of flags
elif cache is None:
return DataQualityDict.query(flags, segments, url=url, **kwargs)
# read one flag
elif flags is None or isinstance(flags, str):
segs = DataQualityFlag.read(cache, flags, coalesce=False, **kwargs)
if segs.known:
segs.known &= segments
else:
segs.known = segments
segs.active &= segments
return segs
# read lots of flags
else:
segs = DataQualityDict.read(cache, flags, coalesce=True, **kwargs)
for name, flag in segs.items():
flag.known &= segments
flag.active &= segments
return segs
def process(self,
nds=None,
nproc=1,
config=GWSummConfigParser(),
datacache=None,
segmentcache=Cache(),
datafind_error='raise',
**kwargs):
"""Process data for the given state.
"""
# finalize state information
self.finalize_states(config=config,
segdb_error=kwargs.get('segdb_error', 'raise'),
datafind_error=datafind_error)
vprint("States finalised [%d total]\n" % len(self.states))
vprint(" Default state: %r\n" % str(self.defaultstate))
# remove plots that have already been generated
for p in self.plots:
if p.outputfile in globalv.WRITTEN_PLOTS:
p.new = False
# --------------------------------------------------------------------
# work out which channels are needed
prefix = '%s:GRD-%s_%%s' % (self.ifo, self.node)
state = sorted(self.states, key=lambda s: abs(s.active))[-1]
prefices = ['STATE_N', 'REQUEST_N', 'NOMINAL_N', 'OK', 'MODE', 'OP']
alldata = list(
get_timeseries_dict([prefix % x for x in prefices],
state,
config=config,
nds=nds,
nproc=nproc,
cache=datacache,
datafind_error=datafind_error,
dtype='int32').values())
vprint(" All time-series data loaded\n")
# --------------------------------------------------------------------
# find segments and transitions
self.transitions = dict((v, []) for v in self.grdstates)
for sdata, rdata, ndata, okdata in zip(*alldata[:4]):
ssegs = DataQualityDict()
rsegs = DataQualityDict()
nsegs = DataQualityDict()
oksegs = (okdata == 1).to_dqflag(name='Node OK')
for v, name in self.grdstates.items():
# get segments for state
tag = self.segmenttag % name
instate = sdata == v
ssegs[tag] = instate.to_dqflag(name=name)
diff_ = numpy.diff(instate.value.astype(int))
transin = (diff_ == 1).nonzero()[0] + 1
transout = (diff_ == -1).nonzero()[0] + 1
for i, j in zip(transin, transout):
t = sdata.times[i].value
from_ = sdata[i - 1].value
to_ = sdata[j].value
self.transitions[v].append((t, from_, to_))
# get segments for request
tag = self.segmenttag % name + REQUESTSTUB
instate = rdata == v
rsegs[tag] = instate.to_dqflag(name=name)
# get segments for nominal
tag = self.segmenttag % name + NOMINALSTUB
nom = ndata == v
nsegs[tag] = nom.to_dqflag(name=name)
globalv.SEGMENTS += ssegs
globalv.SEGMENTS += rsegs
globalv.SEGMENTS += nsegs
globalv.SEGMENTS += {self.segmenttag % 'OK': oksegs}
super(GuardianTab, self).process(config=config,
nds=nds,
nproc=nproc,
datacache=datacache,
segmentcache=segmentcache,
**kwargs)
