def test_cache_segments(self):
# check empty input
sl = cache_segments()
self.assertIsInstance(sl, SegmentList)
self.assertEquals(len(sl), 0)
cache, segs = self.make_cache()
try:
# check good cache
sl = cache_segments(cache)
self.assertNotEquals(sl, segs)
self.assertEquals(sl, type(segs)(segs).coalesce())
# check bad cache
os.remove(cache[0].path)
sl = cache_segments(cache)
self.assertEquals(sl, segs[1:])
# check cache with no existing files
sl = cache_segments(cache[:1])
self.assertEquals(sl, SegmentList())
# check errors
self.assertRaises(TypeError, cache_segments, blah='blah')
self.assertRaises(ValueError, cache_segments, cache,
on_missing='error')
self.assertRaises(ValueError, cache_segments, cache,
on_missing='blah')
# clean up
finally:
self.destroy_cache(cache)
def find_best_frames(ifo, frametype, start, end, **kwargs):
"""Find frames for the given type, replacing with a better type if needed
"""
# find cache for this frametype
cache = find_frames(ifo, frametype, start, end, **kwargs)
# check for gaps in current cache
span = SegmentList([Segment(start, end)])
gaps = span - cache_segments(cache)
# if gaps and using aggregated h(t), check short files
if abs(gaps) and frametype in SHORT_HOFT_TYPES:
f2 = SHORT_HOFT_TYPES[frametype]
vprint(" Gaps discovered in aggregated h(t) type "
"%s, checking %s\n" % (frametype, f2))
kwargs['gaps'] = 'ignore'
cache.extend(filter(lambda e: file_segment(e) in gaps,
find_frames(ifo, f2, start, end, **kwargs)))
new = int(abs(gaps - cache_segments(cache)))
if new:
vprint(" %ss extra coverage with frametype %s\n" % (new, f2))
else:
vprint(" No extra coverage with frametype %s\n" % f2)
return cache, frametype
def find_best_frames(ifo, frametype, start, end, **kwargs):
"""Find frames for the given type, replacing with a better type if needed
"""
# find cache for this frametype
cache = find_frames(ifo, frametype, start, end, **kwargs)
# check for gaps in current cache
span = SegmentList([Segment(start, end)])
gaps = span - cache_segments(cache)
# if gaps and using aggregated h(t), check short files
if abs(gaps) and frametype in SHORT_HOFT_TYPES:
f2 = SHORT_HOFT_TYPES[frametype]
vprint(" Gaps discovered in aggregated h(t) type "
"%s, checking %s\n" % (frametype, f2))
kwargs['gaps'] = 'ignore'
cache.extend(
filter(lambda e: file_segment(e) in gaps,
find_frames(ifo, f2, start, end, **kwargs)))
new = int(abs(gaps - cache_segments(cache)))
if new:
vprint(" %ss extra coverage with frametype %s\n" % (new, f2))
else:
vprint(" No extra coverage with frametype %s\n" % f2)
return cache, frametype
def test_cache_segments(self):
# check empty input
sl = cache_segments()
self.assertIsInstance(sl, SegmentList)
self.assertEquals(len(sl), 0)
cache, segs = self.make_cache()
try:
# check good cache
sl = cache_segments(cache)
self.assertNotEquals(sl, segs)
self.assertEquals(sl, type(segs)(segs).coalesce())
# check bad cache
os.remove(cache[0].path)
sl = cache_segments(cache)
self.assertEquals(sl, segs[1:])
# check cache with no existing files
sl = cache_segments(cache[:1])
self.assertEquals(sl, SegmentList())
# check errors
self.assertRaises(TypeError, cache_segments, blah='blah')
self.assertRaises(ValueError,
cache_segments,
cache,
on_missing='error')
self.assertRaises(ValueError,
cache_segments,
cache,
on_missing='blah')
# clean up
finally:
self.destroy_cache(cache)
def test_find_contiguous(self):
"""Test :func:`gwpy.io.cache.find_contiguous`
"""
a, segs = self.make_cache()
segs.coalesce()
for i, cache in enumerate(io_cache.find_contiguous(a)):
io_cache.cache_segments(cache).extent() == segs[i]
assert not list(io_cache.find_contiguous())
def test_cache_segments(self):
# check empty input
sl = cache_segments()
self.assertIsInstance(sl, SegmentList)
self.assertEquals(len(sl), 0)
cache, segs = self.make_cache()
segs.coalesce()
sl = cache_segments(cache)
self.assertEquals(sl, segs)
sl = cache_segments(cache[:2], cache[2:])
self.assertEquals(sl, segs)
def test_cache_segments(self):
# check empty input
sl = cache_segments()
self.assertIsInstance(sl, SegmentList)
self.assertEquals(len(sl), 0)
cache, segs = self.make_cache()
segs.coalesce()
sl = cache_segments(cache)
self.assertEquals(sl, segs)
sl = cache_segments(cache[:2], cache[2:])
self.assertEquals(sl, segs)
def test_cache_segments(self):
"""Test :func:`gwpy.io.cache.cache_segments`
"""
# check empty input
sl = io_cache.cache_segments()
assert isinstance(sl, SegmentList)
assert len(sl) == 0
# check simple cache
cache, segs = self.make_cache()
segs.coalesce()
sl = io_cache.cache_segments(cache)
assert sl == segs
# check multiple caches produces the same result
sl = io_cache.cache_segments(cache[:2], cache[2:])
assert sl == segs
def test_cache_segments(self):
"""Test :func:`gwpy.io.cache.cache_segments`
"""
# check empty input
sl = io_cache.cache_segments()
assert isinstance(sl, SegmentList)
assert len(sl) == 0
# check simple cache
cache, segs = self.make_cache()
segs.coalesce()
sl = io_cache.cache_segments(cache)
assert sl == segs
# check multiple caches produces the same result
sl = io_cache.cache_segments(cache[:2], cache[2:])
assert sl == segs
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs):
"""Read a table of events from a cache
This function is mainly meant for use from the `get_triggers` method
Parameters
----------
cache : :class:`glue.lal.Cache`
the formatted list of files to read
segments : `~gwpy.segments.SegmentList`
the list of segments to read
etg : `str`
the name of the trigger generator that created the files
nproc : `int`, optional
the number of parallel processes to use when reading
**kwargs
other keyword arguments are passed to the `EventTable.read` or
`{tableclass}.read` methods
Returns
-------
table : `~gwpy.table.EventTable`, `None`
a table of events, or `None` if the cache has no overlap with
the segments
"""
if isinstance(cache, Cache):
cache = cache.sieve(segmentlist=segments)
cache = cache.checkfilesexist()[0]
cache.sort(key=lambda x: x.segment[0])
cache = cache.pfnlist() # some readers only like filenames
else:
cache = [urlparse(url).path for url in cache]
if etg == 'pycbc_live': # remove empty HDF5 files
cache = filter_pycbc_live_files(cache, ifo=kwargs['ifo'])
if len(cache) == 0:
return
# read triggers
table = EventTable.read(cache, **kwargs)
# store read keywords in the meta table
if timecolumn:
table.meta['timecolumn'] = timecolumn
# get back from cache entry
if isinstance(cache, CacheEntry):
cache = Cache([cache])
# append new events to existing table
try:
csegs = cache_segments(cache) & segments
except (AttributeError, TypeError, ValueError):
csegs = SegmentList()
table.meta['segments'] = csegs
if timecolumn: # already filtered on-the-fly
return table
# filter now
return keep_in_segments(table, segments, etg)
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs):
"""Read a table of events from a cache
This function is mainly meant for use from the `get_triggers` method
Parameters
----------
cache : :class:`glue.lal.Cache`
the formatted list of files to read
segments : `~gwpy.segments.SegmentList`
the list of segments to read
etg : `str`
the name of the trigger generator that created the files
nproc : `int`, optional
the number of parallel processes to use when reading
**kwargs
other keyword arguments are passed to the `EventTable.read` or
`{tableclass}.read` methods
Returns
-------
table : `~gwpy.table.EventTable`, `None`
a table of events, or `None` if the cache has no overlap with
the segments
"""
if isinstance(cache, Cache):
cache = cache.sieve(segmentlist=segments)
cache = cache.checkfilesexist()[0]
cache.sort(key=lambda x: x.segment[0])
if etg == 'pycbc_live': # remove empty HDF5 files
cache = type(cache)(
filter_pycbc_live_files(cache, ifo=kwargs['ifo']))
# if no files, skip
if len(cache) == 0:
return
# use multiprocessing except for ascii reading
# (since astropy doesn't allow it)
if kwargs.get('format', 'none').startswith('ascii.'):
cache = cache.pfnlist()
else:
kwargs['nproc'] = nproc
if len(cache) == 1:
cache = cache[0]
# read triggers
table = EventTable.read(cache, **kwargs)
if timecolumn:
table.meta['timecolumn'] = timecolumn
# get back from cache entry
if isinstance(cache, CacheEntry):
cache = Cache([cache])
# append new events to existing table
try:
csegs = cache_segments(cache)
except (AttributeError, TypeError):
csegs = SegmentList()
table.meta['segments'] = csegs
return keep_in_segments(table, segments, etg)
def read_and_cache_events(channel,
etg,
cache=None,
trigfind_kw={},
**read_kw):
cfile = create_path(channel)
# read existing cached triggers and work out new segments to query
if args.append and cfile.is_file():
previous = DataQualityFlag.read(
str(cfile),
path='segments',
format='hdf5',
).coalesce()
new = analysis - previous
else:
new = analysis.copy()
# get cache of files
if cache is None:
cache = find_trigger_files(channel, etg, new.active, **trigfind_kw)
else:
cache = list(
filter(
lambda e: new.active.intersects_segment(file_segment(e)),
cache,
))
# restrict 'active' segments to when we have data
try:
new.active &= cache_segments(cache)
except IndexError:
new.active = type(new.active)()
# find new triggers
try:
trigs = get_triggers(channel,
etg,
new.active,
cache=cache,
raw=True,
**read_kw)
# catch error and continue
except ValueError as e:
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
else:
path = write_events(channel, trigs, new)
try:
return path, len(trigs)
except TypeError: # None
return
def _get_timeseries_dict(channels,
segments,
config=None,
cache=None,
query=True,
nds=None,
frametype=None,
nproc=1,
return_=True,
statevector=False,
archive=True,
datafind_error='raise',
dtype=None,
**ioargs):
"""Internal method to retrieve the data for a set of like-typed
channels using the :meth:`TimeSeriesDict.read` accessor.
"""
channels = list(map(get_channel, channels))
# set classes
if statevector:
ListClass = StateVectorList
DictClass = StateVectorDict
else:
ListClass = TimeSeriesList
DictClass = TimeSeriesDict
# check we have a configparser
if config is None:
config = GWSummConfigParser()
# read segments from global memory
keys = dict((c.ndsname, make_globalv_key(c)) for c in channels)
havesegs = reduce(
operator.and_,
(globalv.DATA.get(keys[channel.ndsname], ListClass()).segments
for channel in channels))
new = segments - havesegs
# read channel information
filter_ = dict()
resample = dict()
dtype_ = dict()
for channel in channels:
name = str(channel)
try:
filter_[name] = channel.filter
except AttributeError:
pass
try:
resample[name] = float(channel.resample)
except AttributeError:
pass
if channel.dtype is not None:
dtype_[name] = channel.dtype
elif dtype is not None:
dtype_[name] = dtype
# work out whether to use NDS or not
if nds is None and cache is not None:
nds = False
elif nds is None:
nds = 'LIGO_DATAFIND_SERVER' not in os.environ
# read new data
query &= (abs(new) > 0)
if cache is not None:
query &= len(cache) > 0
if query:
for channel in channels:
globalv.DATA.setdefault(keys[channel.ndsname], ListClass())
ifo = channels[0].ifo
# open NDS connection
if nds:
if config.has_option('nds', 'host'):
host = config.get('nds', 'host')
port = config.getint('nds', 'port')
ndsconnection = io_nds2.connect(host, port)
else:
ndsconnection = None
frametype = source = 'nds'
ndstype = channels[0].type
# get NDS channel segments
if ndsconnection is not None and ndsconnection.get_protocol() > 1:
span = list(map(int, new.extent()))
avail = io_nds2.get_availability(channels,
*span,
connection=ndsconnection)
new &= avail.intersection(avail.keys())
# or find frame type and check cache
else:
frametype = frametype or channels[0].frametype
new = exclude_short_trend_segments(new, ifo, frametype)
if cache is not None:
fcache = sieve_cache(cache, ifo=ifo[0], tag=frametype)
else:
fcache = []
if (cache is None or len(fcache) == 0) and len(new):
span = new.extent()
fcache, frametype = find_best_frames(ifo,
frametype,
span[0],
span[1],
config=config,
gaps='ignore',
onerror=datafind_error)
# parse discontiguous cache blocks and rebuild segment list
new &= cache_segments(fcache)
source = 'files'
# if reading Virgo h(t) GWF data, filter out files that don't
# contain the channel (Virgo state-vector only)
_names = set(map(str, channels))
_virgohoft = _names.intersection(VIRGO_HOFT_CHANNELS)
if _virgohoft:
vprint(" Determining available segments for "
"Virgo h(t) data...")
new &= data_segments(fcache, _virgohoft.pop())
# set channel type if reading with frameCPP
if fcache and all_adc(fcache):
ioargs['type'] = 'adc'
# store frametype for display in Channel Information tables
for channel in channels:
channel.frametype = frametype
# check whether each channel exists for all new times already
qchannels = []
for channel in channels:
oldsegs = globalv.DATA.get(keys[channel.ndsname],
ListClass()).segments
if abs(new - oldsegs) != 0 and nds:
qchannels.append(channel.ndsname)
elif abs(new - oldsegs) != 0:
qchannels.append(str(channel))
# loop through segments, recording data for each
if len(new):
vprint(
" Fetching data (from %s) for %d channels [%s]:\n" %
(source, len(qchannels), nds and ndstype or frametype or ''))
vstr = " [{0[0]}, {0[1]})"
for segment in new:
# force reading integer-precision segments
segment = type(segment)(int(segment[0]), int(segment[1]))
if abs(segment) < 1:
continue
# reset to minute trend sample times
if frame_trend_type(ifo, frametype) == 'minute':
segment = Segment(*io_nds2.minute_trend_times(*segment))
if abs(segment) < 60:
continue
if nds: # fetch
tsd = DictClass.fetch(qchannels,
segment[0],
segment[1],
connection=ndsconnection,
type=ndstype,
verbose=vstr.format(segment),
**ioargs)
else: # read
# NOTE: this sieve explicitly casts our segment to
# ligo.segments.segment to prevent `TypeError` from
# a mismatch with ligo.segments.segment
segcache = sieve_cache(fcache, segment=segment)
segstart, segend = map(float, segment)
tsd = DictClass.read(segcache,
qchannels,
start=segstart,
end=segend,
nproc=nproc,
verbose=vstr.format(segment),
**ioargs)
vprint(" post-processing...\n")
# apply type casting (copy=False means same type just returns)
for chan, ts in tsd.items():
tsd[chan] = ts.astype(dtype_.get(chan, ts.dtype),
casting='unsafe',
copy=False)
# apply resampling
tsd = resample_timeseries_dict(tsd, nproc=1, **resample)
# post-process
for c, data in tsd.items():
channel = get_channel(c)
key = keys[channel.ndsname]
if (key in globalv.DATA
and data.span in globalv.DATA[key].segments):
continue
if data.unit is None:
data.unit = 'undef'
for i, seg in enumerate(globalv.DATA[key].segments):
if seg in data.span:
# new data completely covers existing segment
# (and more), so just remove the old stuff
globalv.DATA[key].pop(i)
break
elif seg.intersects(data.span):
# new data extends existing segment, so only keep
# the really new stuff
data = data.crop(*(data.span - seg))
break
# filter
try:
filt = filter_[str(channel)]
except KeyError:
pass
else:
data = filter_timeseries(data, filt)
if isinstance(data, StateVector) or ':GRD-' in str(channel):
data.override_unit(units.dimensionless_unscaled)
if hasattr(channel, 'bits'):
data.bits = channel.bits
elif data.unit is None:
data.override_unit(channel.unit)
# update channel type for trends
if data.channel.type is None and (data.channel.trend
is not None):
if data.dt.to('s').value == 1:
data.channel.type = 's-trend'
elif data.dt.to('s').value == 60:
data.channel.type = 'm-trend'
# append and coalesce
add_timeseries(data, key=key, coalesce=True)
# rebuilt global channel list with new parameters
update_channel_params()
if not return_:
return
return locate_data(channels, segments, list_class=ListClass)
def _get_timeseries_dict(channels, segments, config=None,
cache=None, query=True, nds=None, frametype=None,
nproc=1, return_=True, statevector=False,
archive=True, datafind_error='raise', dtype=None,
**ioargs):
"""Internal method to retrieve the data for a set of like-typed
channels using the :meth:`TimeSeriesDict.read` accessor.
"""
channels = list(map(get_channel, channels))
# set classes
if statevector:
ListClass = StateVectorList
DictClass = StateVectorDict
else:
ListClass = TimeSeriesList
DictClass = TimeSeriesDict
# check we have a configparser
if config is None:
config = GWSummConfigParser()
# read segments from global memory
keys = dict((c.ndsname, make_globalv_key(c)) for c in channels)
havesegs = reduce(operator.and_,
(globalv.DATA.get(keys[channel.ndsname],
ListClass()).segments
for channel in channels))
new = segments - havesegs
# read channel information
filter_ = dict()
resample = dict()
dtype_ = dict()
for channel in channels:
name = str(channel)
try:
filter_[name] = channel.filter
except AttributeError:
pass
try:
resample[name] = float(channel.resample)
except AttributeError:
pass
if channel.dtype is not None:
dtype_[name] = channel.dtype
elif dtype is not None:
dtype_[name] = dtype
# work out whether to use NDS or not
if nds is None and cache is not None:
nds = False
elif nds is None:
nds = 'LIGO_DATAFIND_SERVER' not in os.environ
# read new data
query &= (abs(new) > 0)
if cache is not None:
query &= len(cache) > 0
if query:
for channel in channels:
globalv.DATA.setdefault(keys[channel.ndsname], ListClass())
ifo = channels[0].ifo
# open NDS connection
if nds:
if config.has_option('nds', 'host'):
host = config.get('nds', 'host')
port = config.getint('nds', 'port')
ndsconnection = io_nds2.connect(host, port)
else:
ndsconnection = None
frametype = source = 'nds'
ndstype = channels[0].type
# get NDS channel segments
if ndsconnection is not None and ndsconnection.get_protocol() > 1:
span = list(map(int, new.extent()))
avail = io_nds2.get_availability(
channels, *span, connection=ndsconnection
)
new &= avail.intersection(avail.keys())
# or find frame type and check cache
else:
frametype = frametype or channels[0].frametype
new = exclude_short_trend_segments(new, ifo, frametype)
if cache is not None:
fcache = sieve_cache(cache, ifo=ifo[0], tag=frametype)
else:
fcache = []
if (cache is None or len(fcache) == 0) and len(new):
span = new.extent()
fcache, frametype = find_best_frames(
ifo, frametype, span[0], span[1],
config=config, gaps='ignore', onerror=datafind_error)
# parse discontiguous cache blocks and rebuild segment list
new &= cache_segments(fcache)
source = 'files'
# if reading Virgo h(t) GWF data, filter out files that don't
# contain the channel (Virgo state-vector only)
_names = set(map(str, channels))
_virgohoft = _names.intersection(VIRGO_HOFT_CHANNELS)
if _virgohoft:
vprint(" Determining available segments for "
"Virgo h(t) data...")
new &= data_segments(fcache, _virgohoft.pop())
# set channel type if reading with frameCPP
if fcache and all_adc(fcache):
ioargs['type'] = 'adc'
# store frametype for display in Channel Information tables
for channel in channels:
channel.frametype = frametype
# check whether each channel exists for all new times already
qchannels = []
for channel in channels:
oldsegs = globalv.DATA.get(keys[channel.ndsname],
ListClass()).segments
if abs(new - oldsegs) != 0 and nds:
qchannels.append(channel.ndsname)
elif abs(new - oldsegs) != 0:
qchannels.append(str(channel))
# loop through segments, recording data for each
if len(new):
vprint(" Fetching data (from %s) for %d channels [%s]:\n"
% (source, len(qchannels),
nds and ndstype or frametype or ''))
vstr = " [{0[0]}, {0[1]})"
for segment in new:
# force reading integer-precision segments
segment = type(segment)(int(segment[0]), int(segment[1]))
if abs(segment) < 1:
continue
# reset to minute trend sample times
if frame_trend_type(ifo, frametype) == 'minute':
segment = Segment(*io_nds2.minute_trend_times(*segment))
if abs(segment) < 60:
continue
if nds: # fetch
tsd = DictClass.fetch(qchannels, segment[0], segment[1],
connection=ndsconnection, type=ndstype,
verbose=vstr.format(segment), **ioargs)
else: # read
# NOTE: this sieve explicitly casts our segment to
# ligo.segments.segment to prevent `TypeError` from
# a mismatch with ligo.segments.segment
segcache = sieve_cache(fcache, segment=segment)
segstart, segend = map(float, segment)
tsd = DictClass.read(segcache, qchannels, start=segstart,
end=segend, nproc=nproc,
verbose=vstr.format(segment), **ioargs)
vprint(" post-processing...\n")
# apply type casting (copy=False means same type just returns)
for chan, ts in tsd.items():
tsd[chan] = ts.astype(dtype_.get(chan, ts.dtype),
casting='unsafe', copy=False)
# apply resampling
tsd = resample_timeseries_dict(tsd, nproc=1, **resample)
# post-process
for c, data in tsd.items():
channel = get_channel(c)
key = keys[channel.ndsname]
if (key in globalv.DATA and
data.span in globalv.DATA[key].segments):
continue
if data.unit is None:
data.unit = 'undef'
for i, seg in enumerate(globalv.DATA[key].segments):
if seg in data.span:
# new data completely covers existing segment
# (and more), so just remove the old stuff
globalv.DATA[key].pop(i)
break
elif seg.intersects(data.span):
# new data extends existing segment, so only keep
# the really new stuff
data = data.crop(*(data.span - seg))
break
# filter
try:
filt = filter_[str(channel)]
except KeyError:
pass
else:
data = filter_timeseries(data, filt)
if isinstance(data, StateVector) or ':GRD-' in str(channel):
data.override_unit(units.dimensionless_unscaled)
if hasattr(channel, 'bits'):
data.bits = channel.bits
elif data.unit is None:
data.override_unit(channel.unit)
# update channel type for trends
if data.channel.type is None and (
data.channel.trend is not None):
if data.dt.to('s').value == 1:
data.channel.type = 's-trend'
elif data.dt.to('s').value == 60:
data.channel.type = 'm-trend'
# append and coalesce
add_timeseries(data, key=key, coalesce=True)
# rebuilt global channel list with new parameters
update_channel_params()
if not return_:
return
return locate_data(channels, segments, list_class=ListClass)
def get_triggers(channel, etg, segments, config=GWSummConfigParser(),
cache=None, columns=None, query=True, multiprocess=False,
ligolwtable=None, return_=True):
"""Read a table of transient event triggers for a given channel.
"""
key = '%s,%s' % (str(channel), etg.lower())
# convert input segments to a segmentlist (for convenience)
if isinstance(segments, DataQualityFlag):
segments = segments.active
segments = SegmentList(segments)
# get processes
if multiprocess is True:
nproc = count_free_cores()
elif multiprocess is False:
nproc = 1
else:
nproc = multiprocess
# find LIGO_LW table for this ETG
try:
TableClass = get_etg_table(etg)
except KeyError:
TableClass = None
# work out columns
if columns is None:
try:
columns = config.get(etg, 'columns').split(',')
except (NoSectionError, NoOptionError):
columns = None
# read segments from global memory
try:
havesegs = globalv.TRIGGERS[key].segments
except KeyError:
new = segments
else:
new = segments - havesegs
# read new triggers
if query and abs(new) != 0:
ntrigs = 0
vprint(" Grabbing %s triggers for %s" % (etg, str(channel)))
# store read kwargs
kwargs = get_etg_read_kwargs(config, etg, exclude=['columns'])
kwargs['columns'] = columns
if etg.lower().replace('-', '_') in ['cwb', 'pycbc_live']:
kwargs['ifo'] = get_channel(channel).ifo
if 'format' not in kwargs and 'ahope' not in etg.lower():
kwargs['format'] = etg.lower()
# set up ligolw options if needed
if TableClass is not None:
contenthandler = get_partial_contenthandler(TableClass)
lsctables.use_in(contenthandler)
# loop over segments
for segment in new:
# find trigger files
if cache is None and etg.lower() == 'hacr':
raise NotImplementedError("HACR parsing has not been "
"implemented.")
if cache is None and etg.lower() in ['kw', 'kleinewelle']:
kwargs['filt'] = lambda t: t.channel == str(channel)
if cache is None:
try:
segcache = trigfind.find_trigger_urls(str(channel), etg,
segment[0],
segment[1])
except ValueError as e:
warnings.warn("Caught %s: %s" % (type(e).__name__, str(e)))
continue
else:
for regex in TRIGFIND_FORMAT:
if regex.match(etg):
kwargs['format'] = TRIGFIND_FORMAT[regex]
if TRIGFIND_FORMAT[regex] in lsctables.TableByName:
kwargs['get_as_columns'] = True
break
if etg.lower() == 'omega':
kwargs.setdefault('format', 'omega')
else:
kwargs.setdefault('format', 'ligolw')
else:
segcache = cache
if isinstance(segcache, Cache):
segcache = segcache.sieve(segment=segment)
segcache = segcache.checkfilesexist()[0]
segcache.sort(key=lambda x: x.segment[0])
if etg == 'pycbc_live': # remove empty HDF5 files
segcache = type(segcache)(
filter_pycbc_live_files(segcache, ifo=kwargs['ifo']))
# if no files, skip
if len(segcache) == 0:
continue
# read triggers
if kwargs.get('format', None) == 'ligolw':
kwargs['contenthandler'] = contenthandler
try: # try directly reading a numpy.recarray
table = GWRecArray.read(segcache, nproc=nproc, **kwargs)
except Exception as e: # back up to LIGO_LW
if TableClass is not None and 'No reader' in str(e):
try:
table = TableClass.read(segcache, **kwargs)
except Exception:
raise e
else:
table = table.to_recarray(get_as_columns=True)
else:
raise
# append new events to existing table
try:
csegs = cache_segments(segcache)
except AttributeError:
csegs = SegmentList()
table.segments = csegs
t2 = keep_in_segments(table, SegmentList([segment]), etg)
add_triggers(t2, key, csegs)
ntrigs += len(t2)
vprint(".")
vprint(" | %d events read\n" % ntrigs)
# if asked to read triggers, but didn't actually read any,
# create an empty table so that subsequent calls don't raise KeyErrors
if query and key not in globalv.TRIGGERS:
if TableClass is not None:
tab = lsctables.New(TableClass, columns=columns).to_recarray(
get_as_columns=True)
else:
tab = GWRecArray((0,), dtype=[(c, float) for c in columns])
tab.segments = SegmentList()
add_triggers(tab, key, tab.segments)
# work out time function
if return_:
return keep_in_segments(globalv.TRIGGERS[key], segments, etg)
else:
return
def get_triggers(channel,
etg,
segments,
cache=None,
snr=None,
frange=None,
raw=False,
trigfind_kwargs={},
**read_kwargs):
"""Get triggers for the given channel
"""
etg = _sanitize_name(etg)
# format arguments
try:
readfmt = read_kwargs.pop("format", DEFAULT_FORMAT[etg])
except KeyError:
raise ValueError("unsupported ETG {!r}".format(etg))
trigfind_kwargs, read_kwargs = _format_params(channel, etg, readfmt,
trigfind_kwargs, read_kwargs)
# find triggers
if cache is None:
cache = find_trigger_files(channel, etg, segments, **trigfind_kwargs)
# read files
tables = []
for segment in segments:
segaslist = SegmentList([segment])
segcache = io_cache.sieve(cache, segment=segment)
# try and work out if cache overextends segment (so we need to crop)
cachesegs = io_cache.cache_segments(segcache)
outofbounds = abs(cachesegs - segaslist)
if segcache:
if len(segcache) == 1: # just pass the single filename
segcache = segcache[0]
new = EventTable.read(segcache, **read_kwargs)
new.meta = {k: new.meta[k] for k in TABLE_META if new.meta.get(k)}
if outofbounds:
new = new[in_segmentlist(new[new.dtype.names[0]], segaslist)]
tables.append(new)
if len(tables):
table = vstack_tables(tables)
else:
table = EventTable(
names=read_kwargs.get('columns', ['time', 'frequency', 'snr']))
# parse time, frequency-like and snr-like column names
columns = table.dtype.names
tcolumn = columns[0]
fcolumn = columns[1]
scolumn = columns[2]
# filter
keep = numpy.ones(len(table), dtype=bool)
if snr is not None:
keep &= table[scolumn] >= snr
if frange is not None:
keep &= table[fcolumn] >= frange[0]
keep &= table[fcolumn] < frange[1]
table = table[keep]
# return basic table if 'raw'
if raw:
return table
# rename time column so that all tables match in at least that
if tcolumn != "time":
table.rename_column(tcolumn, 'time')
# add channel column to identify all triggers
table.add_column(
table.Column(data=numpy.repeat(channel, len(table)), name='channel'))
table.sort('time')
return table
def main(args=None):
"""Run the online Guardian node visualization tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
fec_map = args.fec_map
simulink = args.simulink
daqsvn = args.daqsvn or ('https://daqsvn.ligo-la.caltech.edu/websvn/'
'listing.php?repname=daq_maps')
if args.ifo == 'H1':
if not fec_map:
fec_map = 'https://lhocds.ligo-wa.caltech.edu/exports/detchar/fec/'
if not simulink:
simulink = 'https://lhocds.ligo-wa.caltech.edu/daq/simulink/'
if args.ifo == 'L1':
if not fec_map:
fec_map = 'https://llocds.ligo-la.caltech.edu/exports/detchar/fec/'
if not simulink:
simulink = 'https://llocds.ligo-la.caltech.edu/daq/simulink/'
span = Segment(args.gpsstart, args.gpsend)
# let's go
LOGGER.info('{} Overflows {}-{}'.format(args.ifo, int(args.gpsstart),
int(args.gpsend)))
# get segments
if args.state_flag:
state = DataQualityFlag.query(args.state_flag,
int(args.gpsstart),
int(args.gpsend),
url=const.DEFAULT_SEGMENT_SERVER)
tmp = type(state.active)()
for i, seg in enumerate(state.active):
if abs(seg) < args.segment_end_pad:
continue
tmp.append(type(seg)(seg[0], seg[1] - args.segment_end_pad))
state.active = tmp.coalesce()
statea = state.active
else:
statea = SegmentList([span])
if not args.output_file:
duration = abs(span)
args.output_file = ('%s-OVERFLOWS-%d-%d.h5' %
(args.ifo, int(args.gpsstart), duration))
LOGGER.debug("Set default output file as %s" % args.output_file)
# set up container
overflows = DataQualityDict()
# prepare data access
if args.nds:
from gwpy.io import nds2 as io_nds2
host, port = args.nds.rsplit(':', 1)
ndsconnection = io_nds2.connect(host, port=int(port))
if ndsconnection.get_protocol() == 1:
cachesegs = SegmentList(
[Segment(int(args.gpsstart), int(args.gpsend))])
else:
cachesegs = io_nds2.get_availability(
['{0}:FEC-1_DAC_OVERFLOW_ACC_0_0'.format(args.ifo)],
int(args.gpsstart),
int(args.gpsend),
)
else: # get frame cache
cache = gwdatafind.find_urls(args.ifo[0], args.frametype,
int(args.gpsstart), int(args.gpsend))
cachesegs = statea & cache_segments(cache)
flag_desc = "ADC/DAC Overflow indicated by {0}"
# get channel and find overflows
for dcuid in args.dcuid:
LOGGER.info("Processing DCUID %d" % dcuid)
channel = daq.ligo_accum_overflow_channel(dcuid, args.ifo)
overflows[channel] = DataQualityFlag(channel, known=cachesegs)
if args.deep:
LOGGER.debug(" -- Getting list of overflow channels")
try:
channels = daq.ligo_model_overflow_channels(dcuid,
args.ifo,
args.frametype,
gpstime=span[0],
nds=args.nds)
except IndexError: # no frame found for GPS start, try GPS end
channels = daq.ligo_model_overflow_channels(dcuid,
args.ifo,
args.frametype,
gpstime=span[-1])
for chan in channels: # set up flags early
overflows[chan] = DataQualityFlag(
chan,
known=cachesegs,
description=flag_desc.format(chan),
isgood=False,
)
LOGGER.debug(" -- %d channels found" % len(channel))
for seg in cachesegs:
LOGGER.debug(" -- Processing {}-{}".format(*seg))
if args.nds:
read_kw = dict(connection=ndsconnection)
else:
read_kw = dict(source=cache, nproc=args.nproc)
msg = "Reading ACCUM_OVERFLOW data:".rjust(30)
data = get_data(channel,
seg[0],
seg[1],
pad=0.,
verbose=msg,
**read_kw)
new = daq.find_overflow_segments(
data,
cumulative=True,
)
overflows[channel] += new
LOGGER.info(" -- {} overflows found".format(len(new.active)))
if not new.active:
continue
# go deep!
for s, e in tqdm.tqdm(new.active.protract(2),
unit='ovfl',
desc='Going deep'.rjust(30)):
data = get_data(channels, s, e, **read_kw)
for ch in channels:
try:
overflows[ch] += daq.find_overflow_segments(
data[ch],
cumulative=True,
)
except KeyError:
warnings.warn("Skipping {}".format(ch), UserWarning)
continue
LOGGER.debug(" -- Search complete")
# write output
LOGGER.info("Writing segments to %s" % args.output_file)
table = table_from_segments(
overflows,
sngl_burst=args.output_file.endswith((".xml", ".xml.gz")),
)
if args.integer_segments:
for key in overflows:
overflows[key] = overflows[key].round()
if args.output_file.endswith((".h5", "hdf", ".hdf5")):
with h5py.File(args.output_file, "w") as h5f:
table.write(h5f, path="triggers")
overflows.write(h5f, path="segments")
else:
table.write(args.output_file, overwrite=True)
overflows.write(args.output_file, overwrite=True, append=True)
# write HTML
if args.html:
# get base path
base = os.path.dirname(args.html)
os.chdir(base)
if args.plot:
args.plot = os.path.curdir
if args.output_file:
args.output_file = os.path.relpath(args.output_file,
os.path.dirname(args.html))
if os.path.basename(args.html) == 'index.html':
links = [
'%d-%d' % (int(args.gpsstart), int(args.gpsend)),
('Parameters', '#parameters'),
('Segments', [('Overflows', '#overflows')]),
('Results', '#results'),
]
if args.state_flag:
links[2][1].insert(0, ('State flag', '#state-flag'))
(brand, class_) = htmlio.get_brand(args.ifo, 'Overflows',
args.gpsstart)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(
title='%s Overflows | %d-%d' %
(args.ifo, int(args.gpsstart), int(args.gpsend)),
navbar=navbar)
else:
page = htmlio.markup.page()
page.div(class_='container')
# -- header
page.div(class_='pb-2 mt-3 mb-2 border-bottom')
page.h1('%s ADC/DAC Overflows: %d-%d' %
(args.ifo, int(args.gpsstart), int(args.gpsend)))
page.div.close()
# -- paramters
content = [('DCUIDs', ' '.join(map(str, args.dcuid)))]
if daqsvn:
content.append(('FEC configuration', (
'<a href="{0}" target="_blank" title="{1} FEC configuration">'
'{0}</a>').format(daqsvn, args.ifo)))
if fec_map:
content.append(
('FEC map', '<a href="{0}" target="_blank" title="{1} FEC '
'map">{0}</a>'.format(fec_map, args.ifo)))
if simulink:
content.append(
('Simulink models', '<a href="{0}" target="_blank" title="{1} '
'Simulink models">{0}</a>'.format(simulink, args.ifo)))
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(
htmlio.parameter_table(content,
start=args.gpsstart,
end=args.gpsend,
flag=args.state_flag))
page.div.close() # col-md-9 col-sm-12
# link to summary file
if args.output_file:
ext = ('HDF' if args.output_file.endswith(
(".h5", "hdf", ".hdf5")) else 'XML')
page.div(class_='col-md-3 col-sm-12')
page.add(
htmlio.download_btn(
[('Segments ({})'.format(ext), args.output_file)],
btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(),
))
page.div.close() # col-md-3 col-sm-12
page.div.close() # row
# -- command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
# -- segments
page.h2('Segments', class_='mt-4', id_='segments')
# give contextual information
msg = ("This analysis searched for digital-to-analogue (DAC) or "
"analogue-to-digital (ADC) conversion overflows in the {0} "
"real-time controls system. ").format(
SITE_MAP.get(args.ifo, 'LIGO'))
if args.deep:
msg += (
"A hierarchichal search was performed, with one cumulative "
"overflow counter checked per front-end controller (FEC). "
"For those models that indicated an overflow, the card- and "
"slot-specific channels were then checked. ")
msg += (
"Consant overflow is shown as yellow, while transient overflow "
"is shown as red. If a data-quality flag was loaded for this "
"analysis, it will be displayed in green.")
page.add(htmlio.alert(msg, context=args.ifo.lower()))
# record state segments
if args.state_flag:
page.h3('State flag', class_='mt-3', id_='state-flag')
page.div(id_='accordion1')
page.add(
htmlio.write_flag_html(state,
span,
'state',
parent='accordion1',
context='success',
plotdir=args.plot,
facecolor=(0.2, 0.8, 0.2),
edgecolor='darkgreen',
known={
'facecolor': 'red',
'edgecolor': 'darkred',
'height': 0.4,
}))
page.div.close()
# record overflow segments
if sum(abs(s.active) for s in overflows.values()):
page.h3('Overflows', class_='mt-3', id_='overflows')
page.div(id_='accordion2')
for i, (c, flag) in enumerate(list(overflows.items())):
if abs(flag.active) == 0:
continue
if abs(flag.active) == abs(cachesegs):
context = 'warning'
else:
context = 'danger'
try:
channel = cds.get_real_channel(flag.name)
except Exception:
title = '%s [%d]' % (flag.name, len(flag.active))
else:
title = '%s (%s) [%d]' % (flag.name, channel,
len(flag.active))
page.add(
htmlio.write_flag_html(flag,
span,
i,
parent='accordion2',
title=title,
context=context,
plotdir=args.plot))
page.div.close()
else:
page.add(
htmlio.alert('No overflows were found in this analysis',
context=args.ifo.lower(),
dismiss=False))
# -- results table
page.h2('Results summary', class_='mt-4', id_='results')
page.table(class_='table table-striped table-hover')
# write table header
page.thead()
page.tr()
for header in ['Channel', 'Connected signal', 'Num. overflows']:
page.th(header)
page.thead.close()
# write body
page.tbody()
for c, seglist in overflows.items():
t = abs(seglist.active)
if t == 0:
page.tr()
elif t == abs(cachesegs):
page.tr(class_='table-warning')
else:
page.tr(class_='table-danger')
page.td(c)
try:
page.td(cds.get_real_channel(str(c)))
except Exception:
page.td()
page.td(len(seglist.active))
page.tr.close()
page.tbody.close()
page.table.close()
# -- close and write
htmlio.close_page(page, args.html)
LOGGER.info("HTML written to %s" % args.html)
def main(args=None):
"""Run the zero-crossing counter tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
span = Segment(args.gpsstart, args.gpsend)
LOGGER.info('-- Processing channel %s over span %d - %d' %
(args.channel, args.gpsstart, args.gpsend))
if args.state_flag:
state = DataQualityFlag.query(
args.state_flag,
int(args.gpsstart),
int(args.gpsend),
url=const.DEFAULT_SEGMENT_SERVER,
)
statea = state.active
else:
statea = SegmentList([span])
duration = abs(span)
# initialize output files for each threshold and store them in a dict
outfiles = {}
for thresh in args.threshold:
outfiles[str(thresh)] = (os.path.join(
args.output_path, '%s_%s_DAC-%d-%d.h5' %
(args.channel.replace('-', '_').replace(':', '-'), str(
int(thresh)).replace('-', 'n'), int(args.gpsstart), duration)))
# get frame cache
cache = gwdatafind.find_urls(args.ifo[0], args.frametype,
int(args.gpsstart), int(args.gpsend))
cachesegs = statea & cache_segments(cache)
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
# initialize a ligolw table for each threshold and store them in a dict
names = ("time", "frequency", "snr")
dtypes = ("f8", ) * len(names)
tables = {}
for thresh in args.threshold:
tables[str(thresh)] = EventTable(
names=names,
dtype=dtypes,
meta={"channel": args.channel},
)
# for each science segment, read in the data from frames, check for
# threshold crossings, and if the rate of crossings is less than
# rate_thresh, write to a sngl_burst table
for seg in cachesegs:
LOGGER.debug("Processing {}:".format(seg))
c = sieve_cache(cache, segment=seg)
if not c:
LOGGER.warning(" No {} data files for this segment, "
"skipping".format(args.frametype))
continue
data = get_data(args.channel,
seg[0],
seg[1],
nproc=args.nproc,
source=c,
verbose="Reading data:".rjust(30))
for thresh in args.threshold:
times = find_crossings(data, thresh)
rate = float(times.size) / abs(seg) if times.size else 0
LOGGER.info(" Found {0} crossings of {1}, rate: {2} Hz".format(
times.size,
thresh,
rate,
))
if times.size and rate < args.rate_thresh:
existing = tables[str(thresh)]
tables[str(thresh)] = vstack_tables(
(
existing,
table_from_times(times,
snr=10.,
frequency=100.,
names=existing.colnames),
),
join_type="exact",
)
n = max(map(len, tables.values()))
for thresh, outfile in outfiles.items():
tables[thresh].write(
outfile,
path="triggers",
format="hdf5",
overwrite=True,
)
LOGGER.info("{0} events written to {1}".format(
str(len(tables[thresh])).rjust(len(str(n))),
outfile,
))
def get_triggers(channel, etg, segments, cache=None, snr=None, frange=None,
raw=False, trigfind_kwargs={}, **read_kwargs):
"""Get triggers for the given channel
"""
etg = _sanitize_name(etg)
# format arguments
try:
readfmt = read_kwargs.pop("format", DEFAULT_FORMAT[etg])
except KeyError:
raise ValueError("unsupported ETG {!r}".format(etg))
trigfind_kwargs, read_kwargs = _format_params(
channel,
etg,
readfmt,
trigfind_kwargs,
read_kwargs
)
# find triggers
if cache is None:
cache = find_trigger_files(channel, etg, segments, **trigfind_kwargs)
# read files
tables = []
for segment in segments:
segaslist = SegmentList([segment])
segcache = io_cache.sieve(cache, segment=segment)
# try and work out if cache overextends segment (so we need to crop)
cachesegs = io_cache.cache_segments(segcache)
outofbounds = abs(cachesegs - segaslist)
if segcache:
if len(segcache) == 1: # just pass the single filename
segcache = segcache[0]
new = EventTable.read(segcache, **read_kwargs)
new.meta = {k: new.meta[k] for k in TABLE_META if new.meta.get(k)}
if outofbounds:
new = new[new[new.dtype.names[0]].in_segmentlist(segaslist)]
tables.append(new)
if len(tables):
table = vstack_tables(tables)
else:
table = EventTable(names=read_kwargs.get(
'columns', ['time', 'frequency', 'snr']))
# parse time, frequency-like and snr-like column names
columns = table.dtype.names
tcolumn = columns[0]
fcolumn = columns[1]
scolumn = columns[2]
# filter
keep = numpy.ones(len(table), dtype=bool)
if snr is not None:
keep &= table[scolumn] >= snr
if frange is not None:
keep &= table[fcolumn] >= frange[0]
keep &= table[fcolumn] < frange[1]
table = table[keep]
# return basic table if 'raw'
if raw:
return table
# rename time column so that all tables match in at least that
if tcolumn != "time":
table.rename_column(tcolumn, 'time')
# add channel column to identify all triggers
table.add_column(table.Column(data=numpy.repeat(channel, len(table)),
name='channel'))
table.sort('time')
return table
def get_triggers(channel, etg, segments, config=ConfigParser(), cache=None,
query=True, multiprocess=False, tablename=None,
columns=None, contenthandler=None, return_=True):
"""Read a table of transient event triggers for a given channel.
"""
key = '%s,%s' % (str(channel), etg.lower())
if isinstance(segments, DataQualityFlag):
segments = segments.active
segments = SegmentList(segments)
# get LIGO_LW table for this etg
if tablename:
TableClass = lsctables.TableByName[tablename]
register_etg_table(etg, TableClass, force=True)
elif key in globalv.TRIGGERS:
TableClass = type(globalv.TRIGGERS[key])
else:
TableClass = get_etg_table(etg)
# work out columns
if columns is None:
try:
columns = config.get(etg, 'columns').split(',')
except (NoSectionError, NoOptionError):
if etg.lower() in ['cwb', 'cwb-ascii']:
columns = None
else:
columns = TableClass.validcolumns.keys()
if columns is not None:
for col in ['process_id', 'search', 'channel']:
if col not in columns:
columns.append(col)
# read segments from global memory
try:
havesegs = globalv.TRIGGERS[key].segments
except KeyError:
new = segments
globalv.TRIGGERS.setdefault(
key, lsctables.New(TableClass, columns=columns))
globalv.TRIGGERS[key].segments = type(segments)()
else:
new = segments - havesegs
# read new triggers
query &= (abs(new) != 0)
if query:
# store read kwargs
kwargs = {'columns': columns}
# set content handler
if contenthandler is None:
contenthandler = get_partial_contenthandler(TableClass)
lsctables.use_in(contenthandler)
for segment in new:
kwargs['filt'] = (
lambda t: float(get_row_value(t, 'time')) in segment)
# find trigger files
if cache is None and etg.lower() == 'hacr':
raise NotImplementedError("HACR parsing has not been "
"implemented.")
if cache is None and re.match('dmt(.*)omega', etg.lower()):
segcache = find_dmt_omega(channel, segment[0], segment[1])
kwargs['format'] = 'ligolw'
elif cache is None and etg.lower() in ['kw', 'kleinewelle']:
segcache = find_kw(channel, segment[0], segment[1])
kwargs['format'] = 'ligolw'
kwargs['filt'] = lambda t: (
float(get_row_value(t, 'time')) in segment and
t.channel == str(channel))
elif cache is None:
segcache = trigfind.find_trigger_urls(str(channel), etg,
segment[0],
segment[1])
kwargs['format'] = 'ligolw'
elif isinstance(cache, Cache):
segcache = cache.sieve(segment=segment)
else:
segcache = cache
if isinstance(segcache, Cache):
segcache = segcache.checkfilesexist()[0]
if 'format' not in kwargs and 'ahope' not in etg.lower():
kwargs['format'] = etg.lower()
if (issubclass(TableClass, lsctables.SnglBurstTable) and
etg.lower().startswith('cwb')):
kwargs['ifo'] = get_channel(channel).ifo
# read triggers and store
if len(segcache) == 0:
continue
if kwargs.get('format', None) == 'ligolw':
kwargs['contenthandler'] = contenthandler
table = TableClass.read(segcache, **kwargs)
globalv.TRIGGERS[key].extend(table)
try:
csegs = cache_segments(segcache)
except AttributeError:
csegs = SegmentList()
try:
globalv.TRIGGERS[key].segments.extend(csegs)
except AttributeError:
globalv.TRIGGERS[key].segments = csegs
finally:
globalv.TRIGGERS[key].segments.coalesce()
vprint('\r')
# work out time function
if return_:
times = get_table_column(globalv.TRIGGERS[key], 'time').astype(float)
# return correct triggers
out = lsctables.New(TableClass, columns=columns)
out.channel = str(channel)
out.etg = str(etg)
out.extend(t for (i, t) in enumerate(globalv.TRIGGERS[key]) if
times[i] in segments)
out.segments = segments & globalv.TRIGGERS[key].segments
return out
else:
return