def cache_overlaps(*caches):
"""Find segments of overlap in the given cache sets
"""
cache = [e for c in caches for e in c]
cache.sort(key=lambda e: file_segment(e)[0])
overlap = SegmentList()
segments = SegmentList()
for e in cache:
seg = file_segment(e)
ol = SegmentList([seg]) & segments
if abs(ol):
overlap.extend(ol)
segments.append(seg)
return overlap
def cache_overlaps(*caches):
"""Find segments of overlap in the given cache sets
"""
cache = [e for c in caches for e in c]
cache.sort(key=lambda e: file_segment(e)[0])
overlap = SegmentList()
segments = SegmentList()
for e in cache:
seg = file_segment(e)
ol = SegmentList([seg]) & segments
if abs(ol):
overlap.extend(ol)
segments.append(seg)
return overlap
def find_best_frames(ifo, frametype, start, end, **kwargs):
"""Find frames for the given type, replacing with a better type if needed
"""
# find cache for this frametype
cache = find_frames(ifo, frametype, start, end, **kwargs)
# check for gaps in current cache
span = SegmentList([Segment(start, end)])
gaps = span - cache_segments(cache)
# if gaps and using aggregated h(t), check short files
if abs(gaps) and frametype in SHORT_HOFT_TYPES:
f2 = SHORT_HOFT_TYPES[frametype]
vprint(" Gaps discovered in aggregated h(t) type "
"%s, checking %s\n" % (frametype, f2))
kwargs['gaps'] = 'ignore'
cache.extend(
filter(lambda e: file_segment(e) in gaps,
find_frames(ifo, f2, start, end, **kwargs)))
new = int(abs(gaps - cache_segments(cache)))
if new:
vprint(" %ss extra coverage with frametype %s\n" % (new, f2))
else:
vprint(" No extra coverage with frametype %s\n" % f2)
return cache, frametype
def test_file_segment(self):
"""Test :func:`gwpy.io.cache.file_segment`
"""
# check basic
fs = io_cache.file_segment('A-B-1-2.ext')
assert isinstance(fs, Segment)
assert fs == Segment(1, 3)
# check mutliple file extensions
assert io_cache.file_segment('A-B-1-2.ext.gz') == (1, 3)
# check floats (and multiple file extensions)
assert io_cache.file_segment('A-B-1.23-4.ext.gz') == (1.23, 5.23)
# test errors
with pytest.raises(ValueError) as exc:
io_cache.file_segment('blah')
assert str(exc.value) == ('Failed to parse \'blah\' as '
'LIGO-T050017-compatible filename')
def find_best_frames(ifo, frametype, start, end, **kwargs):
"""Find frames for the given type, replacing with a better type if needed
"""
# find cache for this frametype
cache = find_frames(ifo, frametype, start, end, **kwargs)
# check for gaps in current cache
span = SegmentList([Segment(start, end)])
gaps = span - cache_segments(cache)
# if gaps and using aggregated h(t), check short files
if abs(gaps) and frametype in SHORT_HOFT_TYPES:
f2 = SHORT_HOFT_TYPES[frametype]
vprint(" Gaps discovered in aggregated h(t) type "
"%s, checking %s\n" % (frametype, f2))
kwargs['gaps'] = 'ignore'
cache.extend(filter(lambda e: file_segment(e) in gaps,
find_frames(ifo, f2, start, end, **kwargs)))
new = int(abs(gaps - cache_segments(cache)))
if new:
vprint(" %ss extra coverage with frametype %s\n" % (new, f2))
else:
vprint(" No extra coverage with frametype %s\n" % f2)
return cache, frametype
def test_file_segment(self):
"""Test :func:`gwpy.io.cache.file_segment`
"""
# check basic
fs = io_cache.file_segment('A-B-1-2.ext')
assert isinstance(fs, Segment)
assert fs == Segment(1, 3)
# check mutliple file extensions
assert io_cache.file_segment('A-B-1-2.ext.gz') == (1, 3)
# check floats (and multiple file extensions)
assert io_cache.file_segment('A-B-1.23-4.ext.gz') == (1.23, 5.23)
# test errors
with pytest.raises(ValueError) as exc:
io_cache.file_segment('blah')
assert str(exc.value) == ('Failed to parse \'blah\' as '
'LIGO-T050017-compatible filename')
def find_omicron_files(channel, start, end, basepath, ext='xml.gz',
filetag=const.OMICRON_FILETAG.upper()):
"""Find Omicron files under a given starting directory
"""
gps5 = int(str(start)[:5])-1
cache = list()
span = Segment(start, end)
while gps5 <= int(str(end)[:5]):
new = _iter_files_in_gps_directory(channel, basepath, gps5,
ext, filetag=filetag)
cache.extend(path for path in new if
file_segment(path).intersects(span))
gps5 += 1
return cache
def read_and_cache_events(channel,
etg,
cache=None,
trigfind_kw={},
**read_kw):
cfile = create_path(channel)
# read existing cached triggers and work out new segments to query
if args.append and cfile.is_file():
previous = DataQualityFlag.read(
str(cfile),
path='segments',
format='hdf5',
).coalesce()
new = analysis - previous
else:
new = analysis.copy()
# get cache of files
if cache is None:
cache = find_trigger_files(channel, etg, new.active, **trigfind_kw)
else:
cache = list(
filter(
lambda e: new.active.intersects_segment(file_segment(e)),
cache,
))
# restrict 'active' segments to when we have data
try:
new.active &= cache_segments(cache)
except IndexError:
new.active = type(new.active)()
# find new triggers
try:
trigs = get_triggers(channel,
etg,
new.active,
cache=cache,
raw=True,
**read_kw)
# catch error and continue
except ValueError as e:
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
else:
path = write_events(channel, trigs, new)
try:
return path, len(trigs)
except TypeError: # None
return
def find_archive_latency(channel,
padding,
frametype=None,
state=None,
base=const.OMICRON_ARCHIVE):
"""Find the latency of Omicron file archival for the given channel
Parameters
----------
channel : `str`
name of channel
padding : `int`
padding parameter for Omicron processing
frametype : `str`, optional
frame type ID for data frame files
state : `str`, optional
name of DQSegDB flag defining operational state for this channel
base : `str`, optional
base directory for Omicron archive
Returns
-------
latency : `dict`
a `dict` of `(ext, latency)` pairs for each file extension stored in
the archive ('root', 'xml.gz')
"""
ifo = channel[:2]
obs = ifo[0]
# find latest GPS time
if state is None and frametype is None:
raise ValueError("Please give one of `state` or `frametype`")
if state is None:
target = get_latest_data_gps(obs, frametype)
else:
target = get_latest_active_gps(state)
target -= padding
# find latest file
latency = {}
for ext in ['root', 'xml.gz']:
f = find_latest_omicron_file(channel, base, ext=ext)
end = file_segment(f)[1]
latency[ext] = (int(target - end), f)
return latency
def find_omicron_files(channel,
start,
end,
basepath,
ext='xml.gz',
filetag=const.OMICRON_FILETAG.upper()):
"""Find Omicron files under a given starting directory
"""
gps5 = int(str(start)[:5]) - 1
cache = list()
span = Segment(start, end)
while gps5 <= int(str(end)[:5]):
new = _iter_files_in_gps_directory(channel,
basepath,
gps5,
ext,
filetag=filetag)
cache.extend(path for path in new
if file_segment(path).intersects(span))
gps5 += 1
return cache
def find_frames(ifo,
frametype,
gpsstart,
gpsend,
config=GWSummConfigParser(),
urltype='file',
gaps='warn',
onerror='raise'):
"""Query the datafind server for GWF files for the given type
Parameters
----------
ifo : `str`
prefix for the IFO of interest (either one or two characters)
frametype : `str`
name of the frametype to find
gpsstart : `int`
GPS start time of the query
gpsend : `int`
GPS end time of the query
config : `~ConfigParser.ConfigParser`, optional
configuration with `[datafind]` section containing `server`
specification, otherwise taken from the environment
urltype : `str`, optional
what type of file paths to return, default: `file`
gaps : `str`, optional
what to do when gaps are detected, one of
- `ignore` : do nothing
- `warn` : display the existence of gaps but carry on
- `raise` : raise an exception
onerror : `str`, optional
what to do when the `gwdatafind` query itself fails, same
options as for ``gaps``
Returns
-------
cache : `list` of `str`
a list of file paths pointing at GWF files matching the request
"""
vprint(' Finding %s-%s frames for [%d, %d)...' %
(ifo[0], frametype, int(gpsstart), int(gpsend)))
# find datafind host:port
try:
host = config.get('datafind', 'server')
except (NoOptionError, NoSectionError):
host = None
port = None
else:
port = config.getint('datafind', 'port')
# XXX HACK: LLO changed frame types on Dec 6 2013:
LLOCHANGE = 1070291904
if re.match(r'L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
frametype = frametype[-1]
# query frames
ifo = ifo[0].upper()
gpsstart = int(floor(gpsstart))
gpsend = int(ceil(min(globalv.NOW, gpsend)))
if gpsend <= gpsstart:
return []
# parse match
try:
frametype, match = frametype.split('|', 1)
except ValueError:
match = None
def _query():
return gwdatafind.find_urls(ifo[0].upper(),
frametype,
gpsstart,
gpsend,
urltype=urltype,
on_gaps=gaps,
match=match,
host=host,
port=port)
try:
cache = _query()
except RuntimeError as e:
sleep(1)
try:
cache = _query()
except RuntimeError:
if 'Invalid GPS times' in str(e):
e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), )
if onerror in ['ignore', None]:
pass
elif onerror in ['warn']:
warnings.warn('Caught %s: %s' % (type(e).__name__, str(e)))
else:
raise
cache = []
# XXX: if querying for day of LLO frame type change, do both
if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T']
and gpsstart < LLOCHANGE < gpsend):
start = len(cache) and cache[-1].segment[1] or gpsstart
if start < gpsend:
cache.extend(
gwdatafind.find_urls(ifo[0].upper(),
'L1_%s' % frametype,
start,
gpsend,
urltype=urltype,
on_gaps=gaps,
host=host,
port=port)[1:])
# extend cache beyond datafind's knowledge to reduce latency
try:
latest = cache[-1]
ngps = len(
re_gwf_gps_epoch.search(
os.path.dirname(latest)).groupdict()['gpsepoch'])
except (IndexError, AttributeError):
pass
else:
while True:
s, e = file_segment(latest)
if s >= gpsend:
break
# replace GPS time of file basename
new = latest.replace('-%d-' % s, '-%d-' % e)
# replace GPS epoch in dirname
new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps])
if os.path.isfile(new):
cache.append(new)
else:
break
# validate files existing and return
cache = list(filter(os.path.exists, map(_urlpath, cache)))
vprint(' %d found.\n' % len(cache))
return cache
def is_saturated(channel,
cache,
start=None,
end=None,
indicator='LIMEN',
nproc=DEFAULT_NPROC):
"""Check whether a channel has saturated its software limit
Parameters
----------
channel : `str`, or `list` of `str`
either a single channel name, or a list of channel names
cache : `list`
a `list` of file paths, the cache must be contiguous
start : `~gwpy.time.LIGOTimeGPS`, `int`
the GPS start time of the check
end : `~gwpy.time.LIGOTimeGPS`, `int`
the GPS end time of the check
indicator : `str`
the suffix of the indicator channel, either `'LIMEN'` or `'SWSTAT'`
nproc : `int`
the number of parallel processes to use for frame reading
Returns
-------
saturated : `bool`, `None`, or `DataQualityFlag`, or `list` of the same
one of the following given the conditions
- `None` : if the channel doesn't have a software limit
- `False` : if the channel didn't saturate
- `~gwpy.segments.DataQualityFlag` : otherwise
OR, a `list` of the above if a `list` of channels was given in the
first place
"""
if isinstance(channel, (list, tuple)):
channels = channel
else:
channels = [channel]
# parse prefix
for i, c in enumerate(channels):
if c.endswith('_LIMIT'):
channels[i] = c[:-6]
# check limit if set
indicators = ['{}_{}'.format(c, indicator) for c in channels]
gps = file_segment(cache[0])[0]
data = get_data(indicators, gps, gps + 1, source=cache, nproc=nproc)
# check limits for returned channels
if len(data) < len(channels): # exclude nonexistent channels
channels = [
c for c in channels if '{}_{}'.format(c, indicator) in data
]
indicators = ['{}_{}'.format(c, indicator) for c in channels]
if indicator.upper() == 'LIMEN':
active = dict(
(c, data[indicators[i]].value[0]) for i, c in enumerate(channels))
elif indicator.upper() == 'SWSTAT':
active = dict(
(c, data[indicators[i]].astype('uint32').value[0] >> 13 & 1)
for i, c in enumerate(channels))
else:
raise ValueError("Don't know how to determine if limit is set for "
"indicator %r" % indicator)
# get output/limit data for all with active limits
activechans = [c for c in channels if active[c]]
datachans = [
'%s_%s' % (c, s) for c in activechans for s in ('LIMIT', 'OUTPUT')
]
data = get_data(datachans, start, end, source=cache, nproc=nproc)
# find saturations of the limit for each channel
dataiter = ((data['%s_OUTPUT' % c], data['%s_LIMIT' % c])
for c in activechans)
if nproc > 1:
with Pool(processes=nproc) as pool:
saturations = list(pool.map(_find_saturations, dataiter))
else:
saturations = list(map(_find_saturations, dataiter))
# return many or one (based on input)
if isinstance(channel, (list, tuple)):
return saturations
else:
return saturations[0]
def is_saturated(channel, cache, start=None, end=None, indicator='LIMEN',
nproc=DEFAULT_NPROC):
"""Check whether a channel has saturated its software limit
Parameters
----------
channel : `str`, or `list` of `str`
either a single channel name, or a list of channel names
cache : `list`
a `list` of file paths, the cache must be contiguous
start : `~gwpy.time.LIGOTimeGPS`, `int`
the GPS start time of the check
end : `~gwpy.time.LIGOTimeGPS`, `int`
the GPS end time of the check
indicator : `str`
the suffix of the indicator channel, either `'LIMEN'` or `'SWSTAT'`
nproc : `int`
the number of parallel processes to use for frame reading
Returns
-------
saturated : `bool`, `None`, or `DataQualityFlag`, or `list` of the same
one of the following given the conditions
- `None` : if the channel doesn't have a software limit
- `False` : if the channel didn't saturate
- `~gwpy.segments.DataQualityFlag` : otherwise
OR, a `list` of the above if a `list` of channels was given in the
first place
"""
if isinstance(channel, (list, tuple)):
channels = channel
else:
channels = [channel]
# parse prefix
for i, c in enumerate(channels):
if c.endswith('_LIMIT'):
channels[i] = c[:-6]
# check limit if set
indicators = ['{}_{}'.format(c, indicator) for c in channels]
gps = file_segment(cache[0])[0]
data = get_data(indicators, gps, gps+1, source=cache, nproc=nproc)
# check limits for returned channels
if len(data) < len(channels): # exclude nonexistent channels
channels = [
c for c in channels if '{}_{}'.format(c, indicator) in data]
indicators = ['{}_{}'.format(c, indicator) for c in channels]
if indicator.upper() == 'LIMEN':
active = dict((c, data[indicators[i]].value[0]) for
i, c in enumerate(channels))
elif indicator.upper() == 'SWSTAT':
active = dict(
(c, data[indicators[i]].astype('uint32').value[0] >> 13 & 1) for
i, c in enumerate(channels))
else:
raise ValueError("Don't know how to determine if limit is set for "
"indicator %r" % indicator)
# get output/limit data for all with active limits
activechans = [c for c in channels if active[c]]
datachans = ['%s_%s' % (c, s) for c in activechans for
s in ('LIMIT', 'OUTPUT')]
data = get_data(datachans, start, end, source=cache, nproc=nproc)
# find saturations of the limit for each channel
dataiter = ((data['%s_OUTPUT' % c], data['%s_LIMIT' % c])
for c in activechans)
if nproc > 1:
pool = Pool(processes=nproc)
saturations = list(pool.map(_find_saturations, dataiter))
pool.close()
else:
saturations = list(map(_find_saturations, dataiter))
# return many or one (based on input)
if isinstance(channel, (list, tuple)):
return saturations
else:
return saturations[0]
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(),
urltype='file', gaps='warn', onerror='raise'):
"""Query the datafind server for GWF files for the given type
Parameters
----------
ifo : `str`
prefix for the IFO of interest (either one or two characters)
frametype : `str`
name of the frametype to find
gpsstart : `int`
GPS start time of the query
gpsend : `int`
GPS end time of the query
config : `~ConfigParser.ConfigParser`, optional
configuration with `[datafind]` section containing `server`
specification, otherwise taken from the environment
urltype : `str`, optional
what type of file paths to return, default: `file`
gaps : `str`, optional
what to do when gaps are detected, one of
- `ignore` : do nothing
- `warn` : display the existence of gaps but carry on
- `raise` : raise an exception
onerror : `str`, optional
what to do when the `gwdatafind` query itself fails, same
options as for ``gaps``
Returns
-------
cache : `list` of `str`
a list of file paths pointing at GWF files matching the request
"""
vprint(' Finding %s-%s frames for [%d, %d)...'
% (ifo[0], frametype, int(gpsstart), int(gpsend)))
# find datafind host:port
try:
host = config.get('datafind', 'server')
except (NoOptionError, NoSectionError):
host = None
port = None
else:
port = config.getint('datafind', 'port')
# XXX HACK: LLO changed frame types on Dec 6 2013:
LLOCHANGE = 1070291904
if re.match(r'L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
frametype = frametype[-1]
# query frames
ifo = ifo[0].upper()
gpsstart = int(floor(gpsstart))
gpsend = int(ceil(min(globalv.NOW, gpsend)))
if gpsend <= gpsstart:
return []
# parse match
try:
frametype, match = frametype.split('|', 1)
except ValueError:
match = None
def _query():
return gwdatafind.find_urls(ifo[0].upper(), frametype, gpsstart,
gpsend, urltype=urltype, on_gaps=gaps,
match=match, host=host, port=port)
try:
cache = _query()
except RuntimeError as e:
sleep(1)
try:
cache = _query()
except RuntimeError:
if 'Invalid GPS times' in str(e):
e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),)
if onerror in ['ignore', None]:
pass
elif onerror in ['warn']:
warnings.warn('Caught %s: %s'
% (type(e).__name__, str(e)))
else:
raise
cache = []
# XXX: if querying for day of LLO frame type change, do both
if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and
gpsstart < LLOCHANGE < gpsend):
start = len(cache) and cache[-1].segment[1] or gpsstart
if start < gpsend:
cache.extend(gwdatafind.find_urls(
ifo[0].upper(), 'L1_%s' % frametype, start, gpsend,
urltype=urltype, on_gaps=gaps, host=host, port=port)[1:])
# extend cache beyond datafind's knowledge to reduce latency
try:
latest = cache[-1]
ngps = len(re_gwf_gps_epoch.search(
os.path.dirname(latest)).groupdict()['gpsepoch'])
except (IndexError, AttributeError):
pass
else:
while True:
s, e = file_segment(latest)
if s >= gpsend:
break
# replace GPS time of file basename
new = latest.replace('-%d-' % s, '-%d-' % e)
# replace GPS epoch in dirname
new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps])
if os.path.isfile(new):
cache.append(new)
else:
break
# validate files existing and return
cache = list(filter(os.path.exists, map(_urlpath, cache)))
vprint(' %d found.\n' % len(cache))
return cache