def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
if etg.lower() == 'omicron':
etg = '?micron'
# construct search
span = Segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1)
trigform = ('%s-%s_%s-%s-*.xml*'
% (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10))
# perform and cache results
out = Cache()
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
gprint("Searching %s..." % os.path.split(gpssearchpath)[0],
end =' ')
gpscache = Cache(map(CacheEntry.from_T050017,
glob.glob(os.path.join(searchbase, str(gpsdir),
trigform))))
out.extend(gpscache.sieve(segment=span))
if verbose:
gprint("%d found" % len(gpscache.sieve(segment=span)))
out.sort(key=lambda e: e.path)
return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
if etg.lower() == 'omicron':
etg = '?micron'
# construct search
span = Segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1)
trigform = ('%s-%s_%s-%s-*.xml*' %
(ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10))
# perform and cache results
out = Cache()
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
gprint("Searching %s..." % os.path.split(gpssearchpath)[0],
end=' ')
gpscache = Cache(
map(CacheEntry.from_T050017,
glob.glob(os.path.join(searchbase, str(gpsdir), trigform))))
out.extend(gpscache.sieve(segment=span))
if verbose:
gprint("%d found" % len(gpscache.sieve(segment=span)))
out.sort(key=lambda e: e.path)
return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False, **kwargs):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
# special case for KW
if etg.lower() in ['kw', 'kleinewelle']:
from .kw import find_dmt_cache
ifo = channel.split(':')[0]
kwargs.setdefault('extension', 'xml')
kwargs.setdefault('check_files', True)
return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs)
elif etg.lower() == 'omega':
from .omega import find_dmt_cache
ifo = channel.split(':')[0]
kwargs.setdefault('check_files', True)
return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs)
elif etg.lower() == 'omicron':
etg = '?micron'
# construct search
span = segments.segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = numpy.arange(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1)
trigform = ('%s-%s_%s-%s-*.xml*'
% (ifo, re.sub('-', '_', channel), etg.lower(), '[0-9]'*10))
# perform and cache results
out = Cache()
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
sys.stdout.write("Searching %s..."
% os.path.split(gpssearchpath)[0])
sys.stdout.flush()
gpscache = Cache(map(CacheEntry.from_T050017,
glob.glob(os.path.join(searchbase, str(gpsdir),
trigform))))
out.extend(gpscache.sieve(segment=span))
if verbose:
sys.stdout.write(" %d found\n" % len(gpscache.sieve(segment=span)))
out.sort(key=lambda e: e.path)
return out
def find_contiguous(*caches):
"""Separate one or more caches into sets of contiguous caches
Parameters
----------
*caches
one or more :class:`~glue.lal.Cache` objects
Returns
-------
caches : `iter` of :class:`~glue.lal.Cache`
an interable yielding each contiguous cache
"""
try:
flat = flatten(*caches)
except IndexError:
flat = Cache()
for segment in cache_segments(flat):
yield flat.sieve(segment=segment)
def _get_timeseries_dict(channels,
segments,
config=None,
cache=None,
query=True,
nds=None,
frametype=None,
multiprocess=True,
return_=True,
statevector=False,
archive=True,
datafind_error='raise',
**ioargs):
"""Internal method to retrieve the data for a set of like-typed
channels using the :meth:`TimeSeriesDict.read` accessor.
"""
channels = map(get_channel, channels)
# set classes
if statevector:
ListClass = StateVectorList
DictClass = StateVectorDict
else:
ListClass = TimeSeriesList
DictClass = TimeSeriesDict
# check we have a configparser
if config is None:
config = GWSummConfigParser()
# read segments from global memory
keys = dict((c.ndsname, make_globalv_key(c)) for c in channels)
havesegs = reduce(
operator.and_,
(globalv.DATA.get(keys[channel.ndsname], ListClass()).segments
for channel in channels))
new = segments - havesegs
# get processes
if multiprocess is True:
nproc = count_free_cores()
elif multiprocess is False:
nproc = 1
else:
nproc = multiprocess
# read channel information
filter_ = dict()
resample = dict()
dtype_ = dict()
for channel in channels:
try:
filter_[channel.ndsname] = channel.filter
except AttributeError:
pass
try:
resample[channel] = float(channel.resample)
except AttributeError:
pass
if channel.dtype is None:
dtype_[channel] = ioargs.get('dtype')
else:
dtype_[channel] = channel.dtype
# work out whether to use NDS or not
if nds is None and cache is not None:
nds = False
elif nds is None:
nds = 'LIGO_DATAFIND_SERVER' not in os.environ
# read new data
query &= (abs(new) > 0)
if cache is not None:
query &= len(cache) > 0
if query:
for channel in channels:
globalv.DATA.setdefault(keys[channel.ndsname], ListClass())
# open NDS connection
if nds and config.has_option('nds', 'host'):
host = config.get('nds', 'host')
port = config.getint('nds', 'port')
try:
ndsconnection = nds2.connection(host, port)
except RuntimeError as e:
if 'SASL authentication' in str(e):
from gwpy.io.nds import kinit
kinit()
ndsconnection = nds2.connection(host, port)
else:
raise
frametype = source = 'nds'
ndstype = channels[0].type
elif nds:
ndsconnection = None
frametype = source = 'nds'
ndstype = channels[0].type
# or find frame type and check cache
else:
ifo = channels[0].ifo
frametype = frametype or channels[0].frametype
if frametype is not None and frametype.endswith('%s_M' % ifo):
new = type(new)([s for s in new if abs(s) >= 60.])
elif frametype is not None and frametype.endswith('%s_T' % ifo):
new = type(new)([s for s in new if abs(s) >= 1.])
if cache is not None:
fcache = cache.sieve(ifos=ifo[0],
description=frametype,
exact_match=True)
else:
fcache = Cache()
if (cache is None or len(fcache) == 0) and len(new):
span = new.extent().protract(8)
fcache = find_frames(ifo,
frametype,
span[0],
span[1],
config=config,
gaps='ignore',
onerror=datafind_error)
cachesegments = find_cache_segments(fcache)
gaps = SegmentList([span]) - cachesegments
if abs(gaps) and frametype == '%s_HOFT_C00' % ifo:
f2 = '%s_DMT_C00' % ifo
vprint(" Gaps discovered in aggregated h(t) type "
"%s, checking %s\n" % (frametype, f2))
c2 = find_frames(ifo,
f2,
span[0],
span[1],
config=config,
gaps='ignore',
onerror=datafind_error)
g2 = SegmentList([span]) - find_cache_segments(c2)
if abs(g2) < abs(gaps):
vprint(" Greater coverage with frametype %s\n" % f2)
fcache = c2
frametype = f2
else:
vprint(" No extra coverage with frametype %s\n" %
f2)
# parse discontiguous cache blocks and rebuild segment list
cachesegments = find_cache_segments(fcache)
new &= cachesegments
source = 'frames'
# set ctype if reading with framecpp
if cache is None and frametype in ADC_TYPES and HAS_FRAMECPP:
ioargs['type'] = 'adc'
for channel in channels:
channel.frametype = frametype
# check whether each channel exists for all new times already
qchannels = []
qresample = {}
qdtype = {}
for channel in channels:
oldsegs = globalv.DATA.get(channel.ndsname, ListClass()).segments
if abs(new - oldsegs) != 0:
qchannels.append(channel)
if channel in resample:
qresample[channel] = resample[channel]
qdtype[channel] = dtype_.get(channel, ioargs.get('dtype'))
ioargs['dtype'] = qdtype
# loop through segments, recording data for each
if len(new) and nproc > 1:
vprint(" Fetching data (from %s) for %d channels [%s]" %
(source, len(qchannels), nds and ndstype or frametype))
for segment in new:
# force reading integer-precision segments
segment = type(segment)(int(segment[0]), int(segment[1]))
if abs(segment) < 1:
continue
if nds:
tsd = DictClass.fetch(qchannels,
segment[0],
segment[1],
connection=ndsconnection,
type=ndstype,
**ioargs)
else:
# pad resampling
if segment[1] == cachesegments[-1][1] and qresample:
resamplepad = 8
if abs(segment) <= resamplepad:
continue
segment = type(segment)(segment[0],
segment[1] - resamplepad)
segcache = fcache.sieve(
segment=segment.protract(resamplepad))
else:
segcache = fcache.sieve(segment=segment)
# set minute trend times modulo 60 from GPS 0
if (re.match('(?:(.*)_)?[A-Z]\d_M', str(frametype))
or (ifo == 'C1' and frametype == 'M')):
segstart = int(segment[0]) // 60 * 60
segend = int(segment[1]) // 60 * 60
if segend >= segment[1]:
segend -= 60
# and ignore segments shorter than 1 full average
if (segend - segstart) < 60:
continue
segcache = segcache.sieve(
segment=type(segment)(segstart, segend))
else:
segstart, segend = map(float, segment)
# pull filters out because they can break multiprocessing
if nproc > 1:
for c in qchannels:
if c.ndsname in filter_:
del c.filter
# read data
tsd = DictClass.read(segcache,
qchannels,
start=segstart,
end=segend,
nproc=nproc,
resample=qresample,
**ioargs)
# put filters back
for c in qchannels:
if c.ndsname in filter_:
c.filter = filter_[c.ndsname]
for (channel, data) in tsd.iteritems():
key = keys[channel.ndsname]
if (key in globalv.DATA
and data.span in globalv.DATA[key].segments):
continue
if data.unit is None:
data.unit = 'undef'
for seg in globalv.DATA[key].segments:
if seg.intersects(data.span):
data = data.crop(*(data.span - seg))
break
try:
filt = filter_[channel.ndsname]
except KeyError:
pass
else:
# filter with function
if callable(filt):
try:
data = filt(data)
except TypeError as e:
if 'Can only apply' in str(e):
data.value[:] = filt(data.value)
else:
raise
# filter with gain
elif (isinstance(filt, tuple) and len(filt) == 3
and len(filt[0] + filt[1]) == 0):
try:
data *= filt[2]
except TypeError:
data = data * filt[2]
# filter zpk
elif isinstance(filt, tuple):
data = data.filter(*filt)
# filter fail
else:
raise ValueError("Cannot parse filter for %s: %r" %
(channel.ndsname, filt))
if isinstance(data, StateVector) or ':GRD-' in str(channel):
try:
data.unit = units.dimensionless_unscaled
except AttributeError:
data._unit = units.dimensionless_unscaled
if hasattr(channel, 'bits'):
data.bits = channel.bits
elif data.unit is None:
data._unit = channel.unit
# XXX: HACK for failing unit check
if len(globalv.DATA[key]):
data._unit = globalv.DATA[key][-1].unit
# update channel type for trends
if data.channel.type is None and (data.channel.trend
is not None):
if data.dt.to('s').value == 1:
data.channel.type = 's-trend'
elif data.dt.to('s').value == 60:
data.channel.type = 'm-trend'
# append and coalesce
add_timeseries(data, key=key, coalesce=True)
if multiprocess:
vprint('.')
if len(new):
vprint("\n")
if not return_:
return
# return correct data
out = OrderedDict()
for channel in channels:
data = ListClass()
if keys[channel.ndsname] not in globalv.DATA:
out[channel.ndsname] = ListClass()
else:
for ts in globalv.DATA[keys[channel.ndsname]]:
for seg in segments:
if abs(seg) == 0 or abs(seg) < ts.dt.value:
continue
if ts.span.intersects(seg):
common = map(float, ts.span & seg)
cropped = ts.crop(*common, copy=False)
if cropped.size:
data.append(cropped)
out[channel.ndsname] = data.coalesce()
return out
def _get_timeseries_dict(channels, segments, config=None,
cache=None, query=True, nds=None, frametype=None,
multiprocess=True, return_=True, statevector=False,
archive=True, datafind_error='raise', **ioargs):
"""Internal method to retrieve the data for a set of like-typed
channels using the :meth:`TimeSeriesDict.read` accessor.
"""
channels = map(get_channel, channels)
# set classes
if statevector:
ListClass = StateVectorList
DictClass = StateVectorDict
else:
ListClass = TimeSeriesList
DictClass = TimeSeriesDict
# check we have a configparser
if config is None:
config = GWSummConfigParser()
# read segments from global memory
keys = dict((c.ndsname, make_globalv_key(c)) for c in channels)
havesegs = reduce(operator.and_,
(globalv.DATA.get(keys[channel.ndsname],
ListClass()).segments
for channel in channels))
new = segments - havesegs
# get processes
if multiprocess is True:
nproc = count_free_cores()
elif multiprocess is False:
nproc = 1
else:
nproc = multiprocess
if globalv.VERBOSE and not multiprocess:
verbose = ' '
else:
verbose = False
# read channel information
filter_ = dict()
resample = dict()
dtype_ = dict()
for channel in channels:
try:
filter_[channel.ndsname] = channel.filter
except AttributeError:
pass
try:
resample[channel] = float(channel.resample)
except AttributeError:
pass
if channel.dtype is None:
dtype_[channel] = ioargs.get('dtype')
else:
dtype_[channel] = channel.dtype
# work out whether to use NDS or not
if nds is None and cache is not None:
nds = False
elif nds is None:
nds = 'LIGO_DATAFIND_SERVER' not in os.environ
# read new data
query &= (abs(new) > 0)
if cache is not None:
query &= len(cache) > 0
if query:
for channel in channels:
globalv.DATA.setdefault(keys[channel.ndsname], ListClass())
# open NDS connection
if nds and config.has_option('nds', 'host'):
host = config.get('nds', 'host')
port = config.getint('nds', 'port')
try:
ndsconnection = nds2.connection(host, port)
except RuntimeError as e:
if 'SASL authentication' in str(e):
from gwpy.io.nds import kinit
kinit()
ndsconnection = nds2.connection(host, port)
else:
raise
frametype = source = 'nds'
ndstype = channels[0].type
elif nds:
ndsconnection = None
frametype = source = 'nds'
ndstype = channels[0].type
# or find frame type and check cache
else:
ifo = channels[0].ifo
frametype = frametype or channels[0].frametype
if frametype is not None and frametype.endswith('%s_M' % ifo):
new = type(new)([s for s in new if abs(s) >= 60.])
elif frametype is not None and frametype.endswith('%s_T' % ifo):
new = type(new)([s for s in new if abs(s) >= 1.])
#elif ((globalv.NOW - new[0][0]) < 86400 * 10 and
# frametype == '%s_R' % ifo and
# find_types(site=ifo[0], match='_C\Z')):
# frametype = '%s_C' % ifo
if cache is not None:
fcache = cache.sieve(ifos=ifo[0], description=frametype,
exact_match=True)
else:
fcache = Cache()
if (cache is None or len(fcache) == 0) and len(new):
span = new.extent().protract(8)
fcache = find_frames(ifo, frametype, span[0], span[1],
config=config, gaps='ignore',
onerror=datafind_error)
if len(fcache) == 0 and frametype == '%s_R' % ifo:
frametype = '%s_C' % ifo
vprint(" Moving to backup frametype %s\n" % frametype)
fcache = find_frames(ifo, frametype, span[0], span[1],
config=config, gaps='ignore',
onerror=datafind_error)
# parse discontiguous cache blocks and rebuild segment list
cachesegments = find_cache_segments(fcache)
new &= cachesegments
source = 'frames'
for channel in channels:
channel.frametype = frametype
# check whether each channel exists for all new times already
qchannels = []
qresample = {}
qdtype = {}
for channel in channels:
oldsegs = globalv.DATA.get(channel.ndsname,
ListClass()).segments
if abs(new - oldsegs) != 0:
qchannels.append(channel)
if channel in resample:
qresample[channel] = resample[channel]
qdtype[channel] = dtype_.get(channel, ioargs.get('dtype'))
ioargs['dtype'] = qdtype
# find channel type
if not nds:
ctype = set()
for channel in qchannels:
try:
ctype.add(channel.ctype)
except AttributeError:
ctype.add(get_channel_type(channel))
if len(ctype) == 1:
ctype = list(ctype)[0]
else:
ctype = None
# loop through segments, recording data for each
if len(new) and nproc > 1:
vprint(" Fetching data (from %s) for %d channels [%s]"
% (source, len(qchannels), nds and ndstype or frametype))
for segment in new:
# force reading integer-precision segments
segment = type(segment)(int(segment[0]), int(segment[1]))
if abs(segment) < 1:
continue
if nds:
tsd = DictClass.fetch(qchannels, segment[0], segment[1],
connection=ndsconnection, type=ndstype,
**ioargs)
else:
# pad resampling
if segment[1] == cachesegments[-1][1] and qresample:
resamplepad = 8
if abs(segment) <= resamplepad:
continue
segment = type(segment)(segment[0],
segment[1] - resamplepad)
segcache = fcache.sieve(
segment=segment.protract(resamplepad))
else:
segcache = fcache.sieve(segment=segment)
# set minute trend times modulo 60 from GPS 0
if (re.match('(?:(.*)_)?[A-Z]\d_M', str(frametype)) or
(ifo == 'C1' and frametype == 'M')):
segstart = int(segment[0]) // 60 * 60
segend = int(segment[1]) // 60 * 60
if segend >= segment[1]:
segend -= 60
# and ignore segments shorter than 1 full average
if (segend - segstart) < 60:
continue
else:
segstart, segend = map(float, segment)
# pull filters out because they can break multiprocessing
if nproc > 1:
for c in qchannels:
if c.ndsname in filter_:
del c.filter
# read data
tsd = DictClass.read(segcache, qchannels,
start=segstart, end=segend, type=ctype,
nproc=nproc, resample=qresample,
verbose=verbose, **ioargs)
# put filters back
for c in qchannels:
if c.ndsname in filter_:
c.filter = filter_[c.ndsname]
for (channel, data) in tsd.iteritems():
key = keys[channel.ndsname]
if (key in globalv.DATA and
data.span in globalv.DATA[key].segments):
continue
if data.unit is None:
data.unit = 'undef'
for seg in globalv.DATA[key].segments:
if seg.intersects(data.span):
data = data.crop(*(data.span - seg))
break
try:
filt = filter_[channel.ndsname]
except KeyError:
pass
else:
# filter with function
if callable(filt):
try:
data = filt(data)
except TypeError as e:
if 'Can only apply' in str(e):
data.value[:] = filt(data.value)
else:
raise
# filter with gain
elif (isinstance(filt, tuple) and len(filt) == 3 and
len(filt[0] + filt[1]) == 0):
try:
data *= filt[2]
except TypeError:
data = data * filt[2]
# filter zpk
elif isinstance(filt, tuple):
data = data.filter(*filt)
# filter fail
else:
raise ValueError("Cannot parse filter for %s: %r"
% (channel.ndsname,
filt))
if isinstance(data, StateVector) or ':GRD-' in str(channel):
try:
data.unit = units.dimensionless_unscaled
except AttributeError:
data._unit = units.dimensionless_unscaled
if hasattr(channel, 'bits'):
data.bits = channel.bits
elif data.unit is None:
data._unit = channel.unit
# XXX: HACK for failing unit check
if len(globalv.DATA[key]):
data._unit = globalv.DATA[key][-1].unit
# update channel type for trends
if (data.channel.type is None and
data.channel.trend is not None):
if data.dt.to('s').value == 1:
data.channel.type = 's-trend'
elif data.dt.to('s').value == 60:
data.channel.type = 'm-trend'
# append and coalesce
add_timeseries(data, key=key, coalesce=True)
if multiprocess:
vprint('.')
if len(new):
vprint("\n")
if not return_:
return
# return correct data
out = OrderedDict()
for channel in channels:
data = ListClass()
if keys[channel.ndsname] not in globalv.DATA:
out[channel.ndsname] = ListClass()
else:
for ts in globalv.DATA[keys[channel.ndsname]]:
for seg in segments:
if abs(seg) == 0 or abs(seg) < ts.dt.value:
continue
if ts.span.intersects(seg):
common = map(float, ts.span & seg)
cropped = ts.crop(*common, copy=False)
if cropped.size:
data.append(cropped)
out[channel.ndsname] = data.coalesce()
return out
