def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
if etg.lower() == 'omicron':
etg = '?micron'
# construct search
span = Segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1)
trigform = ('%s-%s_%s-%s-*.xml*'
% (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10))
# perform and cache results
out = Cache()
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
gprint("Searching %s..." % os.path.split(gpssearchpath)[0],
end =' ')
gpscache = Cache(map(CacheEntry.from_T050017,
glob.glob(os.path.join(searchbase, str(gpsdir),
trigform))))
out.extend(gpscache.sieve(segment=span))
if verbose:
gprint("%d found" % len(gpscache.sieve(segment=span)))
out.sort(key=lambda e: e.path)
return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
if etg.lower() == 'omicron':
etg = '?micron'
# construct search
span = Segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1)
trigform = ('%s-%s_%s-%s-*.xml*' %
(ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10))
# perform and cache results
out = Cache()
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
gprint("Searching %s..." % os.path.split(gpssearchpath)[0],
end=' ')
gpscache = Cache(
map(CacheEntry.from_T050017,
glob.glob(os.path.join(searchbase, str(gpsdir), trigform))))
out.extend(gpscache.sieve(segment=span))
if verbose:
gprint("%d found" % len(gpscache.sieve(segment=span)))
out.sort(key=lambda e: e.path)
return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False, **kwargs):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
# special case for KW
if etg.lower() in ['kw', 'kleinewelle']:
from .kw import find_dmt_cache
ifo = channel.split(':')[0]
kwargs.setdefault('extension', 'xml')
kwargs.setdefault('check_files', True)
return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs)
elif etg.lower() == 'omega':
from .omega import find_dmt_cache
ifo = channel.split(':')[0]
kwargs.setdefault('check_files', True)
return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs)
elif etg.lower() == 'omicron':
etg = '?micron'
# construct search
span = segments.segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = numpy.arange(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1)
trigform = ('%s-%s_%s-%s-*.xml*'
% (ifo, re.sub('-', '_', channel), etg.lower(), '[0-9]'*10))
# perform and cache results
out = Cache()
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
sys.stdout.write("Searching %s..."
% os.path.split(gpssearchpath)[0])
sys.stdout.flush()
gpscache = Cache(map(CacheEntry.from_T050017,
glob.glob(os.path.join(searchbase, str(gpsdir),
trigform))))
out.extend(gpscache.sieve(segment=span))
if verbose:
sys.stdout.write(" %d found\n" % len(gpscache.sieve(segment=span)))
out.sort(key=lambda e: e.path)
return out
def find_trigger_files(channel, etg, segments, **kwargs):
"""Find trigger files for a given channel and ETG
Parameters
----------
channel : `str`
name of channel to find
etg : `str`
name of event trigger generator to find
segments : :class:`~glue.segments.segmentlist`
list of segments to find
**kwargs
all other keyword arguments are passed to
`trigfind.find_trigger_urls`
Returns
-------
cache : :class:`~glue.lal.Cache`
cache of trigger file paths
See Also
--------
trigfind.find_trigger_urls
for details on file discovery
"""
cache = Cache()
for start, end in segments:
try:
cache.extend(trigfind.find_trigger_urls(channel, etg, start,
end, **kwargs))
except ValueError as e:
if str(e).lower().startswith('no channel-level directory'):
warnings.warn(str(e))
else:
raise
return cache.unique()
def find_frames(ifo,
frametype,
gpsstart,
gpsend,
config=GWSummConfigParser(),
urltype='file',
gaps='warn',
onerror='raise'):
"""Query the datafind server for GWF files for the given type
Parameters
----------
ifo : `str`
prefix for the IFO of interest (either one or two characters)
frametype : `str`
name of the frametype to find
gpsstart : `int`
GPS start time of the query
gpsend : `int`
GPS end time of the query
config : `~ConfigParser.ConfigParser`, optional
configuration with `[datafind]` section containing `server`
specification, otherwise taken from the environment
urltype : `str`, optional
what type of file paths to return, default: `file`
gaps : `str`, optional
what to do when gaps are detected, one of
- `ignore` : do nothing
- `warn` : display the existence of gaps but carry on
- `raise` : raise an exception
onerror : `str`, optional
what to do when the `~glue.datafind` query itself fails, same
options as for ``gaps``
Returns
-------
cache : `~glue.lal.Cache`
a list of structured frame file descriptions matching the ifo and
frametype requested
"""
vprint(' Finding %s-%s frames for [%d, %d)...' %
(ifo[0], frametype, int(gpsstart), int(gpsend)))
# find datafind host:port
try:
host = config.get('datafind', 'server')
except (NoOptionError, NoSectionError):
try:
host = os.environ['LIGO_DATAFIND_SERVER']
except KeyError:
host = None
port = None
else:
try:
host, port = host.rsplit(':', 1)
except ValueError:
port = None
else:
port = int(port)
else:
port = config.getint('datafind', 'port')
# get credentials
if port == 80:
cert = None
key = None
else:
cert, key = datafind.find_credential()
# XXX HACK: LLO changed frame types on Dec 6 2013:
LLOCHANGE = 1070291904
if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
frametype = frametype[-1]
# query frames
ifo = ifo[0].upper()
gpsstart = int(floor(gpsstart))
gpsend = int(ceil(min(globalv.NOW, gpsend)))
if gpsend <= gpsstart:
return Cache()
# parse match
try:
frametype, match = frametype.split('|', 1)
except ValueError:
match = None
def _query():
if cert is not None:
dfconn = datafind.GWDataFindHTTPSConnection(host=host,
port=port,
cert_file=cert,
key_file=key)
else:
dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port)
return dfconn.find_frame_urls(ifo[0].upper(),
frametype,
gpsstart,
gpsend,
urltype=urltype,
on_gaps=gaps,
match=match)
try:
cache = _query()
except RuntimeError as e:
sleep(1)
try:
cache = _query()
except RuntimeError:
if 'Invalid GPS times' in str(e):
e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), )
if onerror in ['ignore', None]:
pass
elif onerror in ['warn']:
warnings.warn('Caught %s: %s' % (type(e).__name__, str(e)))
else:
raise
cache = Cache()
# XXX: if querying for day of LLO frame type change, do both
if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T']
and gpsstart < LLOCHANGE < gpsend):
start = len(cache) and cache[-1].segment[1] or gpsstart
if start < gpsend:
cache.extend(
dfconn.find_frame_urls(ifo[0].upper(),
'L1_%s' % frametype,
start,
gpsend,
urltype=urltype,
on_gaps=gaps)[1:])
# extend cache beyond datafind's knowledge to reduce latency
try:
latest = cache[-1]
ngps = len(
re_gwf_gps_epoch.search(os.path.dirname(
latest.path)).groupdict()['gpsepoch'])
except (IndexError, AttributeError):
pass
else:
while True:
s, e = latest.segment
if s >= gpsend:
break
# replace GPS time of file basename
new = latest.path.replace('-%d-' % s, '-%d-' % e)
# replace GPS epoch in dirname
new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps])
if os.path.isfile(new):
latest = CacheEntry.from_T050017(new)
cache.append(latest)
else:
break
# validate files existing and return
cache, _ = cache.checkfilesexist()
vprint(' %d found.\n' % len(cache))
return cache
def find_frames(ifo, frametype, gpsstart, gpsend, config=ConfigParser(),
urltype='file', gaps='warn', onerror='raise'):
"""Query the datafind server for GWF files for the given type
"""
vprint(' Finding %s-%s frames for [%d, %d)...'
% (ifo[0], frametype, int(gpsstart), int(gpsend)))
# find datafind host:port
try:
host = config.get('datafind', 'server')
except (NoOptionError, NoSectionError):
try:
host = os.environ['LIGO_DATAFIND_SERVER']
except KeyError:
host = None
port = None
else:
try:
host, port = host.rsplit(':', 1)
except ValueError:
port = None
else:
port = int(port)
else:
port = config.getint('datafind', 'port')
# get credentials
if port == 80:
cert = None
key = None
else:
cert, key = datafind.find_credential()
# XXX HACK: LLO changed frame types on Dec 6 2013:
LLOCHANGE = 1070291904
if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
frametype = frametype[-1]
# query frames
ifo = ifo[0].upper()
gpsstart = int(floor(gpsstart))
gpsend = int(ceil(min(globalv.NOW, gpsend)))
if gpsend <= gpsstart:
return Cache()
def _query():
if cert is not None:
dfconn = datafind.GWDataFindHTTPSConnection(
host=host, port=port, cert_file=cert, key_file=key)
else:
dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port)
return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart,
gpsend, urltype=urltype, on_gaps=gaps)
try:
cache = _query()
except RuntimeError as e:
sleep(1)
try:
cache = _query()
except RuntimeError:
if 'Invalid GPS times' in str(e):
e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),)
if onerror in ['ignore', None]:
pass
elif onerror in ['warn']:
warnings.warn('Caught %s: %s'
% (type(e).__name__, str(e)))
else:
raise
cache = Cache()
# XXX: if querying for day of LLO frame type change, do both
if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and
gpsstart < LLOCHANGE < gpsend):
start = len(cache) and cache[-1].segment[1] or gpsstart
if start < gpsend:
cache.extend(dfconn.find_frame_urls(ifo[0].upper(),
'L1_%s' % frametype, start,
gpsend, urltype=urltype,
on_gaps=gaps)[1:])
cache, _ = cache.checkfilesexist()
vprint(' %d found.\n' % len(cache))
return cache
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(),
urltype='file', gaps='warn', onerror='raise'):
"""Query the datafind server for GWF files for the given type
Parameters
----------
ifo : `str`
prefix for the IFO of interest (either one or two characters)
frametype : `str`
name of the frametype to find
gpsstart : `int`
GPS start time of the query
gpsend : `int`
GPS end time of the query
config : `~ConfigParser.ConfigParser`, optional
configuration with `[datafind]` section containing `server`
specification, otherwise taken from the environment
urltype : `str`, optional
what type of file paths to return, default: `file`
gaps : `str`, optional
what to do when gaps are detected, one of
- `ignore` : do nothing
- `warn` : display the existence of gaps but carry on
- `raise` : raise an exception
onerror : `str`, optional
what to do when the `~glue.datafind` query itself fails, same
options as for ``gaps``
Returns
-------
cache : `~glue.lal.Cache`
a list of structured frame file descriptions matching the ifo and
frametype requested
"""
vprint(' Finding %s-%s frames for [%d, %d)...'
% (ifo[0], frametype, int(gpsstart), int(gpsend)))
# find datafind host:port
try:
host = config.get('datafind', 'server')
except (NoOptionError, NoSectionError):
try:
host = os.environ['LIGO_DATAFIND_SERVER']
except KeyError:
host = None
port = None
else:
try:
host, port = host.rsplit(':', 1)
except ValueError:
port = None
else:
port = int(port)
else:
port = config.getint('datafind', 'port')
# get credentials
if port == 80:
cert = None
key = None
else:
cert, key = datafind.find_credential()
# XXX HACK: LLO changed frame types on Dec 6 2013:
LLOCHANGE = 1070291904
if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
frametype = frametype[-1]
# query frames
ifo = ifo[0].upper()
gpsstart = int(floor(gpsstart))
gpsend = int(ceil(min(globalv.NOW, gpsend)))
if gpsend <= gpsstart:
return Cache()
# parse match
try:
frametype, match = frametype.split('|', 1)
except ValueError:
match = None
def _query():
if cert is not None:
dfconn = datafind.GWDataFindHTTPSConnection(
host=host, port=port, cert_file=cert, key_file=key)
else:
dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port)
return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart,
gpsend, urltype=urltype, on_gaps=gaps,
match=match)
try:
cache = _query()
except RuntimeError as e:
sleep(1)
try:
cache = _query()
except RuntimeError:
if 'Invalid GPS times' in str(e):
e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),)
if onerror in ['ignore', None]:
pass
elif onerror in ['warn']:
warnings.warn('Caught %s: %s'
% (type(e).__name__, str(e)))
else:
raise
cache = Cache()
# XXX: if querying for day of LLO frame type change, do both
if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and
gpsstart < LLOCHANGE < gpsend):
start = len(cache) and cache[-1].segment[1] or gpsstart
if start < gpsend:
cache.extend(dfconn.find_frame_urls(ifo[0].upper(),
'L1_%s' % frametype, start,
gpsend, urltype=urltype,
on_gaps=gaps)[1:])
cache, _ = cache.checkfilesexist()
vprint(' %d found.\n' % len(cache))
return cache
