def read_cache(lcf, coltype=LIGOTimeGPS):
"""Read a LAL-format cache file into memory as a `Cache`
Parameters
----------
lcf : `str`, `file`
input file or file path to read
coltype : `LIGOTimeGPS`, `int`, optional
`type` for GPS times
Returns
-------
cache : :class:`glue.lal.Cache`
a cache object, representing each line in the file as a
:class:`~lal.utils.CacheEntry`
"""
from glue.lal import Cache
# open file
if not isinstance(lcf, FILE_LIKE):
with open(lcf, 'r') as f:
return open_cache(f, coltype=coltype)
# read file
out = Cache()
for line in lcf:
if isinstance(line, bytes):
line = line.decode('utf-8')
out.append(out.entry_class(line, coltype=coltype))
return out
def FrameCachetoLALCache(fcache):
lcache = LALCache()
files = fcache.get_files()
for f in files:
lcache.append(LALCacheEntry.from_T050017(f))
return lcache
def make_cache():
try:
from lal.utils import CacheEntry
except ImportError as e:
pytest.skip(str(e))
segs = SegmentList()
cache = Cache()
for seg in [(0, 1), (1, 2), (4, 5)]:
d = seg[1] - seg[0]
f = 'A-B-%d-%d.tmp' % (seg[0], d)
cache.append(CacheEntry.from_T050017(f, coltype=int))
segs.append(Segment(*seg))
return cache, segs
def find_frames(ifo,
frametype,
gpsstart,
gpsend,
config=GWSummConfigParser(),
urltype='file',
gaps='warn',
onerror='raise'):
"""Query the datafind server for GWF files for the given type
Parameters
----------
ifo : `str`
prefix for the IFO of interest (either one or two characters)
frametype : `str`
name of the frametype to find
gpsstart : `int`
GPS start time of the query
gpsend : `int`
GPS end time of the query
config : `~ConfigParser.ConfigParser`, optional
configuration with `[datafind]` section containing `server`
specification, otherwise taken from the environment
urltype : `str`, optional
what type of file paths to return, default: `file`
gaps : `str`, optional
what to do when gaps are detected, one of
- `ignore` : do nothing
- `warn` : display the existence of gaps but carry on
- `raise` : raise an exception
onerror : `str`, optional
what to do when the `~glue.datafind` query itself fails, same
options as for ``gaps``
Returns
-------
cache : `~glue.lal.Cache`
a list of structured frame file descriptions matching the ifo and
frametype requested
"""
vprint(' Finding %s-%s frames for [%d, %d)...' %
(ifo[0], frametype, int(gpsstart), int(gpsend)))
# find datafind host:port
try:
host = config.get('datafind', 'server')
except (NoOptionError, NoSectionError):
try:
host = os.environ['LIGO_DATAFIND_SERVER']
except KeyError:
host = None
port = None
else:
try:
host, port = host.rsplit(':', 1)
except ValueError:
port = None
else:
port = int(port)
else:
port = config.getint('datafind', 'port')
# get credentials
if port == 80:
cert = None
key = None
else:
cert, key = datafind.find_credential()
# XXX HACK: LLO changed frame types on Dec 6 2013:
LLOCHANGE = 1070291904
if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
frametype = frametype[-1]
# query frames
ifo = ifo[0].upper()
gpsstart = int(floor(gpsstart))
gpsend = int(ceil(min(globalv.NOW, gpsend)))
if gpsend <= gpsstart:
return Cache()
# parse match
try:
frametype, match = frametype.split('|', 1)
except ValueError:
match = None
def _query():
if cert is not None:
dfconn = datafind.GWDataFindHTTPSConnection(host=host,
port=port,
cert_file=cert,
key_file=key)
else:
dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port)
return dfconn.find_frame_urls(ifo[0].upper(),
frametype,
gpsstart,
gpsend,
urltype=urltype,
on_gaps=gaps,
match=match)
try:
cache = _query()
except RuntimeError as e:
sleep(1)
try:
cache = _query()
except RuntimeError:
if 'Invalid GPS times' in str(e):
e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), )
if onerror in ['ignore', None]:
pass
elif onerror in ['warn']:
warnings.warn('Caught %s: %s' % (type(e).__name__, str(e)))
else:
raise
cache = Cache()
# XXX: if querying for day of LLO frame type change, do both
if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T']
and gpsstart < LLOCHANGE < gpsend):
start = len(cache) and cache[-1].segment[1] or gpsstart
if start < gpsend:
cache.extend(
dfconn.find_frame_urls(ifo[0].upper(),
'L1_%s' % frametype,
start,
gpsend,
urltype=urltype,
on_gaps=gaps)[1:])
# extend cache beyond datafind's knowledge to reduce latency
try:
latest = cache[-1]
ngps = len(
re_gwf_gps_epoch.search(os.path.dirname(
latest.path)).groupdict()['gpsepoch'])
except (IndexError, AttributeError):
pass
else:
while True:
s, e = latest.segment
if s >= gpsend:
break
# replace GPS time of file basename
new = latest.path.replace('-%d-' % s, '-%d-' % e)
# replace GPS epoch in dirname
new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps])
if os.path.isfile(new):
latest = CacheEntry.from_T050017(new)
cache.append(latest)
else:
break
# validate files existing and return
cache, _ = cache.checkfilesexist()
vprint(' %d found.\n' % len(cache))
return cache
def get_cache(start, end, ifo, ftype, framecache=False, server=None,\
verbose=False):
"""
Queries the LSC datafind server and returns a glue.lal.Cache object
containing the frame file paths in the given GPS (start, end) interval
for the given ifo and type (can be lists).
framecache=True returns a pylal.dq.dqFrameUTils.FrameCache object in stead.
Arguments:
start : float
GPS start time (s).
end : float
GPS end time (s).
ifo : [ string | list ]
ifo (or list of) to find, e.g. 'G1'.
ftype : [ string | list ]
frame data type (or list of) to find, e.g. 'G1_RDS_C01_L3'.
"""
# set lists
if isinstance(ftype, str):
types = [ftype]
else:
types = ftype
if isinstance(ifo, str):
ifos = [ifo]
else:
ifos = ifo
# construct span
span = segments.segment(start,end)
# set options
cache = LALCache()
entry_class = LALCacheEntry
# try querying the ligo_data_find server
if not server:
server = _find_datafind_server()
if verbose: sys.stdout.write("Opening connection to %s...\n" % server)
if re.search(':', server):
port = int(server.split(':')[-1])
else:
port = None
cert, key = _get_grid_proxy()
# if we have a credential then use it when setting up the connection
if cert and key and port!=80:
h = httplib.HTTPSConnection(server, key_file=key, cert_file=cert)
else:
h = httplib.HTTPConnection(server)
if verbose: sys.stdout.write("Querying server for frames...\n")
# loop over ifos and types
for ifo in ifos:
for t in types:
# construct the URL for a simple data find query
url = "/LDR/services/data/v1/gwf/%s/%s/%s,%s/file.json" % (ifo[0], t,\
str(start),\
str(end))
# query the server
h.request("GET", url)
response = h.getresponse()
_verify_response(response)
# unravel the response
urlList = cjson.decode(response.read())
for url in urlList:
cache.append(entry_class.from_T050017(url))
# close the server connection
h.close()
if verbose: sys.stdout.write("Connection to %s closed.\n" % server)
# convert to FrameCache if needed
if framecache:
cache = LALCachetoFrameCache(cache)
return cache