def read_cache(lcf, coltype=LIGOTimeGPS): """Read a LAL-format cache file into memory as a `Cache` Parameters ---------- lcf : `str`, `file` input file or file path to read coltype : `LIGOTimeGPS`, `int`, optional `type` for GPS times Returns ------- cache : :class:`glue.lal.Cache` a cache object, representing each line in the file as a :class:`~lal.utils.CacheEntry` """ from glue.lal import Cache # open file if not isinstance(lcf, FILE_LIKE): with open(lcf, 'r') as f: return open_cache(f, coltype=coltype) # read file out = Cache() for line in lcf: if isinstance(line, bytes): line = line.decode('utf-8') out.append(out.entry_class(line, coltype=coltype)) return out
def FrameCachetoLALCache(fcache): lcache = LALCache() files = fcache.get_files() for f in files: lcache.append(LALCacheEntry.from_T050017(f)) return lcache
def make_cache(): try: from lal.utils import CacheEntry except ImportError as e: pytest.skip(str(e)) segs = SegmentList() cache = Cache() for seg in [(0, 1), (1, 2), (4, 5)]: d = seg[1] - seg[0] f = 'A-B-%d-%d.tmp' % (seg[0], d) cache.append(CacheEntry.from_T050017(f, coltype=int)) segs.append(Segment(*seg)) return cache, segs
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `~glue.datafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `~glue.lal.Cache` a list of structured frame file descriptions matching the ifo and frametype requested """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection(host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), ) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend( dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) # extend cache beyond datafind's knowledge to reduce latency try: latest = cache[-1] ngps = len( re_gwf_gps_epoch.search(os.path.dirname( latest.path)).groupdict()['gpsepoch']) except (IndexError, AttributeError): pass else: while True: s, e = latest.segment if s >= gpsend: break # replace GPS time of file basename new = latest.path.replace('-%d-' % s, '-%d-' % e) # replace GPS epoch in dirname new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps]) if os.path.isfile(new): latest = CacheEntry.from_T050017(new) cache.append(latest) else: break # validate files existing and return cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache
def get_cache(start, end, ifo, ftype, framecache=False, server=None,\ verbose=False): """ Queries the LSC datafind server and returns a glue.lal.Cache object containing the frame file paths in the given GPS (start, end) interval for the given ifo and type (can be lists). framecache=True returns a pylal.dq.dqFrameUTils.FrameCache object in stead. Arguments: start : float GPS start time (s). end : float GPS end time (s). ifo : [ string | list ] ifo (or list of) to find, e.g. 'G1'. ftype : [ string | list ] frame data type (or list of) to find, e.g. 'G1_RDS_C01_L3'. """ # set lists if isinstance(ftype, str): types = [ftype] else: types = ftype if isinstance(ifo, str): ifos = [ifo] else: ifos = ifo # construct span span = segments.segment(start,end) # set options cache = LALCache() entry_class = LALCacheEntry # try querying the ligo_data_find server if not server: server = _find_datafind_server() if verbose: sys.stdout.write("Opening connection to %s...\n" % server) if re.search(':', server): port = int(server.split(':')[-1]) else: port = None cert, key = _get_grid_proxy() # if we have a credential then use it when setting up the connection if cert and key and port!=80: h = httplib.HTTPSConnection(server, key_file=key, cert_file=cert) else: h = httplib.HTTPConnection(server) if verbose: sys.stdout.write("Querying server for frames...\n") # loop over ifos and types for ifo in ifos: for t in types: # construct the URL for a simple data find query url = "/LDR/services/data/v1/gwf/%s/%s/%s,%s/file.json" % (ifo[0], t,\ str(start),\ str(end)) # query the server h.request("GET", url) response = h.getresponse() _verify_response(response) # unravel the response urlList = cjson.decode(response.read()) for url in urlList: cache.append(entry_class.from_T050017(url)) # close the server connection h.close() if verbose: sys.stdout.write("Connection to %s closed.\n" % server) # convert to FrameCache if needed if framecache: cache = LALCachetoFrameCache(cache) return cache