def find_frames(ifo, frametype, gpsstart, gpsend, config=ConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection( host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend(dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `~glue.datafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `~glue.lal.Cache` a list of structured frame file descriptions matching the ifo and frametype requested """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection(host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), ) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend( dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) # extend cache beyond datafind's knowledge to reduce latency try: latest = cache[-1] ngps = len( re_gwf_gps_epoch.search(os.path.dirname( latest.path)).groupdict()['gpsepoch']) except (IndexError, AttributeError): pass else: while True: s, e = latest.segment if s >= gpsend: break # replace GPS time of file basename new = latest.path.replace('-%d-' % s, '-%d-' % e) # replace GPS epoch in dirname new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps]) if os.path.isfile(new): latest = CacheEntry.from_T050017(new) cache.append(latest) else: break # validate files existing and return cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `~glue.datafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `~glue.lal.Cache` a list of structured frame file descriptions matching the ifo and frametype requested """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection( host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend(dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache