def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower() == 'omicron': etg = '?micron' # construct search span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end =' ') gpscache = Cache(map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: gprint("%d found" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower() == 'omicron': etg = '?micron' # construct search span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end=' ') gpscache = Cache( map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: gprint("%d found" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False, **kwargs): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ # special case for KW if etg.lower() in ['kw', 'kleinewelle']: from .kw import find_dmt_cache ifo = channel.split(':')[0] kwargs.setdefault('extension', 'xml') kwargs.setdefault('check_files', True) return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs) elif etg.lower() == 'omega': from .omega import find_dmt_cache ifo = channel.split(':')[0] kwargs.setdefault('check_files', True) return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs) elif etg.lower() == 'omicron': etg = '?micron' # construct search span = segments.segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = numpy.arange(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re.sub('-', '_', channel), etg.lower(), '[0-9]'*10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: sys.stdout.write("Searching %s..." % os.path.split(gpssearchpath)[0]) sys.stdout.flush() gpscache = Cache(map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: sys.stdout.write(" %d found\n" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_trigger_files(channel, etg, segments, **kwargs): """Find trigger files for a given channel and ETG Parameters ---------- channel : `str` name of channel to find etg : `str` name of event trigger generator to find segments : :class:`~glue.segments.segmentlist` list of segments to find **kwargs all other keyword arguments are passed to `trigfind.find_trigger_urls` Returns ------- cache : :class:`~glue.lal.Cache` cache of trigger file paths See Also -------- trigfind.find_trigger_urls for details on file discovery """ cache = Cache() for start, end in segments: try: cache.extend(trigfind.find_trigger_urls(channel, etg, start, end, **kwargs)) except ValueError as e: if str(e).lower().startswith('no channel-level directory'): warnings.warn(str(e)) else: raise return cache.unique()
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `~glue.datafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `~glue.lal.Cache` a list of structured frame file descriptions matching the ifo and frametype requested """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection(host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), ) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend( dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) # extend cache beyond datafind's knowledge to reduce latency try: latest = cache[-1] ngps = len( re_gwf_gps_epoch.search(os.path.dirname( latest.path)).groupdict()['gpsepoch']) except (IndexError, AttributeError): pass else: while True: s, e = latest.segment if s >= gpsend: break # replace GPS time of file basename new = latest.path.replace('-%d-' % s, '-%d-' % e) # replace GPS epoch in dirname new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps]) if os.path.isfile(new): latest = CacheEntry.from_T050017(new) cache.append(latest) else: break # validate files existing and return cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache
def find_frames(ifo, frametype, gpsstart, gpsend, config=ConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection( host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend(dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `~glue.datafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `~glue.lal.Cache` a list of structured frame file descriptions matching the ifo and frametype requested """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection( host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend(dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache