def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower() == 'omicron': etg = '?micron' # construct search span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end =' ') gpscache = Cache(map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: gprint("%d found" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower() == 'omicron': etg = '?micron' # construct search span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end=' ') gpscache = Cache( map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: gprint("%d found" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False, **kwargs): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ # special case for KW if etg.lower() in ['kw', 'kleinewelle']: from .kw import find_dmt_cache ifo = channel.split(':')[0] kwargs.setdefault('extension', 'xml') kwargs.setdefault('check_files', True) return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs) elif etg.lower() == 'omega': from .omega import find_dmt_cache ifo = channel.split(':')[0] kwargs.setdefault('check_files', True) return find_dmt_cache(gpsstart, gpsend, ifo, **kwargs) elif etg.lower() == 'omicron': etg = '?micron' # construct search span = segments.segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = numpy.arange(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re.sub('-', '_', channel), etg.lower(), '[0-9]'*10)) # perform and cache results out = Cache() for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: sys.stdout.write("Searching %s..." % os.path.split(gpssearchpath)[0]) sys.stdout.flush() gpscache = Cache(map(CacheEntry.from_T050017, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) out.extend(gpscache.sieve(segment=span)) if verbose: sys.stdout.write(" %d found\n" % len(gpscache.sieve(segment=span))) out.sort(key=lambda e: e.path) return out
def find_contiguous(*caches): """Separate one or more caches into sets of contiguous caches Parameters ---------- *caches one or more :class:`~glue.lal.Cache` objects Returns ------- caches : `iter` of :class:`~glue.lal.Cache` an interable yielding each contiguous cache """ try: flat = flatten(*caches) except IndexError: flat = Cache() for segment in cache_segments(flat): yield flat.sieve(segment=segment)
def _get_timeseries_dict(channels, segments, config=None, cache=None, query=True, nds=None, frametype=None, multiprocess=True, return_=True, statevector=False, archive=True, datafind_error='raise', **ioargs): """Internal method to retrieve the data for a set of like-typed channels using the :meth:`TimeSeriesDict.read` accessor. """ channels = map(get_channel, channels) # set classes if statevector: ListClass = StateVectorList DictClass = StateVectorDict else: ListClass = TimeSeriesList DictClass = TimeSeriesDict # check we have a configparser if config is None: config = GWSummConfigParser() # read segments from global memory keys = dict((c.ndsname, make_globalv_key(c)) for c in channels) havesegs = reduce( operator.and_, (globalv.DATA.get(keys[channel.ndsname], ListClass()).segments for channel in channels)) new = segments - havesegs # get processes if multiprocess is True: nproc = count_free_cores() elif multiprocess is False: nproc = 1 else: nproc = multiprocess # read channel information filter_ = dict() resample = dict() dtype_ = dict() for channel in channels: try: filter_[channel.ndsname] = channel.filter except AttributeError: pass try: resample[channel] = float(channel.resample) except AttributeError: pass if channel.dtype is None: dtype_[channel] = ioargs.get('dtype') else: dtype_[channel] = channel.dtype # work out whether to use NDS or not if nds is None and cache is not None: nds = False elif nds is None: nds = 'LIGO_DATAFIND_SERVER' not in os.environ # read new data query &= (abs(new) > 0) if cache is not None: query &= len(cache) > 0 if query: for channel in channels: globalv.DATA.setdefault(keys[channel.ndsname], ListClass()) # open NDS connection if nds and config.has_option('nds', 'host'): host = config.get('nds', 'host') port = config.getint('nds', 'port') try: ndsconnection = nds2.connection(host, port) except RuntimeError as e: if 'SASL authentication' in str(e): from gwpy.io.nds import kinit kinit() ndsconnection = nds2.connection(host, port) else: raise frametype = source = 'nds' ndstype = channels[0].type elif nds: ndsconnection = None frametype = source = 'nds' ndstype = channels[0].type # or find frame type and check cache else: ifo = channels[0].ifo frametype = frametype or channels[0].frametype if frametype is not None and frametype.endswith('%s_M' % ifo): new = type(new)([s for s in new if abs(s) >= 60.]) elif frametype is not None and frametype.endswith('%s_T' % ifo): new = type(new)([s for s in new if abs(s) >= 1.]) if cache is not None: fcache = cache.sieve(ifos=ifo[0], description=frametype, exact_match=True) else: fcache = Cache() if (cache is None or len(fcache) == 0) and len(new): span = new.extent().protract(8) fcache = find_frames(ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) cachesegments = find_cache_segments(fcache) gaps = SegmentList([span]) - cachesegments if abs(gaps) and frametype == '%s_HOFT_C00' % ifo: f2 = '%s_DMT_C00' % ifo vprint(" Gaps discovered in aggregated h(t) type " "%s, checking %s\n" % (frametype, f2)) c2 = find_frames(ifo, f2, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) g2 = SegmentList([span]) - find_cache_segments(c2) if abs(g2) < abs(gaps): vprint(" Greater coverage with frametype %s\n" % f2) fcache = c2 frametype = f2 else: vprint(" No extra coverage with frametype %s\n" % f2) # parse discontiguous cache blocks and rebuild segment list cachesegments = find_cache_segments(fcache) new &= cachesegments source = 'frames' # set ctype if reading with framecpp if cache is None and frametype in ADC_TYPES and HAS_FRAMECPP: ioargs['type'] = 'adc' for channel in channels: channel.frametype = frametype # check whether each channel exists for all new times already qchannels = [] qresample = {} qdtype = {} for channel in channels: oldsegs = globalv.DATA.get(channel.ndsname, ListClass()).segments if abs(new - oldsegs) != 0: qchannels.append(channel) if channel in resample: qresample[channel] = resample[channel] qdtype[channel] = dtype_.get(channel, ioargs.get('dtype')) ioargs['dtype'] = qdtype # loop through segments, recording data for each if len(new) and nproc > 1: vprint(" Fetching data (from %s) for %d channels [%s]" % (source, len(qchannels), nds and ndstype or frametype)) for segment in new: # force reading integer-precision segments segment = type(segment)(int(segment[0]), int(segment[1])) if abs(segment) < 1: continue if nds: tsd = DictClass.fetch(qchannels, segment[0], segment[1], connection=ndsconnection, type=ndstype, **ioargs) else: # pad resampling if segment[1] == cachesegments[-1][1] and qresample: resamplepad = 8 if abs(segment) <= resamplepad: continue segment = type(segment)(segment[0], segment[1] - resamplepad) segcache = fcache.sieve( segment=segment.protract(resamplepad)) else: segcache = fcache.sieve(segment=segment) # set minute trend times modulo 60 from GPS 0 if (re.match('(?:(.*)_)?[A-Z]\d_M', str(frametype)) or (ifo == 'C1' and frametype == 'M')): segstart = int(segment[0]) // 60 * 60 segend = int(segment[1]) // 60 * 60 if segend >= segment[1]: segend -= 60 # and ignore segments shorter than 1 full average if (segend - segstart) < 60: continue segcache = segcache.sieve( segment=type(segment)(segstart, segend)) else: segstart, segend = map(float, segment) # pull filters out because they can break multiprocessing if nproc > 1: for c in qchannels: if c.ndsname in filter_: del c.filter # read data tsd = DictClass.read(segcache, qchannels, start=segstart, end=segend, nproc=nproc, resample=qresample, **ioargs) # put filters back for c in qchannels: if c.ndsname in filter_: c.filter = filter_[c.ndsname] for (channel, data) in tsd.iteritems(): key = keys[channel.ndsname] if (key in globalv.DATA and data.span in globalv.DATA[key].segments): continue if data.unit is None: data.unit = 'undef' for seg in globalv.DATA[key].segments: if seg.intersects(data.span): data = data.crop(*(data.span - seg)) break try: filt = filter_[channel.ndsname] except KeyError: pass else: # filter with function if callable(filt): try: data = filt(data) except TypeError as e: if 'Can only apply' in str(e): data.value[:] = filt(data.value) else: raise # filter with gain elif (isinstance(filt, tuple) and len(filt) == 3 and len(filt[0] + filt[1]) == 0): try: data *= filt[2] except TypeError: data = data * filt[2] # filter zpk elif isinstance(filt, tuple): data = data.filter(*filt) # filter fail else: raise ValueError("Cannot parse filter for %s: %r" % (channel.ndsname, filt)) if isinstance(data, StateVector) or ':GRD-' in str(channel): try: data.unit = units.dimensionless_unscaled except AttributeError: data._unit = units.dimensionless_unscaled if hasattr(channel, 'bits'): data.bits = channel.bits elif data.unit is None: data._unit = channel.unit # XXX: HACK for failing unit check if len(globalv.DATA[key]): data._unit = globalv.DATA[key][-1].unit # update channel type for trends if data.channel.type is None and (data.channel.trend is not None): if data.dt.to('s').value == 1: data.channel.type = 's-trend' elif data.dt.to('s').value == 60: data.channel.type = 'm-trend' # append and coalesce add_timeseries(data, key=key, coalesce=True) if multiprocess: vprint('.') if len(new): vprint("\n") if not return_: return # return correct data out = OrderedDict() for channel in channels: data = ListClass() if keys[channel.ndsname] not in globalv.DATA: out[channel.ndsname] = ListClass() else: for ts in globalv.DATA[keys[channel.ndsname]]: for seg in segments: if abs(seg) == 0 or abs(seg) < ts.dt.value: continue if ts.span.intersects(seg): common = map(float, ts.span & seg) cropped = ts.crop(*common, copy=False) if cropped.size: data.append(cropped) out[channel.ndsname] = data.coalesce() return out
def _get_timeseries_dict(channels, segments, config=None, cache=None, query=True, nds=None, frametype=None, multiprocess=True, return_=True, statevector=False, archive=True, datafind_error='raise', **ioargs): """Internal method to retrieve the data for a set of like-typed channels using the :meth:`TimeSeriesDict.read` accessor. """ channels = map(get_channel, channels) # set classes if statevector: ListClass = StateVectorList DictClass = StateVectorDict else: ListClass = TimeSeriesList DictClass = TimeSeriesDict # check we have a configparser if config is None: config = GWSummConfigParser() # read segments from global memory keys = dict((c.ndsname, make_globalv_key(c)) for c in channels) havesegs = reduce(operator.and_, (globalv.DATA.get(keys[channel.ndsname], ListClass()).segments for channel in channels)) new = segments - havesegs # get processes if multiprocess is True: nproc = count_free_cores() elif multiprocess is False: nproc = 1 else: nproc = multiprocess if globalv.VERBOSE and not multiprocess: verbose = ' ' else: verbose = False # read channel information filter_ = dict() resample = dict() dtype_ = dict() for channel in channels: try: filter_[channel.ndsname] = channel.filter except AttributeError: pass try: resample[channel] = float(channel.resample) except AttributeError: pass if channel.dtype is None: dtype_[channel] = ioargs.get('dtype') else: dtype_[channel] = channel.dtype # work out whether to use NDS or not if nds is None and cache is not None: nds = False elif nds is None: nds = 'LIGO_DATAFIND_SERVER' not in os.environ # read new data query &= (abs(new) > 0) if cache is not None: query &= len(cache) > 0 if query: for channel in channels: globalv.DATA.setdefault(keys[channel.ndsname], ListClass()) # open NDS connection if nds and config.has_option('nds', 'host'): host = config.get('nds', 'host') port = config.getint('nds', 'port') try: ndsconnection = nds2.connection(host, port) except RuntimeError as e: if 'SASL authentication' in str(e): from gwpy.io.nds import kinit kinit() ndsconnection = nds2.connection(host, port) else: raise frametype = source = 'nds' ndstype = channels[0].type elif nds: ndsconnection = None frametype = source = 'nds' ndstype = channels[0].type # or find frame type and check cache else: ifo = channels[0].ifo frametype = frametype or channels[0].frametype if frametype is not None and frametype.endswith('%s_M' % ifo): new = type(new)([s for s in new if abs(s) >= 60.]) elif frametype is not None and frametype.endswith('%s_T' % ifo): new = type(new)([s for s in new if abs(s) >= 1.]) #elif ((globalv.NOW - new[0][0]) < 86400 * 10 and # frametype == '%s_R' % ifo and # find_types(site=ifo[0], match='_C\Z')): # frametype = '%s_C' % ifo if cache is not None: fcache = cache.sieve(ifos=ifo[0], description=frametype, exact_match=True) else: fcache = Cache() if (cache is None or len(fcache) == 0) and len(new): span = new.extent().protract(8) fcache = find_frames(ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) if len(fcache) == 0 and frametype == '%s_R' % ifo: frametype = '%s_C' % ifo vprint(" Moving to backup frametype %s\n" % frametype) fcache = find_frames(ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) # parse discontiguous cache blocks and rebuild segment list cachesegments = find_cache_segments(fcache) new &= cachesegments source = 'frames' for channel in channels: channel.frametype = frametype # check whether each channel exists for all new times already qchannels = [] qresample = {} qdtype = {} for channel in channels: oldsegs = globalv.DATA.get(channel.ndsname, ListClass()).segments if abs(new - oldsegs) != 0: qchannels.append(channel) if channel in resample: qresample[channel] = resample[channel] qdtype[channel] = dtype_.get(channel, ioargs.get('dtype')) ioargs['dtype'] = qdtype # find channel type if not nds: ctype = set() for channel in qchannels: try: ctype.add(channel.ctype) except AttributeError: ctype.add(get_channel_type(channel)) if len(ctype) == 1: ctype = list(ctype)[0] else: ctype = None # loop through segments, recording data for each if len(new) and nproc > 1: vprint(" Fetching data (from %s) for %d channels [%s]" % (source, len(qchannels), nds and ndstype or frametype)) for segment in new: # force reading integer-precision segments segment = type(segment)(int(segment[0]), int(segment[1])) if abs(segment) < 1: continue if nds: tsd = DictClass.fetch(qchannels, segment[0], segment[1], connection=ndsconnection, type=ndstype, **ioargs) else: # pad resampling if segment[1] == cachesegments[-1][1] and qresample: resamplepad = 8 if abs(segment) <= resamplepad: continue segment = type(segment)(segment[0], segment[1] - resamplepad) segcache = fcache.sieve( segment=segment.protract(resamplepad)) else: segcache = fcache.sieve(segment=segment) # set minute trend times modulo 60 from GPS 0 if (re.match('(?:(.*)_)?[A-Z]\d_M', str(frametype)) or (ifo == 'C1' and frametype == 'M')): segstart = int(segment[0]) // 60 * 60 segend = int(segment[1]) // 60 * 60 if segend >= segment[1]: segend -= 60 # and ignore segments shorter than 1 full average if (segend - segstart) < 60: continue else: segstart, segend = map(float, segment) # pull filters out because they can break multiprocessing if nproc > 1: for c in qchannels: if c.ndsname in filter_: del c.filter # read data tsd = DictClass.read(segcache, qchannels, start=segstart, end=segend, type=ctype, nproc=nproc, resample=qresample, verbose=verbose, **ioargs) # put filters back for c in qchannels: if c.ndsname in filter_: c.filter = filter_[c.ndsname] for (channel, data) in tsd.iteritems(): key = keys[channel.ndsname] if (key in globalv.DATA and data.span in globalv.DATA[key].segments): continue if data.unit is None: data.unit = 'undef' for seg in globalv.DATA[key].segments: if seg.intersects(data.span): data = data.crop(*(data.span - seg)) break try: filt = filter_[channel.ndsname] except KeyError: pass else: # filter with function if callable(filt): try: data = filt(data) except TypeError as e: if 'Can only apply' in str(e): data.value[:] = filt(data.value) else: raise # filter with gain elif (isinstance(filt, tuple) and len(filt) == 3 and len(filt[0] + filt[1]) == 0): try: data *= filt[2] except TypeError: data = data * filt[2] # filter zpk elif isinstance(filt, tuple): data = data.filter(*filt) # filter fail else: raise ValueError("Cannot parse filter for %s: %r" % (channel.ndsname, filt)) if isinstance(data, StateVector) or ':GRD-' in str(channel): try: data.unit = units.dimensionless_unscaled except AttributeError: data._unit = units.dimensionless_unscaled if hasattr(channel, 'bits'): data.bits = channel.bits elif data.unit is None: data._unit = channel.unit # XXX: HACK for failing unit check if len(globalv.DATA[key]): data._unit = globalv.DATA[key][-1].unit # update channel type for trends if (data.channel.type is None and data.channel.trend is not None): if data.dt.to('s').value == 1: data.channel.type = 's-trend' elif data.dt.to('s').value == 60: data.channel.type = 'm-trend' # append and coalesce add_timeseries(data, key=key, coalesce=True) if multiprocess: vprint('.') if len(new): vprint("\n") if not return_: return # return correct data out = OrderedDict() for channel in channels: data = ListClass() if keys[channel.ndsname] not in globalv.DATA: out[channel.ndsname] = ListClass() else: for ts in globalv.DATA[keys[channel.ndsname]]: for seg in segments: if abs(seg) == 0 or abs(seg) < ts.dt.value: continue if ts.span.intersects(seg): common = map(float, ts.span & seg) cropped = ts.crop(*common, copy=False) if cropped.size: data.append(cropped) out[channel.ndsname] = data.coalesce() return out