def test_cache_segments(self): # check empty input sl = cache_segments() self.assertIsInstance(sl, SegmentList) self.assertEquals(len(sl), 0) cache, segs = self.make_cache() try: # check good cache sl = cache_segments(cache) self.assertNotEquals(sl, segs) self.assertEquals(sl, type(segs)(segs).coalesce()) # check bad cache os.remove(cache[0].path) sl = cache_segments(cache) self.assertEquals(sl, segs[1:]) # check cache with no existing files sl = cache_segments(cache[:1]) self.assertEquals(sl, SegmentList()) # check errors self.assertRaises(TypeError, cache_segments, blah='blah') self.assertRaises(ValueError, cache_segments, cache, on_missing='error') self.assertRaises(ValueError, cache_segments, cache, on_missing='blah') # clean up finally: self.destroy_cache(cache)
def find_best_frames(ifo, frametype, start, end, **kwargs): """Find frames for the given type, replacing with a better type if needed """ # find cache for this frametype cache = find_frames(ifo, frametype, start, end, **kwargs) # check for gaps in current cache span = SegmentList([Segment(start, end)]) gaps = span - cache_segments(cache) # if gaps and using aggregated h(t), check short files if abs(gaps) and frametype in SHORT_HOFT_TYPES: f2 = SHORT_HOFT_TYPES[frametype] vprint(" Gaps discovered in aggregated h(t) type " "%s, checking %s\n" % (frametype, f2)) kwargs['gaps'] = 'ignore' cache.extend(filter(lambda e: file_segment(e) in gaps, find_frames(ifo, f2, start, end, **kwargs))) new = int(abs(gaps - cache_segments(cache))) if new: vprint(" %ss extra coverage with frametype %s\n" % (new, f2)) else: vprint(" No extra coverage with frametype %s\n" % f2) return cache, frametype
def find_best_frames(ifo, frametype, start, end, **kwargs): """Find frames for the given type, replacing with a better type if needed """ # find cache for this frametype cache = find_frames(ifo, frametype, start, end, **kwargs) # check for gaps in current cache span = SegmentList([Segment(start, end)]) gaps = span - cache_segments(cache) # if gaps and using aggregated h(t), check short files if abs(gaps) and frametype in SHORT_HOFT_TYPES: f2 = SHORT_HOFT_TYPES[frametype] vprint(" Gaps discovered in aggregated h(t) type " "%s, checking %s\n" % (frametype, f2)) kwargs['gaps'] = 'ignore' cache.extend( filter(lambda e: file_segment(e) in gaps, find_frames(ifo, f2, start, end, **kwargs))) new = int(abs(gaps - cache_segments(cache))) if new: vprint(" %ss extra coverage with frametype %s\n" % (new, f2)) else: vprint(" No extra coverage with frametype %s\n" % f2) return cache, frametype
def test_find_contiguous(self): """Test :func:`gwpy.io.cache.find_contiguous` """ a, segs = self.make_cache() segs.coalesce() for i, cache in enumerate(io_cache.find_contiguous(a)): io_cache.cache_segments(cache).extent() == segs[i] assert not list(io_cache.find_contiguous())
def test_cache_segments(self): # check empty input sl = cache_segments() self.assertIsInstance(sl, SegmentList) self.assertEquals(len(sl), 0) cache, segs = self.make_cache() segs.coalesce() sl = cache_segments(cache) self.assertEquals(sl, segs) sl = cache_segments(cache[:2], cache[2:]) self.assertEquals(sl, segs)
def test_cache_segments(self): """Test :func:`gwpy.io.cache.cache_segments` """ # check empty input sl = io_cache.cache_segments() assert isinstance(sl, SegmentList) assert len(sl) == 0 # check simple cache cache, segs = self.make_cache() segs.coalesce() sl = io_cache.cache_segments(cache) assert sl == segs # check multiple caches produces the same result sl = io_cache.cache_segments(cache[:2], cache[2:]) assert sl == segs
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs): """Read a table of events from a cache This function is mainly meant for use from the `get_triggers` method Parameters ---------- cache : :class:`glue.lal.Cache` the formatted list of files to read segments : `~gwpy.segments.SegmentList` the list of segments to read etg : `str` the name of the trigger generator that created the files nproc : `int`, optional the number of parallel processes to use when reading **kwargs other keyword arguments are passed to the `EventTable.read` or `{tableclass}.read` methods Returns ------- table : `~gwpy.table.EventTable`, `None` a table of events, or `None` if the cache has no overlap with the segments """ if isinstance(cache, Cache): cache = cache.sieve(segmentlist=segments) cache = cache.checkfilesexist()[0] cache.sort(key=lambda x: x.segment[0]) cache = cache.pfnlist() # some readers only like filenames else: cache = [urlparse(url).path for url in cache] if etg == 'pycbc_live': # remove empty HDF5 files cache = filter_pycbc_live_files(cache, ifo=kwargs['ifo']) if len(cache) == 0: return # read triggers table = EventTable.read(cache, **kwargs) # store read keywords in the meta table if timecolumn: table.meta['timecolumn'] = timecolumn # get back from cache entry if isinstance(cache, CacheEntry): cache = Cache([cache]) # append new events to existing table try: csegs = cache_segments(cache) & segments except (AttributeError, TypeError, ValueError): csegs = SegmentList() table.meta['segments'] = csegs if timecolumn: # already filtered on-the-fly return table # filter now return keep_in_segments(table, segments, etg)
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs): """Read a table of events from a cache This function is mainly meant for use from the `get_triggers` method Parameters ---------- cache : :class:`glue.lal.Cache` the formatted list of files to read segments : `~gwpy.segments.SegmentList` the list of segments to read etg : `str` the name of the trigger generator that created the files nproc : `int`, optional the number of parallel processes to use when reading **kwargs other keyword arguments are passed to the `EventTable.read` or `{tableclass}.read` methods Returns ------- table : `~gwpy.table.EventTable`, `None` a table of events, or `None` if the cache has no overlap with the segments """ if isinstance(cache, Cache): cache = cache.sieve(segmentlist=segments) cache = cache.checkfilesexist()[0] cache.sort(key=lambda x: x.segment[0]) if etg == 'pycbc_live': # remove empty HDF5 files cache = type(cache)( filter_pycbc_live_files(cache, ifo=kwargs['ifo'])) # if no files, skip if len(cache) == 0: return # use multiprocessing except for ascii reading # (since astropy doesn't allow it) if kwargs.get('format', 'none').startswith('ascii.'): cache = cache.pfnlist() else: kwargs['nproc'] = nproc if len(cache) == 1: cache = cache[0] # read triggers table = EventTable.read(cache, **kwargs) if timecolumn: table.meta['timecolumn'] = timecolumn # get back from cache entry if isinstance(cache, CacheEntry): cache = Cache([cache]) # append new events to existing table try: csegs = cache_segments(cache) except (AttributeError, TypeError): csegs = SegmentList() table.meta['segments'] = csegs return keep_in_segments(table, segments, etg)
def read_and_cache_events(channel, etg, cache=None, trigfind_kw={}, **read_kw): cfile = create_path(channel) # read existing cached triggers and work out new segments to query if args.append and cfile.is_file(): previous = DataQualityFlag.read( str(cfile), path='segments', format='hdf5', ).coalesce() new = analysis - previous else: new = analysis.copy() # get cache of files if cache is None: cache = find_trigger_files(channel, etg, new.active, **trigfind_kw) else: cache = list( filter( lambda e: new.active.intersects_segment(file_segment(e)), cache, )) # restrict 'active' segments to when we have data try: new.active &= cache_segments(cache) except IndexError: new.active = type(new.active)() # find new triggers try: trigs = get_triggers(channel, etg, new.active, cache=cache, raw=True, **read_kw) # catch error and continue except ValueError as e: warnings.warn('%s: %s' % (type(e).__name__, str(e))) else: path = write_events(channel, trigs, new) try: return path, len(trigs) except TypeError: # None return
def _get_timeseries_dict(channels, segments, config=None, cache=None, query=True, nds=None, frametype=None, nproc=1, return_=True, statevector=False, archive=True, datafind_error='raise', dtype=None, **ioargs): """Internal method to retrieve the data for a set of like-typed channels using the :meth:`TimeSeriesDict.read` accessor. """ channels = list(map(get_channel, channels)) # set classes if statevector: ListClass = StateVectorList DictClass = StateVectorDict else: ListClass = TimeSeriesList DictClass = TimeSeriesDict # check we have a configparser if config is None: config = GWSummConfigParser() # read segments from global memory keys = dict((c.ndsname, make_globalv_key(c)) for c in channels) havesegs = reduce( operator.and_, (globalv.DATA.get(keys[channel.ndsname], ListClass()).segments for channel in channels)) new = segments - havesegs # read channel information filter_ = dict() resample = dict() dtype_ = dict() for channel in channels: name = str(channel) try: filter_[name] = channel.filter except AttributeError: pass try: resample[name] = float(channel.resample) except AttributeError: pass if channel.dtype is not None: dtype_[name] = channel.dtype elif dtype is not None: dtype_[name] = dtype # work out whether to use NDS or not if nds is None and cache is not None: nds = False elif nds is None: nds = 'LIGO_DATAFIND_SERVER' not in os.environ # read new data query &= (abs(new) > 0) if cache is not None: query &= len(cache) > 0 if query: for channel in channels: globalv.DATA.setdefault(keys[channel.ndsname], ListClass()) ifo = channels[0].ifo # open NDS connection if nds: if config.has_option('nds', 'host'): host = config.get('nds', 'host') port = config.getint('nds', 'port') ndsconnection = io_nds2.connect(host, port) else: ndsconnection = None frametype = source = 'nds' ndstype = channels[0].type # get NDS channel segments if ndsconnection is not None and ndsconnection.get_protocol() > 1: span = list(map(int, new.extent())) avail = io_nds2.get_availability(channels, *span, connection=ndsconnection) new &= avail.intersection(avail.keys()) # or find frame type and check cache else: frametype = frametype or channels[0].frametype new = exclude_short_trend_segments(new, ifo, frametype) if cache is not None: fcache = sieve_cache(cache, ifo=ifo[0], tag=frametype) else: fcache = [] if (cache is None or len(fcache) == 0) and len(new): span = new.extent() fcache, frametype = find_best_frames(ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) # parse discontiguous cache blocks and rebuild segment list new &= cache_segments(fcache) source = 'files' # if reading Virgo h(t) GWF data, filter out files that don't # contain the channel (Virgo state-vector only) _names = set(map(str, channels)) _virgohoft = _names.intersection(VIRGO_HOFT_CHANNELS) if _virgohoft: vprint(" Determining available segments for " "Virgo h(t) data...") new &= data_segments(fcache, _virgohoft.pop()) # set channel type if reading with frameCPP if fcache and all_adc(fcache): ioargs['type'] = 'adc' # store frametype for display in Channel Information tables for channel in channels: channel.frametype = frametype # check whether each channel exists for all new times already qchannels = [] for channel in channels: oldsegs = globalv.DATA.get(keys[channel.ndsname], ListClass()).segments if abs(new - oldsegs) != 0 and nds: qchannels.append(channel.ndsname) elif abs(new - oldsegs) != 0: qchannels.append(str(channel)) # loop through segments, recording data for each if len(new): vprint( " Fetching data (from %s) for %d channels [%s]:\n" % (source, len(qchannels), nds and ndstype or frametype or '')) vstr = " [{0[0]}, {0[1]})" for segment in new: # force reading integer-precision segments segment = type(segment)(int(segment[0]), int(segment[1])) if abs(segment) < 1: continue # reset to minute trend sample times if frame_trend_type(ifo, frametype) == 'minute': segment = Segment(*io_nds2.minute_trend_times(*segment)) if abs(segment) < 60: continue if nds: # fetch tsd = DictClass.fetch(qchannels, segment[0], segment[1], connection=ndsconnection, type=ndstype, verbose=vstr.format(segment), **ioargs) else: # read # NOTE: this sieve explicitly casts our segment to # ligo.segments.segment to prevent `TypeError` from # a mismatch with ligo.segments.segment segcache = sieve_cache(fcache, segment=segment) segstart, segend = map(float, segment) tsd = DictClass.read(segcache, qchannels, start=segstart, end=segend, nproc=nproc, verbose=vstr.format(segment), **ioargs) vprint(" post-processing...\n") # apply type casting (copy=False means same type just returns) for chan, ts in tsd.items(): tsd[chan] = ts.astype(dtype_.get(chan, ts.dtype), casting='unsafe', copy=False) # apply resampling tsd = resample_timeseries_dict(tsd, nproc=1, **resample) # post-process for c, data in tsd.items(): channel = get_channel(c) key = keys[channel.ndsname] if (key in globalv.DATA and data.span in globalv.DATA[key].segments): continue if data.unit is None: data.unit = 'undef' for i, seg in enumerate(globalv.DATA[key].segments): if seg in data.span: # new data completely covers existing segment # (and more), so just remove the old stuff globalv.DATA[key].pop(i) break elif seg.intersects(data.span): # new data extends existing segment, so only keep # the really new stuff data = data.crop(*(data.span - seg)) break # filter try: filt = filter_[str(channel)] except KeyError: pass else: data = filter_timeseries(data, filt) if isinstance(data, StateVector) or ':GRD-' in str(channel): data.override_unit(units.dimensionless_unscaled) if hasattr(channel, 'bits'): data.bits = channel.bits elif data.unit is None: data.override_unit(channel.unit) # update channel type for trends if data.channel.type is None and (data.channel.trend is not None): if data.dt.to('s').value == 1: data.channel.type = 's-trend' elif data.dt.to('s').value == 60: data.channel.type = 'm-trend' # append and coalesce add_timeseries(data, key=key, coalesce=True) # rebuilt global channel list with new parameters update_channel_params() if not return_: return return locate_data(channels, segments, list_class=ListClass)
def _get_timeseries_dict(channels, segments, config=None, cache=None, query=True, nds=None, frametype=None, nproc=1, return_=True, statevector=False, archive=True, datafind_error='raise', dtype=None, **ioargs): """Internal method to retrieve the data for a set of like-typed channels using the :meth:`TimeSeriesDict.read` accessor. """ channels = list(map(get_channel, channels)) # set classes if statevector: ListClass = StateVectorList DictClass = StateVectorDict else: ListClass = TimeSeriesList DictClass = TimeSeriesDict # check we have a configparser if config is None: config = GWSummConfigParser() # read segments from global memory keys = dict((c.ndsname, make_globalv_key(c)) for c in channels) havesegs = reduce(operator.and_, (globalv.DATA.get(keys[channel.ndsname], ListClass()).segments for channel in channels)) new = segments - havesegs # read channel information filter_ = dict() resample = dict() dtype_ = dict() for channel in channels: name = str(channel) try: filter_[name] = channel.filter except AttributeError: pass try: resample[name] = float(channel.resample) except AttributeError: pass if channel.dtype is not None: dtype_[name] = channel.dtype elif dtype is not None: dtype_[name] = dtype # work out whether to use NDS or not if nds is None and cache is not None: nds = False elif nds is None: nds = 'LIGO_DATAFIND_SERVER' not in os.environ # read new data query &= (abs(new) > 0) if cache is not None: query &= len(cache) > 0 if query: for channel in channels: globalv.DATA.setdefault(keys[channel.ndsname], ListClass()) ifo = channels[0].ifo # open NDS connection if nds: if config.has_option('nds', 'host'): host = config.get('nds', 'host') port = config.getint('nds', 'port') ndsconnection = io_nds2.connect(host, port) else: ndsconnection = None frametype = source = 'nds' ndstype = channels[0].type # get NDS channel segments if ndsconnection is not None and ndsconnection.get_protocol() > 1: span = list(map(int, new.extent())) avail = io_nds2.get_availability( channels, *span, connection=ndsconnection ) new &= avail.intersection(avail.keys()) # or find frame type and check cache else: frametype = frametype or channels[0].frametype new = exclude_short_trend_segments(new, ifo, frametype) if cache is not None: fcache = sieve_cache(cache, ifo=ifo[0], tag=frametype) else: fcache = [] if (cache is None or len(fcache) == 0) and len(new): span = new.extent() fcache, frametype = find_best_frames( ifo, frametype, span[0], span[1], config=config, gaps='ignore', onerror=datafind_error) # parse discontiguous cache blocks and rebuild segment list new &= cache_segments(fcache) source = 'files' # if reading Virgo h(t) GWF data, filter out files that don't # contain the channel (Virgo state-vector only) _names = set(map(str, channels)) _virgohoft = _names.intersection(VIRGO_HOFT_CHANNELS) if _virgohoft: vprint(" Determining available segments for " "Virgo h(t) data...") new &= data_segments(fcache, _virgohoft.pop()) # set channel type if reading with frameCPP if fcache and all_adc(fcache): ioargs['type'] = 'adc' # store frametype for display in Channel Information tables for channel in channels: channel.frametype = frametype # check whether each channel exists for all new times already qchannels = [] for channel in channels: oldsegs = globalv.DATA.get(keys[channel.ndsname], ListClass()).segments if abs(new - oldsegs) != 0 and nds: qchannels.append(channel.ndsname) elif abs(new - oldsegs) != 0: qchannels.append(str(channel)) # loop through segments, recording data for each if len(new): vprint(" Fetching data (from %s) for %d channels [%s]:\n" % (source, len(qchannels), nds and ndstype or frametype or '')) vstr = " [{0[0]}, {0[1]})" for segment in new: # force reading integer-precision segments segment = type(segment)(int(segment[0]), int(segment[1])) if abs(segment) < 1: continue # reset to minute trend sample times if frame_trend_type(ifo, frametype) == 'minute': segment = Segment(*io_nds2.minute_trend_times(*segment)) if abs(segment) < 60: continue if nds: # fetch tsd = DictClass.fetch(qchannels, segment[0], segment[1], connection=ndsconnection, type=ndstype, verbose=vstr.format(segment), **ioargs) else: # read # NOTE: this sieve explicitly casts our segment to # ligo.segments.segment to prevent `TypeError` from # a mismatch with ligo.segments.segment segcache = sieve_cache(fcache, segment=segment) segstart, segend = map(float, segment) tsd = DictClass.read(segcache, qchannels, start=segstart, end=segend, nproc=nproc, verbose=vstr.format(segment), **ioargs) vprint(" post-processing...\n") # apply type casting (copy=False means same type just returns) for chan, ts in tsd.items(): tsd[chan] = ts.astype(dtype_.get(chan, ts.dtype), casting='unsafe', copy=False) # apply resampling tsd = resample_timeseries_dict(tsd, nproc=1, **resample) # post-process for c, data in tsd.items(): channel = get_channel(c) key = keys[channel.ndsname] if (key in globalv.DATA and data.span in globalv.DATA[key].segments): continue if data.unit is None: data.unit = 'undef' for i, seg in enumerate(globalv.DATA[key].segments): if seg in data.span: # new data completely covers existing segment # (and more), so just remove the old stuff globalv.DATA[key].pop(i) break elif seg.intersects(data.span): # new data extends existing segment, so only keep # the really new stuff data = data.crop(*(data.span - seg)) break # filter try: filt = filter_[str(channel)] except KeyError: pass else: data = filter_timeseries(data, filt) if isinstance(data, StateVector) or ':GRD-' in str(channel): data.override_unit(units.dimensionless_unscaled) if hasattr(channel, 'bits'): data.bits = channel.bits elif data.unit is None: data.override_unit(channel.unit) # update channel type for trends if data.channel.type is None and ( data.channel.trend is not None): if data.dt.to('s').value == 1: data.channel.type = 's-trend' elif data.dt.to('s').value == 60: data.channel.type = 'm-trend' # append and coalesce add_timeseries(data, key=key, coalesce=True) # rebuilt global channel list with new parameters update_channel_params() if not return_: return return locate_data(channels, segments, list_class=ListClass)
def get_triggers(channel, etg, segments, config=GWSummConfigParser(), cache=None, columns=None, query=True, multiprocess=False, ligolwtable=None, return_=True): """Read a table of transient event triggers for a given channel. """ key = '%s,%s' % (str(channel), etg.lower()) # convert input segments to a segmentlist (for convenience) if isinstance(segments, DataQualityFlag): segments = segments.active segments = SegmentList(segments) # get processes if multiprocess is True: nproc = count_free_cores() elif multiprocess is False: nproc = 1 else: nproc = multiprocess # find LIGO_LW table for this ETG try: TableClass = get_etg_table(etg) except KeyError: TableClass = None # work out columns if columns is None: try: columns = config.get(etg, 'columns').split(',') except (NoSectionError, NoOptionError): columns = None # read segments from global memory try: havesegs = globalv.TRIGGERS[key].segments except KeyError: new = segments else: new = segments - havesegs # read new triggers if query and abs(new) != 0: ntrigs = 0 vprint(" Grabbing %s triggers for %s" % (etg, str(channel))) # store read kwargs kwargs = get_etg_read_kwargs(config, etg, exclude=['columns']) kwargs['columns'] = columns if etg.lower().replace('-', '_') in ['cwb', 'pycbc_live']: kwargs['ifo'] = get_channel(channel).ifo if 'format' not in kwargs and 'ahope' not in etg.lower(): kwargs['format'] = etg.lower() # set up ligolw options if needed if TableClass is not None: contenthandler = get_partial_contenthandler(TableClass) lsctables.use_in(contenthandler) # loop over segments for segment in new: # find trigger files if cache is None and etg.lower() == 'hacr': raise NotImplementedError("HACR parsing has not been " "implemented.") if cache is None and etg.lower() in ['kw', 'kleinewelle']: kwargs['filt'] = lambda t: t.channel == str(channel) if cache is None: try: segcache = trigfind.find_trigger_urls(str(channel), etg, segment[0], segment[1]) except ValueError as e: warnings.warn("Caught %s: %s" % (type(e).__name__, str(e))) continue else: for regex in TRIGFIND_FORMAT: if regex.match(etg): kwargs['format'] = TRIGFIND_FORMAT[regex] if TRIGFIND_FORMAT[regex] in lsctables.TableByName: kwargs['get_as_columns'] = True break if etg.lower() == 'omega': kwargs.setdefault('format', 'omega') else: kwargs.setdefault('format', 'ligolw') else: segcache = cache if isinstance(segcache, Cache): segcache = segcache.sieve(segment=segment) segcache = segcache.checkfilesexist()[0] segcache.sort(key=lambda x: x.segment[0]) if etg == 'pycbc_live': # remove empty HDF5 files segcache = type(segcache)( filter_pycbc_live_files(segcache, ifo=kwargs['ifo'])) # if no files, skip if len(segcache) == 0: continue # read triggers if kwargs.get('format', None) == 'ligolw': kwargs['contenthandler'] = contenthandler try: # try directly reading a numpy.recarray table = GWRecArray.read(segcache, nproc=nproc, **kwargs) except Exception as e: # back up to LIGO_LW if TableClass is not None and 'No reader' in str(e): try: table = TableClass.read(segcache, **kwargs) except Exception: raise e else: table = table.to_recarray(get_as_columns=True) else: raise # append new events to existing table try: csegs = cache_segments(segcache) except AttributeError: csegs = SegmentList() table.segments = csegs t2 = keep_in_segments(table, SegmentList([segment]), etg) add_triggers(t2, key, csegs) ntrigs += len(t2) vprint(".") vprint(" | %d events read\n" % ntrigs) # if asked to read triggers, but didn't actually read any, # create an empty table so that subsequent calls don't raise KeyErrors if query and key not in globalv.TRIGGERS: if TableClass is not None: tab = lsctables.New(TableClass, columns=columns).to_recarray( get_as_columns=True) else: tab = GWRecArray((0,), dtype=[(c, float) for c in columns]) tab.segments = SegmentList() add_triggers(tab, key, tab.segments) # work out time function if return_: return keep_in_segments(globalv.TRIGGERS[key], segments, etg) else: return
def get_triggers(channel, etg, segments, cache=None, snr=None, frange=None, raw=False, trigfind_kwargs={}, **read_kwargs): """Get triggers for the given channel """ etg = _sanitize_name(etg) # format arguments try: readfmt = read_kwargs.pop("format", DEFAULT_FORMAT[etg]) except KeyError: raise ValueError("unsupported ETG {!r}".format(etg)) trigfind_kwargs, read_kwargs = _format_params(channel, etg, readfmt, trigfind_kwargs, read_kwargs) # find triggers if cache is None: cache = find_trigger_files(channel, etg, segments, **trigfind_kwargs) # read files tables = [] for segment in segments: segaslist = SegmentList([segment]) segcache = io_cache.sieve(cache, segment=segment) # try and work out if cache overextends segment (so we need to crop) cachesegs = io_cache.cache_segments(segcache) outofbounds = abs(cachesegs - segaslist) if segcache: if len(segcache) == 1: # just pass the single filename segcache = segcache[0] new = EventTable.read(segcache, **read_kwargs) new.meta = {k: new.meta[k] for k in TABLE_META if new.meta.get(k)} if outofbounds: new = new[in_segmentlist(new[new.dtype.names[0]], segaslist)] tables.append(new) if len(tables): table = vstack_tables(tables) else: table = EventTable( names=read_kwargs.get('columns', ['time', 'frequency', 'snr'])) # parse time, frequency-like and snr-like column names columns = table.dtype.names tcolumn = columns[0] fcolumn = columns[1] scolumn = columns[2] # filter keep = numpy.ones(len(table), dtype=bool) if snr is not None: keep &= table[scolumn] >= snr if frange is not None: keep &= table[fcolumn] >= frange[0] keep &= table[fcolumn] < frange[1] table = table[keep] # return basic table if 'raw' if raw: return table # rename time column so that all tables match in at least that if tcolumn != "time": table.rename_column(tcolumn, 'time') # add channel column to identify all triggers table.add_column( table.Column(data=numpy.repeat(channel, len(table)), name='channel')) table.sort('time') return table
def main(args=None): """Run the online Guardian node visualization tool """ parser = create_parser() args = parser.parse_args(args=args) fec_map = args.fec_map simulink = args.simulink daqsvn = args.daqsvn or ('https://daqsvn.ligo-la.caltech.edu/websvn/' 'listing.php?repname=daq_maps') if args.ifo == 'H1': if not fec_map: fec_map = 'https://lhocds.ligo-wa.caltech.edu/exports/detchar/fec/' if not simulink: simulink = 'https://lhocds.ligo-wa.caltech.edu/daq/simulink/' if args.ifo == 'L1': if not fec_map: fec_map = 'https://llocds.ligo-la.caltech.edu/exports/detchar/fec/' if not simulink: simulink = 'https://llocds.ligo-la.caltech.edu/daq/simulink/' span = Segment(args.gpsstart, args.gpsend) # let's go LOGGER.info('{} Overflows {}-{}'.format(args.ifo, int(args.gpsstart), int(args.gpsend))) # get segments if args.state_flag: state = DataQualityFlag.query(args.state_flag, int(args.gpsstart), int(args.gpsend), url=const.DEFAULT_SEGMENT_SERVER) tmp = type(state.active)() for i, seg in enumerate(state.active): if abs(seg) < args.segment_end_pad: continue tmp.append(type(seg)(seg[0], seg[1] - args.segment_end_pad)) state.active = tmp.coalesce() statea = state.active else: statea = SegmentList([span]) if not args.output_file: duration = abs(span) args.output_file = ('%s-OVERFLOWS-%d-%d.h5' % (args.ifo, int(args.gpsstart), duration)) LOGGER.debug("Set default output file as %s" % args.output_file) # set up container overflows = DataQualityDict() # prepare data access if args.nds: from gwpy.io import nds2 as io_nds2 host, port = args.nds.rsplit(':', 1) ndsconnection = io_nds2.connect(host, port=int(port)) if ndsconnection.get_protocol() == 1: cachesegs = SegmentList( [Segment(int(args.gpsstart), int(args.gpsend))]) else: cachesegs = io_nds2.get_availability( ['{0}:FEC-1_DAC_OVERFLOW_ACC_0_0'.format(args.ifo)], int(args.gpsstart), int(args.gpsend), ) else: # get frame cache cache = gwdatafind.find_urls(args.ifo[0], args.frametype, int(args.gpsstart), int(args.gpsend)) cachesegs = statea & cache_segments(cache) flag_desc = "ADC/DAC Overflow indicated by {0}" # get channel and find overflows for dcuid in args.dcuid: LOGGER.info("Processing DCUID %d" % dcuid) channel = daq.ligo_accum_overflow_channel(dcuid, args.ifo) overflows[channel] = DataQualityFlag(channel, known=cachesegs) if args.deep: LOGGER.debug(" -- Getting list of overflow channels") try: channels = daq.ligo_model_overflow_channels(dcuid, args.ifo, args.frametype, gpstime=span[0], nds=args.nds) except IndexError: # no frame found for GPS start, try GPS end channels = daq.ligo_model_overflow_channels(dcuid, args.ifo, args.frametype, gpstime=span[-1]) for chan in channels: # set up flags early overflows[chan] = DataQualityFlag( chan, known=cachesegs, description=flag_desc.format(chan), isgood=False, ) LOGGER.debug(" -- %d channels found" % len(channel)) for seg in cachesegs: LOGGER.debug(" -- Processing {}-{}".format(*seg)) if args.nds: read_kw = dict(connection=ndsconnection) else: read_kw = dict(source=cache, nproc=args.nproc) msg = "Reading ACCUM_OVERFLOW data:".rjust(30) data = get_data(channel, seg[0], seg[1], pad=0., verbose=msg, **read_kw) new = daq.find_overflow_segments( data, cumulative=True, ) overflows[channel] += new LOGGER.info(" -- {} overflows found".format(len(new.active))) if not new.active: continue # go deep! for s, e in tqdm.tqdm(new.active.protract(2), unit='ovfl', desc='Going deep'.rjust(30)): data = get_data(channels, s, e, **read_kw) for ch in channels: try: overflows[ch] += daq.find_overflow_segments( data[ch], cumulative=True, ) except KeyError: warnings.warn("Skipping {}".format(ch), UserWarning) continue LOGGER.debug(" -- Search complete") # write output LOGGER.info("Writing segments to %s" % args.output_file) table = table_from_segments( overflows, sngl_burst=args.output_file.endswith((".xml", ".xml.gz")), ) if args.integer_segments: for key in overflows: overflows[key] = overflows[key].round() if args.output_file.endswith((".h5", "hdf", ".hdf5")): with h5py.File(args.output_file, "w") as h5f: table.write(h5f, path="triggers") overflows.write(h5f, path="segments") else: table.write(args.output_file, overwrite=True) overflows.write(args.output_file, overwrite=True, append=True) # write HTML if args.html: # get base path base = os.path.dirname(args.html) os.chdir(base) if args.plot: args.plot = os.path.curdir if args.output_file: args.output_file = os.path.relpath(args.output_file, os.path.dirname(args.html)) if os.path.basename(args.html) == 'index.html': links = [ '%d-%d' % (int(args.gpsstart), int(args.gpsend)), ('Parameters', '#parameters'), ('Segments', [('Overflows', '#overflows')]), ('Results', '#results'), ] if args.state_flag: links[2][1].insert(0, ('State flag', '#state-flag')) (brand, class_) = htmlio.get_brand(args.ifo, 'Overflows', args.gpsstart) navbar = htmlio.navbar(links, class_=class_, brand=brand) page = htmlio.new_bootstrap_page( title='%s Overflows | %d-%d' % (args.ifo, int(args.gpsstart), int(args.gpsend)), navbar=navbar) else: page = htmlio.markup.page() page.div(class_='container') # -- header page.div(class_='pb-2 mt-3 mb-2 border-bottom') page.h1('%s ADC/DAC Overflows: %d-%d' % (args.ifo, int(args.gpsstart), int(args.gpsend))) page.div.close() # -- paramters content = [('DCUIDs', ' '.join(map(str, args.dcuid)))] if daqsvn: content.append(('FEC configuration', ( '<a href="{0}" target="_blank" title="{1} FEC configuration">' '{0}</a>').format(daqsvn, args.ifo))) if fec_map: content.append( ('FEC map', '<a href="{0}" target="_blank" title="{1} FEC ' 'map">{0}</a>'.format(fec_map, args.ifo))) if simulink: content.append( ('Simulink models', '<a href="{0}" target="_blank" title="{1} ' 'Simulink models">{0}</a>'.format(simulink, args.ifo))) page.h2('Parameters', class_='mt-4 mb-4', id_='parameters') page.div(class_='row') page.div(class_='col-md-9 col-sm-12') page.add( htmlio.parameter_table(content, start=args.gpsstart, end=args.gpsend, flag=args.state_flag)) page.div.close() # col-md-9 col-sm-12 # link to summary file if args.output_file: ext = ('HDF' if args.output_file.endswith( (".h5", "hdf", ".hdf5")) else 'XML') page.div(class_='col-md-3 col-sm-12') page.add( htmlio.download_btn( [('Segments ({})'.format(ext), args.output_file)], btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(), )) page.div.close() # col-md-3 col-sm-12 page.div.close() # row # -- command-line page.h5('Command-line:') page.add(htmlio.get_command_line(about=False, prog=PROG)) # -- segments page.h2('Segments', class_='mt-4', id_='segments') # give contextual information msg = ("This analysis searched for digital-to-analogue (DAC) or " "analogue-to-digital (ADC) conversion overflows in the {0} " "real-time controls system. ").format( SITE_MAP.get(args.ifo, 'LIGO')) if args.deep: msg += ( "A hierarchichal search was performed, with one cumulative " "overflow counter checked per front-end controller (FEC). " "For those models that indicated an overflow, the card- and " "slot-specific channels were then checked. ") msg += ( "Consant overflow is shown as yellow, while transient overflow " "is shown as red. If a data-quality flag was loaded for this " "analysis, it will be displayed in green.") page.add(htmlio.alert(msg, context=args.ifo.lower())) # record state segments if args.state_flag: page.h3('State flag', class_='mt-3', id_='state-flag') page.div(id_='accordion1') page.add( htmlio.write_flag_html(state, span, 'state', parent='accordion1', context='success', plotdir=args.plot, facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen', known={ 'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4, })) page.div.close() # record overflow segments if sum(abs(s.active) for s in overflows.values()): page.h3('Overflows', class_='mt-3', id_='overflows') page.div(id_='accordion2') for i, (c, flag) in enumerate(list(overflows.items())): if abs(flag.active) == 0: continue if abs(flag.active) == abs(cachesegs): context = 'warning' else: context = 'danger' try: channel = cds.get_real_channel(flag.name) except Exception: title = '%s [%d]' % (flag.name, len(flag.active)) else: title = '%s (%s) [%d]' % (flag.name, channel, len(flag.active)) page.add( htmlio.write_flag_html(flag, span, i, parent='accordion2', title=title, context=context, plotdir=args.plot)) page.div.close() else: page.add( htmlio.alert('No overflows were found in this analysis', context=args.ifo.lower(), dismiss=False)) # -- results table page.h2('Results summary', class_='mt-4', id_='results') page.table(class_='table table-striped table-hover') # write table header page.thead() page.tr() for header in ['Channel', 'Connected signal', 'Num. overflows']: page.th(header) page.thead.close() # write body page.tbody() for c, seglist in overflows.items(): t = abs(seglist.active) if t == 0: page.tr() elif t == abs(cachesegs): page.tr(class_='table-warning') else: page.tr(class_='table-danger') page.td(c) try: page.td(cds.get_real_channel(str(c))) except Exception: page.td() page.td(len(seglist.active)) page.tr.close() page.tbody.close() page.table.close() # -- close and write htmlio.close_page(page, args.html) LOGGER.info("HTML written to %s" % args.html)
def main(args=None): """Run the zero-crossing counter tool """ parser = create_parser() args = parser.parse_args(args=args) span = Segment(args.gpsstart, args.gpsend) LOGGER.info('-- Processing channel %s over span %d - %d' % (args.channel, args.gpsstart, args.gpsend)) if args.state_flag: state = DataQualityFlag.query( args.state_flag, int(args.gpsstart), int(args.gpsend), url=const.DEFAULT_SEGMENT_SERVER, ) statea = state.active else: statea = SegmentList([span]) duration = abs(span) # initialize output files for each threshold and store them in a dict outfiles = {} for thresh in args.threshold: outfiles[str(thresh)] = (os.path.join( args.output_path, '%s_%s_DAC-%d-%d.h5' % (args.channel.replace('-', '_').replace(':', '-'), str( int(thresh)).replace('-', 'n'), int(args.gpsstart), duration))) # get frame cache cache = gwdatafind.find_urls(args.ifo[0], args.frametype, int(args.gpsstart), int(args.gpsend)) cachesegs = statea & cache_segments(cache) if not os.path.exists(args.output_path): os.makedirs(args.output_path) # initialize a ligolw table for each threshold and store them in a dict names = ("time", "frequency", "snr") dtypes = ("f8", ) * len(names) tables = {} for thresh in args.threshold: tables[str(thresh)] = EventTable( names=names, dtype=dtypes, meta={"channel": args.channel}, ) # for each science segment, read in the data from frames, check for # threshold crossings, and if the rate of crossings is less than # rate_thresh, write to a sngl_burst table for seg in cachesegs: LOGGER.debug("Processing {}:".format(seg)) c = sieve_cache(cache, segment=seg) if not c: LOGGER.warning(" No {} data files for this segment, " "skipping".format(args.frametype)) continue data = get_data(args.channel, seg[0], seg[1], nproc=args.nproc, source=c, verbose="Reading data:".rjust(30)) for thresh in args.threshold: times = find_crossings(data, thresh) rate = float(times.size) / abs(seg) if times.size else 0 LOGGER.info(" Found {0} crossings of {1}, rate: {2} Hz".format( times.size, thresh, rate, )) if times.size and rate < args.rate_thresh: existing = tables[str(thresh)] tables[str(thresh)] = vstack_tables( ( existing, table_from_times(times, snr=10., frequency=100., names=existing.colnames), ), join_type="exact", ) n = max(map(len, tables.values())) for thresh, outfile in outfiles.items(): tables[thresh].write( outfile, path="triggers", format="hdf5", overwrite=True, ) LOGGER.info("{0} events written to {1}".format( str(len(tables[thresh])).rjust(len(str(n))), outfile, ))
def get_triggers(channel, etg, segments, cache=None, snr=None, frange=None, raw=False, trigfind_kwargs={}, **read_kwargs): """Get triggers for the given channel """ etg = _sanitize_name(etg) # format arguments try: readfmt = read_kwargs.pop("format", DEFAULT_FORMAT[etg]) except KeyError: raise ValueError("unsupported ETG {!r}".format(etg)) trigfind_kwargs, read_kwargs = _format_params( channel, etg, readfmt, trigfind_kwargs, read_kwargs ) # find triggers if cache is None: cache = find_trigger_files(channel, etg, segments, **trigfind_kwargs) # read files tables = [] for segment in segments: segaslist = SegmentList([segment]) segcache = io_cache.sieve(cache, segment=segment) # try and work out if cache overextends segment (so we need to crop) cachesegs = io_cache.cache_segments(segcache) outofbounds = abs(cachesegs - segaslist) if segcache: if len(segcache) == 1: # just pass the single filename segcache = segcache[0] new = EventTable.read(segcache, **read_kwargs) new.meta = {k: new.meta[k] for k in TABLE_META if new.meta.get(k)} if outofbounds: new = new[new[new.dtype.names[0]].in_segmentlist(segaslist)] tables.append(new) if len(tables): table = vstack_tables(tables) else: table = EventTable(names=read_kwargs.get( 'columns', ['time', 'frequency', 'snr'])) # parse time, frequency-like and snr-like column names columns = table.dtype.names tcolumn = columns[0] fcolumn = columns[1] scolumn = columns[2] # filter keep = numpy.ones(len(table), dtype=bool) if snr is not None: keep &= table[scolumn] >= snr if frange is not None: keep &= table[fcolumn] >= frange[0] keep &= table[fcolumn] < frange[1] table = table[keep] # return basic table if 'raw' if raw: return table # rename time column so that all tables match in at least that if tcolumn != "time": table.rename_column(tcolumn, 'time') # add channel column to identify all triggers table.add_column(table.Column(data=numpy.repeat(channel, len(table)), name='channel')) table.sort('time') return table
def get_triggers(channel, etg, segments, config=ConfigParser(), cache=None, query=True, multiprocess=False, tablename=None, columns=None, contenthandler=None, return_=True): """Read a table of transient event triggers for a given channel. """ key = '%s,%s' % (str(channel), etg.lower()) if isinstance(segments, DataQualityFlag): segments = segments.active segments = SegmentList(segments) # get LIGO_LW table for this etg if tablename: TableClass = lsctables.TableByName[tablename] register_etg_table(etg, TableClass, force=True) elif key in globalv.TRIGGERS: TableClass = type(globalv.TRIGGERS[key]) else: TableClass = get_etg_table(etg) # work out columns if columns is None: try: columns = config.get(etg, 'columns').split(',') except (NoSectionError, NoOptionError): if etg.lower() in ['cwb', 'cwb-ascii']: columns = None else: columns = TableClass.validcolumns.keys() if columns is not None: for col in ['process_id', 'search', 'channel']: if col not in columns: columns.append(col) # read segments from global memory try: havesegs = globalv.TRIGGERS[key].segments except KeyError: new = segments globalv.TRIGGERS.setdefault( key, lsctables.New(TableClass, columns=columns)) globalv.TRIGGERS[key].segments = type(segments)() else: new = segments - havesegs # read new triggers query &= (abs(new) != 0) if query: # store read kwargs kwargs = {'columns': columns} # set content handler if contenthandler is None: contenthandler = get_partial_contenthandler(TableClass) lsctables.use_in(contenthandler) for segment in new: kwargs['filt'] = ( lambda t: float(get_row_value(t, 'time')) in segment) # find trigger files if cache is None and etg.lower() == 'hacr': raise NotImplementedError("HACR parsing has not been " "implemented.") if cache is None and re.match('dmt(.*)omega', etg.lower()): segcache = find_dmt_omega(channel, segment[0], segment[1]) kwargs['format'] = 'ligolw' elif cache is None and etg.lower() in ['kw', 'kleinewelle']: segcache = find_kw(channel, segment[0], segment[1]) kwargs['format'] = 'ligolw' kwargs['filt'] = lambda t: ( float(get_row_value(t, 'time')) in segment and t.channel == str(channel)) elif cache is None: segcache = trigfind.find_trigger_urls(str(channel), etg, segment[0], segment[1]) kwargs['format'] = 'ligolw' elif isinstance(cache, Cache): segcache = cache.sieve(segment=segment) else: segcache = cache if isinstance(segcache, Cache): segcache = segcache.checkfilesexist()[0] if 'format' not in kwargs and 'ahope' not in etg.lower(): kwargs['format'] = etg.lower() if (issubclass(TableClass, lsctables.SnglBurstTable) and etg.lower().startswith('cwb')): kwargs['ifo'] = get_channel(channel).ifo # read triggers and store if len(segcache) == 0: continue if kwargs.get('format', None) == 'ligolw': kwargs['contenthandler'] = contenthandler table = TableClass.read(segcache, **kwargs) globalv.TRIGGERS[key].extend(table) try: csegs = cache_segments(segcache) except AttributeError: csegs = SegmentList() try: globalv.TRIGGERS[key].segments.extend(csegs) except AttributeError: globalv.TRIGGERS[key].segments = csegs finally: globalv.TRIGGERS[key].segments.coalesce() vprint('\r') # work out time function if return_: times = get_table_column(globalv.TRIGGERS[key], 'time').astype(float) # return correct triggers out = lsctables.New(TableClass, columns=columns) out.channel = str(channel) out.etg = str(etg) out.extend(t for (i, t) in enumerate(globalv.TRIGGERS[key]) if times[i] in segments) out.segments = segments & globalv.TRIGGERS[key].segments return out else: return