Exemplo n.º 1
0
 def test_cache_segments(self):
     # check empty input
     sl = cache_segments()
     self.assertIsInstance(sl, SegmentList)
     self.assertEquals(len(sl), 0)
     cache, segs = self.make_cache()
     try:
         # check good cache
         sl = cache_segments(cache)
         self.assertNotEquals(sl, segs)
         self.assertEquals(sl, type(segs)(segs).coalesce())
         # check bad cache
         os.remove(cache[0].path)
         sl = cache_segments(cache)
         self.assertEquals(sl, segs[1:])
         # check cache with no existing files
         sl = cache_segments(cache[:1])
         self.assertEquals(sl, SegmentList())
         # check errors
         self.assertRaises(TypeError, cache_segments, blah='blah')
         self.assertRaises(ValueError, cache_segments, cache,
                           on_missing='error')
         self.assertRaises(ValueError, cache_segments, cache,
                          on_missing='blah')
     # clean up
     finally:
         self.destroy_cache(cache)
Exemplo n.º 2
0
def find_best_frames(ifo, frametype, start, end, **kwargs):
    """Find frames for the given type, replacing with a better type if needed
    """
    # find cache for this frametype
    cache = find_frames(ifo, frametype, start, end, **kwargs)

    # check for gaps in current cache
    span = SegmentList([Segment(start, end)])
    gaps = span - cache_segments(cache)

    # if gaps and using aggregated h(t), check short files
    if abs(gaps) and frametype in SHORT_HOFT_TYPES:
        f2 = SHORT_HOFT_TYPES[frametype]
        vprint("    Gaps discovered in aggregated h(t) type "
               "%s, checking %s\n" % (frametype, f2))
        kwargs['gaps'] = 'ignore'
        cache.extend(filter(lambda e: file_segment(e) in gaps,
                            find_frames(ifo, f2, start, end, **kwargs)))
        new = int(abs(gaps - cache_segments(cache)))
        if new:
            vprint("    %ss extra coverage with frametype %s\n" % (new, f2))
        else:
            vprint("    No extra coverage with frametype %s\n" % f2)

    return cache, frametype
Exemplo n.º 3
0
def find_best_frames(ifo, frametype, start, end, **kwargs):
    """Find frames for the given type, replacing with a better type if needed
    """
    # find cache for this frametype
    cache = find_frames(ifo, frametype, start, end, **kwargs)

    # check for gaps in current cache
    span = SegmentList([Segment(start, end)])
    gaps = span - cache_segments(cache)

    # if gaps and using aggregated h(t), check short files
    if abs(gaps) and frametype in SHORT_HOFT_TYPES:
        f2 = SHORT_HOFT_TYPES[frametype]
        vprint("    Gaps discovered in aggregated h(t) type "
               "%s, checking %s\n" % (frametype, f2))
        kwargs['gaps'] = 'ignore'
        cache.extend(
            filter(lambda e: file_segment(e) in gaps,
                   find_frames(ifo, f2, start, end, **kwargs)))
        new = int(abs(gaps - cache_segments(cache)))
        if new:
            vprint("    %ss extra coverage with frametype %s\n" % (new, f2))
        else:
            vprint("    No extra coverage with frametype %s\n" % f2)

    return cache, frametype
Exemplo n.º 4
0
 def test_cache_segments(self):
     # check empty input
     sl = cache_segments()
     self.assertIsInstance(sl, SegmentList)
     self.assertEquals(len(sl), 0)
     cache, segs = self.make_cache()
     try:
         # check good cache
         sl = cache_segments(cache)
         self.assertNotEquals(sl, segs)
         self.assertEquals(sl, type(segs)(segs).coalesce())
         # check bad cache
         os.remove(cache[0].path)
         sl = cache_segments(cache)
         self.assertEquals(sl, segs[1:])
         # check cache with no existing files
         sl = cache_segments(cache[:1])
         self.assertEquals(sl, SegmentList())
         # check errors
         self.assertRaises(TypeError, cache_segments, blah='blah')
         self.assertRaises(ValueError,
                           cache_segments,
                           cache,
                           on_missing='error')
         self.assertRaises(ValueError,
                           cache_segments,
                           cache,
                           on_missing='blah')
     # clean up
     finally:
         self.destroy_cache(cache)
Exemplo n.º 5
0
    def test_find_contiguous(self):
        """Test :func:`gwpy.io.cache.find_contiguous`
        """
        a, segs = self.make_cache()
        segs.coalesce()
        for i, cache in enumerate(io_cache.find_contiguous(a)):
            io_cache.cache_segments(cache).extent() == segs[i]

        assert not list(io_cache.find_contiguous())
Exemplo n.º 6
0
 def test_cache_segments(self):
     # check empty input
     sl = cache_segments()
     self.assertIsInstance(sl, SegmentList)
     self.assertEquals(len(sl), 0)
     cache, segs = self.make_cache()
     segs.coalesce()
     sl = cache_segments(cache)
     self.assertEquals(sl, segs)
     sl = cache_segments(cache[:2], cache[2:])
     self.assertEquals(sl, segs)
Exemplo n.º 7
0
 def test_cache_segments(self):
     # check empty input
     sl = cache_segments()
     self.assertIsInstance(sl, SegmentList)
     self.assertEquals(len(sl), 0)
     cache, segs = self.make_cache()
     segs.coalesce()
     sl = cache_segments(cache)
     self.assertEquals(sl, segs)
     sl = cache_segments(cache[:2], cache[2:])
     self.assertEquals(sl, segs)
Exemplo n.º 8
0
 def test_cache_segments(self):
     """Test :func:`gwpy.io.cache.cache_segments`
     """
     # check empty input
     sl = io_cache.cache_segments()
     assert isinstance(sl, SegmentList)
     assert len(sl) == 0
     # check simple cache
     cache, segs = self.make_cache()
     segs.coalesce()
     sl = io_cache.cache_segments(cache)
     assert sl == segs
     # check multiple caches produces the same result
     sl = io_cache.cache_segments(cache[:2], cache[2:])
     assert sl == segs
Exemplo n.º 9
0
 def test_cache_segments(self):
     """Test :func:`gwpy.io.cache.cache_segments`
     """
     # check empty input
     sl = io_cache.cache_segments()
     assert isinstance(sl, SegmentList)
     assert len(sl) == 0
     # check simple cache
     cache, segs = self.make_cache()
     segs.coalesce()
     sl = io_cache.cache_segments(cache)
     assert sl == segs
     # check multiple caches produces the same result
     sl = io_cache.cache_segments(cache[:2], cache[2:])
     assert sl == segs
Exemplo n.º 10
0
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs):
    """Read a table of events from a cache

    This function is mainly meant for use from the `get_triggers` method

    Parameters
    ----------
    cache : :class:`glue.lal.Cache`
        the formatted list of files to read
    segments : `~gwpy.segments.SegmentList`
        the list of segments to read
    etg : `str`
        the name of the trigger generator that created the files
    nproc : `int`, optional
        the number of parallel processes to use when reading
    **kwargs
        other keyword arguments are passed to the `EventTable.read` or
        `{tableclass}.read` methods

    Returns
    -------
    table : `~gwpy.table.EventTable`, `None`
        a table of events, or `None` if the cache has no overlap with
        the segments
    """
    if isinstance(cache, Cache):
        cache = cache.sieve(segmentlist=segments)
        cache = cache.checkfilesexist()[0]
        cache.sort(key=lambda x: x.segment[0])
        cache = cache.pfnlist()  # some readers only like filenames
    else:
        cache = [urlparse(url).path for url in cache]
    if etg == 'pycbc_live':  # remove empty HDF5 files
        cache = filter_pycbc_live_files(cache, ifo=kwargs['ifo'])

    if len(cache) == 0:
        return

    # read triggers
    table = EventTable.read(cache, **kwargs)

    # store read keywords in the meta table
    if timecolumn:
        table.meta['timecolumn'] = timecolumn

    # get back from cache entry
    if isinstance(cache, CacheEntry):
        cache = Cache([cache])

    # append new events to existing table
    try:
        csegs = cache_segments(cache) & segments
    except (AttributeError, TypeError, ValueError):
        csegs = SegmentList()
    table.meta['segments'] = csegs

    if timecolumn:  # already filtered on-the-fly
        return table
    # filter now
    return keep_in_segments(table, segments, etg)
Exemplo n.º 11
0
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs):
    """Read a table of events from a cache

    This function is mainly meant for use from the `get_triggers` method

    Parameters
    ----------
    cache : :class:`glue.lal.Cache`
        the formatted list of files to read
    segments : `~gwpy.segments.SegmentList`
        the list of segments to read
    etg : `str`
        the name of the trigger generator that created the files
    nproc : `int`, optional
        the number of parallel processes to use when reading
    **kwargs
        other keyword arguments are passed to the `EventTable.read` or
        `{tableclass}.read` methods

    Returns
    -------
    table : `~gwpy.table.EventTable`, `None`
        a table of events, or `None` if the cache has no overlap with
        the segments
    """
    if isinstance(cache, Cache):
        cache = cache.sieve(segmentlist=segments)
        cache = cache.checkfilesexist()[0]
        cache.sort(key=lambda x: x.segment[0])
        if etg == 'pycbc_live':  # remove empty HDF5 files
            cache = type(cache)(
                filter_pycbc_live_files(cache, ifo=kwargs['ifo']))
    # if no files, skip
    if len(cache) == 0:
        return
    # use multiprocessing except for ascii reading
    # (since astropy doesn't allow it)
    if kwargs.get('format', 'none').startswith('ascii.'):
        cache = cache.pfnlist()
    else:
        kwargs['nproc'] = nproc
    if len(cache) == 1:
        cache = cache[0]

    # read triggers
    table = EventTable.read(cache, **kwargs)
    if timecolumn:
        table.meta['timecolumn'] = timecolumn

    # get back from cache entry
    if isinstance(cache, CacheEntry):
        cache = Cache([cache])
    # append new events to existing table
    try:
        csegs = cache_segments(cache)
    except (AttributeError, TypeError):
        csegs = SegmentList()
    table.meta['segments'] = csegs
    return keep_in_segments(table, segments, etg)
Exemplo n.º 12
0
 def read_and_cache_events(channel,
                           etg,
                           cache=None,
                           trigfind_kw={},
                           **read_kw):
     cfile = create_path(channel)
     # read existing cached triggers and work out new segments to query
     if args.append and cfile.is_file():
         previous = DataQualityFlag.read(
             str(cfile),
             path='segments',
             format='hdf5',
         ).coalesce()
         new = analysis - previous
     else:
         new = analysis.copy()
     # get cache of files
     if cache is None:
         cache = find_trigger_files(channel, etg, new.active, **trigfind_kw)
     else:
         cache = list(
             filter(
                 lambda e: new.active.intersects_segment(file_segment(e)),
                 cache,
             ))
     # restrict 'active' segments to when we have data
     try:
         new.active &= cache_segments(cache)
     except IndexError:
         new.active = type(new.active)()
     # find new triggers
     try:
         trigs = get_triggers(channel,
                              etg,
                              new.active,
                              cache=cache,
                              raw=True,
                              **read_kw)
     # catch error and continue
     except ValueError as e:
         warnings.warn('%s: %s' % (type(e).__name__, str(e)))
     else:
         path = write_events(channel, trigs, new)
         try:
             return path, len(trigs)
         except TypeError:  # None
             return
Exemplo n.º 13
0
def _get_timeseries_dict(channels,
                         segments,
                         config=None,
                         cache=None,
                         query=True,
                         nds=None,
                         frametype=None,
                         nproc=1,
                         return_=True,
                         statevector=False,
                         archive=True,
                         datafind_error='raise',
                         dtype=None,
                         **ioargs):
    """Internal method to retrieve the data for a set of like-typed
    channels using the :meth:`TimeSeriesDict.read` accessor.
    """
    channels = list(map(get_channel, channels))

    # set classes
    if statevector:
        ListClass = StateVectorList
        DictClass = StateVectorDict
    else:
        ListClass = TimeSeriesList
        DictClass = TimeSeriesDict

    # check we have a configparser
    if config is None:
        config = GWSummConfigParser()

    # read segments from global memory
    keys = dict((c.ndsname, make_globalv_key(c)) for c in channels)
    havesegs = reduce(
        operator.and_,
        (globalv.DATA.get(keys[channel.ndsname], ListClass()).segments
         for channel in channels))
    new = segments - havesegs

    # read channel information
    filter_ = dict()
    resample = dict()
    dtype_ = dict()
    for channel in channels:
        name = str(channel)
        try:
            filter_[name] = channel.filter
        except AttributeError:
            pass
        try:
            resample[name] = float(channel.resample)
        except AttributeError:
            pass
        if channel.dtype is not None:
            dtype_[name] = channel.dtype
        elif dtype is not None:
            dtype_[name] = dtype

    # work out whether to use NDS or not
    if nds is None and cache is not None:
        nds = False
    elif nds is None:
        nds = 'LIGO_DATAFIND_SERVER' not in os.environ

    # read new data
    query &= (abs(new) > 0)
    if cache is not None:
        query &= len(cache) > 0
    if query:
        for channel in channels:
            globalv.DATA.setdefault(keys[channel.ndsname], ListClass())

        ifo = channels[0].ifo

        # open NDS connection
        if nds:
            if config.has_option('nds', 'host'):
                host = config.get('nds', 'host')
                port = config.getint('nds', 'port')
                ndsconnection = io_nds2.connect(host, port)
            else:
                ndsconnection = None
            frametype = source = 'nds'
            ndstype = channels[0].type

            # get NDS channel segments
            if ndsconnection is not None and ndsconnection.get_protocol() > 1:
                span = list(map(int, new.extent()))
                avail = io_nds2.get_availability(channels,
                                                 *span,
                                                 connection=ndsconnection)
                new &= avail.intersection(avail.keys())

        # or find frame type and check cache
        else:
            frametype = frametype or channels[0].frametype
            new = exclude_short_trend_segments(new, ifo, frametype)

            if cache is not None:
                fcache = sieve_cache(cache, ifo=ifo[0], tag=frametype)
            else:
                fcache = []

            if (cache is None or len(fcache) == 0) and len(new):
                span = new.extent()
                fcache, frametype = find_best_frames(ifo,
                                                     frametype,
                                                     span[0],
                                                     span[1],
                                                     config=config,
                                                     gaps='ignore',
                                                     onerror=datafind_error)

            # parse discontiguous cache blocks and rebuild segment list
            new &= cache_segments(fcache)
            source = 'files'

            # if reading Virgo h(t) GWF data, filter out files that don't
            # contain the channel (Virgo state-vector only)
            _names = set(map(str, channels))
            _virgohoft = _names.intersection(VIRGO_HOFT_CHANNELS)
            if _virgohoft:
                vprint("    Determining available segments for "
                       "Virgo h(t) data...")
                new &= data_segments(fcache, _virgohoft.pop())

            # set channel type if reading with frameCPP
            if fcache and all_adc(fcache):
                ioargs['type'] = 'adc'

        # store frametype for display in Channel Information tables
        for channel in channels:
            channel.frametype = frametype

        # check whether each channel exists for all new times already
        qchannels = []
        for channel in channels:
            oldsegs = globalv.DATA.get(keys[channel.ndsname],
                                       ListClass()).segments
            if abs(new - oldsegs) != 0 and nds:
                qchannels.append(channel.ndsname)
            elif abs(new - oldsegs) != 0:
                qchannels.append(str(channel))

        # loop through segments, recording data for each
        if len(new):
            vprint(
                "    Fetching data (from %s) for %d channels [%s]:\n" %
                (source, len(qchannels), nds and ndstype or frametype or ''))
        vstr = "        [{0[0]}, {0[1]})"
        for segment in new:
            # force reading integer-precision segments
            segment = type(segment)(int(segment[0]), int(segment[1]))
            if abs(segment) < 1:
                continue

            # reset to minute trend sample times
            if frame_trend_type(ifo, frametype) == 'minute':
                segment = Segment(*io_nds2.minute_trend_times(*segment))
                if abs(segment) < 60:
                    continue

            if nds:  # fetch
                tsd = DictClass.fetch(qchannels,
                                      segment[0],
                                      segment[1],
                                      connection=ndsconnection,
                                      type=ndstype,
                                      verbose=vstr.format(segment),
                                      **ioargs)
            else:  # read
                # NOTE: this sieve explicitly casts our segment to
                #       ligo.segments.segment to prevent `TypeError` from
                #       a mismatch with ligo.segments.segment
                segcache = sieve_cache(fcache, segment=segment)
                segstart, segend = map(float, segment)
                tsd = DictClass.read(segcache,
                                     qchannels,
                                     start=segstart,
                                     end=segend,
                                     nproc=nproc,
                                     verbose=vstr.format(segment),
                                     **ioargs)

            vprint("        post-processing...\n")

            # apply type casting (copy=False means same type just returns)
            for chan, ts in tsd.items():
                tsd[chan] = ts.astype(dtype_.get(chan, ts.dtype),
                                      casting='unsafe',
                                      copy=False)

            # apply resampling
            tsd = resample_timeseries_dict(tsd, nproc=1, **resample)

            # post-process
            for c, data in tsd.items():
                channel = get_channel(c)
                key = keys[channel.ndsname]
                if (key in globalv.DATA
                        and data.span in globalv.DATA[key].segments):
                    continue
                if data.unit is None:
                    data.unit = 'undef'
                for i, seg in enumerate(globalv.DATA[key].segments):
                    if seg in data.span:
                        # new data completely covers existing segment
                        # (and more), so just remove the old stuff
                        globalv.DATA[key].pop(i)
                        break
                    elif seg.intersects(data.span):
                        # new data extends existing segment, so only keep
                        # the really new stuff
                        data = data.crop(*(data.span - seg))
                        break

                # filter
                try:
                    filt = filter_[str(channel)]
                except KeyError:
                    pass
                else:
                    data = filter_timeseries(data, filt)

                if isinstance(data, StateVector) or ':GRD-' in str(channel):
                    data.override_unit(units.dimensionless_unscaled)
                    if hasattr(channel, 'bits'):
                        data.bits = channel.bits
                elif data.unit is None:
                    data.override_unit(channel.unit)

                # update channel type for trends
                if data.channel.type is None and (data.channel.trend
                                                  is not None):
                    if data.dt.to('s').value == 1:
                        data.channel.type = 's-trend'
                    elif data.dt.to('s').value == 60:
                        data.channel.type = 'm-trend'

                # append and coalesce
                add_timeseries(data, key=key, coalesce=True)

    # rebuilt global channel list with new parameters
    update_channel_params()

    if not return_:
        return

    return locate_data(channels, segments, list_class=ListClass)
Exemplo n.º 14
0
def _get_timeseries_dict(channels, segments, config=None,
                         cache=None, query=True, nds=None, frametype=None,
                         nproc=1, return_=True, statevector=False,
                         archive=True, datafind_error='raise', dtype=None,
                         **ioargs):
    """Internal method to retrieve the data for a set of like-typed
    channels using the :meth:`TimeSeriesDict.read` accessor.
    """
    channels = list(map(get_channel, channels))

    # set classes
    if statevector:
        ListClass = StateVectorList
        DictClass = StateVectorDict
    else:
        ListClass = TimeSeriesList
        DictClass = TimeSeriesDict

    # check we have a configparser
    if config is None:
        config = GWSummConfigParser()

    # read segments from global memory
    keys = dict((c.ndsname, make_globalv_key(c)) for c in channels)
    havesegs = reduce(operator.and_,
                      (globalv.DATA.get(keys[channel.ndsname],
                                        ListClass()).segments
                       for channel in channels))
    new = segments - havesegs

    # read channel information
    filter_ = dict()
    resample = dict()
    dtype_ = dict()
    for channel in channels:
        name = str(channel)
        try:
            filter_[name] = channel.filter
        except AttributeError:
            pass
        try:
            resample[name] = float(channel.resample)
        except AttributeError:
            pass
        if channel.dtype is not None:
            dtype_[name] = channel.dtype
        elif dtype is not None:
            dtype_[name] = dtype

    # work out whether to use NDS or not
    if nds is None and cache is not None:
        nds = False
    elif nds is None:
        nds = 'LIGO_DATAFIND_SERVER' not in os.environ

    # read new data
    query &= (abs(new) > 0)
    if cache is not None:
        query &= len(cache) > 0
    if query:
        for channel in channels:
            globalv.DATA.setdefault(keys[channel.ndsname], ListClass())

        ifo = channels[0].ifo

        # open NDS connection
        if nds:
            if config.has_option('nds', 'host'):
                host = config.get('nds', 'host')
                port = config.getint('nds', 'port')
                ndsconnection = io_nds2.connect(host, port)
            else:
                ndsconnection = None
            frametype = source = 'nds'
            ndstype = channels[0].type

            # get NDS channel segments
            if ndsconnection is not None and ndsconnection.get_protocol() > 1:
                span = list(map(int, new.extent()))
                avail = io_nds2.get_availability(
                    channels, *span, connection=ndsconnection
                )
                new &= avail.intersection(avail.keys())

        # or find frame type and check cache
        else:
            frametype = frametype or channels[0].frametype
            new = exclude_short_trend_segments(new, ifo, frametype)

            if cache is not None:
                fcache = sieve_cache(cache, ifo=ifo[0], tag=frametype)
            else:
                fcache = []

            if (cache is None or len(fcache) == 0) and len(new):
                span = new.extent()
                fcache, frametype = find_best_frames(
                    ifo, frametype, span[0], span[1],
                    config=config, gaps='ignore', onerror=datafind_error)

            # parse discontiguous cache blocks and rebuild segment list
            new &= cache_segments(fcache)
            source = 'files'

            # if reading Virgo h(t) GWF data, filter out files that don't
            # contain the channel (Virgo state-vector only)
            _names = set(map(str, channels))
            _virgohoft = _names.intersection(VIRGO_HOFT_CHANNELS)
            if _virgohoft:
                vprint("    Determining available segments for "
                       "Virgo h(t) data...")
                new &= data_segments(fcache, _virgohoft.pop())

            # set channel type if reading with frameCPP
            if fcache and all_adc(fcache):
                ioargs['type'] = 'adc'

        # store frametype for display in Channel Information tables
        for channel in channels:
            channel.frametype = frametype

        # check whether each channel exists for all new times already
        qchannels = []
        for channel in channels:
            oldsegs = globalv.DATA.get(keys[channel.ndsname],
                                       ListClass()).segments
            if abs(new - oldsegs) != 0 and nds:
                qchannels.append(channel.ndsname)
            elif abs(new - oldsegs) != 0:
                qchannels.append(str(channel))

        # loop through segments, recording data for each
        if len(new):
            vprint("    Fetching data (from %s) for %d channels [%s]:\n"
                   % (source, len(qchannels),
                      nds and ndstype or frametype or ''))
        vstr = "        [{0[0]}, {0[1]})"
        for segment in new:
            # force reading integer-precision segments
            segment = type(segment)(int(segment[0]), int(segment[1]))
            if abs(segment) < 1:
                continue

            # reset to minute trend sample times
            if frame_trend_type(ifo, frametype) == 'minute':
                segment = Segment(*io_nds2.minute_trend_times(*segment))
                if abs(segment) < 60:
                    continue

            if nds:  # fetch
                tsd = DictClass.fetch(qchannels, segment[0], segment[1],
                                      connection=ndsconnection, type=ndstype,
                                      verbose=vstr.format(segment), **ioargs)
            else:  # read
                # NOTE: this sieve explicitly casts our segment to
                #       ligo.segments.segment to prevent `TypeError` from
                #       a mismatch with ligo.segments.segment
                segcache = sieve_cache(fcache, segment=segment)
                segstart, segend = map(float, segment)
                tsd = DictClass.read(segcache, qchannels, start=segstart,
                                     end=segend, nproc=nproc,
                                     verbose=vstr.format(segment), **ioargs)

            vprint("        post-processing...\n")

            # apply type casting (copy=False means same type just returns)
            for chan, ts in tsd.items():
                tsd[chan] = ts.astype(dtype_.get(chan, ts.dtype),
                                      casting='unsafe', copy=False)

            # apply resampling
            tsd = resample_timeseries_dict(tsd, nproc=1, **resample)

            # post-process
            for c, data in tsd.items():
                channel = get_channel(c)
                key = keys[channel.ndsname]
                if (key in globalv.DATA and
                        data.span in globalv.DATA[key].segments):
                    continue
                if data.unit is None:
                    data.unit = 'undef'
                for i, seg in enumerate(globalv.DATA[key].segments):
                    if seg in data.span:
                        # new data completely covers existing segment
                        # (and more), so just remove the old stuff
                        globalv.DATA[key].pop(i)
                        break
                    elif seg.intersects(data.span):
                        # new data extends existing segment, so only keep
                        # the really new stuff
                        data = data.crop(*(data.span - seg))
                        break

                # filter
                try:
                    filt = filter_[str(channel)]
                except KeyError:
                    pass
                else:
                    data = filter_timeseries(data, filt)

                if isinstance(data, StateVector) or ':GRD-' in str(channel):
                    data.override_unit(units.dimensionless_unscaled)
                    if hasattr(channel, 'bits'):
                        data.bits = channel.bits
                elif data.unit is None:
                    data.override_unit(channel.unit)

                # update channel type for trends
                if data.channel.type is None and (
                        data.channel.trend is not None):
                    if data.dt.to('s').value == 1:
                        data.channel.type = 's-trend'
                    elif data.dt.to('s').value == 60:
                        data.channel.type = 'm-trend'

                # append and coalesce
                add_timeseries(data, key=key, coalesce=True)

    # rebuilt global channel list with new parameters
    update_channel_params()

    if not return_:
        return

    return locate_data(channels, segments, list_class=ListClass)
Exemplo n.º 15
0
def get_triggers(channel, etg, segments, config=GWSummConfigParser(),
                 cache=None, columns=None, query=True, multiprocess=False,
                 ligolwtable=None, return_=True):
    """Read a table of transient event triggers for a given channel.
    """
    key = '%s,%s' % (str(channel), etg.lower())

    # convert input segments to a segmentlist (for convenience)
    if isinstance(segments, DataQualityFlag):
        segments = segments.active
    segments = SegmentList(segments)

    # get processes
    if multiprocess is True:
        nproc = count_free_cores()
    elif multiprocess is False:
        nproc = 1
    else:
        nproc = multiprocess

    # find LIGO_LW table for this ETG
    try:
        TableClass = get_etg_table(etg)
    except KeyError:
        TableClass = None

    # work out columns
    if columns is None:
        try:
            columns = config.get(etg, 'columns').split(',')
        except (NoSectionError, NoOptionError):
            columns = None

    # read segments from global memory
    try:
        havesegs = globalv.TRIGGERS[key].segments
    except KeyError:
        new = segments
    else:
        new = segments - havesegs

    # read new triggers
    if query and abs(new) != 0:
        ntrigs = 0
        vprint("    Grabbing %s triggers for %s" % (etg, str(channel)))

        # store read kwargs
        kwargs = get_etg_read_kwargs(config, etg, exclude=['columns'])
        kwargs['columns'] = columns
        if etg.lower().replace('-', '_') in ['cwb', 'pycbc_live']:
            kwargs['ifo'] = get_channel(channel).ifo
        if 'format' not in kwargs and 'ahope' not in etg.lower():
            kwargs['format'] = etg.lower()

        # set up ligolw options if needed
        if TableClass is not None:
            contenthandler = get_partial_contenthandler(TableClass)
            lsctables.use_in(contenthandler)

        # loop over segments
        for segment in new:
            # find trigger files
            if cache is None and etg.lower() == 'hacr':
                raise NotImplementedError("HACR parsing has not been "
                                          "implemented.")
            if cache is None and etg.lower() in ['kw', 'kleinewelle']:
                kwargs['filt'] = lambda t: t.channel == str(channel)
            if cache is None:
                try:
                    segcache = trigfind.find_trigger_urls(str(channel), etg,
                                                          segment[0],
                                                          segment[1])
                except ValueError as e:
                    warnings.warn("Caught %s: %s" % (type(e).__name__, str(e)))
                    continue
                else:
                    for regex in TRIGFIND_FORMAT:
                        if regex.match(etg):
                            kwargs['format'] = TRIGFIND_FORMAT[regex]
                            if TRIGFIND_FORMAT[regex] in lsctables.TableByName:
                                kwargs['get_as_columns'] = True
                            break
                if etg.lower() == 'omega':
                    kwargs.setdefault('format', 'omega')
                else:
                    kwargs.setdefault('format', 'ligolw')
            else:
                segcache = cache
            if isinstance(segcache, Cache):
                segcache = segcache.sieve(segment=segment)
                segcache = segcache.checkfilesexist()[0]
                segcache.sort(key=lambda x: x.segment[0])
                if etg == 'pycbc_live':  # remove empty HDF5 files
                    segcache = type(segcache)(
                        filter_pycbc_live_files(segcache, ifo=kwargs['ifo']))
            # if no files, skip
            if len(segcache) == 0:
                continue
            # read triggers
            if kwargs.get('format', None) == 'ligolw':
                kwargs['contenthandler'] = contenthandler
            try:  # try directly reading a numpy.recarray
                table = GWRecArray.read(segcache, nproc=nproc, **kwargs)
            except Exception as e:  # back up to LIGO_LW
                if TableClass is not None and 'No reader' in str(e):
                    try:
                        table = TableClass.read(segcache, **kwargs)
                    except Exception:
                        raise e
                    else:
                        table = table.to_recarray(get_as_columns=True)
                else:
                    raise
            # append new events to existing table
            try:
                csegs = cache_segments(segcache)
            except AttributeError:
                csegs = SegmentList()
            table.segments = csegs
            t2 = keep_in_segments(table, SegmentList([segment]), etg)
            add_triggers(t2, key, csegs)
            ntrigs += len(t2)
            vprint(".")
        vprint(" | %d events read\n" % ntrigs)

    # if asked to read triggers, but didn't actually read any,
    # create an empty table so that subsequent calls don't raise KeyErrors
    if query and key not in globalv.TRIGGERS:
        if TableClass is not None:
            tab = lsctables.New(TableClass, columns=columns).to_recarray(
                get_as_columns=True)
        else:
            tab = GWRecArray((0,), dtype=[(c, float) for c in columns])
        tab.segments = SegmentList()
        add_triggers(tab, key, tab.segments)

    # work out time function
    if return_:
        return keep_in_segments(globalv.TRIGGERS[key], segments, etg)
    else:
        return
Exemplo n.º 16
0
def get_triggers(channel,
                 etg,
                 segments,
                 cache=None,
                 snr=None,
                 frange=None,
                 raw=False,
                 trigfind_kwargs={},
                 **read_kwargs):
    """Get triggers for the given channel
    """
    etg = _sanitize_name(etg)
    # format arguments
    try:
        readfmt = read_kwargs.pop("format", DEFAULT_FORMAT[etg])
    except KeyError:
        raise ValueError("unsupported ETG {!r}".format(etg))
    trigfind_kwargs, read_kwargs = _format_params(channel, etg, readfmt,
                                                  trigfind_kwargs, read_kwargs)

    # find triggers
    if cache is None:
        cache = find_trigger_files(channel, etg, segments, **trigfind_kwargs)

    # read files
    tables = []
    for segment in segments:
        segaslist = SegmentList([segment])
        segcache = io_cache.sieve(cache, segment=segment)
        # try and work out if cache overextends segment (so we need to crop)
        cachesegs = io_cache.cache_segments(segcache)
        outofbounds = abs(cachesegs - segaslist)
        if segcache:
            if len(segcache) == 1:  # just pass the single filename
                segcache = segcache[0]
            new = EventTable.read(segcache, **read_kwargs)
            new.meta = {k: new.meta[k] for k in TABLE_META if new.meta.get(k)}
            if outofbounds:
                new = new[in_segmentlist(new[new.dtype.names[0]], segaslist)]
            tables.append(new)
    if len(tables):
        table = vstack_tables(tables)
    else:
        table = EventTable(
            names=read_kwargs.get('columns', ['time', 'frequency', 'snr']))

    # parse time, frequency-like and snr-like column names
    columns = table.dtype.names
    tcolumn = columns[0]
    fcolumn = columns[1]
    scolumn = columns[2]

    # filter
    keep = numpy.ones(len(table), dtype=bool)
    if snr is not None:
        keep &= table[scolumn] >= snr
    if frange is not None:
        keep &= table[fcolumn] >= frange[0]
        keep &= table[fcolumn] < frange[1]
    table = table[keep]

    # return basic table if 'raw'
    if raw:
        return table

    # rename time column so that all tables match in at least that
    if tcolumn != "time":
        table.rename_column(tcolumn, 'time')

    # add channel column to identify all triggers
    table.add_column(
        table.Column(data=numpy.repeat(channel, len(table)), name='channel'))

    table.sort('time')
    return table
Exemplo n.º 17
0
def main(args=None):
    """Run the online Guardian node visualization tool
    """
    parser = create_parser()
    args = parser.parse_args(args=args)

    fec_map = args.fec_map
    simulink = args.simulink
    daqsvn = args.daqsvn or ('https://daqsvn.ligo-la.caltech.edu/websvn/'
                             'listing.php?repname=daq_maps')
    if args.ifo == 'H1':
        if not fec_map:
            fec_map = 'https://lhocds.ligo-wa.caltech.edu/exports/detchar/fec/'
        if not simulink:
            simulink = 'https://lhocds.ligo-wa.caltech.edu/daq/simulink/'
    if args.ifo == 'L1':
        if not fec_map:
            fec_map = 'https://llocds.ligo-la.caltech.edu/exports/detchar/fec/'
        if not simulink:
            simulink = 'https://llocds.ligo-la.caltech.edu/daq/simulink/'

    span = Segment(args.gpsstart, args.gpsend)

    # let's go
    LOGGER.info('{} Overflows {}-{}'.format(args.ifo, int(args.gpsstart),
                                            int(args.gpsend)))

    # get segments
    if args.state_flag:
        state = DataQualityFlag.query(args.state_flag,
                                      int(args.gpsstart),
                                      int(args.gpsend),
                                      url=const.DEFAULT_SEGMENT_SERVER)
        tmp = type(state.active)()
        for i, seg in enumerate(state.active):
            if abs(seg) < args.segment_end_pad:
                continue
            tmp.append(type(seg)(seg[0], seg[1] - args.segment_end_pad))
        state.active = tmp.coalesce()
        statea = state.active
    else:
        statea = SegmentList([span])

    if not args.output_file:
        duration = abs(span)
        args.output_file = ('%s-OVERFLOWS-%d-%d.h5' %
                            (args.ifo, int(args.gpsstart), duration))
        LOGGER.debug("Set default output file as %s" % args.output_file)

    # set up container
    overflows = DataQualityDict()

    # prepare data access
    if args.nds:
        from gwpy.io import nds2 as io_nds2
        host, port = args.nds.rsplit(':', 1)
        ndsconnection = io_nds2.connect(host, port=int(port))
        if ndsconnection.get_protocol() == 1:
            cachesegs = SegmentList(
                [Segment(int(args.gpsstart), int(args.gpsend))])
        else:
            cachesegs = io_nds2.get_availability(
                ['{0}:FEC-1_DAC_OVERFLOW_ACC_0_0'.format(args.ifo)],
                int(args.gpsstart),
                int(args.gpsend),
            )
    else:  # get frame cache
        cache = gwdatafind.find_urls(args.ifo[0], args.frametype,
                                     int(args.gpsstart), int(args.gpsend))
        cachesegs = statea & cache_segments(cache)

    flag_desc = "ADC/DAC Overflow indicated by {0}"

    # get channel and find overflows
    for dcuid in args.dcuid:
        LOGGER.info("Processing DCUID %d" % dcuid)
        channel = daq.ligo_accum_overflow_channel(dcuid, args.ifo)
        overflows[channel] = DataQualityFlag(channel, known=cachesegs)
        if args.deep:
            LOGGER.debug(" -- Getting list of overflow channels")
            try:
                channels = daq.ligo_model_overflow_channels(dcuid,
                                                            args.ifo,
                                                            args.frametype,
                                                            gpstime=span[0],
                                                            nds=args.nds)
            except IndexError:  # no frame found for GPS start, try GPS end
                channels = daq.ligo_model_overflow_channels(dcuid,
                                                            args.ifo,
                                                            args.frametype,
                                                            gpstime=span[-1])
            for chan in channels:  # set up flags early
                overflows[chan] = DataQualityFlag(
                    chan,
                    known=cachesegs,
                    description=flag_desc.format(chan),
                    isgood=False,
                )
            LOGGER.debug(" -- %d channels found" % len(channel))
        for seg in cachesegs:
            LOGGER.debug(" -- Processing {}-{}".format(*seg))
            if args.nds:
                read_kw = dict(connection=ndsconnection)
            else:
                read_kw = dict(source=cache, nproc=args.nproc)
            msg = "Reading ACCUM_OVERFLOW data:".rjust(30)
            data = get_data(channel,
                            seg[0],
                            seg[1],
                            pad=0.,
                            verbose=msg,
                            **read_kw)
            new = daq.find_overflow_segments(
                data,
                cumulative=True,
            )
            overflows[channel] += new
            LOGGER.info(" -- {} overflows found".format(len(new.active)))
            if not new.active:
                continue
            # go deep!
            for s, e in tqdm.tqdm(new.active.protract(2),
                                  unit='ovfl',
                                  desc='Going deep'.rjust(30)):
                data = get_data(channels, s, e, **read_kw)
                for ch in channels:
                    try:
                        overflows[ch] += daq.find_overflow_segments(
                            data[ch],
                            cumulative=True,
                        )
                    except KeyError:
                        warnings.warn("Skipping {}".format(ch), UserWarning)
                        continue
        LOGGER.debug(" -- Search complete")

    # write output
    LOGGER.info("Writing segments to %s" % args.output_file)
    table = table_from_segments(
        overflows,
        sngl_burst=args.output_file.endswith((".xml", ".xml.gz")),
    )
    if args.integer_segments:
        for key in overflows:
            overflows[key] = overflows[key].round()
    if args.output_file.endswith((".h5", "hdf", ".hdf5")):
        with h5py.File(args.output_file, "w") as h5f:
            table.write(h5f, path="triggers")
            overflows.write(h5f, path="segments")
    else:
        table.write(args.output_file, overwrite=True)
        overflows.write(args.output_file, overwrite=True, append=True)

    # write HTML
    if args.html:
        # get base path
        base = os.path.dirname(args.html)
        os.chdir(base)
        if args.plot:
            args.plot = os.path.curdir
        if args.output_file:
            args.output_file = os.path.relpath(args.output_file,
                                               os.path.dirname(args.html))
        if os.path.basename(args.html) == 'index.html':
            links = [
                '%d-%d' % (int(args.gpsstart), int(args.gpsend)),
                ('Parameters', '#parameters'),
                ('Segments', [('Overflows', '#overflows')]),
                ('Results', '#results'),
            ]
            if args.state_flag:
                links[2][1].insert(0, ('State flag', '#state-flag'))
            (brand, class_) = htmlio.get_brand(args.ifo, 'Overflows',
                                               args.gpsstart)
            navbar = htmlio.navbar(links, class_=class_, brand=brand)
            page = htmlio.new_bootstrap_page(
                title='%s Overflows | %d-%d' %
                (args.ifo, int(args.gpsstart), int(args.gpsend)),
                navbar=navbar)
        else:
            page = htmlio.markup.page()
            page.div(class_='container')

        # -- header
        page.div(class_='pb-2 mt-3 mb-2 border-bottom')
        page.h1('%s ADC/DAC Overflows: %d-%d' %
                (args.ifo, int(args.gpsstart), int(args.gpsend)))
        page.div.close()

        # -- paramters
        content = [('DCUIDs', ' '.join(map(str, args.dcuid)))]
        if daqsvn:
            content.append(('FEC configuration', (
                '<a href="{0}" target="_blank" title="{1} FEC configuration">'
                '{0}</a>').format(daqsvn, args.ifo)))
        if fec_map:
            content.append(
                ('FEC map', '<a href="{0}" target="_blank" title="{1} FEC '
                 'map">{0}</a>'.format(fec_map, args.ifo)))
        if simulink:
            content.append(
                ('Simulink models', '<a href="{0}" target="_blank" title="{1} '
                 'Simulink models">{0}</a>'.format(simulink, args.ifo)))
        page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
        page.div(class_='row')
        page.div(class_='col-md-9 col-sm-12')
        page.add(
            htmlio.parameter_table(content,
                                   start=args.gpsstart,
                                   end=args.gpsend,
                                   flag=args.state_flag))
        page.div.close()  # col-md-9 col-sm-12

        # link to summary file
        if args.output_file:
            ext = ('HDF' if args.output_file.endswith(
                (".h5", "hdf", ".hdf5")) else 'XML')
            page.div(class_='col-md-3 col-sm-12')
            page.add(
                htmlio.download_btn(
                    [('Segments ({})'.format(ext), args.output_file)],
                    btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(),
                ))
            page.div.close()  # col-md-3 col-sm-12
        page.div.close()  # row

        # -- command-line
        page.h5('Command-line:')
        page.add(htmlio.get_command_line(about=False, prog=PROG))

        # -- segments
        page.h2('Segments', class_='mt-4', id_='segments')

        # give contextual information
        msg = ("This analysis searched for digital-to-analogue (DAC) or "
               "analogue-to-digital (ADC) conversion overflows in the {0} "
               "real-time controls system. ").format(
                   SITE_MAP.get(args.ifo, 'LIGO'))
        if args.deep:
            msg += (
                "A hierarchichal search was performed, with one cumulative "
                "overflow counter checked per front-end controller (FEC). "
                "For those models that indicated an overflow, the card- and "
                "slot-specific channels were then checked. ")
        msg += (
            "Consant overflow is shown as yellow, while transient overflow "
            "is shown as red. If a data-quality flag was loaded for this "
            "analysis, it will be displayed in green.")
        page.add(htmlio.alert(msg, context=args.ifo.lower()))
        # record state segments
        if args.state_flag:
            page.h3('State flag', class_='mt-3', id_='state-flag')
            page.div(id_='accordion1')
            page.add(
                htmlio.write_flag_html(state,
                                       span,
                                       'state',
                                       parent='accordion1',
                                       context='success',
                                       plotdir=args.plot,
                                       facecolor=(0.2, 0.8, 0.2),
                                       edgecolor='darkgreen',
                                       known={
                                           'facecolor': 'red',
                                           'edgecolor': 'darkred',
                                           'height': 0.4,
                                       }))
            page.div.close()
        # record overflow segments
        if sum(abs(s.active) for s in overflows.values()):
            page.h3('Overflows', class_='mt-3', id_='overflows')
            page.div(id_='accordion2')
            for i, (c, flag) in enumerate(list(overflows.items())):
                if abs(flag.active) == 0:
                    continue
                if abs(flag.active) == abs(cachesegs):
                    context = 'warning'
                else:
                    context = 'danger'
                try:
                    channel = cds.get_real_channel(flag.name)
                except Exception:
                    title = '%s [%d]' % (flag.name, len(flag.active))
                else:
                    title = '%s (%s) [%d]' % (flag.name, channel,
                                              len(flag.active))
                page.add(
                    htmlio.write_flag_html(flag,
                                           span,
                                           i,
                                           parent='accordion2',
                                           title=title,
                                           context=context,
                                           plotdir=args.plot))
            page.div.close()
        else:
            page.add(
                htmlio.alert('No overflows were found in this analysis',
                             context=args.ifo.lower(),
                             dismiss=False))

        # -- results table
        page.h2('Results summary', class_='mt-4', id_='results')
        page.table(class_='table table-striped table-hover')
        # write table header
        page.thead()
        page.tr()
        for header in ['Channel', 'Connected signal', 'Num. overflows']:
            page.th(header)
        page.thead.close()
        # write body
        page.tbody()
        for c, seglist in overflows.items():
            t = abs(seglist.active)
            if t == 0:
                page.tr()
            elif t == abs(cachesegs):
                page.tr(class_='table-warning')
            else:
                page.tr(class_='table-danger')
            page.td(c)
            try:
                page.td(cds.get_real_channel(str(c)))
            except Exception:
                page.td()
            page.td(len(seglist.active))
            page.tr.close()
        page.tbody.close()
        page.table.close()

        # -- close and write
        htmlio.close_page(page, args.html)
        LOGGER.info("HTML written to %s" % args.html)
Exemplo n.º 18
0
def main(args=None):
    """Run the zero-crossing counter tool
    """
    parser = create_parser()
    args = parser.parse_args(args=args)

    span = Segment(args.gpsstart, args.gpsend)
    LOGGER.info('-- Processing channel %s over span %d - %d' %
                (args.channel, args.gpsstart, args.gpsend))

    if args.state_flag:
        state = DataQualityFlag.query(
            args.state_flag,
            int(args.gpsstart),
            int(args.gpsend),
            url=const.DEFAULT_SEGMENT_SERVER,
        )
        statea = state.active
    else:
        statea = SegmentList([span])

    duration = abs(span)

    # initialize output files for each threshold and store them in a dict
    outfiles = {}
    for thresh in args.threshold:
        outfiles[str(thresh)] = (os.path.join(
            args.output_path, '%s_%s_DAC-%d-%d.h5' %
            (args.channel.replace('-', '_').replace(':', '-'), str(
                int(thresh)).replace('-', 'n'), int(args.gpsstart), duration)))

    # get frame cache
    cache = gwdatafind.find_urls(args.ifo[0], args.frametype,
                                 int(args.gpsstart), int(args.gpsend))

    cachesegs = statea & cache_segments(cache)

    if not os.path.exists(args.output_path):
        os.makedirs(args.output_path)

    # initialize a ligolw table for each threshold and store them in a dict
    names = ("time", "frequency", "snr")
    dtypes = ("f8", ) * len(names)
    tables = {}
    for thresh in args.threshold:
        tables[str(thresh)] = EventTable(
            names=names,
            dtype=dtypes,
            meta={"channel": args.channel},
        )

    # for each science segment, read in the data from frames, check for
    # threshold crossings, and if the rate of crossings is less than
    # rate_thresh, write to a sngl_burst table
    for seg in cachesegs:
        LOGGER.debug("Processing {}:".format(seg))
        c = sieve_cache(cache, segment=seg)
        if not c:
            LOGGER.warning("    No {} data files for this segment, "
                           "skipping".format(args.frametype))
            continue
        data = get_data(args.channel,
                        seg[0],
                        seg[1],
                        nproc=args.nproc,
                        source=c,
                        verbose="Reading data:".rjust(30))
        for thresh in args.threshold:
            times = find_crossings(data, thresh)
            rate = float(times.size) / abs(seg) if times.size else 0
            LOGGER.info("    Found {0} crossings of {1}, rate: {2} Hz".format(
                times.size,
                thresh,
                rate,
            ))
            if times.size and rate < args.rate_thresh:
                existing = tables[str(thresh)]
                tables[str(thresh)] = vstack_tables(
                    (
                        existing,
                        table_from_times(times,
                                         snr=10.,
                                         frequency=100.,
                                         names=existing.colnames),
                    ),
                    join_type="exact",
                )

    n = max(map(len, tables.values()))
    for thresh, outfile in outfiles.items():
        tables[thresh].write(
            outfile,
            path="triggers",
            format="hdf5",
            overwrite=True,
        )
        LOGGER.info("{0} events written to {1}".format(
            str(len(tables[thresh])).rjust(len(str(n))),
            outfile,
        ))
Exemplo n.º 19
0
def get_triggers(channel, etg, segments, cache=None, snr=None, frange=None,
                 raw=False, trigfind_kwargs={}, **read_kwargs):
    """Get triggers for the given channel
    """
    etg = _sanitize_name(etg)
    # format arguments
    try:
        readfmt = read_kwargs.pop("format", DEFAULT_FORMAT[etg])
    except KeyError:
        raise ValueError("unsupported ETG {!r}".format(etg))
    trigfind_kwargs, read_kwargs = _format_params(
        channel,
        etg,
        readfmt,
        trigfind_kwargs,
        read_kwargs
    )

    # find triggers
    if cache is None:
        cache = find_trigger_files(channel, etg, segments, **trigfind_kwargs)

    # read files
    tables = []
    for segment in segments:
        segaslist = SegmentList([segment])
        segcache = io_cache.sieve(cache, segment=segment)
        # try and work out if cache overextends segment (so we need to crop)
        cachesegs = io_cache.cache_segments(segcache)
        outofbounds = abs(cachesegs - segaslist)
        if segcache:
            if len(segcache) == 1:  # just pass the single filename
                segcache = segcache[0]
            new = EventTable.read(segcache, **read_kwargs)
            new.meta = {k: new.meta[k] for k in TABLE_META if new.meta.get(k)}
            if outofbounds:
                new = new[new[new.dtype.names[0]].in_segmentlist(segaslist)]
            tables.append(new)
    if len(tables):
        table = vstack_tables(tables)
    else:
        table = EventTable(names=read_kwargs.get(
            'columns', ['time', 'frequency', 'snr']))

    # parse time, frequency-like and snr-like column names
    columns = table.dtype.names
    tcolumn = columns[0]
    fcolumn = columns[1]
    scolumn = columns[2]

    # filter
    keep = numpy.ones(len(table), dtype=bool)
    if snr is not None:
        keep &= table[scolumn] >= snr
    if frange is not None:
        keep &= table[fcolumn] >= frange[0]
        keep &= table[fcolumn] < frange[1]
    table = table[keep]

    # return basic table if 'raw'
    if raw:
        return table

    # rename time column so that all tables match in at least that
    if tcolumn != "time":
        table.rename_column(tcolumn, 'time')

    # add channel column to identify all triggers
    table.add_column(table.Column(data=numpy.repeat(channel, len(table)),
                                  name='channel'))

    table.sort('time')
    return table
Exemplo n.º 20
0
def get_triggers(channel, etg, segments, config=ConfigParser(), cache=None,
                 query=True, multiprocess=False, tablename=None,
                 columns=None, contenthandler=None, return_=True):
    """Read a table of transient event triggers for a given channel.
    """
    key = '%s,%s' % (str(channel), etg.lower())
    if isinstance(segments, DataQualityFlag):
        segments = segments.active
    segments = SegmentList(segments)

    # get LIGO_LW table for this etg
    if tablename:
        TableClass = lsctables.TableByName[tablename]
        register_etg_table(etg, TableClass, force=True)
    elif key in globalv.TRIGGERS:
        TableClass = type(globalv.TRIGGERS[key])
    else:
        TableClass = get_etg_table(etg)

    # work out columns
    if columns is None:
        try:
            columns = config.get(etg, 'columns').split(',')
        except (NoSectionError, NoOptionError):
            if etg.lower() in ['cwb', 'cwb-ascii']:
                columns = None
            else:
                columns = TableClass.validcolumns.keys()
    if columns is not None:
        for col in ['process_id', 'search', 'channel']:
            if col not in columns:
                columns.append(col)

    # read segments from global memory
    try:
        havesegs = globalv.TRIGGERS[key].segments
    except KeyError:
        new = segments
        globalv.TRIGGERS.setdefault(
            key, lsctables.New(TableClass, columns=columns))
        globalv.TRIGGERS[key].segments = type(segments)()
    else:
        new = segments - havesegs

    # read new triggers
    query &= (abs(new) != 0)
    if query:
        # store read kwargs
        kwargs = {'columns': columns}

        # set content handler
        if contenthandler is None:
            contenthandler = get_partial_contenthandler(TableClass)
        lsctables.use_in(contenthandler)

        for segment in new:
            kwargs['filt'] = (
                lambda t: float(get_row_value(t, 'time')) in segment)
            # find trigger files
            if cache is None and etg.lower() == 'hacr':
                raise NotImplementedError("HACR parsing has not been "
                                          "implemented.")
            if cache is None and re.match('dmt(.*)omega', etg.lower()):
                segcache = find_dmt_omega(channel, segment[0], segment[1])
                kwargs['format'] = 'ligolw'
            elif cache is None and etg.lower() in ['kw', 'kleinewelle']:
                segcache = find_kw(channel, segment[0], segment[1])
                kwargs['format'] = 'ligolw'
                kwargs['filt'] = lambda t: (
                    float(get_row_value(t, 'time')) in segment and
                    t.channel == str(channel))
            elif cache is None:
                segcache = trigfind.find_trigger_urls(str(channel), etg,
                                                      segment[0],
                                                      segment[1])
                kwargs['format'] = 'ligolw'
            elif isinstance(cache, Cache):
                segcache = cache.sieve(segment=segment)
            else:
                segcache = cache
            if isinstance(segcache, Cache):
                segcache = segcache.checkfilesexist()[0]
            if 'format' not in kwargs and 'ahope' not in etg.lower():
                kwargs['format'] = etg.lower()
            if (issubclass(TableClass, lsctables.SnglBurstTable) and
                    etg.lower().startswith('cwb')):
                kwargs['ifo'] = get_channel(channel).ifo
            # read triggers and store
            if len(segcache) == 0:
                continue
            if kwargs.get('format', None) == 'ligolw':
                kwargs['contenthandler'] = contenthandler
            table = TableClass.read(segcache, **kwargs)
            globalv.TRIGGERS[key].extend(table)
            try:
                csegs = cache_segments(segcache)
            except AttributeError:
                csegs = SegmentList()
            try:
                globalv.TRIGGERS[key].segments.extend(csegs)
            except AttributeError:
                globalv.TRIGGERS[key].segments = csegs
            finally:
                globalv.TRIGGERS[key].segments.coalesce()
            vprint('\r')

    # work out time function
    if return_:
        times = get_table_column(globalv.TRIGGERS[key], 'time').astype(float)

        # return correct triggers
        out = lsctables.New(TableClass, columns=columns)
        out.channel = str(channel)
        out.etg = str(etg)
        out.extend(t for (i, t) in enumerate(globalv.TRIGGERS[key]) if
                   times[i] in segments)
        out.segments = segments & globalv.TRIGGERS[key].segments
        return out
    else:
        return