Esempio n. 1
0
def test_expand_versions(engine):
    res = segmentdb_utils.expand_version_number( engine, ('H1','DMT-TESTSEG_1','*',924900000,924900016,0,0) )

    if len(res) != 3:
        return False

    values = [('H1', 'DMT-TESTSEG_1', 2, 924900008, 924900010, 0, 0), 
              ('H1', 'DMT-TESTSEG_1', 1, 924900000, 924900008, 0, 0), 
              ('H1', 'DMT-TESTSEG_1', 1, 924900010, 924900016, 0, 0)]

    for v in values:
        found = False
        for r in res:
            found = found or v == r

        if not found:
            return False

    return True
def test_expand_versions(engine):
    res = segmentdb_utils.expand_version_number(
        engine, ('H1', 'DMT-TESTSEG_1', '*', 924900000, 924900016, 0, 0))

    if len(res) != 3:
        return False

    values = [('H1', 'DMT-TESTSEG_1', 2, 924900008, 924900010, 0, 0),
              ('H1', 'DMT-TESTSEG_1', 1, 924900000, 924900008, 0, 0),
              ('H1', 'DMT-TESTSEG_1', 1, 924900010, 924900016, 0, 0)]

    for v in values:
        found = False
        for r in res:
            found = found or v == r

        if not found:
            return False

    return True
Esempio n. 3
0
    def query_segdb(cls, flags, *args, **kwargs):
        """Query the inital LIGO segment database for a list of flags.

        Parameters
        ----------
        flags : `iterable`
            A list of flag names for which to query.
        *args
            Either, two `float`-like numbers indicating the
            GPS [start, stop) interval, or a `SegmentList`
            defining a number of summary segments.
        url : `str`, optional, default: ``'https://segments.ligo.org'``
            URL of the segment database.

        Returns
        -------
        flagdict : `DataQualityDict`
            An ordered `DataQualityDict` of (name, `DataQualityFlag`)
            pairs.
        """
        # given segmentlist
        if len(args) == 1 and isinstance(args[0], SegmentList):
            qsegs = args[0]
        elif len(args) == 1 and len(args[0]) == 2:
            qsegs = SegmentList(Segment(to_gps(args[0][0]),
                                        to_gps(args[0][1])))
        elif len(args) == 2:
            qsegs = SegmentList([Segment(to_gps(args[0]), to_gps(args[1]))])
        else:
            raise ValueError("DataQualityDict.query_segdb must be called with "
                             "a list of flag names, and either GPS start and "
                             "stop times, or a SegmentList of query segments")
        url = kwargs.pop('url', 'https://segments.ligo.org')
        if kwargs.pop('on_error', None) is not None:
            warnings.warn("DataQualityDict.query_segdb doesn't accept the "
                          "on_error keyword argument")
        if kwargs.keys():
            raise TypeError("DataQualityDict.query_segdb has no keyword "
                            "argument '%s'" % list(kwargs.keys()[0]))

        # process query
        from glue.segmentdb import (segmentdb_utils as segdb_utils,
                                    query_engine as segdb_engine)
        connection = segdb_utils.setup_database(url)
        engine = segdb_engine.LdbdQueryEngine(connection)
        segdefs = []
        for flag in flags:
            dqflag = DataQualityFlag(name=flag)
            ifo = dqflag.ifo
            name = dqflag.tag
            if dqflag.version is None:
                vers = '*'
            else:
                vers = dqflag.version
            for gpsstart, gpsend in qsegs:
                if float(gpsend) == +inf:
                    gpsend = to_gps('now').seconds
                gpsstart = float(gpsstart)
                if not gpsstart.is_integer():
                    raise ValueError("Segment database queries can only"
                                     "operate on integer GPS times")
                gpsend = float(gpsend)
                if not gpsend.is_integer():
                    raise ValueError("Segment database queries can only"
                                     "operate on integer GPS times")
                segdefs += segdb_utils.expand_version_number(
                    engine, (ifo, name, vers, gpsstart, gpsend, 0, 0))
        segs = segdb_utils.query_segments(engine, 'segment', segdefs)
        segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
        # build output
        out = cls()
        for definition, segments, summary in zip(segdefs, segs, segsum):
            # parse flag name
            flag = ':'.join(map(str, definition[:3]))
            name = flag.rsplit(':', 1)[0]
            # if versionless
            if flag.endswith('*'):
                flag = name
                key = name
            # if asked for versionless, but returned a version
            elif flag not in flags and name in flags:
                key = name
            # other
            else:
                key = flag
            # define flag
            if key not in out:
                out[key] = DataQualityFlag(name=flag)
            # add segments
            out[key].known.extend(summary)
            out[key].active.extend(segments)
        return out
Esempio n. 4
0
def grab_segments(start, end, flag,\
                  segment_url='https://segdb.ligo.caltech.edu',\
                  segment_summary=False):

  """
    Returns a segmentlist containing the segments during which the given flag
    was active in the given period.

    Arguments:

      start : int
        GPS start time
      end : int
        GPS end time
      flag : string
        'IFO:NAME:VERSION' format string

    Keyword arguments:

      segment_url : string
        url of segment database to query, default https://segdb.ligo.caltech.edu
      segment_summary : [ True | False ]
        also return the glue.segments.segmentlist defining the valid span of the
        returned segments
  """

  # set times
  start = int(math.floor(start))
  end   = int(math.ceil(end))

  # set query engine
  connection        = segmentdb_utils.setup_database(segment_url)
  engine            = query_engine.LdbdQueryEngine(connection)

  # format flag name
  if isinstance(flag, basestring):
    flags = flag.split(',')
  else:
    flags = flag

  segdefs = []
  for f in flags:
    spec = f.split(':')
    if len(spec) < 2 or len(spec) > 3:
      raise AttributeError, "Included segements must be of the form "+\
                            "ifo:name:version or ifo:name:*"

    ifo  = spec[0]
    name = spec[1]

    if len(spec) is 3 and spec[2] is not '*':
      version = int(spec[2])
      if version < 1:
        raise AttributeError, "Segment version numbers must be greater than zero"
    else:
      version = '*'

    # expand segment definer
    segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, \
                                                            start, end, 0, 0))

  # query database and return
  segs = segmentdb_utils.query_segments(engine, 'segment', segdefs)
  segs = [s.coalesce() for s in segs]
  if segment_summary:
    segsums = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs)
    #segsums = reduce(operator.or_, segsums).coalesce()
    segsums = [s.coalesce() for s in segsums]
    segsummap = [segments.segmentlist() for f in flags]
    for segdef,segsum in zip(segdefs, segsums):
      try:
        fidx = flags.index(':'.join(map(str, segdef[:3])))
      except ValueError:
        fidx = flags.index(':'.join(segdef[:2]))
      segsummap[fidx].extend(segsum)
    if flag == flags[0]:
      return segs[0],segsummap[0]
    else:
      return segs,segsummap
  if flag == flags[0]:
    return segs[0]
  else:
    return segs
Esempio n. 5
0
    def query(cls, flags, *args, **kwargs):
        """Query the segment database URL as given for the listed
        `DataQualityFlag` names

        Parameters
        ----------
        flag : str
            The name of the flag for which to query
        *args
            Either, two `float`-like numbers indicating the
            GPS [start, stop) interval, or a
            :class:`~gwpy.segments.segments.SegmentList`
            defining a number of summary segments
        url : `str`, optional, default: ``'https://segdb.ligo.caltech.edu'``
            URL of the segment database, defaults to segdb.ligo.caltech.edu

        Returns
        -------
        flag : `DataQualityFlag`
            A new `DataQualityFlag`, with the `valid` and `active` lists
            filled appropriately.
        """
        # parse arguments
        if len(args) == 1:
            qsegs = args[0]
        elif len(args) == 2:
            qsegs = [args]
        else:
            raise ValueError("DataQualityDict.query must be called with a "
                             "flag name, and either GPS start and stop times, "
                             "or a SegmentList of query segments")
        url = kwargs.pop('url', 'https://segdb.ligo.caltech.edu')
        if kwargs.keys():
            TypeError("DataQualityDict.query has no keyword argument '%s'"
                      % kwargs.keys()[0])
        # parse flags
        if isinstance(flags, basestring):
            flags = flags.split(',')
        else:
            flags = flags
        # process query
        from glue.segmentdb import (segmentdb_utils as segdb_utils,
                                    query_engine as segdb_engine)
        connection = segdb_utils.setup_database(url)
        engine = segdb_engine.LdbdQueryEngine(connection)
        segdefs = []
        for flag in flags:
            ifo, name, version = parse_flag_name(flag)
            if not version:
                version = '*'
            for gpsstart, gpsend in qsegs:
                gpsstart = int(float(gpsstart))
                gpsend = int(ceil(float(gpsend)))
                segdefs += segdb_utils.expand_version_number(
                               engine, (ifo, name, version,
                                        gpsstart, gpsend, 0, 0))
        segs = segdb_utils.query_segments(engine, 'segment', segdefs)
        segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
        segs = [s.coalesce() for s in segs]
        segsum = [s.coalesce() for s in segsum]
        # build output
        out = cls()
        for definition, segments, summary in zip(segdefs, segs, segsum):
            # parse flag name
            flag = ':'.join(map(str, definition[:3]))
            if flag.endswith('*'):
                flag = flag.rsplit(':', 1)[0]
            # define flag
            if not flag in out:
                out[flag] = DataQualityFlag(name=flag)
            # add segments
            out[flag].valid.extend(summary)
            out[flag].active.extend(segments)
        return out
Esempio n. 6
0
def run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time, include_segments, exclude_segments, result_name):
    segdefs = []

    for included in include_segments.split(','):
        spec = included.split(':')

        if len(spec) < 2 or len(spec) > 3:
            print("Included segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr)
            sys.exit(1)

        ifo     = spec[0]
        name    = spec[1]
        if len(spec) is 3 and spec[2] is not '*':
            version = int(spec[2])
            if version < 1:
                print("Segment version numbers must be greater than zero", file=sys.stderr)
                sys.exit(1)
        else:
            version = '*'

        segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0) )

    found_segments = segmentdb_utils.query_segments(engine, 'segment', segdefs)
    found_segments = reduce(operator.or_, found_segments).coalesce()

    # We could also do:
    segment_summaries = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs)

    # And we could write out everything we found
    segmentdb_utils.add_segment_info(doc, process_id, segdefs, None, segment_summaries)


    # Do the same for excluded
    if exclude_segments:
        ex_segdefs = []

        for excluded in exclude_segments.split(','):
            spec = excluded.split(':')

            if len(spec) < 2:
                print("Excluded segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr)
                sys.exit(1)

            ifo     = spec[0]
            name    = spec[1]
            version = len(spec) > 2 and spec[2] or '*'

            ex_segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0) )


        excluded_segments = segmentdb_utils.query_segments(engine, 'segment', ex_segdefs)
        excluded_segments = reduce(operator.or_, excluded_segments).coalesce()

        found_segments.coalesce()
        found_segments -= excluded_segments



    # Add the result type to the segment definer table
    seg_name   = result_name
    seg_def_id = segmentdb_utils.add_to_segment_definer(doc, process_id, ifo, seg_name, 1)

    # and segment summary
    segmentdb_utils.add_to_segment_summary(doc, process_id, seg_def_id, [[gps_start_time, gps_end_time]])

    # and store the segments
    segmentdb_utils.add_to_segment(doc, process_id, seg_def_id, found_segments)
    print("Made it to the end of the query code")
    print(doc)
Esempio n. 7
0
    def query_segdb(cls, flags, *args, **kwargs):
        """Query the inital LIGO segment database for a list of flags.

        Parameters
        ----------
        flags : `iterable`
            A list of flag names for which to query.
        *args
            Either, two `float`-like numbers indicating the
            GPS [start, stop) interval, or a `SegmentList`
            defining a number of summary segments.
        url : `str`, optional, default: ``'https://segments.ligo.org'``
            URL of the segment database.

        Returns
        -------
        flagdict : `DataQualityDict
            An ordered `DataQualityDict` of (name, `DataQualityFlag`)
            pairs.
        """
        # given segmentlist
        if len(args) == 1 and isinstance(args[0], SegmentList):
            qsegs = args[0]
        elif len(args) == 1 and len(args[0]) == 2:
            qsegs = SegmentList(Segment(map(to_gps, args[0])))
        elif len(args) == 2:
            qsegs = SegmentList([Segment(map(to_gps, args))])
        else:
            raise ValueError("DataQualityDict.query_segdb must be called with "
                             "a list of flag names, and either GPS start and "
                             "stop times, or a SegmentList of query segments")
        url = kwargs.pop('url', 'https://segments.ligo.org')
        if kwargs.pop('on_error', None) is not None:
            warnings.warn("DataQualityDict.query_segdb doesn't accept the "
                          "on_error keyword argument")
        if kwargs.keys():
            raise TypeError("DataQualityDict.query_segdb has no keyword "
                            "argument '%s'" % kwargs.keys()[0])
        # parse flags
        if isinstance(flags, (str, unicode)):
            flags = flags.split(',')
        else:
            flags = flags
        # process query
        from glue.segmentdb import (segmentdb_utils as segdb_utils,
                                    query_engine as segdb_engine)
        connection = segdb_utils.setup_database(url)
        engine = segdb_engine.LdbdQueryEngine(connection)
        segdefs = []
        for flag in flags:
            dqflag = DataQualityFlag(name=flag)
            ifo = dqflag.ifo
            name = dqflag.tag
            if dqflag.version is None:
                vers = '*'
            else:
                vers = dqflag.version
            for gpsstart, gpsend in qsegs:
                if gpsend == PosInfinity or float(gpsend) == +inf:
                    gpsend = to_gps('now').seconds
                gpsstart = float(gpsstart)
                if not gpsstart.is_integer():
                    raise ValueError("Segment database queries can only"
                                     "operate on integer GPS times")
                gpsend = float(gpsend)
                if not gpsend.is_integer():
                    raise ValueError("Segment database queries can only"
                                     "operate on integer GPS times")
                segdefs += segdb_utils.expand_version_number(
                    engine, (ifo, name, vers, gpsstart, gpsend, 0, 0))
        segs = segdb_utils.query_segments(engine, 'segment', segdefs)
        segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
        # build output
        out = cls()
        for definition, segments, summary in zip(segdefs, segs, segsum):
            # parse flag name
            flag = ':'.join(map(str, definition[:3]))
            name = flag.rsplit(':', 1)[0]
            # if versionless
            if flag.endswith('*'):
                flag = name
                key = name
            # if asked for versionless, but returned a version
            elif flag not in flags and name in flags:
                key = name
            # other
            else:
                key = flag
            # define flag
            if key not in out:
                out[key] = DataQualityFlag(name=flag)
            # add segments
            out[key].known.extend(summary)
            out[key].active.extend(segments)
        return out
Esempio n. 8
0
def run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time,
                       include_segments, exclude_segments, result_name):
    segdefs = []

    for included in include_segments.split(','):
        spec = included.split(':')

        if len(spec) < 2 or len(spec) > 3:
            print >> sys.stderr, "Included segements must be of the form ifo:name:version or ifo:name:*"
            sys.exit(1)

        ifo = spec[0]
        name = spec[1]
        if len(spec) is 3 and spec[2] is not '*':
            version = int(spec[2])
            if version < 1:
                print >> sys.stderr, "Segment version numbers must be greater than zero"
                sys.exit(1)
        else:
            version = '*'

        segdefs += segmentdb_utils.expand_version_number(
            engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0))

    found_segments = segmentdb_utils.query_segments(engine, 'segment', segdefs)
    found_segments = reduce(operator.or_, found_segments).coalesce()

    # We could also do:
    segment_summaries = segmentdb_utils.query_segments(engine,
                                                       'segment_summary',
                                                       segdefs)

    # And we could write out everything we found
    segmentdb_utils.add_segment_info(doc, process_id, segdefs, None,
                                     segment_summaries)

    # Do the same for excluded
    if exclude_segments:
        ex_segdefs = []

        for excluded in exclude_segments.split(','):
            spec = excluded.split(':')

            if len(spec) < 2:
                print >> sys.stderr, "Excluded segements must be of the form ifo:name:version or ifo:name:*"
                sys.exit(1)

            ifo = spec[0]
            name = spec[1]
            version = len(spec) > 2 and spec[2] or '*'

            ex_segdefs += segmentdb_utils.expand_version_number(
                engine,
                (ifo, name, version, gps_start_time, gps_end_time, 0, 0))

        excluded_segments = segmentdb_utils.query_segments(
            engine, 'segment', ex_segdefs)
        excluded_segments = reduce(operator.or_, excluded_segments).coalesce()

        found_segments.coalesce()
        found_segments -= excluded_segments

    # Add the result type to the segment definer table
    seg_name = result_name
    seg_def_id = segmentdb_utils.add_to_segment_definer(
        doc, process_id, ifo, seg_name, 1)

    # and segment summary
    segmentdb_utils.add_to_segment_summary(doc, process_id, seg_def_id,
                                           [[gps_start_time, gps_end_time]])

    # and store the segments
    segmentdb_utils.add_to_segment(doc, process_id, seg_def_id, found_segments)
    print "Made it to the end of the query code"
    print doc
def grab_segments(start, end, flag,\
                  segment_url='https://segdb.ligo.caltech.edu',\
                  segment_summary=False):
    """
    Returns a segmentlist containing the segments during which the given flag
    was active in the given period.

    Arguments:

      start : int
        GPS start time
      end : int
        GPS end time
      flag : string
        'IFO:NAME:VERSION' format string

    Keyword arguments:

      segment_url : string
        url of segment database to query, default https://segdb.ligo.caltech.edu
      segment_summary : [ True | False ]
        also return the glue.segments.segmentlist defining the valid span of the
        returned segments
  """

    # set times
    start = int(math.floor(start))
    end = int(math.ceil(end))

    # set query engine
    connection = segmentdb_utils.setup_database(segment_url)
    engine = query_engine.LdbdQueryEngine(connection)

    # format flag name
    if isinstance(flag, basestring):
        flags = flag.split(',')
    else:
        flags = flag

    segdefs = []
    for f in flags:
        spec = f.split(':')
        if len(spec) < 2 or len(spec) > 3:
            raise AttributeError, "Included segements must be of the form "+\
                                  "ifo:name:version or ifo:name:*"

        ifo = spec[0]
        name = spec[1]

        if len(spec) is 3 and spec[2] is not '*':
            version = int(spec[2])
            if version < 1:
                raise AttributeError, "Segment version numbers must be greater than zero"
        else:
            version = '*'

        # expand segment definer
        segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, \
                                                                start, end, 0, 0))

    # query database and return
    segs = segmentdb_utils.query_segments(engine, 'segment', segdefs)
    segs = [s.coalesce() for s in segs]
    if segment_summary:
        segsums = segmentdb_utils.query_segments(engine, 'segment_summary',
                                                 segdefs)
        #segsums = reduce(operator.or_, segsums).coalesce()
        segsums = [s.coalesce() for s in segsums]
        segsummap = [segments.segmentlist() for f in flags]
        for segdef, segsum in zip(segdefs, segsums):
            try:
                fidx = flags.index(':'.join(map(str, segdef[:3])))
            except ValueError:
                fidx = flags.index(':'.join(segdef[:2]))
            segsummap[fidx].extend(segsum)
        if flag == flags[0]:
            return segs[0], segsummap[0]
        else:
            return segs, segsummap
    if flag == flags[0]:
        return segs[0]
    else:
        return segs
Esempio n. 10
0
    def query(cls, flags, *args, **kwargs):
        """Query the segment database for a set of `DataQualityFlags`.

        Parameters
        ----------
        flags : `iterable`
            A list of flag names for which to query.
        *args
            Either, two `float`-like numbers indicating the
            GPS [start, stop) interval, or a
            :class:`~gwpy.segments.segments.SegmentList`
            defining a number of summary segments.
        url : `str`, optional, default: ``'https://segdb.ligo.caltech.edu'``
            URL of the segment database.

        Returns
        -------
        flagdict : `DataQualityDict
            An ordered `DataQualityDict` of (name, `DataQualityFlag`)
            pairs.
        """
        # given segmentlist
        if len(args) == 1 and isinstance(args[0], SegmentList):
            qsegs = args[0]
        elif len(args) == 1 and len(args[0]) == 2:
            qsegs = SegmentList(Segment(args[0]))
        elif len(args) == 2:
            qsegs = SegmentList([Segment(args)])
        else:
            raise ValueError("DataQualityDict.query must be called with a "
                             "flag name, and either GPS start and stop times, "
                             "or a SegmentList of query segments")
        url = kwargs.pop('url', 'https://segdb.ligo.caltech.edu')
        if kwargs.keys():
            raise TypeError("DataQualityDict.query has no keyword argument "
                            "'%s'" % kwargs.keys()[0])
        # parse flags
        if isinstance(flags, (str, unicode)):
            flags = flags.split(',')
        else:
            flags = flags
        # process query
        from glue.segmentdb import (segmentdb_utils as segdb_utils,
                                    query_engine as segdb_engine)
        connection = segdb_utils.setup_database(url)
        engine = segdb_engine.LdbdQueryEngine(connection)
        segdefs = []
        for flag in flags:
            dqflag = DataQualityFlag(name=flag)
            ifo = dqflag.ifo
            name = dqflag.tag
            if dqflag.version is None:
                vers = '*'
            else:
                vers = dqflag.version
            for gpsstart, gpsend in qsegs:
                gpsstart = float(gpsstart)
                if not gpsstart.is_integer():
                    raise ValueError("Segment database queries can only"
                                     "operate on integer GPS times")
                gpsend = float(gpsend)
                if not gpsend.is_integer():
                    raise ValueError("Segment database queries can only"
                                     "operate on integer GPS times")
                segdefs += segdb_utils.expand_version_number(
                    engine, (ifo, name, vers, gpsstart, gpsend, 0, 0))
        segs = segdb_utils.query_segments(engine, 'segment', segdefs)
        segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
        segs = [s.coalesce() for s in segs]
        segsum = [s.coalesce() for s in segsum]
        # build output
        out = cls()
        for definition, segments, summary in zip(segdefs, segs, segsum):
            # parse flag name
            flag = ':'.join(map(str, definition[:3]))
            name = flag.rsplit(':', 1)[0]
            # if versionless
            if flag.endswith('*'):
                flag = name
                key = name
            # if asked for versionless, but returned a version
            elif flag not in flags and name in flags:
                key = name
            # other
            else:
                key = flag
            # define flag
            if not key in out:
                out[key] = DataQualityFlag(name=flag)
            # add segments
            out[key].valid.extend(summary)
            out[key].active.extend(segments)
        return out