def test_basic_seg_summary(engine): res = segmentdb_utils.query_segments( engine, 'segment_summary', [ ('H1','DMT-TESTSEG_1',1,924900000,924900016,0,0) ] ) if res != [ segmentlist( [segment(924900000, 924900016)] ) ]: return False res = segmentdb_utils.query_segments( engine, 'segment_summary', [ ('H1','DMT-TESTSEG_1',2,924900000,924900016,0,0) ] ) if res != [ segmentlist( [segment(924900008, 924900010)] ) ]: return False return True
def test_basic_seg_summary(engine): res = segmentdb_utils.query_segments( engine, 'segment_summary', [('H1', 'DMT-TESTSEG_1', 1, 924900000, 924900016, 0, 0)]) if res != [segmentlist([segment(924900000, 924900016)])]: return False res = segmentdb_utils.query_segments( engine, 'segment_summary', [('H1', 'DMT-TESTSEG_1', 2, 924900000, 924900016, 0, 0)]) if res != [segmentlist([segment(924900008, 924900010)])]: return False return True
def query_segments_db(db_location, gps_start, gps_end, spec): """ Query db_location to get segments between (gps_start, gps_end), with definer spec. """ engine = LdbdQueryEngine(segmentdb_utils.setup_database(db_location)) ifo, definer, version = spec.split(":") definer_args = [[ifo, definer, int(version), gps_start, gps_end, 0, 0]] result = segmentdb_utils.query_segments(engine, "segment", definer_args) return segmentlist(result[0])
def test_optimized_query(engine): res = segmentdb_utils.query_segments( engine, 'segment_summary', [ ('H1','DMT-TESTSEG_2',1,924900000,924900016,0,0), ('H1','DMT-TESTSEG_3',1,924900000,924900016,0,0) ] ) if res[0] != segmentlist([segment(924900000, 924900010)]): return False if res[1] != segmentlist([segment(924900008, 924900016)]): return False return True
def test_optimized_query(engine): res = segmentdb_utils.query_segments( engine, 'segment_summary', [('H1', 'DMT-TESTSEG_2', 1, 924900000, 924900016, 0, 0), ('H1', 'DMT-TESTSEG_3', 1, 924900000, 924900016, 0, 0)]) if res[0] != segmentlist([segment(924900000, 924900010)]): return False if res[1] != segmentlist([segment(924900008, 924900016)]): return False return True
def query_segdb(cls, flags, *args, **kwargs): """Query the inital LIGO segment database for a list of flags. Parameters ---------- flags : `iterable` A list of flag names for which to query. *args Either, two `float`-like numbers indicating the GPS [start, stop) interval, or a `SegmentList` defining a number of summary segments. url : `str`, optional, default: ``'https://segments.ligo.org'`` URL of the segment database. Returns ------- flagdict : `DataQualityDict` An ordered `DataQualityDict` of (name, `DataQualityFlag`) pairs. """ # given segmentlist if len(args) == 1 and isinstance(args[0], SegmentList): qsegs = args[0] elif len(args) == 1 and len(args[0]) == 2: qsegs = SegmentList(Segment(to_gps(args[0][0]), to_gps(args[0][1]))) elif len(args) == 2: qsegs = SegmentList([Segment(to_gps(args[0]), to_gps(args[1]))]) else: raise ValueError("DataQualityDict.query_segdb must be called with " "a list of flag names, and either GPS start and " "stop times, or a SegmentList of query segments") url = kwargs.pop('url', 'https://segments.ligo.org') if kwargs.pop('on_error', None) is not None: warnings.warn("DataQualityDict.query_segdb doesn't accept the " "on_error keyword argument") if kwargs.keys(): raise TypeError("DataQualityDict.query_segdb has no keyword " "argument '%s'" % list(kwargs.keys()[0])) # process query from glue.segmentdb import (segmentdb_utils as segdb_utils, query_engine as segdb_engine) connection = segdb_utils.setup_database(url) engine = segdb_engine.LdbdQueryEngine(connection) segdefs = [] for flag in flags: dqflag = DataQualityFlag(name=flag) ifo = dqflag.ifo name = dqflag.tag if dqflag.version is None: vers = '*' else: vers = dqflag.version for gpsstart, gpsend in qsegs: if float(gpsend) == +inf: gpsend = to_gps('now').seconds gpsstart = float(gpsstart) if not gpsstart.is_integer(): raise ValueError("Segment database queries can only" "operate on integer GPS times") gpsend = float(gpsend) if not gpsend.is_integer(): raise ValueError("Segment database queries can only" "operate on integer GPS times") segdefs += segdb_utils.expand_version_number( engine, (ifo, name, vers, gpsstart, gpsend, 0, 0)) segs = segdb_utils.query_segments(engine, 'segment', segdefs) segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs) # build output out = cls() for definition, segments, summary in zip(segdefs, segs, segsum): # parse flag name flag = ':'.join(map(str, definition[:3])) name = flag.rsplit(':', 1)[0] # if versionless if flag.endswith('*'): flag = name key = name # if asked for versionless, but returned a version elif flag not in flags and name in flags: key = name # other else: key = flag # define flag if key not in out: out[key] = DataQualityFlag(name=flag) # add segments out[key].known.extend(summary) out[key].active.extend(segments) return out
def grab_segments(start, end, flag,\ segment_url='https://segdb.ligo.caltech.edu',\ segment_summary=False): """ Returns a segmentlist containing the segments during which the given flag was active in the given period. Arguments: start : int GPS start time end : int GPS end time flag : string 'IFO:NAME:VERSION' format string Keyword arguments: segment_url : string url of segment database to query, default https://segdb.ligo.caltech.edu segment_summary : [ True | False ] also return the glue.segments.segmentlist defining the valid span of the returned segments """ # set times start = int(math.floor(start)) end = int(math.ceil(end)) # set query engine connection = segmentdb_utils.setup_database(segment_url) engine = query_engine.LdbdQueryEngine(connection) # format flag name if isinstance(flag, basestring): flags = flag.split(',') else: flags = flag segdefs = [] for f in flags: spec = f.split(':') if len(spec) < 2 or len(spec) > 3: raise AttributeError, "Included segements must be of the form "+\ "ifo:name:version or ifo:name:*" ifo = spec[0] name = spec[1] if len(spec) is 3 and spec[2] is not '*': version = int(spec[2]) if version < 1: raise AttributeError, "Segment version numbers must be greater than zero" else: version = '*' # expand segment definer segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, \ start, end, 0, 0)) # query database and return segs = segmentdb_utils.query_segments(engine, 'segment', segdefs) segs = [s.coalesce() for s in segs] if segment_summary: segsums = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs) #segsums = reduce(operator.or_, segsums).coalesce() segsums = [s.coalesce() for s in segsums] segsummap = [segments.segmentlist() for f in flags] for segdef,segsum in zip(segdefs, segsums): try: fidx = flags.index(':'.join(map(str, segdef[:3]))) except ValueError: fidx = flags.index(':'.join(segdef[:2])) segsummap[fidx].extend(segsum) if flag == flags[0]: return segs[0],segsummap[0] else: return segs,segsummap if flag == flags[0]: return segs[0] else: return segs
def query(cls, flags, *args, **kwargs): """Query the segment database URL as given for the listed `DataQualityFlag` names Parameters ---------- flag : str The name of the flag for which to query *args Either, two `float`-like numbers indicating the GPS [start, stop) interval, or a :class:`~gwpy.segments.segments.SegmentList` defining a number of summary segments url : `str`, optional, default: ``'https://segdb.ligo.caltech.edu'`` URL of the segment database, defaults to segdb.ligo.caltech.edu Returns ------- flag : `DataQualityFlag` A new `DataQualityFlag`, with the `valid` and `active` lists filled appropriately. """ # parse arguments if len(args) == 1: qsegs = args[0] elif len(args) == 2: qsegs = [args] else: raise ValueError("DataQualityDict.query must be called with a " "flag name, and either GPS start and stop times, " "or a SegmentList of query segments") url = kwargs.pop('url', 'https://segdb.ligo.caltech.edu') if kwargs.keys(): TypeError("DataQualityDict.query has no keyword argument '%s'" % kwargs.keys()[0]) # parse flags if isinstance(flags, basestring): flags = flags.split(',') else: flags = flags # process query from glue.segmentdb import (segmentdb_utils as segdb_utils, query_engine as segdb_engine) connection = segdb_utils.setup_database(url) engine = segdb_engine.LdbdQueryEngine(connection) segdefs = [] for flag in flags: ifo, name, version = parse_flag_name(flag) if not version: version = '*' for gpsstart, gpsend in qsegs: gpsstart = int(float(gpsstart)) gpsend = int(ceil(float(gpsend))) segdefs += segdb_utils.expand_version_number( engine, (ifo, name, version, gpsstart, gpsend, 0, 0)) segs = segdb_utils.query_segments(engine, 'segment', segdefs) segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs) segs = [s.coalesce() for s in segs] segsum = [s.coalesce() for s in segsum] # build output out = cls() for definition, segments, summary in zip(segdefs, segs, segsum): # parse flag name flag = ':'.join(map(str, definition[:3])) if flag.endswith('*'): flag = flag.rsplit(':', 1)[0] # define flag if not flag in out: out[flag] = DataQualityFlag(name=flag) # add segments out[flag].valid.extend(summary) out[flag].active.extend(segments) return out
def run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time, include_segments, exclude_segments, result_name): segdefs = [] for included in include_segments.split(','): spec = included.split(':') if len(spec) < 2 or len(spec) > 3: print("Included segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr) sys.exit(1) ifo = spec[0] name = spec[1] if len(spec) is 3 and spec[2] is not '*': version = int(spec[2]) if version < 1: print("Segment version numbers must be greater than zero", file=sys.stderr) sys.exit(1) else: version = '*' segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0) ) found_segments = segmentdb_utils.query_segments(engine, 'segment', segdefs) found_segments = reduce(operator.or_, found_segments).coalesce() # We could also do: segment_summaries = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs) # And we could write out everything we found segmentdb_utils.add_segment_info(doc, process_id, segdefs, None, segment_summaries) # Do the same for excluded if exclude_segments: ex_segdefs = [] for excluded in exclude_segments.split(','): spec = excluded.split(':') if len(spec) < 2: print("Excluded segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr) sys.exit(1) ifo = spec[0] name = spec[1] version = len(spec) > 2 and spec[2] or '*' ex_segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0) ) excluded_segments = segmentdb_utils.query_segments(engine, 'segment', ex_segdefs) excluded_segments = reduce(operator.or_, excluded_segments).coalesce() found_segments.coalesce() found_segments -= excluded_segments # Add the result type to the segment definer table seg_name = result_name seg_def_id = segmentdb_utils.add_to_segment_definer(doc, process_id, ifo, seg_name, 1) # and segment summary segmentdb_utils.add_to_segment_summary(doc, process_id, seg_def_id, [[gps_start_time, gps_end_time]]) # and store the segments segmentdb_utils.add_to_segment(doc, process_id, seg_def_id, found_segments) print("Made it to the end of the query code") print(doc)
def query_segdb(cls, flags, *args, **kwargs): """Query the inital LIGO segment database for a list of flags. Parameters ---------- flags : `iterable` A list of flag names for which to query. *args Either, two `float`-like numbers indicating the GPS [start, stop) interval, or a `SegmentList` defining a number of summary segments. url : `str`, optional, default: ``'https://segments.ligo.org'`` URL of the segment database. Returns ------- flagdict : `DataQualityDict An ordered `DataQualityDict` of (name, `DataQualityFlag`) pairs. """ # given segmentlist if len(args) == 1 and isinstance(args[0], SegmentList): qsegs = args[0] elif len(args) == 1 and len(args[0]) == 2: qsegs = SegmentList(Segment(map(to_gps, args[0]))) elif len(args) == 2: qsegs = SegmentList([Segment(map(to_gps, args))]) else: raise ValueError("DataQualityDict.query_segdb must be called with " "a list of flag names, and either GPS start and " "stop times, or a SegmentList of query segments") url = kwargs.pop('url', 'https://segments.ligo.org') if kwargs.pop('on_error', None) is not None: warnings.warn("DataQualityDict.query_segdb doesn't accept the " "on_error keyword argument") if kwargs.keys(): raise TypeError("DataQualityDict.query_segdb has no keyword " "argument '%s'" % kwargs.keys()[0]) # parse flags if isinstance(flags, (str, unicode)): flags = flags.split(',') else: flags = flags # process query from glue.segmentdb import (segmentdb_utils as segdb_utils, query_engine as segdb_engine) connection = segdb_utils.setup_database(url) engine = segdb_engine.LdbdQueryEngine(connection) segdefs = [] for flag in flags: dqflag = DataQualityFlag(name=flag) ifo = dqflag.ifo name = dqflag.tag if dqflag.version is None: vers = '*' else: vers = dqflag.version for gpsstart, gpsend in qsegs: if gpsend == PosInfinity or float(gpsend) == +inf: gpsend = to_gps('now').seconds gpsstart = float(gpsstart) if not gpsstart.is_integer(): raise ValueError("Segment database queries can only" "operate on integer GPS times") gpsend = float(gpsend) if not gpsend.is_integer(): raise ValueError("Segment database queries can only" "operate on integer GPS times") segdefs += segdb_utils.expand_version_number( engine, (ifo, name, vers, gpsstart, gpsend, 0, 0)) segs = segdb_utils.query_segments(engine, 'segment', segdefs) segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs) # build output out = cls() for definition, segments, summary in zip(segdefs, segs, segsum): # parse flag name flag = ':'.join(map(str, definition[:3])) name = flag.rsplit(':', 1)[0] # if versionless if flag.endswith('*'): flag = name key = name # if asked for versionless, but returned a version elif flag not in flags and name in flags: key = name # other else: key = flag # define flag if key not in out: out[key] = DataQualityFlag(name=flag) # add segments out[key].known.extend(summary) out[key].active.extend(segments) return out
def run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time, include_segments, exclude_segments, result_name): segdefs = [] for included in include_segments.split(','): spec = included.split(':') if len(spec) < 2 or len(spec) > 3: print >> sys.stderr, "Included segements must be of the form ifo:name:version or ifo:name:*" sys.exit(1) ifo = spec[0] name = spec[1] if len(spec) is 3 and spec[2] is not '*': version = int(spec[2]) if version < 1: print >> sys.stderr, "Segment version numbers must be greater than zero" sys.exit(1) else: version = '*' segdefs += segmentdb_utils.expand_version_number( engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0)) found_segments = segmentdb_utils.query_segments(engine, 'segment', segdefs) found_segments = reduce(operator.or_, found_segments).coalesce() # We could also do: segment_summaries = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs) # And we could write out everything we found segmentdb_utils.add_segment_info(doc, process_id, segdefs, None, segment_summaries) # Do the same for excluded if exclude_segments: ex_segdefs = [] for excluded in exclude_segments.split(','): spec = excluded.split(':') if len(spec) < 2: print >> sys.stderr, "Excluded segements must be of the form ifo:name:version or ifo:name:*" sys.exit(1) ifo = spec[0] name = spec[1] version = len(spec) > 2 and spec[2] or '*' ex_segdefs += segmentdb_utils.expand_version_number( engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0)) excluded_segments = segmentdb_utils.query_segments( engine, 'segment', ex_segdefs) excluded_segments = reduce(operator.or_, excluded_segments).coalesce() found_segments.coalesce() found_segments -= excluded_segments # Add the result type to the segment definer table seg_name = result_name seg_def_id = segmentdb_utils.add_to_segment_definer( doc, process_id, ifo, seg_name, 1) # and segment summary segmentdb_utils.add_to_segment_summary(doc, process_id, seg_def_id, [[gps_start_time, gps_end_time]]) # and store the segments segmentdb_utils.add_to_segment(doc, process_id, seg_def_id, found_segments) print "Made it to the end of the query code" print doc
def grab_segments(start, end, flag,\ segment_url='https://segdb.ligo.caltech.edu',\ segment_summary=False): """ Returns a segmentlist containing the segments during which the given flag was active in the given period. Arguments: start : int GPS start time end : int GPS end time flag : string 'IFO:NAME:VERSION' format string Keyword arguments: segment_url : string url of segment database to query, default https://segdb.ligo.caltech.edu segment_summary : [ True | False ] also return the glue.segments.segmentlist defining the valid span of the returned segments """ # set times start = int(math.floor(start)) end = int(math.ceil(end)) # set query engine connection = segmentdb_utils.setup_database(segment_url) engine = query_engine.LdbdQueryEngine(connection) # format flag name if isinstance(flag, basestring): flags = flag.split(',') else: flags = flag segdefs = [] for f in flags: spec = f.split(':') if len(spec) < 2 or len(spec) > 3: raise AttributeError, "Included segements must be of the form "+\ "ifo:name:version or ifo:name:*" ifo = spec[0] name = spec[1] if len(spec) is 3 and spec[2] is not '*': version = int(spec[2]) if version < 1: raise AttributeError, "Segment version numbers must be greater than zero" else: version = '*' # expand segment definer segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, \ start, end, 0, 0)) # query database and return segs = segmentdb_utils.query_segments(engine, 'segment', segdefs) segs = [s.coalesce() for s in segs] if segment_summary: segsums = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs) #segsums = reduce(operator.or_, segsums).coalesce() segsums = [s.coalesce() for s in segsums] segsummap = [segments.segmentlist() for f in flags] for segdef, segsum in zip(segdefs, segsums): try: fidx = flags.index(':'.join(map(str, segdef[:3]))) except ValueError: fidx = flags.index(':'.join(segdef[:2])) segsummap[fidx].extend(segsum) if flag == flags[0]: return segs[0], segsummap[0] else: return segs, segsummap if flag == flags[0]: return segs[0] else: return segs
def query(cls, flags, *args, **kwargs): """Query the segment database for a set of `DataQualityFlags`. Parameters ---------- flags : `iterable` A list of flag names for which to query. *args Either, two `float`-like numbers indicating the GPS [start, stop) interval, or a :class:`~gwpy.segments.segments.SegmentList` defining a number of summary segments. url : `str`, optional, default: ``'https://segdb.ligo.caltech.edu'`` URL of the segment database. Returns ------- flagdict : `DataQualityDict An ordered `DataQualityDict` of (name, `DataQualityFlag`) pairs. """ # given segmentlist if len(args) == 1 and isinstance(args[0], SegmentList): qsegs = args[0] elif len(args) == 1 and len(args[0]) == 2: qsegs = SegmentList(Segment(args[0])) elif len(args) == 2: qsegs = SegmentList([Segment(args)]) else: raise ValueError("DataQualityDict.query must be called with a " "flag name, and either GPS start and stop times, " "or a SegmentList of query segments") url = kwargs.pop('url', 'https://segdb.ligo.caltech.edu') if kwargs.keys(): raise TypeError("DataQualityDict.query has no keyword argument " "'%s'" % kwargs.keys()[0]) # parse flags if isinstance(flags, (str, unicode)): flags = flags.split(',') else: flags = flags # process query from glue.segmentdb import (segmentdb_utils as segdb_utils, query_engine as segdb_engine) connection = segdb_utils.setup_database(url) engine = segdb_engine.LdbdQueryEngine(connection) segdefs = [] for flag in flags: dqflag = DataQualityFlag(name=flag) ifo = dqflag.ifo name = dqflag.tag if dqflag.version is None: vers = '*' else: vers = dqflag.version for gpsstart, gpsend in qsegs: gpsstart = float(gpsstart) if not gpsstart.is_integer(): raise ValueError("Segment database queries can only" "operate on integer GPS times") gpsend = float(gpsend) if not gpsend.is_integer(): raise ValueError("Segment database queries can only" "operate on integer GPS times") segdefs += segdb_utils.expand_version_number( engine, (ifo, name, vers, gpsstart, gpsend, 0, 0)) segs = segdb_utils.query_segments(engine, 'segment', segdefs) segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs) segs = [s.coalesce() for s in segs] segsum = [s.coalesce() for s in segsum] # build output out = cls() for definition, segments, summary in zip(segdefs, segs, segsum): # parse flag name flag = ':'.join(map(str, definition[:3])) name = flag.rsplit(':', 1)[0] # if versionless if flag.endswith('*'): flag = name key = name # if asked for versionless, but returned a version elif flag not in flags and name in flags: key = name # other else: key = flag # define flag if not key in out: out[key] = DataQualityFlag(name=flag) # add segments out[key].valid.extend(summary) out[key].active.extend(segments) return out