def test_expand_versions(engine):
res = segmentdb_utils.expand_version_number( engine, ('H1','DMT-TESTSEG_1','*',924900000,924900016,0,0) )
if len(res) != 3:
return False
values = [('H1', 'DMT-TESTSEG_1', 2, 924900008, 924900010, 0, 0),
('H1', 'DMT-TESTSEG_1', 1, 924900000, 924900008, 0, 0),
('H1', 'DMT-TESTSEG_1', 1, 924900010, 924900016, 0, 0)]
for v in values:
found = False
for r in res:
found = found or v == r
if not found:
return False
return True
def test_expand_versions(engine):
res = segmentdb_utils.expand_version_number(
engine, ('H1', 'DMT-TESTSEG_1', '*', 924900000, 924900016, 0, 0))
if len(res) != 3:
return False
values = [('H1', 'DMT-TESTSEG_1', 2, 924900008, 924900010, 0, 0),
('H1', 'DMT-TESTSEG_1', 1, 924900000, 924900008, 0, 0),
('H1', 'DMT-TESTSEG_1', 1, 924900010, 924900016, 0, 0)]
for v in values:
found = False
for r in res:
found = found or v == r
if not found:
return False
return True
def query_segdb(cls, flags, *args, **kwargs):
"""Query the inital LIGO segment database for a list of flags.
Parameters
----------
flags : `iterable`
A list of flag names for which to query.
*args
Either, two `float`-like numbers indicating the
GPS [start, stop) interval, or a `SegmentList`
defining a number of summary segments.
url : `str`, optional, default: ``'https://segments.ligo.org'``
URL of the segment database.
Returns
-------
flagdict : `DataQualityDict`
An ordered `DataQualityDict` of (name, `DataQualityFlag`)
pairs.
"""
# given segmentlist
if len(args) == 1 and isinstance(args[0], SegmentList):
qsegs = args[0]
elif len(args) == 1 and len(args[0]) == 2:
qsegs = SegmentList(Segment(to_gps(args[0][0]),
to_gps(args[0][1])))
elif len(args) == 2:
qsegs = SegmentList([Segment(to_gps(args[0]), to_gps(args[1]))])
else:
raise ValueError("DataQualityDict.query_segdb must be called with "
"a list of flag names, and either GPS start and "
"stop times, or a SegmentList of query segments")
url = kwargs.pop('url', 'https://segments.ligo.org')
if kwargs.pop('on_error', None) is not None:
warnings.warn("DataQualityDict.query_segdb doesn't accept the "
"on_error keyword argument")
if kwargs.keys():
raise TypeError("DataQualityDict.query_segdb has no keyword "
"argument '%s'" % list(kwargs.keys()[0]))
# process query
from glue.segmentdb import (segmentdb_utils as segdb_utils,
query_engine as segdb_engine)
connection = segdb_utils.setup_database(url)
engine = segdb_engine.LdbdQueryEngine(connection)
segdefs = []
for flag in flags:
dqflag = DataQualityFlag(name=flag)
ifo = dqflag.ifo
name = dqflag.tag
if dqflag.version is None:
vers = '*'
else:
vers = dqflag.version
for gpsstart, gpsend in qsegs:
if float(gpsend) == +inf:
gpsend = to_gps('now').seconds
gpsstart = float(gpsstart)
if not gpsstart.is_integer():
raise ValueError("Segment database queries can only"
"operate on integer GPS times")
gpsend = float(gpsend)
if not gpsend.is_integer():
raise ValueError("Segment database queries can only"
"operate on integer GPS times")
segdefs += segdb_utils.expand_version_number(
engine, (ifo, name, vers, gpsstart, gpsend, 0, 0))
segs = segdb_utils.query_segments(engine, 'segment', segdefs)
segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
# build output
out = cls()
for definition, segments, summary in zip(segdefs, segs, segsum):
# parse flag name
flag = ':'.join(map(str, definition[:3]))
name = flag.rsplit(':', 1)[0]
# if versionless
if flag.endswith('*'):
flag = name
key = name
# if asked for versionless, but returned a version
elif flag not in flags and name in flags:
key = name
# other
else:
key = flag
# define flag
if key not in out:
out[key] = DataQualityFlag(name=flag)
# add segments
out[key].known.extend(summary)
out[key].active.extend(segments)
return out
def grab_segments(start, end, flag,\
segment_url='https://segdb.ligo.caltech.edu',\
segment_summary=False):
"""
Returns a segmentlist containing the segments during which the given flag
was active in the given period.
Arguments:
start : int
GPS start time
end : int
GPS end time
flag : string
'IFO:NAME:VERSION' format string
Keyword arguments:
segment_url : string
url of segment database to query, default https://segdb.ligo.caltech.edu
segment_summary : [ True | False ]
also return the glue.segments.segmentlist defining the valid span of the
returned segments
"""
# set times
start = int(math.floor(start))
end = int(math.ceil(end))
# set query engine
connection = segmentdb_utils.setup_database(segment_url)
engine = query_engine.LdbdQueryEngine(connection)
# format flag name
if isinstance(flag, basestring):
flags = flag.split(',')
else:
flags = flag
segdefs = []
for f in flags:
spec = f.split(':')
if len(spec) < 2 or len(spec) > 3:
raise AttributeError, "Included segements must be of the form "+\
"ifo:name:version or ifo:name:*"
ifo = spec[0]
name = spec[1]
if len(spec) is 3 and spec[2] is not '*':
version = int(spec[2])
if version < 1:
raise AttributeError, "Segment version numbers must be greater than zero"
else:
version = '*'
# expand segment definer
segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, \
start, end, 0, 0))
# query database and return
segs = segmentdb_utils.query_segments(engine, 'segment', segdefs)
segs = [s.coalesce() for s in segs]
if segment_summary:
segsums = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs)
#segsums = reduce(operator.or_, segsums).coalesce()
segsums = [s.coalesce() for s in segsums]
segsummap = [segments.segmentlist() for f in flags]
for segdef,segsum in zip(segdefs, segsums):
try:
fidx = flags.index(':'.join(map(str, segdef[:3])))
except ValueError:
fidx = flags.index(':'.join(segdef[:2]))
segsummap[fidx].extend(segsum)
if flag == flags[0]:
return segs[0],segsummap[0]
else:
return segs,segsummap
if flag == flags[0]:
return segs[0]
else:
return segs
def query(cls, flags, *args, **kwargs):
"""Query the segment database URL as given for the listed
`DataQualityFlag` names
Parameters
----------
flag : str
The name of the flag for which to query
*args
Either, two `float`-like numbers indicating the
GPS [start, stop) interval, or a
:class:`~gwpy.segments.segments.SegmentList`
defining a number of summary segments
url : `str`, optional, default: ``'https://segdb.ligo.caltech.edu'``
URL of the segment database, defaults to segdb.ligo.caltech.edu
Returns
-------
flag : `DataQualityFlag`
A new `DataQualityFlag`, with the `valid` and `active` lists
filled appropriately.
"""
# parse arguments
if len(args) == 1:
qsegs = args[0]
elif len(args) == 2:
qsegs = [args]
else:
raise ValueError("DataQualityDict.query must be called with a "
"flag name, and either GPS start and stop times, "
"or a SegmentList of query segments")
url = kwargs.pop('url', 'https://segdb.ligo.caltech.edu')
if kwargs.keys():
TypeError("DataQualityDict.query has no keyword argument '%s'"
% kwargs.keys()[0])
# parse flags
if isinstance(flags, basestring):
flags = flags.split(',')
else:
flags = flags
# process query
from glue.segmentdb import (segmentdb_utils as segdb_utils,
query_engine as segdb_engine)
connection = segdb_utils.setup_database(url)
engine = segdb_engine.LdbdQueryEngine(connection)
segdefs = []
for flag in flags:
ifo, name, version = parse_flag_name(flag)
if not version:
version = '*'
for gpsstart, gpsend in qsegs:
gpsstart = int(float(gpsstart))
gpsend = int(ceil(float(gpsend)))
segdefs += segdb_utils.expand_version_number(
engine, (ifo, name, version,
gpsstart, gpsend, 0, 0))
segs = segdb_utils.query_segments(engine, 'segment', segdefs)
segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
segs = [s.coalesce() for s in segs]
segsum = [s.coalesce() for s in segsum]
# build output
out = cls()
for definition, segments, summary in zip(segdefs, segs, segsum):
# parse flag name
flag = ':'.join(map(str, definition[:3]))
if flag.endswith('*'):
flag = flag.rsplit(':', 1)[0]
# define flag
if not flag in out:
out[flag] = DataQualityFlag(name=flag)
# add segments
out[flag].valid.extend(summary)
out[flag].active.extend(segments)
return out
def run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time, include_segments, exclude_segments, result_name):
segdefs = []
for included in include_segments.split(','):
spec = included.split(':')
if len(spec) < 2 or len(spec) > 3:
print("Included segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr)
sys.exit(1)
ifo = spec[0]
name = spec[1]
if len(spec) is 3 and spec[2] is not '*':
version = int(spec[2])
if version < 1:
print("Segment version numbers must be greater than zero", file=sys.stderr)
sys.exit(1)
else:
version = '*'
segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0) )
found_segments = segmentdb_utils.query_segments(engine, 'segment', segdefs)
found_segments = reduce(operator.or_, found_segments).coalesce()
# We could also do:
segment_summaries = segmentdb_utils.query_segments(engine, 'segment_summary', segdefs)
# And we could write out everything we found
segmentdb_utils.add_segment_info(doc, process_id, segdefs, None, segment_summaries)
# Do the same for excluded
if exclude_segments:
ex_segdefs = []
for excluded in exclude_segments.split(','):
spec = excluded.split(':')
if len(spec) < 2:
print("Excluded segements must be of the form ifo:name:version or ifo:name:*", file=sys.stderr)
sys.exit(1)
ifo = spec[0]
name = spec[1]
version = len(spec) > 2 and spec[2] or '*'
ex_segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0) )
excluded_segments = segmentdb_utils.query_segments(engine, 'segment', ex_segdefs)
excluded_segments = reduce(operator.or_, excluded_segments).coalesce()
found_segments.coalesce()
found_segments -= excluded_segments
# Add the result type to the segment definer table
seg_name = result_name
seg_def_id = segmentdb_utils.add_to_segment_definer(doc, process_id, ifo, seg_name, 1)
# and segment summary
segmentdb_utils.add_to_segment_summary(doc, process_id, seg_def_id, [[gps_start_time, gps_end_time]])
# and store the segments
segmentdb_utils.add_to_segment(doc, process_id, seg_def_id, found_segments)
print("Made it to the end of the query code")
print(doc)
def query_segdb(cls, flags, *args, **kwargs):
"""Query the inital LIGO segment database for a list of flags.
Parameters
----------
flags : `iterable`
A list of flag names for which to query.
*args
Either, two `float`-like numbers indicating the
GPS [start, stop) interval, or a `SegmentList`
defining a number of summary segments.
url : `str`, optional, default: ``'https://segments.ligo.org'``
URL of the segment database.
Returns
-------
flagdict : `DataQualityDict
An ordered `DataQualityDict` of (name, `DataQualityFlag`)
pairs.
"""
# given segmentlist
if len(args) == 1 and isinstance(args[0], SegmentList):
qsegs = args[0]
elif len(args) == 1 and len(args[0]) == 2:
qsegs = SegmentList(Segment(map(to_gps, args[0])))
elif len(args) == 2:
qsegs = SegmentList([Segment(map(to_gps, args))])
else:
raise ValueError("DataQualityDict.query_segdb must be called with "
"a list of flag names, and either GPS start and "
"stop times, or a SegmentList of query segments")
url = kwargs.pop('url', 'https://segments.ligo.org')
if kwargs.pop('on_error', None) is not None:
warnings.warn("DataQualityDict.query_segdb doesn't accept the "
"on_error keyword argument")
if kwargs.keys():
raise TypeError("DataQualityDict.query_segdb has no keyword "
"argument '%s'" % kwargs.keys()[0])
# parse flags
if isinstance(flags, (str, unicode)):
flags = flags.split(',')
else:
flags = flags
# process query
from glue.segmentdb import (segmentdb_utils as segdb_utils,
query_engine as segdb_engine)
connection = segdb_utils.setup_database(url)
engine = segdb_engine.LdbdQueryEngine(connection)
segdefs = []
for flag in flags:
dqflag = DataQualityFlag(name=flag)
ifo = dqflag.ifo
name = dqflag.tag
if dqflag.version is None:
vers = '*'
else:
vers = dqflag.version
for gpsstart, gpsend in qsegs:
if gpsend == PosInfinity or float(gpsend) == +inf:
gpsend = to_gps('now').seconds
gpsstart = float(gpsstart)
if not gpsstart.is_integer():
raise ValueError("Segment database queries can only"
"operate on integer GPS times")
gpsend = float(gpsend)
if not gpsend.is_integer():
raise ValueError("Segment database queries can only"
"operate on integer GPS times")
segdefs += segdb_utils.expand_version_number(
engine, (ifo, name, vers, gpsstart, gpsend, 0, 0))
segs = segdb_utils.query_segments(engine, 'segment', segdefs)
segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
# build output
out = cls()
for definition, segments, summary in zip(segdefs, segs, segsum):
# parse flag name
flag = ':'.join(map(str, definition[:3]))
name = flag.rsplit(':', 1)[0]
# if versionless
if flag.endswith('*'):
flag = name
key = name
# if asked for versionless, but returned a version
elif flag not in flags and name in flags:
key = name
# other
else:
key = flag
# define flag
if key not in out:
out[key] = DataQualityFlag(name=flag)
# add segments
out[key].known.extend(summary)
out[key].active.extend(segments)
return out
def run_query_segments(doc, process_id, engine, gps_start_time, gps_end_time,
include_segments, exclude_segments, result_name):
segdefs = []
for included in include_segments.split(','):
spec = included.split(':')
if len(spec) < 2 or len(spec) > 3:
print >> sys.stderr, "Included segements must be of the form ifo:name:version or ifo:name:*"
sys.exit(1)
ifo = spec[0]
name = spec[1]
if len(spec) is 3 and spec[2] is not '*':
version = int(spec[2])
if version < 1:
print >> sys.stderr, "Segment version numbers must be greater than zero"
sys.exit(1)
else:
version = '*'
segdefs += segmentdb_utils.expand_version_number(
engine, (ifo, name, version, gps_start_time, gps_end_time, 0, 0))
found_segments = segmentdb_utils.query_segments(engine, 'segment', segdefs)
found_segments = reduce(operator.or_, found_segments).coalesce()
# We could also do:
segment_summaries = segmentdb_utils.query_segments(engine,
'segment_summary',
segdefs)
# And we could write out everything we found
segmentdb_utils.add_segment_info(doc, process_id, segdefs, None,
segment_summaries)
# Do the same for excluded
if exclude_segments:
ex_segdefs = []
for excluded in exclude_segments.split(','):
spec = excluded.split(':')
if len(spec) < 2:
print >> sys.stderr, "Excluded segements must be of the form ifo:name:version or ifo:name:*"
sys.exit(1)
ifo = spec[0]
name = spec[1]
version = len(spec) > 2 and spec[2] or '*'
ex_segdefs += segmentdb_utils.expand_version_number(
engine,
(ifo, name, version, gps_start_time, gps_end_time, 0, 0))
excluded_segments = segmentdb_utils.query_segments(
engine, 'segment', ex_segdefs)
excluded_segments = reduce(operator.or_, excluded_segments).coalesce()
found_segments.coalesce()
found_segments -= excluded_segments
# Add the result type to the segment definer table
seg_name = result_name
seg_def_id = segmentdb_utils.add_to_segment_definer(
doc, process_id, ifo, seg_name, 1)
# and segment summary
segmentdb_utils.add_to_segment_summary(doc, process_id, seg_def_id,
[[gps_start_time, gps_end_time]])
# and store the segments
segmentdb_utils.add_to_segment(doc, process_id, seg_def_id, found_segments)
print "Made it to the end of the query code"
print doc
def grab_segments(start, end, flag,\
segment_url='https://segdb.ligo.caltech.edu',\
segment_summary=False):
"""
Returns a segmentlist containing the segments during which the given flag
was active in the given period.
Arguments:
start : int
GPS start time
end : int
GPS end time
flag : string
'IFO:NAME:VERSION' format string
Keyword arguments:
segment_url : string
url of segment database to query, default https://segdb.ligo.caltech.edu
segment_summary : [ True | False ]
also return the glue.segments.segmentlist defining the valid span of the
returned segments
"""
# set times
start = int(math.floor(start))
end = int(math.ceil(end))
# set query engine
connection = segmentdb_utils.setup_database(segment_url)
engine = query_engine.LdbdQueryEngine(connection)
# format flag name
if isinstance(flag, basestring):
flags = flag.split(',')
else:
flags = flag
segdefs = []
for f in flags:
spec = f.split(':')
if len(spec) < 2 or len(spec) > 3:
raise AttributeError, "Included segements must be of the form "+\
"ifo:name:version or ifo:name:*"
ifo = spec[0]
name = spec[1]
if len(spec) is 3 and spec[2] is not '*':
version = int(spec[2])
if version < 1:
raise AttributeError, "Segment version numbers must be greater than zero"
else:
version = '*'
# expand segment definer
segdefs += segmentdb_utils.expand_version_number(engine, (ifo, name, version, \
start, end, 0, 0))
# query database and return
segs = segmentdb_utils.query_segments(engine, 'segment', segdefs)
segs = [s.coalesce() for s in segs]
if segment_summary:
segsums = segmentdb_utils.query_segments(engine, 'segment_summary',
segdefs)
#segsums = reduce(operator.or_, segsums).coalesce()
segsums = [s.coalesce() for s in segsums]
segsummap = [segments.segmentlist() for f in flags]
for segdef, segsum in zip(segdefs, segsums):
try:
fidx = flags.index(':'.join(map(str, segdef[:3])))
except ValueError:
fidx = flags.index(':'.join(segdef[:2]))
segsummap[fidx].extend(segsum)
if flag == flags[0]:
return segs[0], segsummap[0]
else:
return segs, segsummap
if flag == flags[0]:
return segs[0]
else:
return segs
def query(cls, flags, *args, **kwargs):
"""Query the segment database for a set of `DataQualityFlags`.
Parameters
----------
flags : `iterable`
A list of flag names for which to query.
*args
Either, two `float`-like numbers indicating the
GPS [start, stop) interval, or a
:class:`~gwpy.segments.segments.SegmentList`
defining a number of summary segments.
url : `str`, optional, default: ``'https://segdb.ligo.caltech.edu'``
URL of the segment database.
Returns
-------
flagdict : `DataQualityDict
An ordered `DataQualityDict` of (name, `DataQualityFlag`)
pairs.
"""
# given segmentlist
if len(args) == 1 and isinstance(args[0], SegmentList):
qsegs = args[0]
elif len(args) == 1 and len(args[0]) == 2:
qsegs = SegmentList(Segment(args[0]))
elif len(args) == 2:
qsegs = SegmentList([Segment(args)])
else:
raise ValueError("DataQualityDict.query must be called with a "
"flag name, and either GPS start and stop times, "
"or a SegmentList of query segments")
url = kwargs.pop('url', 'https://segdb.ligo.caltech.edu')
if kwargs.keys():
raise TypeError("DataQualityDict.query has no keyword argument "
"'%s'" % kwargs.keys()[0])
# parse flags
if isinstance(flags, (str, unicode)):
flags = flags.split(',')
else:
flags = flags
# process query
from glue.segmentdb import (segmentdb_utils as segdb_utils,
query_engine as segdb_engine)
connection = segdb_utils.setup_database(url)
engine = segdb_engine.LdbdQueryEngine(connection)
segdefs = []
for flag in flags:
dqflag = DataQualityFlag(name=flag)
ifo = dqflag.ifo
name = dqflag.tag
if dqflag.version is None:
vers = '*'
else:
vers = dqflag.version
for gpsstart, gpsend in qsegs:
gpsstart = float(gpsstart)
if not gpsstart.is_integer():
raise ValueError("Segment database queries can only"
"operate on integer GPS times")
gpsend = float(gpsend)
if not gpsend.is_integer():
raise ValueError("Segment database queries can only"
"operate on integer GPS times")
segdefs += segdb_utils.expand_version_number(
engine, (ifo, name, vers, gpsstart, gpsend, 0, 0))
segs = segdb_utils.query_segments(engine, 'segment', segdefs)
segsum = segdb_utils.query_segments(engine, 'segment_summary', segdefs)
segs = [s.coalesce() for s in segs]
segsum = [s.coalesce() for s in segsum]
# build output
out = cls()
for definition, segments, summary in zip(segdefs, segs, segsum):
# parse flag name
flag = ':'.join(map(str, definition[:3]))
name = flag.rsplit(':', 1)[0]
# if versionless
if flag.endswith('*'):
flag = name
key = name
# if asked for versionless, but returned a version
elif flag not in flags and name in flags:
key = name
# other
else:
key = flag
# define flag
if not key in out:
out[key] = DataQualityFlag(name=flag)
# add segments
out[key].valid.extend(summary)
out[key].active.extend(segments)
return out