def write_events(channel, tab, segments):
"""Write events to file with a given filename
"""
# get filename
path = create_path(channel)
h5f = h5py.File(str(path), 'a')
# read existing table from file
try:
old = tab.read(h5f["triggers"], format="hdf5")
except KeyError:
pass
else:
tab = vstack(old, tab)
# append event table
tab.write(h5f, path="triggers", append=True, overwrite=True)
# write segments
try:
oldsegs = DataQualityFlag.read(h5f, path="segments", format="hdf5")
except KeyError:
pass
else:
segments = oldsegs + segments
segments.write(h5f, path="segments", append=True, overwrite=True)
# write file to disk
h5f.close()
return path
def test_read_segwizard(self):
flag = DataQualityFlag.read(SEGWIZ, FLAG1, coalesce=False)
self.assertTrue(
flag.active == ACTIVE,
'DataQualityFlag.read(segwizard) mismatch:\n\n%s\n\n%s' %
(ACTIVE, flag.active))
self.assertTrue(flag.known == flag.active)
def test_read_ligolw(self):
flag = DataQualityFlag.read(SEGXML, FLAG1, coalesce=False)
self.assertTrue(flag.active == ACTIVE,
'DataQualityFlag.read(ligol) mismatch:\n\n%s\n\n%s'
% (ACTIVE, flag.active))
self.assertTrue(flag.known == KNOWN,
'DataQualityFlag.read(ligol) mismatch:\n\n%s\n\n%s'
% (KNOWN, flag.known))
def test_read_ligolw(self):
flag = DataQualityFlag.read(SEGXML, FLAG1, coalesce=False)
self.assertTrue(
flag.active == ACTIVE,
'DataQualityFlag.read(ligol) mismatch:\n\n%s\n\n%s' %
(ACTIVE, flag.active))
self.assertTrue(
flag.known == KNOWN,
'DataQualityFlag.read(ligol) mismatch:\n\n%s\n\n%s' %
(KNOWN, flag.known))
def _read_segments(self, filename):
segs = DataQualityFlag.read(filename, self.definition)
# XXX HACK around malformed segment files with no segment_summary table
if segs.active and not segs.known:
segs.known = type(segs.active)(segs.active)
if self.known:
self.known = self.known & segs.known
self.active = self.known & segs.active
else:
self.known = segs.known
self.active = segs.active
return self
def _read_segments(self, filename):
segs = DataQualityFlag.read(filename, self.definition)
# XXX HACK around malformed segment files with no segment_summary table
if segs.active and not segs.known:
segs.known = type(segs.active)(segs.active)
if self.known:
self.known = self.known & segs.known
self.active = self.known & segs.active
else:
self.known = segs.known
self.active = segs.active
return self
def test_read_hdf5(self):
try:
hdfout = self.test_write_hdf5(delete=False)
except ImportError as e:
self.skipTest(str(e))
else:
flag = DataQualityFlag.read(hdfout)
os.remove(hdfout)
self.assertTrue(flag.active == ACTIVE,
'DataQualityFlag.read(hdf5) mismatch:\n\n%s\n\n%s'
% (ACTIVE, flag.active))
self.assertTrue(flag.known == KNOWN,
'DataQualityFlag.read(hdf5) mismatch:\n\n%s\n\n%s'
% (KNOWN, flag.known))
def test_read_hdf5(self):
try:
hdfout = self.test_write_hdf5(delete=False)
except ImportError as e:
self.skipTest(str(e))
else:
flag = DataQualityFlag.read(hdfout)
os.remove(hdfout)
self.assertTrue(
flag.active == ACTIVE,
'DataQualityFlag.read(hdf5) mismatch:\n\n%s\n\n%s' %
(ACTIVE, flag.active))
self.assertTrue(
flag.known == KNOWN,
'DataQualityFlag.read(hdf5) mismatch:\n\n%s\n\n%s' %
(KNOWN, flag.known))
def read_and_cache_events(channel,
etg,
cache=None,
trigfind_kw={},
**read_kw):
cfile = create_path(channel)
# read existing cached triggers and work out new segments to query
if args.append and cfile.is_file():
previous = DataQualityFlag.read(
str(cfile),
path='segments',
format='hdf5',
).coalesce()
new = analysis - previous
else:
new = analysis.copy()
# get cache of files
if cache is None:
cache = find_trigger_files(channel, etg, new.active, **trigfind_kw)
else:
cache = list(
filter(
lambda e: new.active.intersects_segment(file_segment(e)),
cache,
))
# restrict 'active' segments to when we have data
try:
new.active &= cache_segments(cache)
except IndexError:
new.active = type(new.active)()
# find new triggers
try:
trigs = get_triggers(channel,
etg,
new.active,
cache=cache,
raw=True,
**read_kw)
# catch error and continue
except ValueError as e:
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
else:
path = write_events(channel, trigs, new)
try:
return path, len(trigs)
except TypeError: # None
return
def process(self, *args, **kwargs):
# read the segment files
if os.path.isfile(self.segmentfile):
segs = DataQualityFlag.read(self.segmentfile, coalesce=False)
self.states[0].known = segs.known
self.states[0].active = segs.active
self.states[0].ready = True
else:
warn('Segment file %s not found.' % self.segmentfile)
return
if len(self.states[0].active) == 0:
warn('No segments analysed by daily ahope.')
return
# read the cache files
if os.path.isfile(self.inspiralcachefile):
with open(self.inspiralcachefile, 'r') as fobj:
try:
self.inspiralcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError as e:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.inspiralcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.inspiralcachefile)
return
if os.path.isfile(self.tmpltbankcachefile):
with open(self.tmpltbankcachefile, 'r') as fobj:
try:
self.tmpltbankcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.tmpltbankcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.tmpltbankcachefile)
self.tmpltbankcache = Cache()
# only process if the cachfile was found
super(DailyAhopeTab, self).process(*args, **kwargs)
def process(self, *args, **kwargs):
# read the segment files
if os.path.isfile(self.segmentfile):
segs = DataQualityFlag.read(self.segmentfile, coalesce=False)
self.states[0].known = segs.known
self.states[0].active = segs.active
self.states[0].ready = True
else:
warn('Segment file %s not found.' % self.segmentfile)
return
if len(self.states[0].active) == 0:
warn('No segments analysed by daily ahope.')
return
# read the cache files
if os.path.isfile(self.inspiralcachefile):
with open(self.inspiralcachefile, 'r') as fobj:
try:
self.inspiralcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError as e:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.inspiralcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.inspiralcachefile)
return
if os.path.isfile(self.tmpltbankcachefile):
with open(self.tmpltbankcachefile, 'r') as fobj:
try:
self.tmpltbankcache = Cache.fromfile(fobj).sieve(
segment=self.span)
except ValueError:
if "could not convert \'\\n\' to CacheEntry" in str(e):
self.tmpltbankcache = Cache()
else:
raise
else:
warn("Cache file %s not found." % self.tmpltbankcachefile)
self.tmpltbankcache = Cache()
# only process if the cachfile was found
super(DailyAhopeTab, self).process(*args, **kwargs)
def read_data_archive(sourcefile):
"""Read archived data from an HDF5 archive source
This method reads all found data into the data containers defined by
the `gwsumm.globalv` module, then returns nothing.
Parameters
----------
sourcefile : `str`
path to source HDF5 file
"""
from h5py import File
with File(sourcefile, 'r') as h5file:
# -- channels ---------------------------
try:
ctable = Table.read(h5file['channels'])
except KeyError: # no channels table written
pass
else:
for row in ctable:
chan = get_channel(row['name'])
for p in ctable.colnames[1:]:
if row[p]:
setattr(chan, p, row[p])
# -- timeseries -------------------------
for dataset in h5file.get('timeseries', {}).values():
ts = TimeSeries.read(dataset, format='hdf5')
if (re.search(r'\.(rms|min|mean|max|n)\Z', ts.channel.name)
and ts.sample_rate.value == 1.0):
ts.channel.type = 's-trend'
elif re.search(r'\.(rms|min|mean|max|n)\Z', ts.channel.name):
ts.channel.type = 'm-trend'
ts.channel = get_channel(ts.channel)
try:
add_timeseries(ts, key=ts.channel.ndsname)
except ValueError:
if mode.get_mode() != mode.Mode.day:
raise
warnings.warn('Caught ValueError in combining daily archives')
# get end time
globalv.DATA[ts.channel.ndsname].pop(-1)
t = globalv.DATA[ts.channel.ndsname][-1].span[-1]
add_timeseries(ts.crop(start=t), key=ts.channel.ndsname)
# -- statevector -- ---------------------
for dataset in h5file.get('statevector', {}).values():
sv = StateVector.read(dataset, format='hdf5')
sv.channel = get_channel(sv.channel)
add_timeseries(sv, key=sv.channel.ndsname)
# -- spectrogram ------------------------
for tag, add_ in zip(
['spectrogram', 'coherence-components'],
[add_spectrogram, add_coherence_component_spectrogram]):
for key, dataset in h5file.get(tag, {}).items():
key = key.rsplit(',', 1)[0]
spec = Spectrogram.read(dataset, format='hdf5')
spec.channel = get_channel(spec.channel)
add_(spec, key=key)
# -- segments ---------------------------
for name, dataset in h5file.get('segments', {}).items():
dqflag = DataQualityFlag.read(h5file,
path=dataset.name,
format='hdf5')
globalv.SEGMENTS += {name: dqflag}
# -- triggers ---------------------------
for dataset in h5file.get('triggers', {}).values():
load_table(dataset)
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
flag : `str`, `list`
either the name of one flag, or a list of names
validity : `~gwpy.segments.SegmentList`
the segments over which to search for other segments
query : `bool`, optional, default: `True`
actually execute a read/query operation (if needed), otherwise
just retrieve segments that have already been cached
config : `~configparser.ConfigParser`, optional
the configuration for your analysis, if you have one. If
present the ``[segment-database]`` section will be queried
for the following options
- ``gps-start-time``, and ``gps-end-time``, if ``validity`` is
not given
- ``url`` (the remote hostname for the segment database) if
the ``url`` keyword is not given
cache : :class:`glue.lal.Cache`, optional
a cache of files from which to read segments, otherwise segments
will be downloaded from the segment database
coalesce : `bool`, optional, default: `True`
coalesce all segmentlists before returning, otherwise just return
segments as they were downloaded/read
padding : `tuple`, or `dict` of `tuples`, optional
`(start, end)` padding with which to pad segments that are
downloaded/read
segdb_error : `str`, optional, default: ``'raise'``
how to handle errors returned from the segment database, one of
- ``'raise'`` (default) : raise the exception as normal
- ``'warn'`` : print the exception as a warning, but return no
segments
- ``'ignore'`` : silently ignore the error and return no segments
url : `str`, optional
the remote hostname for the target segment database
return_ : `bool`, optional, default: `True`
internal flag to enable (True) or disable (False) actually returning
anything. This is useful if you want to download/read segments now
but not use them until later (e.g. plotting)
Returns
-------
flag : `~gwpy.segments.DataQualityFlag`
the flag object representing segments for the given single flag, OR
flagdict : `~gwpy.segments.DataQualityDict`
the dict of `~gwpy.segments.DataQualityFlag` objects for multiple
flags, if ``flag`` is given as a `list`, OR
None
if ``return_=False``
"""
if isinstance(flag, str):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag,
isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEFAULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(
operator.and_,
(globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, str):
return out[flag]
else:
return out
type=str,
help='observatory',
required=False)
args = parser.parse_args()
# Load channel list; accept FrChannels format
channels = []
with open(args.channels, 'r') as fin:
for line in fin:
channels.append(line.split()[0])
#Get ifo and active sci segs
ifo = args.ifo
if args.sci_seg:
sci_segs = DataQualityFlag.read(args.sci_seg,
path='H1:DMT-ANALYSIS_READY:1')
assert sci_segs.ifo == ifo
segs = sci_segs.active
elif args.start_time and args.end_time:
segs = [Segment(args.start_time, args.end_time)]
else:
print "Either --segment-file, or both start and end time must be provided."
sys.exit(2)
st = segs[0].start
et = segs[-1].end
###################
###DARM Channels###
###################
def test_read_segwizard(self):
flag = DataQualityFlag.read(SEGWIZ, FLAG1, coalesce=False)
self.assertTrue(flag.active == ACTIVE,
'DataQualityFlag.read(segwizard) mismatch:\n\n%s\n\n%s'
% (ACTIVE, flag.active))
self.assertTrue(flag.known == flag.active)
#! /usr/bin/env python
import sys
from gwpy.segments import DataQualityFlag
def add_job(the_file, job_type, job_number, **kwargs):
job_id = "%s%.6u" % (job_type, job_number)
the_file.write("JOB %s %s.sub\n" % (job_id, job_type))
vars_line = " ".join(
['%s="%s"' % (arg, str(val)) for arg, val in kwargs.iteritems()])
the_file.write("VARS %s %s\n" % (job_id, vars_line))
the_file.write("\n")
if __name__ == "__main__":
segment_file = sys.argv[1]
sci_segs = DataQualityFlag.read(segment_file)
ifo = sci_segs.ifo
segs = sci_segs.active
jobtypes = ['target', 'aux']
fdag = open("my.dag", 'w')
for idx, seg in enumerate(segs):
for jobtype in jobtypes:
add_job(fdag, jobtype, idx, ifo=ifo, st=seg.start, et=seg.end)
def read_data_archive(sourcefile):
"""Read archived data from an HDF5 archive source.
Parameters
----------
sourcefile : `str`
path to source HDF5 file
"""
from h5py import File
with File(sourcefile, 'r') as h5file:
# read all time-series data
try:
group = h5file['timeseries']
except KeyError:
group = dict()
for dataset in group.itervalues():
ts = TimeSeries.read(dataset, format='hdf')
if (re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name) and
ts.sample_rate.value == 1.0):
ts.channel.type = 's-trend'
elif re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name):
ts.channel.type = 'm-trend'
ts.channel = get_channel(ts.channel)
try:
add_timeseries(ts, key=ts.channel.ndsname)
except ValueError:
if mode.get_mode() == mode.SUMMARY_MODE_DAY:
raise
warnings.warn('Caught ValueError in combining daily archives')
# get end time
globalv.DATA[ts.channel.ndsname].pop(-1)
t = globalv.DATA[ts.channel.ndsname][-1].span[-1]
add_timeseries(ts.crop(start=t), key=ts.channel.ndsname)
# read all state-vector data
try:
group = h5file['statevector']
except KeyError:
group = dict()
for dataset in group.itervalues():
sv = StateVector.read(dataset, format='hdf')
sv.channel = get_channel(sv.channel)
add_timeseries(sv, key=sv.channel.ndsname)
# read all spectrogram data
try:
group = h5file['spectrogram']
except KeyError:
group = dict()
for key, dataset in group.iteritems():
key = key.rsplit(',', 1)[0]
spec = Spectrogram.read(dataset, format='hdf')
spec.channel = get_channel(spec.channel)
add_spectrogram(spec, key=key)
try:
group = h5file['segments']
except KeyError:
group = dict()
for name, dataset in group.iteritems():
dqflag = DataQualityFlag.read(dataset, format='hdf')
globalv.SEGMENTS += {name: dqflag}
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
FIXME
Returns
-------
FIXME
"""
if isinstance(flag, (unicode, str)):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag, isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEAFULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(operator.and_, (globalv.SEGMENTS.get(
f, DataQualityFlag(f)).known
for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except Exception as e:
if type(e) is not Exception:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= segs.known
out[compound].active &= segs.known
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, basestring):
return out[flag]
else:
return out
# calculations are in progress.
skip_load = True # Set to 'True' if you want to use the secondary intermediate files
# that will let you skip glitch rate generation. Use for repeated runs.
# Define the name of the directory within the directory
# '{year in 3rd millenium}/{month #}/{day #}' that will store script-generated files.
skip_load_direct = 'Wind_Thresh_5'
tablet = ['Observing', 'Transition', 'EQ'] # The list of names for configurations
# defined by their respective entries in 'configsecs'
# below.
histbinwidths = [0.1, 0.1, 0.1] # Desired bin width for glitch rate categories in
# each configuration's histogram plot.
if skip_load:
partstarts = list(np.loadtxt(File_Name(start, end, 'partstarts', 'txt', extras = [skip_load_direct])))
rates = list(np.loadtxt(File_Name(start, end, 'glitchrates', 'txt', extras = [skip_load_direct])))
nomflags = DataQualityFlag.read(File_Name(start, end, '{}_dqflag'.format(tablet[0]), 'hdf5', extras = [skip_load_direct]))
transflags = DataQualityFlag.read(File_Name(start, end, '{}_dqflag'.format(tablet[1]), 'hdf5', extras = [skip_load_direct]))
EQflags = DataQualityFlag.read(File_Name(start, end, '{}_dqflag'.format(tablet[2]), 'hdf5', extras = [skip_load_direct]))
configs = [nomflags, transflags, EQflags]
configsecs = [config.active for config in configs]
else:
# Category-specific cutoffs
print('Grabbinng cat-cutoff info...')
indchan1 = 'L1:ISI-HAM6_SENSCOR_X_FADE_TIME_LEFT_MON'
indchan2 = 'L1:ISI-HAM6_SENSCOR_X_FADE_CUR_CHAN_MON'
# transmodes = Grab_Series(start, end, indchan1, frame, procs)
transmodes = Grab_Sfiles(start, end, obsrun, indchan1, frame)
# EQmodes = Grab_Series(start, end, indchan2, frame, procs)
EQmodes = Grab_Sfiles(start, end, obsrun, indchan2, frame)
transtricts = [[(0, 'min')]]
print('\n--- Total {0}h {1}m ---'.format(
int((time.time() - start_time) / 3600),
int(((time.time() - start_time) / 3600 - int(
(time.time() - start_time) / 3600)) * 60)))
# whole day file should be produced at the end of the day.
end_time = (datetime.now() + timedelta(hours=-9) +
timedelta(minutes=-15)).strftime("%Y-%m-%d")
if utc_date != end_time:
print("date changed.")
for key in keys:
for snr in snrs[key]:
tmp = DataQualityFlag.read(filepath_xml[key + str(snr)])
tmp.write(SEGMENT_DIR + 'K1-' + key + '_SNR' + str(snr) + '/' +
year + '/' + 'K1-' + key + '_SNR' + str(snr) +
'_SEGMENT_UTC_' + utc_date + '.xml',
overwrite=True)
# Check if missing part exist
day = DataQualityFlag(known=[(end_gps_time - 86400, end_gps_time)],
active=[(end_gps_time - 86400, end_gps_time)
],
name=key + str(snr))
missing = day.known - tmp.known
for seg in missing:
mkSegment(seg[0], seg[1], utc_date, txt=False)
tmp = DataQualityFlag.read(filepath_xml[key + str(snr)])
params = parse_command_line()
st_dir = int(str(params.st)[0:5])
et_dir = int(str(params.et)[0:5])
dirs = np.arange(st_dir, et_dir + 1)
pipeline_dict = coh_io.read_pipeline_ini(params.ini)
env_params, run_params = coh_io.check_ini_params(pipeline_dict)
segs = []
for directory in dirs:
seg_files = sorted(
glob.glob('%s/SEGMENTS/%d' %
(env_params['base_directory'], directory)))
for f in seg_files:
temps = DataQualityFlag.read(f)
segs.append(temps)
plots = sorted(glob.glob('%s/*.png' % params.directory))
notches_page = 'webpage/notches.html'
params_page = 'webpage/params.html'
diagnostic_plots_page = 'webpage/diagnostic_plots.html'
index_page = 'webpage/index.html'
os.system('mkdir -p webpage')
os.system('touch %s' % notches_page)
os.system('touch %s' % params_page)
os.system('touch %s' % diagnostic_plots_page)
os.system('touch %s' % index_page)
os.system('cp main.css webpage/')
make_index_page(index_page, params)
make_diagnostic_page(diagnostic_plots_page, params)
def main(args=None):
"""Run the online Guardian node visualization tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# parse command line options
ifo = args.ifo
if not args.ifo:
parser.error('--ifo must be given if not obvious from the host')
start = getattr(args, 'gpsstart')
end = getattr(args, 'gpsend')
duration = int(ceil(end) - floor(start))
categories = args.categories.split(',')
for i, c in enumerate(categories):
try:
categories[i] = int(c)
except (TypeError, ValueError):
pass
vetofile = getattr(args, 'veto-definer-file')
vetofile = (urlparse(vetofile).netloc or os.path.abspath(vetofile))
args.metric = args.metric or DEFAULT_METRICS
# -- setup --------------------------------------
tag = '%d-%d' % (start.seconds, end.seconds)
outdir = os.path.abspath(os.path.join(args.output_directory, tag))
mkdir(outdir)
os.chdir(outdir)
mkdir('etc', 'segments', 'condor')
# -- segment handling ---------------------------
os.chdir('segments')
ALLSEGMENTS = DataQualityDict()
# -- get analysis segments ----------------------
aflags = args.analysis_segments
asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo)
for i, flag in enumerate(aflags):
# use union of segments from a file
if os.path.isfile(flag):
asegments += DataQualityFlag.read(flag)
# or intersection of segments from multiple flags
else:
new = DataQualityFlag.query(flag, start, end, url=args.segdb)
if i:
asegments.known &= new.known
asegments.active &= new.active
else:
asegments.known = new.known
asegments.active = new.active
ALLSEGMENTS[asegments.name] = asegments
if os.path.isfile(aflags[0]):
asegments.filename = aflags
# -- read veto definer and process --------------
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
with open(vetofile, 'w') as f:
f.write(tmp.read())
LOGGER.info('Downloaded veto definer file')
vdf = DataQualityDict.from_veto_definer_file(vetofile,
format='ligolw',
start=start,
end=end,
ifo=ifo)
LOGGER.info('Read %d flags from veto definer' % len(vdf.keys()))
# populate veto definer file from database
vdf.populate(source=args.segdb, on_error=args.on_segdb_error)
ALLSEGMENTS += vdf
# organise flags into categories
flags = dict((c, DataQualityDict()) for c in categories)
for name, flag in vdf.items():
try:
flags[flag.category][name] = flag
except KeyError:
pass
# find the states and segments for each category
states, after, oldtitle = (dict(), None, '')
for i, category in enumerate(categories):
title = isinstance(category, int) and 'Cat %d' % category or category
tag = re_cchar.sub('_', str(title).upper())
states[category] = SummaryState(
'After %s' % oldtitle,
key=tag,
known=after.known,
active=after.active,
definition=after.name,
) if i else SummaryState(
args.analysis_name,
key=args.analysis_name,
definition=asegments.name,
)
try:
segs = flags[category].union()
except TypeError: # no flags
segs = DataQualityFlag()
segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag)
ALLSEGMENTS[segs.name] = segs
after = (after - segs) if i else (asegments - segs)
after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag)
ALLSEGMENTS[after.name] = after
oldtitle = title
# write all segments to disk
segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' %
(ifo, start.seconds, duration))
ALLSEGMENTS.write(segfile)
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("All segments accessed and written to\n%s" % segfile)
# -- job preparation ----------------------------
os.chdir('etc')
configs = []
for category in categories:
title = (isinstance(category, int) and 'Category %d' % category
or category)
tab = 'tab-%s' % title
config = ConfigParser()
# add segment-database configuration
add_config_section(config, 'segment-database', url=args.segdb)
# add plot configurations
pconfig = ConfigParser()
pconfig.read(args.config_file)
for section in pconfig.sections():
if section.startswith('plot-'):
config._sections[section] = pconfig._sections[section].copy()
try:
plots = pconfig.items('plots-%s' % category, raw=True)
except NoSectionError:
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
# add state
if args.independent:
state = states[categories[0]]
else:
state = states[category]
sname = 'state-%s' % state.key
add_config_section(config,
sname,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
# add plugin
add_config_section(config, 'plugins', **{'gwvet.tabs': ''})
# define metrics
if category == 1:
metrics = ['Deadtime']
else:
metrics = args.metric
# define summary tab
if category == 1:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{'veto-name': title})
else:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{
'veto-name': title,
'event-channel': args.event_channel,
'event-generator':
args.event_generator,
})
if len(categories) == 1:
config.set(tab, 'index',
'%(gps-start-time)s-%(gps-end-time)s/index.html')
for key, value in plots:
if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605
config.set(tab, key, '%%(union)s,%s' % value)
if '%s-labels' % key not in plots:
config.set(tab, '%s-labels' % key, 'Union,%(flags)s')
else:
config.set(tab, key, value)
# now a tab for each flag
for flag in flags[category]:
if category == 1:
tab = configure_veto_tab(config, flag, title, state, [flag],
segfile, metrics)
else:
tab = configure_veto_tab(
config, flag, title, state, [flag], segfile, metrics, **{
'event-channel': args.event_channel,
'event-generator': args.event_generator
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
if len(categories) > 1 and category != categories[-1]:
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
# configure summary job
if len(categories) > 1:
state = states[categories[0]]
add_config_section(config,
'state-%s' % state.key,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
flags = [f for c in categories for f in flags[c].keys()]
tab = configure_veto_tab(
config,
'Impact of full veto definer file',
None,
state,
flags,
segfile,
args.metric,
shortname='Summary',
index='%(gps-start-time)s-%(gps-end-time)s/index.html',
**{
'event-channel': args.event_channel,
'event-generator': args.event_generator,
'veto-name': 'All vetoes'
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("Generated configuration files for each category")
# -- condor preparation -------------------------
os.chdir(os.pardir)
# get condor variables
if getuser() == 'detchar':
accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy'
else:
accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy'
gwsumm_args = [
'--gps-start-time',
str(start.seconds),
'--gps-end-time',
str(end.seconds),
'--ifo',
ifo,
'--file-tag',
'gwpy-vet',
'--condor-command',
'accounting_group=%s' % accgroup,
'--condor-command',
'accounting_group_user=%s' % getuser(),
'--on-segdb-error',
args.on_segdb_error,
'--output-dir',
args.output_directory,
]
for cf in args.global_config:
gwsumm_args.extend(('--global-config', cf))
for cf in configs:
gwsumm_args.extend(('--config-file', cf))
if args.verbose:
gwsumm_args.append('--verbose')
if args.verbose:
LOGGER.debug('Generating summary DAG via:\n')
LOGGER.debug(' '.join([batch.PROG] + gwsumm_args))
# execute gwsumm in batch mode
batch.main(args=gwsumm_args)
args = parser.parse_args()
freq = args.freq
year = args.year
month = args.month
day = args.day
if len(month) < 2:
month = "0" + month
if len(day) < 2:
day = "0" + day
#=============Get locked segments=============
locked = DataQualityFlag.read("/home/detchar/Segments/K1-DET_FOR_GRB200415A/" +
year + "/K1-DET_FOR_GRB200415A_UTC_" + year +
"-" + month + "-" + day + ".xml")
# Remove segments shorter than 94 sec
act = SegmentList()
for seg in locked.active:
duration = seg[1] - seg[0]
if duration >= 94:
act.append(seg)
# Remove last 30 sec and margin 2 sec
act = act.contract(17)
act = act.shift(-15)
locked.active = act
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None, **read_kw):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
flag : `str`, `list`
either the name of one flag, or a list of names
validity : `~gwpy.segments.SegmentList`
the segments over which to search for other segments
query : `bool`, optional, default: `True`
actually execute a read/query operation (if needed), otherwise
just retrieve segments that have already been cached
config : `~configparser.ConfigParser`, optional
the configuration for your analysis, if you have one. If
present the ``[segment-database]`` section will be queried
for the following options
- ``gps-start-time``, and ``gps-end-time``, if ``validity`` is
not given
- ``url`` (the remote hostname for the segment database) if
the ``url`` keyword is not given
cache : :class:`glue.lal.Cache`, optional
a cache of files from which to read segments, otherwise segments
will be downloaded from the segment database
coalesce : `bool`, optional, default: `True`
coalesce all segmentlists before returning, otherwise just return
segments as they were downloaded/read
padding : `tuple`, or `dict` of `tuples`, optional
`(start, end)` padding with which to pad segments that are
downloaded/read
segdb_error : `str`, optional, default: ``'raise'``
how to handle errors returned from the segment database, one of
- ``'raise'`` (default) : raise the exception as normal
- ``'warn'`` : print the exception as a warning, but return no
segments
- ``'ignore'`` : silently ignore the error and return no segments
url : `str`, optional
the remote hostname for the target segment database
return_ : `bool`, optional, default: `True`
internal flag to enable (True) or disable (False) actually returning
anything. This is useful if you want to download/read segments now
but not use them until later (e.g. plotting)
**read_kw : `dict`, optional
additional keyword arguments to `~gwpy.segments.DataQualityDict.read`
or `~gwpy.segments.DataQualityFlag.read`
Returns
-------
flag : `~gwpy.segments.DataQualityFlag`
the flag object representing segments for the given single flag, OR
flagdict : `~gwpy.segments.DataQualityDict`
the dict of `~gwpy.segments.DataQualityFlag` objects for multiple
flags, if ``flag`` is given as a `list`, OR
None
if ``return_=False``
"""
if isinstance(flag, str):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag,
isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEFAULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(
operator.and_,
(globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
if isinstance(cache, str) and cache.endswith(
(".h5", ".hdf", ".hdf5")) and (
'path' not in read_kw):
read_kw['path'] = config.get(
'DEFAULT', 'segments-hdf5-path', fallback='segments')
try:
new = DataQualityDict.read(cache, list(allflags), **read_kw)
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(
cache, f, coalesce=False, **read_kw)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, str):
return out[flag]
else:
return out
mkSegment(start_gps_time, end_gps_time, utc_date)
#print(' DQF segment file saved')
except ValueError :
print(' Cannot append discontiguous TimeSeries')
pass
#------------------------------------------------------------
print('\n--- Total {0}h {1}m ---'.format( int((time.time()-start_time)/3600), int(( (time.time()-start_time)/3600 - int((time.time()-start_time)/3600) )*60) ))
# whole day file should be produced at the end of the day.
end_time = (datetime.now() + timedelta(hours=-9)).strftime("%Y-%m-%d")
if utc_date != end_time:
for key in keys:
tmp = DataQualityFlag.read(filepath_xml[key])
#tmp.write(SEGMENT_DIR +key+'/'+year+'/'+key+'_SEGMENT_UTC_' + utc_date + '.xml',overwrite=True)
# Check if missing part exist
day = DataQualityFlag(known=[(end_gps_time-86400,end_gps_time)],active=[(end_gps_time-86400,end_gps_time)],name=key)
missing = day.known - tmp.known
for seg in missing:
mkSegment(seg[0], seg[1], utc_date, txt=False)
tmp = DataQualityFlag.read(filepath_xml[key])
tmp.write(SEGMENT_DIR +key+'/'+year+'/'+key+'_SEGMENT_UTC_' + utc_date + '.xml',overwrite=True)
with open(SEGMENT_DIR +key+'/'+year+'/'+key+'_SEGMENT_UTC_' + utc_date + '.txt', mode='w') as f:
for seg in tmp.active :
f.write('{0} {1}\n'.format(int(seg[0]), int(seg[1])))
os.remove(filepath_xml[key])
import subprocess
import argparse
from gwpy.segments import DataQualityFlag
from gwpy.segments import SegmentList
from gwpy.segments import Segment
parser = argparse.ArgumentParser(description='Get locked segment including the given gps time.')
parser.add_argument('-t','--time',help='input gpstime.',required=True)
parser.add_argument('-d','--date',help='input date',required=True)
args = parser.parse_args()
time = int(args.time)
date = args.date
segmentfile="/users/DET/Segments/SegmentList_FPMI_UTC_"+date+".xml"
Locked = DataQualityFlag.read(segmentfile)
#triggerseg = DataQualityFlag(known=[time-30,time+30], active=[time-1,time+1])
triggerseg=SegmentList([Segment(time-1,time+1)])
IsLocked = Locked.active.intersects(triggerseg)
if IsLocked:
for seg in Locked.active:
if seg[0] < time and time < seg[1]:
print(str(seg[0]) + " " + str(seg[1]))
break
else:
print("Not locked.")
def mkSegment(gst, get, utc_date, txt=True) :
chGRDLSC = 'K1:GRD-LSC_LOCK_STATE_N'
chGRDIFO = 'K1:GRD-IFO_STATE_N'
chGRDEQ = 'K1:GRD-PEM_EARTHQUAKE_STATE_N'
chOMCADC = 'K1:FEC-32_ADC_OVERFLOW_0_0'
channels = [chGRDLSC,chGRDIFO,chGRDEQ,chOMCADC]
if getpass.getuser() == "controls":
gwf_cache = '/users/DET/Cache/latest.cache'
with open(gwf_cache, 'r') as fobj:
cache = Cache.fromfile(fobj)
else:
# add 1sec margin for locked segments contract.
cache = GetFilelist(gst-1, get+1)
#------------------------------------------------------------
#print('Reading {0} timeseries data...'.format(date))
# add 1sec margin for locked segments contract.
channeldata = TimeSeriesDict.read(cache, channels, start=gst-1, end=get+1, format='gwf.lalframe', gap='pad')
channeldataGRDIFO = channeldata[chGRDIFO]
channeldataGRDLSC = channeldata[chGRDLSC]
channeldataGRDEQ = channeldata[chGRDEQ]
channeldataOMCADC = channeldata[chOMCADC]
sv={}
sv['K1-GRD_SCIENCE_MODE'] = channeldataGRDIFO == 1000
# Locked will be defined by inverse of unlocked segments for technical reason.
#sv['K1-GRD_LOCKED'] = channeldataGRDLSC == 1000
sv['K1-GRD_UNLOCKED'] = channeldataGRDLSC != 1000
sv['K1-GRD_PEM_EARTHQUAKE'] = channeldataGRDEQ == 1000
sv['K1-OMC_OVERFLOW_VETO'] = channeldataOMCADC != 0
# OMC_OVERFLOW_OK will be defined by inverse of veto segments for technical reason.
#sv['K1-OMC_OVERFLOW_OK'] = channeldataOMCADC == 0
dqflag = {}
for key in keys:
if key == 'K1-GRD_LOCKED' or key == 'K1-OMC_OVERFLOW_OK':
continue
dqflag[key] = sv[key].to_dqflag(round=True)
# To omit fraction. round=True option is inclusive in default.
dqflag['K1-GRD_SCIENCE_MODE'].active = dqflag['K1-GRD_SCIENCE_MODE'].active.contract(1.0)
dqflag['K1-GRD_LOCKED'] = ~dqflag['K1-GRD_UNLOCKED']
dqflag['K1-GRD_LOCKED'].name = "K1:GRD-LSC_LOCK_STATE_N == 1000"
dqflag['K1-OMC_OVERFLOW_OK'] = ~dqflag['K1-OMC_OVERFLOW_VETO']
dqflag['K1-OMC_OVERFLOW_OK'].name = "K1:FEC-32_ADC_OVERFLOW_0_0 == 0"
dqflag['K1-GRD_SCIENCE_MODE'].description = "Observation mode. K1:GRD-IFO_STATE_N == 1000"
dqflag['K1-GRD_UNLOCKED'].description = "Interferometer is not locked. K1:GRD-LSC_LOCK_STATE_N != 1000"
dqflag['K1-GRD_LOCKED'].description = "Interferometer is locked. K1:GRD-LSC_LOCK_STATE_N == 1000"
dqflag['K1-OMC_OVERFLOW_VETO'].description = "OMC overflow happened. K1:FEC-32_ADC_OVERFLOW_0_0 != 0"
dqflag['K1-OMC_OVERFLOW_OK'].description = "OMC overflow does not happened. K1:FEC-32_ADC_OVERFLOW_0_0 == 0"
for key in keys:
# added 1sec margin for locked segments contract is removed.
margin = DataQualityFlag(known=[(gst,get)],active=[(gst-1,gst),(get,get+1)])
dqflag[key] -= margin
# write down 15 min segments.
if txt:
with open(filepath_txt[key], mode='w') as f:
for seg in dqflag[key].active :
f.write('{0} {1}\n'.format(int(seg[0]), int(seg[1])))
# if accumulated file exists, it is added.
if os.path.exists(filepath_xml[key]):
tmp = DataQualityFlag.read(filepath_xml[key])
dqflag[key] = dqflag[key] + tmp
dqflag[key].write(filepath_xml[key],overwrite=True)
def mkSegment(gst, get, utc_date, txt=True):
for key in keys:
sources = GetFilelist(gst, get, key)
first = True
for source in sources:
events = EventTable.read(
source,
tablename='sngl_burst',
columns=['start_time', 'start_time_ns', 'duration', 'snr'])
#events = EventTable.read(source, tablename='sngl_burst',columns=['peak_time', 'peak_time_ns','start_time', 'start_time_ns', 'duration', 'peak_frequency', 'central_freq', 'bandwidth', 'channel', 'amplitude', 'snr', 'confidence', 'chisq', 'chisq_dof', 'param_one_name', 'param_one_value'])
col = events.get_column('start_time')
if first:
if len(col) > 0:
mergedevents = events
first = False
else:
pass
else:
mergedevents = vstack([mergedevents, events])
for snr in snrs[key]:
Triggered = DataQualityFlag(name="K1:" + key,
known=[(gst, get)],
active=[],
label="Glitch",
description="Glitch veto segment K1:" +
key + " >= SNR" + str(snr))
#Triggered.ifo = "K1"
if not first:
fevents = mergedevents.filter(
('snr', mylib.Islargerequal, snr))
durations = fevents.get_column('duration')
start_times = fevents.get_column('start_time')
for start_time, duration in zip(start_times, durations):
tmpstart = int(start_time)
#tmpend = start_time + duration
tmpend = int(start_time + 1)
tmpsegment = Segment(tmpstart, tmpend)
tmpTriggered = DataQualityFlag(known=[(gst, get)],
active=[(tmpstart, tmpend)])
Triggered |= tmpTriggered
#dqflag['K1-GRD_SCIENCE_MODE'].description = "Observation mode. K1:GRD-IFO_STATE_N == 1000"
#dqflag['K1-GRD_LOCKED'].name = "K1:GRD-LSC_LOCK_STATE_N >= 300 & K1:GRD-LSC_LOCK_STATE_N <= 1000"
# write down 15 min segments.
if txt:
with open(filepath_txt[key + str(snr)], mode='w') as f:
for seg in Triggered.active:
f.write('{0} {1}\n'.format(int(seg[0]), int(seg[1])))
# if accumulated file exists, it is added.
if os.path.exists(filepath_xml[key + str(snr)]):
tmp = DataQualityFlag.read(filepath_xml[key + str(snr)])
Triggered = Triggered + tmp
Triggered.write(filepath_xml[key + str(snr)], overwrite=True)
def read_data_archive(sourcefile):
"""Read archived data from an HDF5 archive source
This method reads all found data into the data containers defined by
the `gwsumm.globalv` module, then returns nothing.
Parameters
----------
sourcefile : `str`
path to source HDF5 file
"""
from h5py import File
with File(sourcefile, 'r') as h5file:
# -- channels ---------------------------
try:
ctable = Table.read(h5file['channels'])
except KeyError: # no channels table written
pass
else:
for row in ctable:
chan = get_channel(row['name'])
for p in ctable.colnames[1:]:
if row[p]:
setattr(chan, p, row[p])
# -- timeseries -------------------------
for dataset in h5file.get('timeseries', {}).values():
ts = TimeSeries.read(dataset, format='hdf5')
if (re.search(r'\.(rms|min|mean|max|n)\Z', ts.channel.name) and
ts.sample_rate.value == 1.0):
ts.channel.type = 's-trend'
elif re.search(r'\.(rms|min|mean|max|n)\Z', ts.channel.name):
ts.channel.type = 'm-trend'
ts.channel = get_channel(ts.channel)
try:
add_timeseries(ts, key=ts.channel.ndsname)
except ValueError:
if mode.get_mode() != mode.Mode.day:
raise
warnings.warn('Caught ValueError in combining daily archives')
# get end time
globalv.DATA[ts.channel.ndsname].pop(-1)
t = globalv.DATA[ts.channel.ndsname][-1].span[-1]
add_timeseries(ts.crop(start=t), key=ts.channel.ndsname)
# -- statevector -- ---------------------
for dataset in h5file.get('statevector', {}).values():
sv = StateVector.read(dataset, format='hdf5')
sv.channel = get_channel(sv.channel)
add_timeseries(sv, key=sv.channel.ndsname)
# -- spectrogram ------------------------
for tag, add_ in zip(
['spectrogram', 'coherence-components'],
[add_spectrogram, add_coherence_component_spectrogram]):
for key, dataset in h5file.get(tag, {}).items():
key = key.rsplit(',', 1)[0]
spec = Spectrogram.read(dataset, format='hdf5')
spec.channel = get_channel(spec.channel)
add_(spec, key=key)
# -- segments ---------------------------
for name, dataset in h5file.get('segments', {}).items():
dqflag = DataQualityFlag.read(h5file, path=dataset.name,
format='hdf5')
globalv.SEGMENTS += {name: dqflag}
# -- triggers ---------------------------
for dataset in h5file.get('triggers', {}).values():
load_table(dataset)
def read_data_archive(sourcefile):
"""Read archived data from an HDF5 archive source.
Parameters
----------
sourcefile : `str`
path to source HDF5 file
"""
from h5py import File
with File(sourcefile, 'r') as h5file:
# read all time-series data
try:
group = h5file['timeseries']
except KeyError:
group = dict()
for dataset in group.itervalues():
ts = TimeSeries.read(dataset, format='hdf5')
if (re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name)
and ts.sample_rate.value == 1.0):
ts.channel.type = 's-trend'
elif re.search('\.(rms|min|mean|max|n)\Z', ts.channel.name):
ts.channel.type = 'm-trend'
ts.channel = get_channel(ts.channel)
try:
add_timeseries(ts, key=ts.channel.ndsname)
except ValueError:
if mode.get_mode() != mode.Mode.day:
raise
warnings.warn('Caught ValueError in combining daily archives')
# get end time
globalv.DATA[ts.channel.ndsname].pop(-1)
t = globalv.DATA[ts.channel.ndsname][-1].span[-1]
add_timeseries(ts.crop(start=t), key=ts.channel.ndsname)
# read all state-vector data
try:
group = h5file['statevector']
except KeyError:
group = dict()
for dataset in group.itervalues():
sv = StateVector.read(dataset, format='hdf5')
sv.channel = get_channel(sv.channel)
add_timeseries(sv, key=sv.channel.ndsname)
# read all spectrogram data
for tag in ['spectrogram', 'coherence-components']:
if tag == 'coherence-components':
add_ = add_coherence_component_spectrogram
else:
add_ = add_spectrogram
try:
group = h5file[tag]
except KeyError:
group = dict()
for key, dataset in group.iteritems():
key = key.rsplit(',', 1)[0]
spec = Spectrogram.read(dataset, format='hdf5')
spec.channel = get_channel(spec.channel)
add_(spec, key=key)
# read all segments
try:
group = h5file['segments']
except KeyError:
group = dict()
for name in group:
dqflag = DataQualityFlag.read(group, path=name, format='hdf5')
globalv.SEGMENTS += {name: dqflag}
# read all triggers
try:
group = h5file['triggers']
except KeyError:
group = dict()
for key in group:
load_table(group[key])
import fir
from utils import chunk_segments
parser = argparse.ArgumentParser(description='Generate target features for machine learning on LIGO data.')
parser.add_argument('--ifo',type=str,required=True)
parser.add_argument('-f','--segment-file',type=str)
parser.add_argument('-s','--start-time',type=int)
parser.add_argument('-e','--end-time',type=int)
parser.add_argument('-p','--path', help='path to output directory', required=False)
args = parser.parse_args()
ifo=args.ifo
if args.segment_file:
sci_segs=DataQualityFlag.read(args.segment_file, path='%s:DMT-ANALYSIS_READY:1' % ifo)
assert sci_segs.ifo == ifo
segs=sci_segs.active
elif args.start_time and args.end_time:
segs=[Segment(args.start_time, args.end_time)]
else:
print "Either --segment-file, or both start and end time must be provided."
exit(2)
st=segs[0].start
et=segs[-1].end
chunk=4096
pad=256
target_chan=ifo+':GDS-CALIB_STRAIN'
def get_segments(flags,
segments,
cache=None,
url='https://segdb-er.ligo.caltech.edu',
**kwargs):
"""Fetch some segments from the segment database
Parameters
----------
flags : `str`, `list`
one of more flags for which to query
segments : `~gwpy.segments.DataQualityFlag`, `~gwpy.segments.SegmentList`
span over which to query for flag segments
cache : `~glue.lal.Cache`, optional
cache of files to use as data source
url : `str`
URL of segment database, if ``cache`` is not given
**kwargs
other keyword arguments to pass to either
`~gwpy.segments.DataQualityFlag.read` (if ``cache`` is given) or
`~gwpy.segments.DataQualityFlag.query` (otherwise)
Returns
-------
segments : `~gwpy.segments.DataQualityFlag`,
`~gwpy.segments.DataQualityDict`
a single `~gwpy.segments.DataQualityFlag` (if ``flags`` is given
as a `str`), or a `~gwpy.segments.DataQualityDict` (if ``flags``
is given as a `list`)
"""
# format segments
if isinstance(segments, DataQualityFlag):
segments = segments.active
elif isinstance(segments, tuple):
segments = [Segment(to_gps(segments[0]), to_gps(segments[1]))]
segments = SegmentList(segments)
# get format for files
if cache is not None and not isinstance(cache, Cache):
kwargs.setdefault(
'format',
_get_valid_format('read', DataQualityFlag, None, None,
(cache[0], ), {}))
# populate an existing set of flags
if isinstance(flags, (DataQualityFlag, DataQualityDict)):
return flags.populate(source=cache or url, segments=segments, **kwargs)
# query one flag
elif cache is None and isinstance(flags, str):
return DataQualityFlag.query(flags, segments, url=url, **kwargs)
# query lots of flags
elif cache is None:
return DataQualityDict.query(flags, segments, url=url, **kwargs)
# read one flag
elif flags is None or isinstance(flags, str):
segs = DataQualityFlag.read(cache, flags, coalesce=False, **kwargs)
if segs.known:
segs.known &= segments
else:
segs.known = segments
segs.active &= segments
return segs
# read lots of flags
else:
segs = DataQualityDict.read(cache, flags, coalesce=True, **kwargs)
for name, flag in segs.items():
flag.known &= segments
flag.active &= segments
return segs
def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
query=True, return_=True, coalesce=True, padding=None,
segdb_error='raise', url=None):
"""Retrieve the segments for a given flag
Segments will be loaded from global memory if already defined,
otherwise they will be loaded from the given
:class:`~glue.lal.Cache`, or finally from the segment database
Parameters
----------
FIXME
Returns
-------
FIXME
"""
if isinstance(flag, (unicode, str)):
flags = flag.split(',')
else:
flags = flag
allflags = set([f for cf in flags for f in
re_flagdiv.split(str(cf))[::2] if f])
if padding is None and isinstance(flag, DataQualityFlag):
padding = {flag: flag.padding}
elif padding is None:
padding = dict((flag, isinstance(flag, DataQualityFlag) and
flag.padding or None) for flag in flags)
# check validity
if validity is None:
start = config.get(DEAFULTSECT, 'gps-start-time')
end = config.get(DEFAULTSECT, 'gps-end-time')
span = SegmentList([Segment(start, end)])
elif isinstance(validity, DataQualityFlag):
validity = validity.active
try:
span = SegmentList([validity.extent()])
except ValueError:
span = SegmentList()
else:
try:
span = SegmentList([SegmentList(validity).extent()])
except ValueError:
span = SegmentList()
validity = SegmentList(validity)
# generate output object
out = DataQualityDict()
for f in flags:
out[f] = DataQualityFlag(f, known=validity, active=validity)
for f in allflags:
globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
# read segments from global memory and get the union of needed times
try:
old = reduce(operator.and_, (globalv.SEGMENTS.get(
f, DataQualityFlag(f)).known
for f in flags))
except TypeError:
old = SegmentList()
newsegs = validity - old
# load new segments
query &= abs(newsegs) != 0
query &= len(allflags) > 0
if cache is not None:
query &= len(cache) != 0
if query:
if cache is not None:
try:
new = DataQualityDict.read(cache, list(allflags))
except IORegistryError as e:
# can remove when astropy >= 1.2 is required
if type(e) is not IORegistryError:
raise
if len(allflags) == 1:
f = list(allflags)[0]
new = DataQualityDict()
new[f] = DataQualityFlag.read(cache, f, coalesce=False)
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Read %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
else:
if len(newsegs) >= 10:
qsegs = span
else:
qsegs = newsegs
# parse configuration for query
kwargs = {}
if url is not None:
kwargs['url'] = url
else:
try:
kwargs['url'] = config.get('segment-database', 'url')
except (NoSectionError, NoOptionError):
pass
if kwargs.get('url', None) in SEGDB_URLS:
query_func = DataQualityDict.query_segdb
else:
query_func = DataQualityDict.query_dqsegdb
try:
new = query_func(allflags, qsegs, on_error=segdb_error,
**kwargs)
except Exception as e:
# ignore error from SegDB
if segdb_error in ['ignore', None]:
pass
# convert to warning
elif segdb_error in ['warn']:
print('%sWARNING: %sCaught %s: %s [gwsumm.segments]'
% (WARNC, ENDC, type(e).__name__, str(e)),
file=sys.stderr)
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
# otherwise raise as normal
else:
raise
new = DataQualityDict()
for f in new:
new[f].known &= newsegs
new[f].active &= newsegs
if coalesce:
new[f].coalesce()
vprint(" Downloaded %d segments for %s (%.2f%% coverage).\n"
% (len(new[f].active), f,
float(abs(new[f].known))/float(abs(newsegs))*100))
# record new segments
globalv.SEGMENTS += new
for f in new:
globalv.SEGMENTS[f].description = str(new[f].description)
# return what was asked for
if return_:
for compound in flags:
union, intersection, exclude, notequal = split_compound_flag(
compound)
if len(union + intersection) == 1:
out[compound].description = globalv.SEGMENTS[f].description
out[compound].padding = padding.get(f, (0, 0))
for flist, op in zip([exclude, intersection, union, notequal],
[operator.sub, operator.and_, operator.or_,
not_equal]):
for f in flist:
pad = padding.get(f, (0, 0))
segs = globalv.SEGMENTS[f].copy()
if isinstance(pad, (float, int)):
segs = segs.pad(pad, pad)
elif pad is not None:
segs = segs.pad(*pad)
if coalesce:
segs = segs.coalesce()
out[compound] = op(out[compound], segs)
out[compound].known &= segs.known
out[compound].active &= segs.known
out[compound].known &= validity
out[compound].active &= validity
if coalesce:
out[compound].coalesce()
if isinstance(flag, basestring):
return out[flag]
else:
return out
