logger.warning('WARNING: conversion from xml to ASCII segment file failed.')
if opts.force:
logger.info(traceback.print_exc())
raise e
else:
gpsstart += stride
continue
### if we aren't building auxmvc vectors, we re-use pat files from realtime job
### this requires us to redefine the 'science-segments' as the intersection of scisegs with realtime segs
### we call this intersection "idq_segs"
else: ### we're re-using pat files!
try:
### determine segments from realtime filenames
realtime_segs = idq.get_idq_segments(realtimedir, gpsstart - lookback, gpsstart + stride, suffix='.pat')
### read in science segments
(scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name)
### take the intersection of these segments
idq_segs = event.andsegments([scisegs, realtime_segs])
### write segment file
idqseg_path = idq.idqsegascii(output_dir, '_%s'%dq_name, gpsstart - lookback, lookback+stride)
f = open(idqseg_path, 'w')
for seg in idq_segs:
print >> f, seg[0], seg[1]
f.close()
### we may want to remove the unsafe channels, but this could be tricky and we don't want to throw away GW channels accidentally
ovlsegs = idqseg_path
seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]
### science segments xml filename
seg_file = idq.segxml(gdbdir, "%s_%s" % (filetag, dq_name), opts.start,
opts.end - opts.start)
if opts.verbose:
print ' writing science segments to file : ' + seg_file
ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(
seg_file, dq_name) ### read in segments from xml file
if opts.verbose:
print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir,
opts.start,
opts.end,
suffix='.dat')
if opts.verbose:
print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments([scisegs, idqsegs])
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start,
opts.end - opts.start)
else:
idqseg_path = idq.idqsegascii(gdbdir, '%s_%s' % (filetag, dq_name),
opts.start, opts.end - opts.start)
if opts.verbose:
print " writing : " + idqseg_path
except Exception as e:
traceback.print_exc()
logger.info(
'ERROR: segment generation failed. Skipping this calibration period.'
)
if opts.force: ### we are require successful training or else we want errors
logger.info(traceback.print_exc())
raise e
else: ### we don't care if any particular training job fails
gpsstart += stride
continue
logger.info('finding idq segments')
idqsegs = idq.get_idq_segments(realtimedir,
gpsstart - lookback,
gpsstart + stride,
suffix='.dat')
logger.info(
'taking intersection between science segments and idq segments')
idqsegs = event.andsegments([scisegs, idqsegs])
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(output_dir, '', gpsstart - lookback,
lookback + stride)
else:
idqseg_path = idq.idqsegascii(output_dir, '_%s' % dq_name,
gpsstart - lookback, lookback + stride)
f = open(idqseg_path, 'w')
for seg in idqsegs:
### write seg_xml_file to disk
lsctables.use_in(ligolw.LIGOLWContentHandler)
xmldoc = ligolw_utils.load_fileobj(seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]
### science segments xml filename
seg_file = idq.segxml(gdbdir, "%s_%s"%(filetag, dq_name), opts.start , opts.end-opts.start )
if opts.verbose:
print ' writing science segments to file : '+seg_file
ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file
if opts.verbose:
print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat')
if opts.verbose:
print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments( [scisegs, idqsegs] )
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end-opts.start)
else:
idqseg_path = idq.idqsegascii(gdbdir, '%s_%s'%(filetag, dq_name), opts.start, opts.end-opts.start)
if opts.verbose:
print " writing : "+idqseg_path
f = open(idqseg_path, 'w')
for seg in idqsegs:
print >> f, seg[0], seg[1]
def check_calibartion(
realtimedir,
start,
end,
classifier,
FAPthrs,
verbose=False,
):
"""
....checks the pipeline's calibration at each "FAPthr in FAPThrs"
"""
# =================================================
# grab idq_segments so we compute meaningful livetimes
# =================================================
if verbose:
print 'getting idq_segements'
idq_segs = idq.get_idq_segments(realtimedir, start, end,
suffix='.npy.gz')
idq_livetime = event.livetime(idq_segs)
# =================================================
# grab relevant data
# =================================================
if verbose:
print 'looking for *_fap_*.npy.gz files in', opts.realtimedir
fapfilenames = [filename for filename in
idq.get_all_files_in_range(realtimedir, start, end,
pad=0, suffix='.npy.gz') if '_fap_' in filename
and classifier in filename]
if opts.verbose:
print 'discovered %d files' % len(fapfilenames)
print 'building time-series'
(times, timeseries) = idq_gdb_utils.combine_ts(fapfilenames)
# =================================================
# check calibration
# =================================================
segments = []
deadtimes = []
statedFAPs = []
for FAPthr in FAPthrs:
if verbose:
print 'computing segements for FAPthr =', FAPthr
segs = []
max_statedFAP = None
for (t, ts) in zip(times, timeseries):
(_segs, _min_ts) = timeseries_to_segments(t, -ts, -FAPthr) # we want FAP <= FAPthr <--> -FAP >= FAPthr
segs += _segs
if _min_ts != None:
statedFAP = -_min_ts
if max_statedFAP < statedFAP:
max_statedFAP = statedFAP
segs = event.andsegments([segs, idq_segs])
segments.append(segs)
deadtimes.append(1.0 * event.livetime(segs) / idq_livetime)
statedFAPs.append(statedFAP)
return (idq_segs, segments, deadtimes, statedFAPs)
xmldoc = ligolw_utils.load_fileobj(seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]
### science segments xml filename
seg_file = idq.segxml(opts.output_dir, "_%s"%dq_name, startgps, stride)
logger.info('writing science segments to file : '+seg_file)
ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file
### modify scisegs to account for shifts
logger.info('modifying scisegs to account for shifts')
modified_scisegs = [(s-opts.right_padding-opts.t_lag, e+opts.left_padding-opts.t_lag) for s, e in scisegs]
### find idq segments
logger.info('finding idq segments')
idqsegs = idq.get_idq_segments(realtimedir, lookup_startgps, lookup_endgps, suffix='.dat')
logger.info('taking intersection between modified science segments and idq segments')
idqsegs = event.andsegments( [modified_scisegs, idqsegs] )
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride)
else:
idqseg_path = idq.idqsegascii(opts.output_dir, '_%s'%dq_name, startgps , stride)
f = open(idqseg_path, 'w')
for seg in idqsegs:
print >> f, seg[0], seg[1]
f.close()
#========================
seg_file = idq.segxml(opts.output_dir, "_%s" % dq_name, startgps, stride)
logger.info('writing science segments to file : ' + seg_file)
ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(
seg_file, dq_name) ### read in segments from xml file
### modify scisegs to account for shifts
logger.info('modifying scisegs to account for shifts')
modified_scisegs = [(s - opts.right_padding - opts.t_lag,
e + opts.left_padding - opts.t_lag) for s, e in scisegs]
### find idq segments
logger.info('finding idq segments')
idqsegs = idq.get_idq_segments(realtimedir,
lookup_startgps,
lookup_endgps,
suffix='.dat')
logger.info(
'taking intersection between modified science segments and idq segments')
idqsegs = event.andsegments([modified_scisegs, idqsegs])
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride)
else:
idqseg_path = idq.idqsegascii(opts.output_dir, '_%s' % dq_name, startgps,
stride)
f = open(idqseg_path, 'w')
for seg in idqsegs:
print >> f, seg[0], seg[1]