logger.info("ignoring science segments")
scisegs = [[gpsstart, gpsstart+stride]]
else:
logger.info("generating science segments")
try:
seg_xml_file = idq.segment_query(config, gpsstart, gpsstart+stride, url=config.get("get_science_segments","segdb"))
lsctables.use_in(ligolw.LIGOLWContentHandler)
xmldoc = utils.load_fileobj(seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]
seg_file = "%s/science_segements-%d-%d.xml.gz"%(this_sumdir, int(gpsstart), int(stride))
logger.info("writting science segments to file : %s"%seg_file)
utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(seg_file, config.get('get_science_segments', 'include'))
except Exception as e:
traceback.print_exc()
logger.info("ERROR: segment generation failed. Skipping this summary period.")
gpsstart += stride
continue
#=============================================
# generating summary datfiles filtered by segments
#=============================================
### get all relevant datfiles
datfiles = idq.get_all_files_in_range(realtimedir, gpsstart, gpsstart+stride, suffix=".dat")
for classifier in classifiers:
except Exception as e:
traceback.print_exc()
logger.info('ERROR: segment generation failed. Skipping this training period.')
if opts.force: ### we are require successful training or else we want errors
logger.info(traceback.print_exc())
raise e
else: ### we don't care if any particular training job fails
gpsstart += stride
continue
if ovl:
if (not mla) or build_auxmvc_vectors or (not os.path.exists(realtimedir)): ### mla will use science segments, so we need to write those for ovl
### if realtimedir doesn't exits, we need to use queried scisegs
try:
(scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file
### write segments to ascii list
sciseg_path = idq.segascii(output_dir, "_%s"%dq_name, gpsstart-lookback, lookback+stride)
logger.info('writing science segments to file : '+sciseg_path)
f = open(sciseg_path, 'w')
for line in scisegs:
print >> f, line[0], line[1]
f.close()
ovlsegs = sciseg_path
except Exception as e:
traceback.print_exc()
logger.warning('WARNING: conversion from xml to ASCII segment file failed.')
if opts.force:
opts.end,
url=segdb_url)
### write seg_xml_file to disk
lsctables.use_in(ligolw.LIGOLWContentHandler)
xmldoc = ligolw_utils.load_fileobj(
seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]
### science segments xml filename
seg_file = idq.segxml(gdbdir, "%s_%s" % (filetag, dq_name), opts.start,
opts.end - opts.start)
if opts.verbose:
print ' writing science segments to file : ' + seg_file
ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(
seg_file, dq_name) ### read in segments from xml file
if opts.verbose:
print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir,
opts.start,
opts.end,
suffix='.dat')
if opts.verbose:
print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments([scisegs, idqsegs])
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start,
print 'querrying science segments'
### this returns a string
seg_xml_file = idq.segment_query(config, opts.start , opts.end, url=segdb_url)
### write seg_xml_file to disk
lsctables.use_in(ligolw.LIGOLWContentHandler)
xmldoc = ligolw_utils.load_fileobj(seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]
### science segments xml filename
seg_file = idq.segxml(gdbdir, "%s_%s"%(filetag, dq_name), opts.start , opts.end-opts.start )
if opts.verbose:
print ' writing science segments to file : '+seg_file
ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file
if opts.verbose:
print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat')
if opts.verbose:
print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments( [scisegs, idqsegs] )
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end-opts.start)
else:
idqseg_path = idq.idqsegascii(gdbdir, '%s_%s'%(filetag, dq_name), opts.start, opts.end-opts.start)
if opts.verbose:
print "Loading triggers ..."
# load triggers from files
trigger_dict = event.loadkwm(trig_files)
if main_channel not in trigger_dict:
trigger_dict[main_channel] = []
if opts.verbose:
print "Done."
if not trigger_dict:
print "Warning: No triggers in the input files, exiting without doing anything."
sys.exit(0)
if opts.dq_segments:
# load dq segments
(dq_segments, covered_segments) = idq.extract_dq_segments(open(opts.dq_segments, "r"), opts.dq_segments_name)
# sort and merge segments
dq_segments = event.fixsegments(dq_segments)
else:
dq_segments = None
# construct auxmvc feature vectors
auxmvc_vectors = idq.build_auxmvc_vectors(trigger_dict, main_channel, opts.time_window, opts.signif_threshold, opts.output_file,\
gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\
science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\
max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples)
# gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\
# science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\
# max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples)
clean_samples = auxmvc_vectors[numpy.nonzero(auxmvc_vectors['i']==0)[0],:]
"program"), config.get("get_science_segments", "segdb"),
config.get("get_science_segments", "include"), gpsstart, gpsstop)
if opts.verbose:
print "getting science segments"
print "\t", args
segfile = "%s/science_segments-%d-%d.xml" % (output_dir, gpsstart,
int(gpsstop - gpsstart))
segfile_obj = open(segfile, "w")
p = subprocess.Popen(args.split(), stdout=segfile_obj)
p.wait()
segfile_obj.close()
### read in segments from xml file
(scisegs, coveredseg) = idq.extract_dq_segments(
segfile, config.get('get_science_segments', 'include'))
else:
scisegs = [[gpsstart, gpsstop]]
### write segments to ascii list
if opts.verbose:
print "writing segments to ascii list"
sciseg_path = "%s/science_segments-%d-%d.seg" % (output_dir, int(gpsstart),
int(gpsstop - gpsstart))
f = open(sciseg_path, 'w')
for line in scisegs:
print >> f, line[0], line[1]
f.close()
#========================
print 'querrying science segments'
### this returns a string
seg_xml_file = idq.segment_query(config, opts.start , opts.end, url=segdb_url)
### write seg_xml_file to disk
lsctables.use_in(ligolw.LIGOLWContentHandler)
xmldoc = ligolw_utils.load_fileobj(seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]
### science segments xml filename
seg_file = idq.segxml(gdbdir, "%s_%s"%(filetag, dq_name), opts.start , opts.end-opts.start )
if opts.verbose:
print ' writing science segments to file : '+seg_file
ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))
(scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file
if opts.verbose:
print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat')
if opts.verbose:
print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments( [scisegs, idqsegs] )
### write segment file
if opts.ignore_science_segments:
idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end-opts.start)
else:
idqseg_path = idq.idqsegascii(gdbdir, '%s_%s'%(filetag, dq_name), opts.start, opts.end-opts.start)
if opts.verbose:
# get all *.pat files in the specififed range
patfiles = idq.get_all_files_in_range(opts.source_directory,
gps_start_time,
gps_end_time,
suffix='.pat')
if opts.verbose:
print "found %d pat files" % len(patfiles)
if opts.dq_segments:
# load dq segments
if opts.verbose:
print "reading segments from %s" % opts.dq_segments
(dq_segments, covered_segments) = \
idq.extract_dq_segments(open(opts.dq_segments, 'r'),
opts.dq_segments_name)
# sort and merge segments
dq_segments = event.fixsegments(dq_segments)
### filter patfiles by scisegs, keep only those events with non-zero overlap with science time
patfiles = [
pat for pat in patfiles if event.livetime(
event.andsegments(
[dq_segments, [idq.extract_start_stop(pat, suffix=".pat")]]))
]
if opts.verbose:
print "%d patfiles remain after taking overlap with segments" % len(
patfiles)
# load triggers from files
trigger_dict = event.loadkwm(trig_files)
if main_channel not in trigger_dict:
trigger_dict[main_channel] = []
if opts.verbose:
print "Done."
if not trigger_dict:
print "Warning: No triggers in the input files, exiting without doing anything."
sys.exit(0)
if opts.dq_segments:
# load dq segments
(dq_segments,
covered_segments) = idq.extract_dq_segments(open(opts.dq_segments, "r"),
opts.dq_segments_name)
# sort and merge segments
dq_segments = event.fixsegments(dq_segments)
else:
dq_segments = None
# construct auxmvc feature vectors
auxmvc_vectors = idq.build_auxmvc_vectors(trigger_dict, main_channel, opts.time_window, opts.signif_threshold, opts.output_file,\
gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\
science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\
max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples)
# gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\
# science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\
# max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples)
clean_samples = auxmvc_vectors[numpy.nonzero(auxmvc_vectors['i'] == 0)[0], :]
print 'No *.pat files found in the gps range ' \
+ str(gps_start_time) + ' - ' + str(gps_end_time)
print 'Exiting with status 2.'
sys.exit(2)
# load auxmvc vector samples
auxmvc_samples = auxmvc_utils.ReadMVSCTriggers(patfiles,
Classified=False)
if opts.dq_segments:
# load dq segments
(dq_segments, covered_segments) = \
idq.extract_dq_segments(open(opts.dq_segments, 'r'),
opts.dq_segments_name)
# sort and merge segments
dq_segments = event.fixsegments(dq_segments)
# keep only samples that fall in the dq_segments
auxmvc_samples = \
auxmvc_utils.get_samples_in_segments(auxmvc_samples,
dq_segments)
# get clean and glitch samples
random_samples = auxmvc_samples[numpy.nonzero(auxmvc_samples['i']
== 0)[0], :]