file_obj = open(summary_dat, "w")
print >> file_obj, " ".join(columns)
for datfile in datfiles:
if classifier not in datfile: ### downselect based on classifier
continue
### load only the required columns from datfile
output = idq.slim_load_datfile(datfile, skip_lines=0, columns=columns)
### convert to list of triggers
triggers = [[output[c][i] for c in columns] for i in xrange(len(output["GPS"]))]
triggers = [[float(l[0])]+l[1:] for l in triggers]
### window triggers with scisegs
triggers = event.include( triggers, scisegs, tcent=0 ) ### tcent hard coded according to columns
### write surviving triggers to datfile
for trigger in triggers:
trigger = [str(trigger[0])] + trigger[1:]
print >> file_obj, " ".join(trigger)
file_obj.close()
logger.info("Done.")
#=============================================
# generate *roc files
#=============================================
### collect *dat files and merge into *roc files
logger.info('generating *.roc files')
roc_paths = idq.datfiles_to_roc(
### go find triggers
if opts.verbose:
print "\tdiscoverying KW triggers within [%.9f, %.9f]" % (gps - opts.window, gps + opts.window)
### figure out which files you want
filenames = []
coverage = []
for gdsdir in kwgdsdirs:
for filename in idq.get_all_files_in_range(gdsdir, gps - opts.window, gps + opts.window, pad=0, suffix=".trg"):
seg = idq.extract_start_stop(filename, suffix=".trg")
if not event.livetime(event.andsegments([coverage, [seg]])):
coverage = event.fixsegments(coverage + [seg])
filenames.append(filename)
### figure out the extent of the coverage
if len(event.include([[gps]], coverage, tcent=0)) == 0:
if opts.force:
if opts.verbose:
print "no triggers found for gps : %.3f" % (gps)
continue
else:
raise ValueError("no triggers found for gps : %.3f" % (gps))
for s, e in coverage:
if s < gps:
if gps - s < minwin:
minwin = gps - s
elif e > gps:
if e - gps < minwin:
minwin = e - gps
else:
print "\t\tminwin = %.3f" % (minwin)
### go find triggers
if opts.verbose:
print "\tdiscoverying KW triggers within [%.9f, %.9f]"%(gps-opts.window, gps+opts.window)
### figure out which files you want
filenames = []
coverage = []
for gdsdir in kwgdsdirs:
for filename in idq.get_all_files_in_range(gdsdir, gps-opts.window, gps+opts.window, pad=0, suffix=".trg"):
seg = idq.extract_start_stop(filename, suffix=".trg")
if not event.livetime(event.andsegments([coverage, [seg]])):
coverage = event.fixsegments( coverage + [seg] )
filenames.append( filename )
### figure out the extent of the coverage
if len(event.include([[gps]], coverage, tcent=0)) == 0:
if opts.force:
if opts.verbose:
print "no triggers found for gps : %.3f"%(gps)
continue
else:
raise ValueError("no triggers found for gps : %.3f"%(gps))
for s, e in coverage:
if s < gps:
if gps-s < minwin:
minwin = gps-s
elif e > gps:
if e-gps < minwin:
minwin = e-gps
if minwin < opts.exclude:
raise ValueError("minwin < opts.exclude caused by gps : %3f"%(gps))
opts.classifier, ifo, opts.start, opts.end)
if opts.verbose:
print " " + message
gracedb.writeLog(opts.gracedb_id,
message=message,
filename=figname,
tagname=['data_quality'])
### discover KW triggers, compute efficiencies, plot ROC curves
if opts.verbose:
print "retrieving gw triggers"
gwtrgs = event.include(idq.retrieve_kwtrigs(GWgdsdir,
GWkwbasename,
opts.start,
opts.end - opts.start,
kwstride,
sleep=0,
ntrials=1,
verbose=False)[gwchannel],
idqsegs,
tcent=event.col_kw['tcent'])
fig = isp.plt.figure()
ax = fig.add_axes(isp.default_axpos)
jsonD = {}
for signif in opts.KWsignifThr:
trgs = [trg for trg in gwtrgs if trg[event.col_kw['signif']] >= signif]
N = len(trgs) * 1.0
if N:
eff = []
for fapThr in opts.FAPthr:
figname = isp.calibfig( gdbdir, ifo, opts.classifier, filetag, opts.start, opts.end-opts.start )
if opts.verbose:
print " %s"%figname
fig.savefig(figname)
isp.plt.close(fig)
if not opts.skip_gracedb_upload:
message = "iDQ calibration sanity check figure for %s at %s within [%.3f, %.3f]"%(opts.classifier, ifo, opts.start, opts.end)
if opts.verbose:
print " "+message
gracedb.writeLog( opts.gracedb_id, message=message, filename=figname, tagname=['data_quality'] )
### discover KW triggers, compute efficiencies, plot ROC curves
if opts.verbose:
print "retrieving gw triggers"
gwtrgs = event.include( idq.retrieve_kwtrigs(GWgdsdir, GWkwbasename, opts.start, opts.end-opts.start, kwstride, sleep=0, ntrials=1, verbose=False)[gwchannel], idqsegs, tcent=event.col_kw['tcent'] )
fig = isp.plt.figure()
ax = fig.add_axes( isp.default_axpos )
jsonD = {}
for signif in opts.KWsignifThr:
trgs = [trg for trg in gwtrgs if trg[event.col_kw['signif']] >= signif]
N = len(trgs)*1.0
if N:
eff = []
for fapThr in opts.FAPthr:
s = segs[fapThr][0]
eff.append( len(event.include( trgs, s, tcent=event.col_kw['tcent'] ))/N )
else:
eff = [0.0 for fapThr in opts.FAPthr]