ax.fill_between(dt, l, h, color=color, alpha=0.25, edgecolor='none')
### add curve from dat files
if opts.verbose:
print "finding all *dat files"
dats = [
dat for dat in idq.get_all_files_in_range(
realtimedir, opts.start, opts.end, pad=0, suffix='.dat')
if (opts.classifier == idq.extract_dat_name(dat)) and event.livetime(
event.andsegments([[idq.extract_start_stop(dat, suffix=".dat")],
idqsegs]))
]
if opts.verbose:
print "reading samples from %d dat files" % len(dats)
output = idq.slim_load_datfiles(dats,
skip_lines=0,
columns=['GPS', 'i', 'rank'])
output = idq.filter_datfile_output(output, idqsegs)
#output['GPS'] = [float(l) for l in output['GPS']] ### not necessary because values are cast appropriately within idq.filter_datfile_output
#output['i'] = [float(l) for l in output['i']]
#output['rank'] = [float(l) for l in output['rank']]
r, c, g, = idq.dat_to_rcg(output)
if g[-1] and c[-1]:
color = ax.step(1.0 * c / c[-1],
1.0 * g / g[-1],
label='datfiles: $N_c=%d$, $N_g=%d$' % (c[-1], g[-1]),
linewidth=2,
where='post')[0].get_color()
for G, C in zip(g, c):
#====================
urocs = {} ### stores uroc files for kde estimation
for classifier in classifiers:
### write list of dats to cache file
cache = idq.cache(output_dir, classifier, "_datcache%s" % usertag)
logger.info('writing list of dat files to %s' % cache)
f = open(cache, 'w')
for dat in datsD[classifier]:
print >> f, dat
f.close()
logger.info(' computing new calibration for %s' % classifier)
### extract data from dat files
output = idq.slim_load_datfiles(datsD[classifier],
skip_lines=0,
columns='GPS i rank'.split() +
[cluster_key])
### filter times by scisegs -> keep only the ones within scisegs
output = idq.filter_datfile_output(output, idqsegs)
### cluster
if not opts.dont_cluster:
output = idq.cluster_datfile_output(output,
cluster_key=cluster_key,
cluster_win=cluster_win)
### downselect to only keep the most recent max_num_gch and max_num_cln
these_columns, glitches, cleans = idq.separate_output(output)
glitches.sort(key=lambda l: l[these_columns['GPS']])
cleans.sort(key=lambda l: l[these_columns['GPS']])
# update uroc for each classifier
#====================
urocs = {} ### stores uroc files for kde estimation
for classifier in classifiers:
### write list of dats to cache file
cache = idq.cache(output_dir, classifier, "_datcache%s"%usertag)
logger.info('writing list of dat files to %s'%cache)
f = open(cache, 'w')
for dat in datsD[classifier]:
print >>f, dat
f.close()
logger.info(' computing new calibration for %s'%classifier)
### extract data from dat files
output = idq.slim_load_datfiles(datsD[classifier], skip_lines=0, columns='GPS i rank'.split()+[cluster_key])
### filter times by scisegs -> keep only the ones within scisegs
output = idq.filter_datfile_output( output, idqsegs )
### cluster
if not opts.dont_cluster:
output = idq.cluster_datfile_output( output, cluster_key=cluster_key, cluster_win=cluster_win)
### downselect to only keep the most recent max_num_gch and max_num_cln
these_columns, glitches, cleans = idq.separate_output( output )
glitches.sort(key=lambda l: l[these_columns['GPS']])
cleans.sort(key=lambda l: l[these_columns['GPS']])
if len(glitches) > max_num_gch:
logger.info(' downselecting to the %d most recent glitches'%max_num_gch)
glitches = glitches[-max_num_gch:]
print( "finding maximum veto window used in vetolist" )
win=0.001 ### use this as an absolute mimimum. Will almost certainly be replaced, unless vetolist is empty
file_obj = open(vetolist, 'r')
for line in file_obj:
if line[0]=="#":
pass
else:
win = max(win, float(line.strip().split()[ovl.vD['vwin']]))
file_obj.close()
#---
### find time ranges of interest from patfile
if opts.verbose:
print( "defining segments in which we need KW triggers" )
gps = idq.slim_load_datfiles(patfiles, columns=['GPS_s', 'GPS_ms'])
gps = np.array(gps['GPS_s'], dtype=float) + 1e-6*np.array(gps['GPS_ms'], dtype=float)
Ngps = len(gps)
if Ngps==0:
raise ValueError, 'please supply at least one GPS time within : '+patfile
elif opts.verbose:
print( "found %d times"%Ngps )
segs = event.fixsegments([[t-win, t+win] for t in gps]) ### the segments in which we need KW triggers
#---
### look up KW trg files that intersect segs
if opts.verbose:
jsonD[signif] = {'observed deadtime':dt, 'observed efficiency':eff, 'number of glitches':N, 'duration':T}
l, h = [], []
for e in eff:
cr = idq.binomialCR( e*N, N, conf=0.68 )
l.append(cr[0])
h.append(cr[1])
ax.fill_between( dt, l, h, color=color, alpha=0.25, edgecolor='none' )
### add curve from dat files
if opts.verbose:
print "finding all *dat files"
dats = [dat for dat in idq.get_all_files_in_range( realtimedir, opts.start, opts.end, pad=0, suffix='.dat') if (opts.classifier==idq.extract_dat_name( dat )) and event.livetime(event.andsegments([[idq.extract_start_stop(dat, suffix=".dat")], idqsegs]))]
if opts.verbose:
print "reading samples from %d dat files"%len(dats)
output = idq.slim_load_datfiles( dats, skip_lines=0, columns=['GPS', 'i', 'rank'])
output = idq.filter_datfile_output( output, idqsegs )
#output['GPS'] = [float(l) for l in output['GPS']] ### not necessary because values are cast appropriately within idq.filter_datfile_output
#output['i'] = [float(l) for l in output['i']]
#output['rank'] = [float(l) for l in output['rank']]
r, c, g, = idq.dat_to_rcg( output )
if g[-1] and c[-1]:
color = ax.step( 1.0*c/c[-1], 1.0*g/g[-1], label='datfiles: $N_c=%d$, $N_g=%d$'%(c[-1], g[-1]), linewidth=2, where='post' )[0].get_color()
for G, C in zip(g, c):
c_cr = idq.binomialCR( C, c[-1], conf=0.68 )
g_cr = idq.binomialCR( G, g[-1], conf=0.68 )
ax.fill_between( c_cr, [g_cr[0]]*2, [g_cr[1]]*2, color=color, alpha=0.25, edgecolor='none' )
jsonD['dat'] = {'rank':list(r), 'cumulative cleans':list(c), 'cumulative glitches':list(g)}
print("finding maximum veto window used in vetolist")
win = 0.001 ### use this as an absolute mimimum. Will almost certainly be replaced, unless vetolist is empty
file_obj = open(vetolist, 'r')
for line in file_obj:
if line[0] == "#":
pass
else:
win = max(win, float(line.strip().split()[ovl.vD['vwin']]))
file_obj.close()
#---
### find time ranges of interest from patfile
if opts.verbose:
print("defining segments in which we need KW triggers")
gps = idq.slim_load_datfiles(patfiles, columns=['GPS_s', 'GPS_ms'])
gps = np.array(gps['GPS_s'],
dtype=float) + 1e-6 * np.array(gps['GPS_ms'], dtype=float)
Ngps = len(gps)
if Ngps == 0:
raise ValueError, 'please supply at least one GPS time within : ' + patfile
elif opts.verbose:
print("found %d times" % Ngps)
segs = event.fixsegments([[t - win, t + win] for t in gps
]) ### the segments in which we need KW triggers
#---