def extract_ovl_vconfigs(rank_frames,
channame,
traindir,
start,
end,
metric='eff/dt'):
"""
returns a dictionary mapping active vconfigs to segments
does NOT include "none" channel
"""
vconfigs = []
for rnkfr in rank_frames:
trained, calib = idq.extract_timeseries_ranges(rnkfr)
classifier = idq.extract_fap_name(rnkfr)
vetolist = glob.glob("%s/%d_%d/ovl/ovl/*vetolist.eval" %
(traindir, trained[0], trained[1]))
if len(vetolist) != 1:
raise ValueError(
"trouble finding a single vetolist file for : %s" % rnkfr)
vetolist = vetolist[0]
v = event.loadstringtable(vetolist)
rankmap = {0: [(None, None, None, None, 0, 0)]}
for line in v:
metric_exp = float(line[ovl.vD['metric_exp']])
if metric == 'eff/dt':
rnk = ovl.effbydt_to_rank(metric_exp)
elif metric == 'vsig':
rnk = ovl.vsig_to_rank(metric_exp)
elif metric == 'useP':
rnk = ovl.useP_to_rank(metric_exp)
else:
raise ValueError("metric=%s not understood" % metric)
if rankmap.has_key(rnk):
rankmap[rnk].append(
(line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]),
float(line[ovl.vD['vwin']]), metric, metric_exp, rnk))
else:
rankmap[rnk] = [
(line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]),
float(line[ovl.vD['vwin']]), metric, metric_exp, rnk)
]
for key, value in rankmap.items():
rankmap[key] = tuple(value)
t, ts = idq.combine_gwf([rnkfr], [channame])
t = t[0]
truth = (start <= t) * (t <= end)
t = t[truth]
ts = ts[0][truth]
if not len(ts):
continue
configs = rankmap[ts[0]]
segStart = t[0]
for T, TS in zip(t, ts):
if rankmap[TS] != configs:
vconfigs.append((configs, [segStart, T]))
segStart = T
configs = rankmap[TS]
else:
pass
vconfigs.append((configs, [segStart, T + t[1] - t[0]]))
configs = {}
for vconfig, seg in vconfigs:
if configs.has_key(vconfig):
configs[vconfig].append(seg)
else:
configs[vconfig] = [seg]
for key, value in configs.items():
value = event.andsegments([event.fixsegments(value), [[start, end]]])
if event.livetime(value):
configs[key] = event.fixsegments(value)
else:
raise ValueError(
"somehow picked up a config with zero livetime...")
return vconfigs, configs, {
"vchan": 0,
"vthr": 1,
"vwin": 2,
"metric": 3,
"metric_exp": 4,
"rank": 5
}
### extract trained, calib ranges and associate them with segments
trainD = defaultdict(list)
calibD = defaultdict(list)
for fap in faps:
seg = idq.extract_start_stop(fap, suffix=".gwf")
trained, calib = idq.extract_timeseries_ranges(fap)
trainD[tuple(trained)].append(seg)
calibD[tuple(calib)].append(seg)
if opts.ignore_science_segments:
dq_name = config.get('get_science_segments', 'include')
### fix segments, extract info
jsonD = {}
for (calib_start, calib_end), segs in calibD.items():
segs = event.fixsegments(segs)
thisD = {'used': segs}
### extract livetime
sciseg_files = glob.glob("%s/*_%d/science_segments*%s-*-*.xml.gz" %
(calibrationdir, calib_end, dq_name))
if len(sciseg_files) > 1:
raise ValueError("something odd with %s scisegs in : %s/*_%d/" %
(dq_name, calibrationdir, calib_end))
elif sciseg_files: ### len == 1
thisD['%s livetime' % dq_name] = event.livetime(
idq.extract_dq_segments(sciseg_files[0], dq_name)[0])
segstart, segend = idq.extract_start_stop(sciseg_files[0],
suffix='.xml.gz')
thisD['start'] = int(segstart)
thisD['end'] = int(segend)
print "gps : %.9f" % (gps)
minwin = opts.window
### go find triggers
if opts.verbose:
print "\tdiscoverying KW triggers within [%.9f, %.9f]" % (gps - opts.window, gps + opts.window)
### figure out which files you want
filenames = []
coverage = []
for gdsdir in kwgdsdirs:
for filename in idq.get_all_files_in_range(gdsdir, gps - opts.window, gps + opts.window, pad=0, suffix=".trg"):
seg = idq.extract_start_stop(filename, suffix=".trg")
if not event.livetime(event.andsegments([coverage, [seg]])):
coverage = event.fixsegments(coverage + [seg])
filenames.append(filename)
### figure out the extent of the coverage
if len(event.include([[gps]], coverage, tcent=0)) == 0:
if opts.force:
if opts.verbose:
print "no triggers found for gps : %.3f" % (gps)
continue
else:
raise ValueError("no triggers found for gps : %.3f" % (gps))
for s, e in coverage:
if s < gps:
if gps - s < minwin:
minwin = gps - s
elif e > gps:
def extract_ovl_vconfigs( rank_frames, channame, traindir, start, end, metric='eff/dt' ):
"""
returns a dictionary mapping active vconfigs to segments
does NOT include "none" channel
"""
vconfigs = []
for rnkfr in rank_frames:
trained, calib = idq.extract_timeseries_ranges( rnkfr )
classifier = idq.extract_fap_name( rnkfr )
vetolist = glob.glob( "%s/%d_%d/ovl/ovl/*vetolist.eval"%(traindir, trained[0], trained[1]) )
if len(vetolist) != 1:
raise ValueError( "trouble finding a single vetolist file for : %s"%rnkfr )
vetolist=vetolist[0]
v = event.loadstringtable( vetolist )
rankmap = { 0:[(None, None, None, None, 0, 0)] }
for line in v:
metric_exp = float(line[ovl.vD['metric_exp']])
if metric == 'eff/dt':
rnk = ovl.effbydt_to_rank( metric_exp )
elif metric == 'vsig':
rnk = ovl.vsig_to_rank( metric_exp )
elif metric == 'useP':
rnk = ovl.useP_to_rank( metric_exp )
else:
raise ValueError("metric=%s not understood"%metric)
if rankmap.has_key(rnk):
rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk ))
else:
rankmap[rnk] = [(line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )]
for key, value in rankmap.items():
rankmap[key] = tuple(value)
t, ts = idq.combine_gwf( [rnkfr], [channame])
t = t[0]
truth = (start <= t)*(t <= end)
t = t[truth]
ts = ts[0][truth]
if not len(ts):
continue
configs = rankmap[ts[0]]
segStart = t[0]
for T, TS in zip(t, ts):
if rankmap[TS] != configs:
vconfigs.append( (configs, [segStart, T] ) )
segStart = T
configs = rankmap[TS]
else:
pass
vconfigs.append( (configs, [segStart, T+t[1]-t[0]] ) )
configs = {}
for vconfig, seg in vconfigs:
if configs.has_key( vconfig ):
configs[vconfig].append( seg )
else:
configs[vconfig] = [ seg ]
for key, value in configs.items():
value = event.andsegments( [event.fixsegments( value ), [[start,end]] ] )
if event.livetime( value ):
configs[key] = event.fixsegments( value )
else:
raise ValueError("somehow picked up a config with zero livetime...")
return vconfigs, configs, {"vchan":0, "vthr":1, "vwin":2, "metric":3, "metric_exp":4, "rank":5}
trigger_dict = event.loadkwm(trig_files)
if main_channel not in trigger_dict:
trigger_dict[main_channel] = []
if opts.verbose:
print "Done."
if not trigger_dict:
print "Warning: No triggers in the input files, exiting without doing anything."
sys.exit(0)
if opts.dq_segments:
# load dq segments
(dq_segments, covered_segments) = idq.extract_dq_segments(open(opts.dq_segments, "r"), opts.dq_segments_name)
# sort and merge segments
dq_segments = event.fixsegments(dq_segments)
else:
dq_segments = None
# construct auxmvc feature vectors
auxmvc_vectors = idq.build_auxmvc_vectors(trigger_dict, main_channel, opts.time_window, opts.signif_threshold, opts.output_file,\
gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\
science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\
max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples)
# gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\
# science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\
# max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples)
clean_samples = auxmvc_vectors[numpy.nonzero(auxmvc_vectors['i']==0)[0],:]
glitch_samples = auxmvc_vectors[numpy.nonzero(auxmvc_vectors['i']==1)[0],:]
#---
### find time ranges of interest from patfile
if opts.verbose:
print( "defining segments in which we need KW triggers" )
gps = idq.slim_load_datfiles(patfiles, columns=['GPS_s', 'GPS_ms'])
gps = np.array(gps['GPS_s'], dtype=float) + 1e-6*np.array(gps['GPS_ms'], dtype=float)
Ngps = len(gps)
if Ngps==0:
raise ValueError, 'please supply at least one GPS time within : '+patfile
elif opts.verbose:
print( "found %d times"%Ngps )
segs = event.fixsegments([[t-win, t+win] for t in gps]) ### the segments in which we need KW triggers
#---
### look up KW trg files that intersect segs
if opts.verbose:
print( "finding relevant kw_trgfiles" )
kw_trgfiles = []
### iterate over different configurations used in training
for kwconf, dirname in eval(config.get('general', 'kw')).items(): ### this is kinda ugly...
if opts.verbose:
print( " searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]"%(kwconf, dirname, segs[0][0], segs[-1][1]) )
### iterate over all trg files found in that directory
for trgfile in idq.get_all_files_in_range(dirname, segs[0][0], segs[-1][1], pad=0, suffix='.trg'):
### check whether there is some overlap
print "gps : %.9f"%(gps)
minwin = opts.window
### go find triggers
if opts.verbose:
print "\tdiscoverying KW triggers within [%.9f, %.9f]"%(gps-opts.window, gps+opts.window)
### figure out which files you want
filenames = []
coverage = []
for gdsdir in kwgdsdirs:
for filename in idq.get_all_files_in_range(gdsdir, gps-opts.window, gps+opts.window, pad=0, suffix=".trg"):
seg = idq.extract_start_stop(filename, suffix=".trg")
if not event.livetime(event.andsegments([coverage, [seg]])):
coverage = event.fixsegments( coverage + [seg] )
filenames.append( filename )
### figure out the extent of the coverage
if len(event.include([[gps]], coverage, tcent=0)) == 0:
if opts.force:
if opts.verbose:
print "no triggers found for gps : %.3f"%(gps)
continue
else:
raise ValueError("no triggers found for gps : %.3f"%(gps))
for s, e in coverage:
if s < gps:
if gps-s < minwin:
minwin = gps-s
elif e > gps:
### extract trained, calib ranges and associate them with segments
trainD = defaultdict( list )
calibD = defaultdict( list )
for fap in faps:
seg = idq.extract_start_stop( fap, suffix=".gwf" )
trained, calib = idq.extract_timeseries_ranges( fap )
trainD[tuple(trained)].append( seg )
calibD[tuple(calib)].append( seg )
if opts.ignore_science_segments:
dq_name = config.get('get_science_segments', 'include')
### fix segments, extract info
jsonD = {}
for (calib_start, calib_end), segs in calibD.items():
segs = event.fixsegments( segs )
thisD = {'used':segs}
### extract livetime
sciseg_files = glob.glob("%s/*_%d/science_segments*%s-*-*.xml.gz"%(calibrationdir, calib_end, dq_name))
if len(sciseg_files) > 1:
raise ValueError("something odd with %s scisegs in : %s/*_%d/"%(dq_name, calibrationdir, calib_end))
elif sciseg_files: ### len == 1
thisD['%s livetime'%dq_name] = event.livetime( idq.extract_dq_segments( sciseg_files[0], dq_name)[0] )
segstart, segend = idq.extract_start_stop( sciseg_files[0], suffix='.xml.gz')
thisD['start'] = int(segstart)
thisD['end'] = int(segend)
else:
thisD['%s livetime'%dq_name] = None
thisD['start'] = None
thisD['end'] = None
suffix='.pat')
if opts.verbose:
print "found %d pat files" % len(patfiles)
if opts.dq_segments:
# load dq segments
if opts.verbose:
print "reading segments from %s" % opts.dq_segments
(dq_segments, covered_segments) = \
idq.extract_dq_segments(open(opts.dq_segments, 'r'),
opts.dq_segments_name)
# sort and merge segments
dq_segments = event.fixsegments(dq_segments)
### filter patfiles by scisegs, keep only those events with non-zero overlap with science time
patfiles = [
pat for pat in patfiles if event.livetime(
event.andsegments(
[dq_segments, [idq.extract_start_stop(pat, suffix=".pat")]]))
]
if opts.verbose:
print "%d patfiles remain after taking overlap with segments" % len(
patfiles)
if len(patfiles) == 0:
print 'No *.pat files found in the gps range ' \
+ str(gps_start_time) + ' - ' + str(gps_end_time)
### find time ranges of interest from patfile
if opts.verbose:
print("defining segments in which we need KW triggers")
gps = idq.slim_load_datfiles(patfiles, columns=['GPS_s', 'GPS_ms'])
gps = np.array(gps['GPS_s'],
dtype=float) + 1e-6 * np.array(gps['GPS_ms'], dtype=float)
Ngps = len(gps)
if Ngps == 0:
raise ValueError, 'please supply at least one GPS time within : ' + patfile
elif opts.verbose:
print("found %d times" % Ngps)
segs = event.fixsegments([[t - win, t + win] for t in gps
]) ### the segments in which we need KW triggers
#---
### look up KW trg files that intersect segs
if opts.verbose:
print("finding relevant kw_trgfiles")
kw_trgfiles = []
### iterate over different configurations used in training
for kwconf, dirname in eval(config.get(
'general', 'kw')).items(): ### this is kinda ugly...
if opts.verbose:
print(
" searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]"
% (kwconf, dirname, segs[0][0], segs[-1][1]))
