def extract_ovl_vconfigs(rank_frames, channame, traindir, start, end, metric='eff/dt'): """ returns a dictionary mapping active vconfigs to segments does NOT include "none" channel """ vconfigs = [] for rnkfr in rank_frames: trained, calib = idq.extract_timeseries_ranges(rnkfr) classifier = idq.extract_fap_name(rnkfr) vetolist = glob.glob("%s/%d_%d/ovl/ovl/*vetolist.eval" % (traindir, trained[0], trained[1])) if len(vetolist) != 1: raise ValueError( "trouble finding a single vetolist file for : %s" % rnkfr) vetolist = vetolist[0] v = event.loadstringtable(vetolist) rankmap = {0: [(None, None, None, None, 0, 0)]} for line in v: metric_exp = float(line[ovl.vD['metric_exp']]) if metric == 'eff/dt': rnk = ovl.effbydt_to_rank(metric_exp) elif metric == 'vsig': rnk = ovl.vsig_to_rank(metric_exp) elif metric == 'useP': rnk = ovl.useP_to_rank(metric_exp) else: raise ValueError("metric=%s not understood" % metric) if rankmap.has_key(rnk): rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk)) else: rankmap[rnk] = [ (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk) ] for key, value in rankmap.items(): rankmap[key] = tuple(value) t, ts = idq.combine_gwf([rnkfr], [channame]) t = t[0] truth = (start <= t) * (t <= end) t = t[truth] ts = ts[0][truth] if not len(ts): continue configs = rankmap[ts[0]] segStart = t[0] for T, TS in zip(t, ts): if rankmap[TS] != configs: vconfigs.append((configs, [segStart, T])) segStart = T configs = rankmap[TS] else: pass vconfigs.append((configs, [segStart, T + t[1] - t[0]])) configs = {} for vconfig, seg in vconfigs: if configs.has_key(vconfig): configs[vconfig].append(seg) else: configs[vconfig] = [seg] for key, value in configs.items(): value = event.andsegments([event.fixsegments(value), [[start, end]]]) if event.livetime(value): configs[key] = event.fixsegments(value) else: raise ValueError( "somehow picked up a config with zero livetime...") return vconfigs, configs, { "vchan": 0, "vthr": 1, "vwin": 2, "metric": 3, "metric_exp": 4, "rank": 5 }
### extract trained, calib ranges and associate them with segments trainD = defaultdict(list) calibD = defaultdict(list) for fap in faps: seg = idq.extract_start_stop(fap, suffix=".gwf") trained, calib = idq.extract_timeseries_ranges(fap) trainD[tuple(trained)].append(seg) calibD[tuple(calib)].append(seg) if opts.ignore_science_segments: dq_name = config.get('get_science_segments', 'include') ### fix segments, extract info jsonD = {} for (calib_start, calib_end), segs in calibD.items(): segs = event.fixsegments(segs) thisD = {'used': segs} ### extract livetime sciseg_files = glob.glob("%s/*_%d/science_segments*%s-*-*.xml.gz" % (calibrationdir, calib_end, dq_name)) if len(sciseg_files) > 1: raise ValueError("something odd with %s scisegs in : %s/*_%d/" % (dq_name, calibrationdir, calib_end)) elif sciseg_files: ### len == 1 thisD['%s livetime' % dq_name] = event.livetime( idq.extract_dq_segments(sciseg_files[0], dq_name)[0]) segstart, segend = idq.extract_start_stop(sciseg_files[0], suffix='.xml.gz') thisD['start'] = int(segstart) thisD['end'] = int(segend)
print "gps : %.9f" % (gps) minwin = opts.window ### go find triggers if opts.verbose: print "\tdiscoverying KW triggers within [%.9f, %.9f]" % (gps - opts.window, gps + opts.window) ### figure out which files you want filenames = [] coverage = [] for gdsdir in kwgdsdirs: for filename in idq.get_all_files_in_range(gdsdir, gps - opts.window, gps + opts.window, pad=0, suffix=".trg"): seg = idq.extract_start_stop(filename, suffix=".trg") if not event.livetime(event.andsegments([coverage, [seg]])): coverage = event.fixsegments(coverage + [seg]) filenames.append(filename) ### figure out the extent of the coverage if len(event.include([[gps]], coverage, tcent=0)) == 0: if opts.force: if opts.verbose: print "no triggers found for gps : %.3f" % (gps) continue else: raise ValueError("no triggers found for gps : %.3f" % (gps)) for s, e in coverage: if s < gps: if gps - s < minwin: minwin = gps - s elif e > gps:
def extract_ovl_vconfigs( rank_frames, channame, traindir, start, end, metric='eff/dt' ): """ returns a dictionary mapping active vconfigs to segments does NOT include "none" channel """ vconfigs = [] for rnkfr in rank_frames: trained, calib = idq.extract_timeseries_ranges( rnkfr ) classifier = idq.extract_fap_name( rnkfr ) vetolist = glob.glob( "%s/%d_%d/ovl/ovl/*vetolist.eval"%(traindir, trained[0], trained[1]) ) if len(vetolist) != 1: raise ValueError( "trouble finding a single vetolist file for : %s"%rnkfr ) vetolist=vetolist[0] v = event.loadstringtable( vetolist ) rankmap = { 0:[(None, None, None, None, 0, 0)] } for line in v: metric_exp = float(line[ovl.vD['metric_exp']]) if metric == 'eff/dt': rnk = ovl.effbydt_to_rank( metric_exp ) elif metric == 'vsig': rnk = ovl.vsig_to_rank( metric_exp ) elif metric == 'useP': rnk = ovl.useP_to_rank( metric_exp ) else: raise ValueError("metric=%s not understood"%metric) if rankmap.has_key(rnk): rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )) else: rankmap[rnk] = [(line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )] for key, value in rankmap.items(): rankmap[key] = tuple(value) t, ts = idq.combine_gwf( [rnkfr], [channame]) t = t[0] truth = (start <= t)*(t <= end) t = t[truth] ts = ts[0][truth] if not len(ts): continue configs = rankmap[ts[0]] segStart = t[0] for T, TS in zip(t, ts): if rankmap[TS] != configs: vconfigs.append( (configs, [segStart, T] ) ) segStart = T configs = rankmap[TS] else: pass vconfigs.append( (configs, [segStart, T+t[1]-t[0]] ) ) configs = {} for vconfig, seg in vconfigs: if configs.has_key( vconfig ): configs[vconfig].append( seg ) else: configs[vconfig] = [ seg ] for key, value in configs.items(): value = event.andsegments( [event.fixsegments( value ), [[start,end]] ] ) if event.livetime( value ): configs[key] = event.fixsegments( value ) else: raise ValueError("somehow picked up a config with zero livetime...") return vconfigs, configs, {"vchan":0, "vthr":1, "vwin":2, "metric":3, "metric_exp":4, "rank":5}
trigger_dict = event.loadkwm(trig_files) if main_channel not in trigger_dict: trigger_dict[main_channel] = [] if opts.verbose: print "Done." if not trigger_dict: print "Warning: No triggers in the input files, exiting without doing anything." sys.exit(0) if opts.dq_segments: # load dq segments (dq_segments, covered_segments) = idq.extract_dq_segments(open(opts.dq_segments, "r"), opts.dq_segments_name) # sort and merge segments dq_segments = event.fixsegments(dq_segments) else: dq_segments = None # construct auxmvc feature vectors auxmvc_vectors = idq.build_auxmvc_vectors(trigger_dict, main_channel, opts.time_window, opts.signif_threshold, opts.output_file,\ gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\ science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\ max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples) # gps_start_time=gps_start_time, gps_end_time=gps_end_time, channels=opts.channels, unsafe_channels=opts.unsafe_channels,\ # science_segments = dq_segments, clean_samples_rate=opts.clean_samples_rate, filter_out_unclean=opts.filter_out_unclean,\ # max_clean_samples = opts.max_clean_samples, max_glitch_samples = opts.max_glitch_samples) clean_samples = auxmvc_vectors[numpy.nonzero(auxmvc_vectors['i']==0)[0],:] glitch_samples = auxmvc_vectors[numpy.nonzero(auxmvc_vectors['i']==1)[0],:]
#--- ### find time ranges of interest from patfile if opts.verbose: print( "defining segments in which we need KW triggers" ) gps = idq.slim_load_datfiles(patfiles, columns=['GPS_s', 'GPS_ms']) gps = np.array(gps['GPS_s'], dtype=float) + 1e-6*np.array(gps['GPS_ms'], dtype=float) Ngps = len(gps) if Ngps==0: raise ValueError, 'please supply at least one GPS time within : '+patfile elif opts.verbose: print( "found %d times"%Ngps ) segs = event.fixsegments([[t-win, t+win] for t in gps]) ### the segments in which we need KW triggers #--- ### look up KW trg files that intersect segs if opts.verbose: print( "finding relevant kw_trgfiles" ) kw_trgfiles = [] ### iterate over different configurations used in training for kwconf, dirname in eval(config.get('general', 'kw')).items(): ### this is kinda ugly... if opts.verbose: print( " searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]"%(kwconf, dirname, segs[0][0], segs[-1][1]) ) ### iterate over all trg files found in that directory for trgfile in idq.get_all_files_in_range(dirname, segs[0][0], segs[-1][1], pad=0, suffix='.trg'): ### check whether there is some overlap
print "gps : %.9f"%(gps) minwin = opts.window ### go find triggers if opts.verbose: print "\tdiscoverying KW triggers within [%.9f, %.9f]"%(gps-opts.window, gps+opts.window) ### figure out which files you want filenames = [] coverage = [] for gdsdir in kwgdsdirs: for filename in idq.get_all_files_in_range(gdsdir, gps-opts.window, gps+opts.window, pad=0, suffix=".trg"): seg = idq.extract_start_stop(filename, suffix=".trg") if not event.livetime(event.andsegments([coverage, [seg]])): coverage = event.fixsegments( coverage + [seg] ) filenames.append( filename ) ### figure out the extent of the coverage if len(event.include([[gps]], coverage, tcent=0)) == 0: if opts.force: if opts.verbose: print "no triggers found for gps : %.3f"%(gps) continue else: raise ValueError("no triggers found for gps : %.3f"%(gps)) for s, e in coverage: if s < gps: if gps-s < minwin: minwin = gps-s elif e > gps:
### extract trained, calib ranges and associate them with segments trainD = defaultdict( list ) calibD = defaultdict( list ) for fap in faps: seg = idq.extract_start_stop( fap, suffix=".gwf" ) trained, calib = idq.extract_timeseries_ranges( fap ) trainD[tuple(trained)].append( seg ) calibD[tuple(calib)].append( seg ) if opts.ignore_science_segments: dq_name = config.get('get_science_segments', 'include') ### fix segments, extract info jsonD = {} for (calib_start, calib_end), segs in calibD.items(): segs = event.fixsegments( segs ) thisD = {'used':segs} ### extract livetime sciseg_files = glob.glob("%s/*_%d/science_segments*%s-*-*.xml.gz"%(calibrationdir, calib_end, dq_name)) if len(sciseg_files) > 1: raise ValueError("something odd with %s scisegs in : %s/*_%d/"%(dq_name, calibrationdir, calib_end)) elif sciseg_files: ### len == 1 thisD['%s livetime'%dq_name] = event.livetime( idq.extract_dq_segments( sciseg_files[0], dq_name)[0] ) segstart, segend = idq.extract_start_stop( sciseg_files[0], suffix='.xml.gz') thisD['start'] = int(segstart) thisD['end'] = int(segend) else: thisD['%s livetime'%dq_name] = None thisD['start'] = None thisD['end'] = None
suffix='.pat') if opts.verbose: print "found %d pat files" % len(patfiles) if opts.dq_segments: # load dq segments if opts.verbose: print "reading segments from %s" % opts.dq_segments (dq_segments, covered_segments) = \ idq.extract_dq_segments(open(opts.dq_segments, 'r'), opts.dq_segments_name) # sort and merge segments dq_segments = event.fixsegments(dq_segments) ### filter patfiles by scisegs, keep only those events with non-zero overlap with science time patfiles = [ pat for pat in patfiles if event.livetime( event.andsegments( [dq_segments, [idq.extract_start_stop(pat, suffix=".pat")]])) ] if opts.verbose: print "%d patfiles remain after taking overlap with segments" % len( patfiles) if len(patfiles) == 0: print 'No *.pat files found in the gps range ' \ + str(gps_start_time) + ' - ' + str(gps_end_time)
### find time ranges of interest from patfile if opts.verbose: print("defining segments in which we need KW triggers") gps = idq.slim_load_datfiles(patfiles, columns=['GPS_s', 'GPS_ms']) gps = np.array(gps['GPS_s'], dtype=float) + 1e-6 * np.array(gps['GPS_ms'], dtype=float) Ngps = len(gps) if Ngps == 0: raise ValueError, 'please supply at least one GPS time within : ' + patfile elif opts.verbose: print("found %d times" % Ngps) segs = event.fixsegments([[t - win, t + win] for t in gps ]) ### the segments in which we need KW triggers #--- ### look up KW trg files that intersect segs if opts.verbose: print("finding relevant kw_trgfiles") kw_trgfiles = [] ### iterate over different configurations used in training for kwconf, dirname in eval(config.get( 'general', 'kw')).items(): ### this is kinda ugly... if opts.verbose: print( " searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]" % (kwconf, dirname, segs[0][0], segs[-1][1]))