def extract_ovl_vconfigs(rank_frames, channame, traindir, start, end, metric='eff/dt'): """ returns a dictionary mapping active vconfigs to segments does NOT include "none" channel """ vconfigs = [] for rnkfr in rank_frames: trained, calib = idq.extract_timeseries_ranges(rnkfr) classifier = idq.extract_fap_name(rnkfr) vetolist = glob.glob("%s/%d_%d/ovl/ovl/*vetolist.eval" % (traindir, trained[0], trained[1])) if len(vetolist) != 1: raise ValueError( "trouble finding a single vetolist file for : %s" % rnkfr) vetolist = vetolist[0] v = event.loadstringtable(vetolist) rankmap = {0: [(None, None, None, None, 0, 0)]} for line in v: metric_exp = float(line[ovl.vD['metric_exp']]) if metric == 'eff/dt': rnk = ovl.effbydt_to_rank(metric_exp) elif metric == 'vsig': rnk = ovl.vsig_to_rank(metric_exp) elif metric == 'useP': rnk = ovl.useP_to_rank(metric_exp) else: raise ValueError("metric=%s not understood" % metric) if rankmap.has_key(rnk): rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk)) else: rankmap[rnk] = [ (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk) ] for key, value in rankmap.items(): rankmap[key] = tuple(value) t, ts = idq.combine_gwf([rnkfr], [channame]) t = t[0] truth = (start <= t) * (t <= end) t = t[truth] ts = ts[0][truth] if not len(ts): continue configs = rankmap[ts[0]] segStart = t[0] for T, TS in zip(t, ts): if rankmap[TS] != configs: vconfigs.append((configs, [segStart, T])) segStart = T configs = rankmap[TS] else: pass vconfigs.append((configs, [segStart, T + t[1] - t[0]])) configs = {} for vconfig, seg in vconfigs: if configs.has_key(vconfig): configs[vconfig].append(seg) else: configs[vconfig] = [seg] for key, value in configs.items(): value = event.andsegments([event.fixsegments(value), [[start, end]]]) if event.livetime(value): configs[key] = event.fixsegments(value) else: raise ValueError( "somehow picked up a config with zero livetime...") return vconfigs, configs, { "vchan": 0, "vthr": 1, "vwin": 2, "metric": 3, "metric_exp": 4, "rank": 5 }
def extract_ovl_vconfigs( rank_frames, channame, traindir, start, end, metric='eff/dt' ): """ returns a dictionary mapping active vconfigs to segments does NOT include "none" channel """ vconfigs = [] for rnkfr in rank_frames: trained, calib = idq.extract_timeseries_ranges( rnkfr ) classifier = idq.extract_fap_name( rnkfr ) vetolist = glob.glob( "%s/%d_%d/ovl/ovl/*vetolist.eval"%(traindir, trained[0], trained[1]) ) if len(vetolist) != 1: raise ValueError( "trouble finding a single vetolist file for : %s"%rnkfr ) vetolist=vetolist[0] v = event.loadstringtable( vetolist ) rankmap = { 0:[(None, None, None, None, 0, 0)] } for line in v: metric_exp = float(line[ovl.vD['metric_exp']]) if metric == 'eff/dt': rnk = ovl.effbydt_to_rank( metric_exp ) elif metric == 'vsig': rnk = ovl.vsig_to_rank( metric_exp ) elif metric == 'useP': rnk = ovl.useP_to_rank( metric_exp ) else: raise ValueError("metric=%s not understood"%metric) if rankmap.has_key(rnk): rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )) else: rankmap[rnk] = [(line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )] for key, value in rankmap.items(): rankmap[key] = tuple(value) t, ts = idq.combine_gwf( [rnkfr], [channame]) t = t[0] truth = (start <= t)*(t <= end) t = t[truth] ts = ts[0][truth] if not len(ts): continue configs = rankmap[ts[0]] segStart = t[0] for T, TS in zip(t, ts): if rankmap[TS] != configs: vconfigs.append( (configs, [segStart, T] ) ) segStart = T configs = rankmap[TS] else: pass vconfigs.append( (configs, [segStart, T+t[1]-t[0]] ) ) configs = {} for vconfig, seg in vconfigs: if configs.has_key( vconfig ): configs[vconfig].append( seg ) else: configs[vconfig] = [ seg ] for key, value in configs.items(): value = event.andsegments( [event.fixsegments( value ), [[start,end]] ] ) if event.livetime( value ): configs[key] = event.fixsegments( value ) else: raise ValueError("somehow picked up a config with zero livetime...") return vconfigs, configs, {"vchan":0, "vthr":1, "vwin":2, "metric":3, "metric_exp":4, "rank":5}
message = "iDQ local ROC curves for %s at %s within [%.3f, %.3f]" % ( opts.classifier, ifo, opts.start, opts.end) if opts.verbose: print " " + message gracedb.writeLog(opts.gracedb_id, message=message, filename=jsonfilename) #================================================= ### determine vital statistics about training and calibration ### extract trained, calib ranges and associate them with segments trainD = defaultdict(list) calibD = defaultdict(list) for fap in faps: seg = idq.extract_start_stop(fap, suffix=".gwf") trained, calib = idq.extract_timeseries_ranges(fap) trainD[tuple(trained)].append(seg) calibD[tuple(calib)].append(seg) if opts.ignore_science_segments: dq_name = config.get('get_science_segments', 'include') ### fix segments, extract info jsonD = {} for (calib_start, calib_end), segs in calibD.items(): segs = event.fixsegments(segs) thisD = {'used': segs} ### extract livetime sciseg_files = glob.glob("%s/*_%d/science_segments*%s-*-*.xml.gz" % (calibrationdir, calib_end, dq_name))
if not opts.skip_gracedb_upload: message = "iDQ local ROC curves for %s at %s within [%.3f, %.3f]"%(opts.classifier, ifo, opts.start, opts.end) if opts.verbose: print " "+message gracedb.writeLog( opts.gracedb_id, message=message, filename=jsonfilename ) #================================================= ### determine vital statistics about training and calibration ### extract trained, calib ranges and associate them with segments trainD = defaultdict( list ) calibD = defaultdict( list ) for fap in faps: seg = idq.extract_start_stop( fap, suffix=".gwf" ) trained, calib = idq.extract_timeseries_ranges( fap ) trainD[tuple(trained)].append( seg ) calibD[tuple(calib)].append( seg ) if opts.ignore_science_segments: dq_name = config.get('get_science_segments', 'include') ### fix segments, extract info jsonD = {} for (calib_start, calib_end), segs in calibD.items(): segs = event.fixsegments( segs ) thisD = {'used':segs} ### extract livetime sciseg_files = glob.glob("%s/*_%d/science_segments*%s-*-*.xml.gz"%(calibrationdir, calib_end, dq_name)) if len(sciseg_files) > 1: