def check_calibration(segs, times, timeseries, FAPthrs): """ checks the pipeline's calibration at each "FAPthr in FAPThrs" this may be sped up with a call to timeseries_to_livetime() instead of idq.timeseries_to_segments() -> event.livetime however, we currently use some segment logic and it is not clear that we can avoid actually generating segments """ idq_livetime = event.livetime(segs) segments = [] deadtimes = [] statedFAPs = [] errs = [] for FAPthr in FAPthrs: SEGS = [] max_statedFAP = 0.0 for (t, ts) in zip(times, timeseries): (_segs, _min_ts) = idq.timeseries_to_segments( t, -ts, -FAPthr) # we want FAP <= FAPthr <--> -FAP >= -FAPthr SEGS += ( _segs ) ### ensure this is a list in case the behavior of timeseries_to_segments changes... if _min_ts != None: statedFAP = -_min_ts if max_statedFAP < statedFAP: max_statedFAP = statedFAP SEGS = event.andsegments([SEGS, segs]) segments.append(SEGS) SEGS_livetime = event.livetime(SEGS) if not idq_livetime: if SEGS_livetime: raise ValueError( "something is weird with segments... idq_livetime is zero but SEGS_livetime is not" ) else: deadtime = 0.0 else: deadtime = 1.0 * SEGS_livetime / idq_livetime if deadtime > 1.0: raise ValueError( "deadtime > 1.0, something is weird...\n SEGS_livetime = %f\n idq_livetime = %f" % (SEGS_livetime, idq_livetime)) deadtimes.append(deadtime) statedFAPs.append(max_statedFAP) if max_statedFAP > 0: err = deadtime / max_statedFAP - 1 elif deadtime: err = 1 else: err = 0 errs.append(err) return segments, deadtimes, statedFAPs, errs
def check_calibration( segs, times, timeseries, FAPthrs): """ checks the pipeline's calibration at each "FAPthr in FAPThrs" this may be sped up with a call to timeseries_to_livetime() instead of idq.timeseries_to_segments() -> event.livetime however, we currently use some segment logic and it is not clear that we can avoid actually generating segments """ idq_livetime = event.livetime(segs) segments = [] deadtimes = [] statedFAPs = [] errs = [] for FAPthr in FAPthrs: SEGS = [] max_statedFAP = 0.0 for (t, ts) in zip(times, timeseries): (_segs, _min_ts) = idq.timeseries_to_segments(t, -ts, -FAPthr) # we want FAP <= FAPthr <--> -FAP >= -FAPthr SEGS += (_segs) ### ensure this is a list in case the behavior of timeseries_to_segments changes... if _min_ts != None: statedFAP = -_min_ts if max_statedFAP < statedFAP: max_statedFAP = statedFAP SEGS = event.andsegments([SEGS, segs]) segments.append(SEGS) SEGS_livetime = event.livetime(SEGS) if not idq_livetime: if SEGS_livetime: raise ValueError("something is weird with segments... idq_livetime is zero but SEGS_livetime is not") else: deadtime = 0.0 else: deadtime = 1.0 * SEGS_livetime / idq_livetime if deadtime > 1.0: raise ValueError("deadtime > 1.0, something is weird...\n SEGS_livetime = %f\n idq_livetime = %f"%(SEGS_livetime, idq_livetime)) deadtimes.append( deadtime ) statedFAPs.append(max_statedFAP) if max_statedFAP > 0: err = deadtime/max_statedFAP - 1 elif deadtime: err = 1 else: err = 0 errs.append( err ) return segments, deadtimes, statedFAPs, errs
if opts.force: ### we are require successful training or else we want errors logger.info(traceback.print_exc()) raise e else: ### we don't care if any particular training job fails gpsstart += stride continue logger.info('finding idq segments') idqsegs = idq.get_idq_segments(realtimedir, gpsstart - lookback, gpsstart + stride, suffix='.dat') logger.info( 'taking intersection between science segments and idq segments') idqsegs = event.andsegments([scisegs, idqsegs]) ### write segment file if opts.ignore_science_segments: idqseg_path = idq.idqsegascii(output_dir, '', gpsstart - lookback, lookback + stride) else: idqseg_path = idq.idqsegascii(output_dir, '_%s' % dq_name, gpsstart - lookback, lookback + stride) f = open(idqseg_path, 'w') for seg in idqsegs: print >> f, seg[0], seg[1] f.close() #=============================================================================================== # update mappings via uroc files
### science segments xml filename seg_file = idq.segxml(gdbdir, "%s_%s"%(filetag, dq_name), opts.start , opts.end-opts.start ) if opts.verbose: print ' writing science segments to file : '+seg_file ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz")) (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file if opts.verbose: print 'finding idq segments' idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat') if opts.verbose: print 'taking intersection between science segments and idq segments' idqsegs = event.andsegments( [scisegs, idqsegs] ) ### write segment file if opts.ignore_science_segments: idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end-opts.start) else: idqseg_path = idq.idqsegascii(gdbdir, '%s_%s'%(filetag, dq_name), opts.start, opts.end-opts.start) if opts.verbose: print " writing : "+idqseg_path f = open(idqseg_path, 'w') for seg in idqsegs: print >> f, seg[0], seg[1] f.close() #=================================================
def extract_ovl_vconfigs( rank_frames, channame, traindir, start, end, metric='eff/dt' ): """ returns a dictionary mapping active vconfigs to segments does NOT include "none" channel """ vconfigs = [] for rnkfr in rank_frames: trained, calib = idq.extract_timeseries_ranges( rnkfr ) classifier = idq.extract_fap_name( rnkfr ) vetolist = glob.glob( "%s/%d_%d/ovl/ovl/*vetolist.eval"%(traindir, trained[0], trained[1]) ) if len(vetolist) != 1: raise ValueError( "trouble finding a single vetolist file for : %s"%rnkfr ) vetolist=vetolist[0] v = event.loadstringtable( vetolist ) rankmap = { 0:[(None, None, None, None, 0, 0)] } for line in v: metric_exp = float(line[ovl.vD['metric_exp']]) if metric == 'eff/dt': rnk = ovl.effbydt_to_rank( metric_exp ) elif metric == 'vsig': rnk = ovl.vsig_to_rank( metric_exp ) elif metric == 'useP': rnk = ovl.useP_to_rank( metric_exp ) else: raise ValueError("metric=%s not understood"%metric) if rankmap.has_key(rnk): rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )) else: rankmap[rnk] = [(line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )] for key, value in rankmap.items(): rankmap[key] = tuple(value) t, ts = idq.combine_gwf( [rnkfr], [channame]) t = t[0] truth = (start <= t)*(t <= end) t = t[truth] ts = ts[0][truth] if not len(ts): continue configs = rankmap[ts[0]] segStart = t[0] for T, TS in zip(t, ts): if rankmap[TS] != configs: vconfigs.append( (configs, [segStart, T] ) ) segStart = T configs = rankmap[TS] else: pass vconfigs.append( (configs, [segStart, T+t[1]-t[0]] ) ) configs = {} for vconfig, seg in vconfigs: if configs.has_key( vconfig ): configs[vconfig].append( seg ) else: configs[vconfig] = [ seg ] for key, value in configs.items(): value = event.andsegments( [event.fixsegments( value ), [[start,end]] ] ) if event.livetime( value ): configs[key] = event.fixsegments( value ) else: raise ValueError("somehow picked up a config with zero livetime...") return vconfigs, configs, {"vchan":0, "vthr":1, "vwin":2, "metric":3, "metric_exp":4, "rank":5}
### this requires us to redefine the 'science-segments' as the intersection of scisegs with realtime segs ### we call this intersection "idq_segs" else: ### we're re-using pat files! try: ### determine segments from realtime filenames realtime_segs = idq.get_idq_segments(realtimedir, gpsstart - lookback, gpsstart + stride, suffix='.pat') ### read in science segments (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### take the intersection of these segments idq_segs = event.andsegments([scisegs, realtime_segs]) ### write segment file idqseg_path = idq.idqsegascii(output_dir, '_%s' % dq_name, gpsstart - lookback, lookback + stride) f = open(idqseg_path, 'w') for seg in idq_segs: print >> f, seg[0], seg[1] f.close() ### we may want to remove the unsafe channels, but this could be tricky and we don't want to throw away GW channels accidentally ovlsegs = idqseg_path except Exception as e: traceback.print_exc() logger.info(
except Exception as e: traceback.print_exc() logger.info('ERROR: segment generation failed. Skipping this calibration period.') if opts.force: ### we are require successful training or else we want errors logger.info(traceback.print_exc()) raise e else: ### we don't care if any particular training job fails gpsstart += stride continue logger.info('finding idq segments') idqsegs = idq.get_idq_segments(realtimedir, gpsstart-lookback, gpsstart+stride, suffix='.dat') logger.info('taking intersection between science segments and idq segments') idqsegs = event.andsegments( [scisegs, idqsegs] ) ### write segment file if opts.ignore_science_segments: idqseg_path = idq.idqsegascii(output_dir, '', gpsstart-lookback, lookback+stride) else: idqseg_path = idq.idqsegascii(output_dir, '_%s'%dq_name, gpsstart - lookback, lookback+stride) f = open(idqseg_path, 'w') for seg in idqsegs: print >> f, seg[0], seg[1] f.close() #=============================================================================================== # update mappings via uroc files #===============================================================================================
#------------------------------------------------- ### query segments segs = [[start, end]] for ifo in opts.observatory: segfilename = "%s/%s1_%s-%d-%d.xml"%(opts.output_dir, ifo, opts.flag.replace(":","_"), start, end-start) cmd = "ligolw_segment_query_dqsegdb -t %s -q -a %s1:%s -s %d -e %d -o %s"%(opts.segdb_url, ifo, opts.flag, start, end, segfilename) if opts.verbose: print "querying %s segments for %s\n %s"%(opts.flag, ifo, cmd) output = sp.Popen( cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE ).communicate() ### iterate over segments, computing PSDs xmldoc = ligolw_utils.load_filename(segfilename, contenthandler=lsctables.use_in(ligolw.LIGOLWContentHandler)) segs = event.andsegments( [segs, [[row.start_time, row.end_time] for row in table.get_table(xmldoc, lsctables.SegmentTable.tableName)]] ) ### take intersection of segments ### write intersection to ascii file segfilename = "%s/intersection-%d-%d.seg"%(opts.output_dir, start, end-start) if opts.verbose: print "found %d sec of joint livetime"%(event.livetime(segs)) print "writing : %s"%(segfilename) file_obj = open(segfilename, 'w') for s, e in segs: print >> file_obj, s, e file_obj.close() #------------------------------------------------- ### iterate through segments and estimte PSD for each IFO if opts.verbose:
for gps in args: print "gps : %.9f" % (gps) minwin = opts.window ### go find triggers if opts.verbose: print "\tdiscoverying KW triggers within [%.9f, %.9f]" % (gps - opts.window, gps + opts.window) ### figure out which files you want filenames = [] coverage = [] for gdsdir in kwgdsdirs: for filename in idq.get_all_files_in_range(gdsdir, gps - opts.window, gps + opts.window, pad=0, suffix=".trg"): seg = idq.extract_start_stop(filename, suffix=".trg") if not event.livetime(event.andsegments([coverage, [seg]])): coverage = event.fixsegments(coverage + [seg]) filenames.append(filename) ### figure out the extent of the coverage if len(event.include([[gps]], coverage, tcent=0)) == 0: if opts.force: if opts.verbose: print "no triggers found for gps : %.3f" % (gps) continue else: raise ValueError("no triggers found for gps : %.3f" % (gps)) for s, e in coverage: if s < gps: if gps - s < minwin: minwin = gps - s
for gps in args: print "gps : %.9f"%(gps) minwin = opts.window ### go find triggers if opts.verbose: print "\tdiscoverying KW triggers within [%.9f, %.9f]"%(gps-opts.window, gps+opts.window) ### figure out which files you want filenames = [] coverage = [] for gdsdir in kwgdsdirs: for filename in idq.get_all_files_in_range(gdsdir, gps-opts.window, gps+opts.window, pad=0, suffix=".trg"): seg = idq.extract_start_stop(filename, suffix=".trg") if not event.livetime(event.andsegments([coverage, [seg]])): coverage = event.fixsegments( coverage + [seg] ) filenames.append( filename ) ### figure out the extent of the coverage if len(event.include([[gps]], coverage, tcent=0)) == 0: if opts.force: if opts.verbose: print "no triggers found for gps : %.3f"%(gps) continue else: raise ValueError("no triggers found for gps : %.3f"%(gps)) for s, e in coverage: if s < gps: if gps-s < minwin: minwin = gps-s
if opts.dq_segments: # load dq segments if opts.verbose: print "reading segments from %s" % opts.dq_segments (dq_segments, covered_segments) = \ idq.extract_dq_segments(open(opts.dq_segments, 'r'), opts.dq_segments_name) # sort and merge segments dq_segments = event.fixsegments(dq_segments) ### filter patfiles by scisegs, keep only those events with non-zero overlap with science time patfiles = [ pat for pat in patfiles if event.livetime( event.andsegments( [dq_segments, [idq.extract_start_stop(pat, suffix=".pat")]])) ] if opts.verbose: print "%d patfiles remain after taking overlap with segments" % len( patfiles) if len(patfiles) == 0: print 'No *.pat files found in the gps range ' \ + str(gps_start_time) + ' - ' + str(gps_end_time) print 'Exiting with status 2.' sys.exit(2) # load auxmvc vector samples auxmvc_samples = auxmvc_utils.ReadMVSCTriggers(patfiles, Classified=False)
def check_calibartion( realtimedir, start, end, classifier, FAPthrs, verbose=False, ): """ ....checks the pipeline's calibration at each "FAPthr in FAPThrs" """ # ================================================= # grab idq_segments so we compute meaningful livetimes # ================================================= if verbose: print 'getting idq_segements' idq_segs = idq.get_idq_segments(realtimedir, start, end, suffix='.npy.gz') idq_livetime = event.livetime(idq_segs) # ================================================= # grab relevant data # ================================================= if verbose: print 'looking for *_fap_*.npy.gz files in', opts.realtimedir fapfilenames = [filename for filename in idq.get_all_files_in_range(realtimedir, start, end, pad=0, suffix='.npy.gz') if '_fap_' in filename and classifier in filename] if opts.verbose: print 'discovered %d files' % len(fapfilenames) print 'building time-series' (times, timeseries) = idq_gdb_utils.combine_ts(fapfilenames) # ================================================= # check calibration # ================================================= segments = [] deadtimes = [] statedFAPs = [] for FAPthr in FAPthrs: if verbose: print 'computing segements for FAPthr =', FAPthr segs = [] max_statedFAP = None for (t, ts) in zip(times, timeseries): (_segs, _min_ts) = timeseries_to_segments(t, -ts, -FAPthr) # we want FAP <= FAPthr <--> -FAP >= FAPthr segs += _segs if _min_ts != None: statedFAP = -_min_ts if max_statedFAP < statedFAP: max_statedFAP = statedFAP segs = event.andsegments([segs, idq_segs]) segments.append(segs) deadtimes.append(1.0 * event.livetime(segs) / idq_livetime) statedFAPs.append(statedFAP) return (idq_segs, segments, deadtimes, statedFAPs)
'general', 'kw')).items(): ### this is kinda ugly... if opts.verbose: print( " searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]" % (kwconf, dirname, segs[0][0], segs[-1][1])) ### iterate over all trg files found in that directory for trgfile in idq.get_all_files_in_range(dirname, segs[0][0], segs[-1][1], pad=0, suffix='.trg'): ### check whether there is some overlap ### not gauranteed if there are gaps between min and max gps times if event.livetime( event.andsegments( [[idq.extract_start_stop(trgfile, suffix='.trg')], segs])): if opts.verbose: print(" kept : " + trgfile) kw_trgfiles.append(trgfile) elif opts.verbose: print(" discarded : " + trgfile) #--- if opts.verbose: print("evaluating %d times using %d KW trgfiles" % (Ngps, len(kw_trgfiles))) ### set up output pointers if opts.output_filename: datfile = os.path.basename(opts.output_filename)
seg_file = idq.segxml(opts.output_dir, "_%s"%dq_name, startgps, stride) logger.info('writing science segments to file : '+seg_file) ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz")) (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file ### modify scisegs to account for shifts logger.info('modifying scisegs to account for shifts') modified_scisegs = [(s-opts.right_padding-opts.t_lag, e+opts.left_padding-opts.t_lag) for s, e in scisegs] ### find idq segments logger.info('finding idq segments') idqsegs = idq.get_idq_segments(realtimedir, lookup_startgps, lookup_endgps, suffix='.dat') logger.info('taking intersection between modified science segments and idq segments') idqsegs = event.andsegments( [modified_scisegs, idqsegs] ) ### write segment file if opts.ignore_science_segments: idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride) else: idqseg_path = idq.idqsegascii(opts.output_dir, '_%s'%dq_name, startgps , stride) f = open(idqseg_path, 'w') for seg in idqsegs: print >> f, seg[0], seg[1] f.close() #======================== # go findeth the frame data #======================== logger.info(' finding all *fap*.gwf files')
gpsstart += stride continue ### if we aren't building auxmvc vectors, we re-use pat files from realtime job ### this requires us to redefine the 'science-segments' as the intersection of scisegs with realtime segs ### we call this intersection "idq_segs" else: ### we're re-using pat files! try: ### determine segments from realtime filenames realtime_segs = idq.get_idq_segments(realtimedir, gpsstart - lookback, gpsstart + stride, suffix='.pat') ### read in science segments (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### take the intersection of these segments idq_segs = event.andsegments([scisegs, realtime_segs]) ### write segment file idqseg_path = idq.idqsegascii(output_dir, '_%s'%dq_name, gpsstart - lookback, lookback+stride) f = open(idqseg_path, 'w') for seg in idq_segs: print >> f, seg[0], seg[1] f.close() ### we may want to remove the unsafe channels, but this could be tricky and we don't want to throw away GW channels accidentally ovlsegs = idqseg_path except Exception as e: traceback.print_exc() logger.info('WARNING: failed to generate iDQ segments from realtime output.') if opts.force:
print ' writing science segments to file : ' + seg_file ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz")) (scisegs, coveredseg) = idq.extract_dq_segments( seg_file, dq_name) ### read in segments from xml file if opts.verbose: print 'finding idq segments' idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat') if opts.verbose: print 'taking intersection between science segments and idq segments' idqsegs = event.andsegments([scisegs, idqsegs]) ### write segment file if opts.ignore_science_segments: idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end - opts.start) else: idqseg_path = idq.idqsegascii(gdbdir, '%s_%s' % (filetag, dq_name), opts.start, opts.end - opts.start) if opts.verbose: print " writing : " + idqseg_path f = open(idqseg_path, 'w') for seg in idqsegs: print >> f, seg[0], seg[1] f.close()
#--- ### look up KW trg files that intersect segs if opts.verbose: print( "finding relevant kw_trgfiles" ) kw_trgfiles = [] ### iterate over different configurations used in training for kwconf, dirname in eval(config.get('general', 'kw')).items(): ### this is kinda ugly... if opts.verbose: print( " searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]"%(kwconf, dirname, segs[0][0], segs[-1][1]) ) ### iterate over all trg files found in that directory for trgfile in idq.get_all_files_in_range(dirname, segs[0][0], segs[-1][1], pad=0, suffix='.trg'): ### check whether there is some overlap ### not gauranteed if there are gaps between min and max gps times if event.livetime(event.andsegments([[idq.extract_start_stop(trgfile, suffix='.trg')], segs])): if opts.verbose: print( " kept : "+trgfile ) kw_trgfiles.append( trgfile ) elif opts.verbose: print( " discarded : "+trgfile ) #--- if opts.verbose: print( "evaluating %d times using %d KW trgfiles"%(Ngps, len(kw_trgfiles) ) ) ### set up output pointers if opts.output_filename: datfile = os.path.basename(opts.output_filename) output_dir = os.path.dirname(opts.output_filename)
def extract_ovl_vconfigs(rank_frames, channame, traindir, start, end, metric='eff/dt'): """ returns a dictionary mapping active vconfigs to segments does NOT include "none" channel """ vconfigs = [] for rnkfr in rank_frames: trained, calib = idq.extract_timeseries_ranges(rnkfr) classifier = idq.extract_fap_name(rnkfr) vetolist = glob.glob("%s/%d_%d/ovl/ovl/*vetolist.eval" % (traindir, trained[0], trained[1])) if len(vetolist) != 1: raise ValueError( "trouble finding a single vetolist file for : %s" % rnkfr) vetolist = vetolist[0] v = event.loadstringtable(vetolist) rankmap = {0: [(None, None, None, None, 0, 0)]} for line in v: metric_exp = float(line[ovl.vD['metric_exp']]) if metric == 'eff/dt': rnk = ovl.effbydt_to_rank(metric_exp) elif metric == 'vsig': rnk = ovl.vsig_to_rank(metric_exp) elif metric == 'useP': rnk = ovl.useP_to_rank(metric_exp) else: raise ValueError("metric=%s not understood" % metric) if rankmap.has_key(rnk): rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk)) else: rankmap[rnk] = [ (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk) ] for key, value in rankmap.items(): rankmap[key] = tuple(value) t, ts = idq.combine_gwf([rnkfr], [channame]) t = t[0] truth = (start <= t) * (t <= end) t = t[truth] ts = ts[0][truth] if not len(ts): continue configs = rankmap[ts[0]] segStart = t[0] for T, TS in zip(t, ts): if rankmap[TS] != configs: vconfigs.append((configs, [segStart, T])) segStart = T configs = rankmap[TS] else: pass vconfigs.append((configs, [segStart, T + t[1] - t[0]])) configs = {} for vconfig, seg in vconfigs: if configs.has_key(vconfig): configs[vconfig].append(seg) else: configs[vconfig] = [seg] for key, value in configs.items(): value = event.andsegments([event.fixsegments(value), [[start, end]]]) if event.livetime(value): configs[key] = event.fixsegments(value) else: raise ValueError( "somehow picked up a config with zero livetime...") return vconfigs, configs, { "vchan": 0, "vthr": 1, "vwin": 2, "metric": 3, "metric_exp": 4, "rank": 5 }
### modify scisegs to account for shifts logger.info('modifying scisegs to account for shifts') modified_scisegs = [(s - opts.right_padding - opts.t_lag, e + opts.left_padding - opts.t_lag) for s, e in scisegs] ### find idq segments logger.info('finding idq segments') idqsegs = idq.get_idq_segments(realtimedir, lookup_startgps, lookup_endgps, suffix='.dat') logger.info( 'taking intersection between modified science segments and idq segments') idqsegs = event.andsegments([modified_scisegs, idqsegs]) ### write segment file if opts.ignore_science_segments: idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride) else: idqseg_path = idq.idqsegascii(opts.output_dir, '_%s' % dq_name, startgps, stride) f = open(idqseg_path, 'w') for seg in idqsegs: print >> f, seg[0], seg[1] f.close() #======================== # go findeth the frame data #========================