def downtime(vetosegs, segments, channel): myflag = DataQualityFlag() myflag.active = vetosegs myflag.known = segments plot = myflag.plot() plot.set_title(r'Active and vetoed segments for %s' %channel) plot.savefig(r'%s_downtime.png'%channel)
def evaluate_flag(flag, triggers=None, metrics=['deadtime'], injections=None, minduration=0, vetotag='', channel=None, etg=None): """Evaluate the performance of a set of a `~gwpy.segments.DataQualityFlag` Parameters ---------- flag : `~gwpy.segments.DataQualityFlag` the data-quality flag to be tested triggers : `~glue.ligolw.table.Table`, optional the set of analysis event triggers against which to test metrics : `list`, optional the list of `Metrics <~gwvet.Metric>` injections : `~glue.ligolw.table.Table`, `~gwpy.segments.SegmentList`, optional a list of injections, or injection segments, against which to test flag safety minduration : `float`, optional the minimum duration of post-veto segments, if applicable, default: 0 Returns ------- results, after : `OrderedDict`, `~glue.ligolw.table.Table` the results of each metric test, and the triggers after vetoes have been applied (or `None` if not given) """ # format as flag if not isinstance(flag, DataQualityFlag): flag = DataQualityFlag(active=flag) else: flag = flag.copy() # get inverse of veto segments if minduration: post = type(flag.known)([s for s in (flag.known - flag.active) if float(abs(s)) >= minduration]) flag.active = flag.known - post # apply vetoes to triggers triggers.etg = etg if triggers is not None: after = veto(triggers, flag, tag=vetotag, channel=channel, etg=etg) else: after = None # test each metric out = OrderedDict() for metric in metrics: if isinstance(metric, Metric): _metric = metric else: _metric = get_metric(metric) if _metric.name.lower() == 'safety': out[metric] = _metric(flag, injections) elif _metric.needs_triggers: out[metric] = _metric(flag, triggers, after=after) else: out[metric] = _metric(flag) return out, after
def summary_stats(statistics, bbh_trigs, omicron_trigs, channel, vetosegs, segments): # Metrics being considered eff = get_metric('efficiency') dt = get_metric('deadtime') eff_over_dt = get_metric('efficiency/deadtime') usep = get_metric('my use percentage') loudbysnr = get_metric('loudest event by snr') get_percentile= get_metric('percentile') myflag = DataQualityFlag() myflag.active = vetosegs myflag.known = segments statistics[i] = (channel, eff(myflag, bbh_trigs).value, dt(myflag).value,\ eff_over_dt(myflag, bbh_trigs).value, usep(myflag, omic_trigs).value,\ loudbysnr(myflag, bbh_trigs).value)
idxhighscat = argwhere(scatf2>=thresh) highscatf2 = scatf2[idxhighscat] highscattimes = times[idxhighscat] highscattimesgps = highscattimes+start_time # save text file with values above threshold [GPS f2 index] outdata = hstack((highscattimesgps,highscatf2,idxhighscat)) savetxt('%s-ALL-TIMES-SCATTER-GT%dHZ-%d-%d.txt' % (ifo,thresh,start_time,dur),outdata,fmt='%f %f %i') # save segments XML file with segments (based off code from Duncan Macleod) from math import (floor, ceil) from gwpy.segments import (Segment, DataQualityFlag) flag = '%s:DCH-SCATTERED_LIGHT_GT%dHZ:1' % (ifo,thresh) flag = DataQualityFlag(flag) segs = [] append = segs.append for gps in highscattimesgps: if len(segs) and gps in segs[-1]: continue seg = Segment(floor(gps), ceil(gps)) append(seg) flag.active = segs flag.known = [Segment(start_time, end_time)] flag.coalesce() flag.write('%s-%s_%d-%d-%d.xml.gz' % (flag.ifo, flag.tag.replace('-', '_'), flag.version,start_time, dur)) #EOF
def main(args=None): """Run the online Guardian node visualization tool """ parser = create_parser() args = parser.parse_args(args=args) # parse command line options ifo = args.ifo if not args.ifo: parser.error('--ifo must be given if not obvious from the host') start = getattr(args, 'gpsstart') end = getattr(args, 'gpsend') duration = int(ceil(end) - floor(start)) categories = args.categories.split(',') for i, c in enumerate(categories): try: categories[i] = int(c) except (TypeError, ValueError): pass vetofile = getattr(args, 'veto-definer-file') vetofile = (urlparse(vetofile).netloc or os.path.abspath(vetofile)) args.metric = args.metric or DEFAULT_METRICS # -- setup -------------------------------------- tag = '%d-%d' % (start.seconds, end.seconds) outdir = os.path.abspath(os.path.join(args.output_directory, tag)) mkdir(outdir) os.chdir(outdir) mkdir('etc', 'segments', 'condor') # -- segment handling --------------------------- os.chdir('segments') ALLSEGMENTS = DataQualityDict() # -- get analysis segments ---------------------- aflags = args.analysis_segments asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo) for i, flag in enumerate(aflags): # use union of segments from a file if os.path.isfile(flag): asegments += DataQualityFlag.read(flag) # or intersection of segments from multiple flags else: new = DataQualityFlag.query(flag, start, end, url=args.segdb) if i: asegments.known &= new.known asegments.active &= new.active else: asegments.known = new.known asegments.active = new.active ALLSEGMENTS[asegments.name] = asegments if os.path.isfile(aflags[0]): asegments.filename = aflags # -- read veto definer and process -------------- if urlparse(vetofile).netloc: tmp = urlopen(vetofile) vetofile = os.path.abspath(os.path.basename(vetofile)) with open(vetofile, 'w') as f: f.write(tmp.read()) LOGGER.info('Downloaded veto definer file') vdf = DataQualityDict.from_veto_definer_file(vetofile, format='ligolw', start=start, end=end, ifo=ifo) LOGGER.info('Read %d flags from veto definer' % len(vdf.keys())) # populate veto definer file from database vdf.populate(source=args.segdb, on_error=args.on_segdb_error) ALLSEGMENTS += vdf # organise flags into categories flags = dict((c, DataQualityDict()) for c in categories) for name, flag in vdf.items(): try: flags[flag.category][name] = flag except KeyError: pass # find the states and segments for each category states, after, oldtitle = (dict(), None, '') for i, category in enumerate(categories): title = isinstance(category, int) and 'Cat %d' % category or category tag = re_cchar.sub('_', str(title).upper()) states[category] = SummaryState( 'After %s' % oldtitle, key=tag, known=after.known, active=after.active, definition=after.name, ) if i else SummaryState( args.analysis_name, key=args.analysis_name, definition=asegments.name, ) try: segs = flags[category].union() except TypeError: # no flags segs = DataQualityFlag() segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag) ALLSEGMENTS[segs.name] = segs after = (after - segs) if i else (asegments - segs) after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag) ALLSEGMENTS[after.name] = after oldtitle = title # write all segments to disk segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' % (ifo, start.seconds, duration)) ALLSEGMENTS.write(segfile) os.chdir(os.pardir) if args.verbose: LOGGER.debug("All segments accessed and written to\n%s" % segfile) # -- job preparation ---------------------------- os.chdir('etc') configs = [] for category in categories: title = (isinstance(category, int) and 'Category %d' % category or category) tab = 'tab-%s' % title config = ConfigParser() # add segment-database configuration add_config_section(config, 'segment-database', url=args.segdb) # add plot configurations pconfig = ConfigParser() pconfig.read(args.config_file) for section in pconfig.sections(): if section.startswith('plot-'): config._sections[section] = pconfig._sections[section].copy() try: plots = pconfig.items('plots-%s' % category, raw=True) except NoSectionError: try: plots = pconfig.items('plots', raw=True) except NoSectionError: plots = [] # add state if args.independent: state = states[categories[0]] else: state = states[category] sname = 'state-%s' % state.key add_config_section(config, sname, key=state.key, name=state.name, definition=state.definition, filename=segfile) # add plugin add_config_section(config, 'plugins', **{'gwvet.tabs': ''}) # define metrics if category == 1: metrics = ['Deadtime'] else: metrics = args.metric # define summary tab if category == 1: tab = configure_veto_tab(config, title, title, state, flags[category].keys(), segfile, metrics, name='Summary', **{'veto-name': title}) else: tab = configure_veto_tab(config, title, title, state, flags[category].keys(), segfile, metrics, name='Summary', **{ 'veto-name': title, 'event-channel': args.event_channel, 'event-generator': args.event_generator, }) if len(categories) == 1: config.set(tab, 'index', '%(gps-start-time)s-%(gps-end-time)s/index.html') for key, value in plots: if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605 config.set(tab, key, '%%(union)s,%s' % value) if '%s-labels' % key not in plots: config.set(tab, '%s-labels' % key, 'Union,%(flags)s') else: config.set(tab, key, value) # now a tab for each flag for flag in flags[category]: if category == 1: tab = configure_veto_tab(config, flag, title, state, [flag], segfile, metrics) else: tab = configure_veto_tab( config, flag, title, state, [flag], segfile, metrics, **{ 'event-channel': args.event_channel, 'event-generator': args.event_generator }) if args.event_file: config.set(tab, 'event-file', args.event_file) for key, value in plots: config.set(tab, key, value) if len(categories) > 1 and category != categories[-1]: with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: config.write(f) configs.append(os.path.abspath(f.name)) # configure summary job if len(categories) > 1: state = states[categories[0]] add_config_section(config, 'state-%s' % state.key, key=state.key, name=state.name, definition=state.definition, filename=segfile) try: plots = pconfig.items('plots', raw=True) except NoSectionError: plots = [] flags = [f for c in categories for f in flags[c].keys()] tab = configure_veto_tab( config, 'Impact of full veto definer file', None, state, flags, segfile, args.metric, shortname='Summary', index='%(gps-start-time)s-%(gps-end-time)s/index.html', **{ 'event-channel': args.event_channel, 'event-generator': args.event_generator, 'veto-name': 'All vetoes' }) if args.event_file: config.set(tab, 'event-file', args.event_file) for key, value in plots: config.set(tab, key, value) with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: config.write(f) configs.append(os.path.abspath(f.name)) os.chdir(os.pardir) if args.verbose: LOGGER.debug("Generated configuration files for each category") # -- condor preparation ------------------------- os.chdir(os.pardir) # get condor variables if getuser() == 'detchar': accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy' else: accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy' gwsumm_args = [ '--gps-start-time', str(start.seconds), '--gps-end-time', str(end.seconds), '--ifo', ifo, '--file-tag', 'gwpy-vet', '--condor-command', 'accounting_group=%s' % accgroup, '--condor-command', 'accounting_group_user=%s' % getuser(), '--on-segdb-error', args.on_segdb_error, '--output-dir', args.output_directory, ] for cf in args.global_config: gwsumm_args.extend(('--global-config', cf)) for cf in configs: gwsumm_args.extend(('--config-file', cf)) if args.verbose: gwsumm_args.append('--verbose') if args.verbose: LOGGER.debug('Generating summary DAG via:\n') LOGGER.debug(' '.join([batch.PROG] + gwsumm_args)) # execute gwsumm in batch mode batch.main(args=gwsumm_args)