def test_populate(self): # read veto definer start, end = VETO_DEFINER_TEST_SEGMENTS[0] vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE, ifo='H1', start=start, end=end) # test query that should fail with 404 try: vdf.populate(url='https://segments.ligo.org') except (ImportError, UnboundLocalError) as e: self.skipTest(str(e)) except URLError as e: if e.code == 401: # 401 is uninformative self.skipTest(str(e)) elif e.code == 404: pass else: raise else: raise AssertionError("URLError not raised") # check reduction to warning vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE, ifo='H1', start=start, end=end) with pytest.warns(UserWarning): vdf.populate(url='https://segments.ligo.org', on_error='warn') # check results self.assertEqual(len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36) # check use of specific segments vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE, ifo='H1') vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
def test_populate(self): # read veto definer start, end = VETO_DEFINER_TEST_SEGMENTS[0] vdf = DataQualityDict.from_veto_definer_file( VETO_DEFINER_FILE, ifo='H1', start=start, end=end) # test query that should fail with 404 try: vdf.populate(url='https://segments.ligo.org') except (ImportError, UnboundLocalError) as e: self.skipTest(str(e)) except URLError as e: if e.code == 401: # 401 is uninformative self.skipTest(str(e)) elif e.code == 404: pass else: raise else: raise AssertionError("URLError not raised") # check reduction to warning vdf = DataQualityDict.from_veto_definer_file( VETO_DEFINER_FILE, ifo='H1', start=start, end=end) with pytest.warns(UserWarning): vdf.populate(url='https://segments.ligo.org', on_error='warn') # check results self.assertEqual( len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36) # check use of specific segments vdf = DataQualityDict.from_veto_definer_file( VETO_DEFINER_FILE, ifo='H1') vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
def test_from_veto_definer_file(self): vdf = DataQualityDict.from_veto_definer_file(self.VETO_DEFINER) self.assertNotEqual(len(vdf.keys()), 0) # test missing h(t) flag self.assertIn('H1:DCH-MISSING_H1_HOFT_C00:1', vdf) self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][0], 1073779216) self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][-1], PosInfinity) self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].category, 1) # test injections padding self.assertEquals(vdf['H1:ODC-INJECTION_CBC:1'].padding, Segment(-8, 8)) # test download URL vdf2 = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE) self.assertEqual(len(vdf.keys()), len(vdf2.keys()))
def test_populate(self): start, end = VETO_DEFINER_TEST_SEGMENTS[0] vdf = DataQualityDict.from_veto_definer_file( VETO_DEFINER_FILE, ifo='H1', start=start, end=end) vdf.pop("H1:FAKE_CAT5:1", None) try: vdf.populate(url='https://segments.ligo.org') except ImportError as e: self.skipTest(str(e)) else: self.assertEqual( len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36) vdf = DataQualityDict.from_veto_definer_file( VETO_DEFINER_FILE, ifo='H1') vdf.pop("H1:FAKE_CAT5:1", None) vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS)
def read_veto_definer_file(vetofile, start=None, end=None, ifo=None): """Read a veto definer file, downloading it if necessary """ if urlparse(vetofile).netloc: tmp = urlopen(vetofile) vetofile = os.path.abspath(os.path.basename(vetofile)) with open(vetofile, 'w') as f: f.write(str(tmp.read())) return DataQualityDict.from_veto_definer_file( vetofile, format='ligolw', start=start, end=end, ifo=ifo)
def query_veto_definer_file(ifo, start, end, cp): # Obtain segments that are analysis ready analysis_ready = DataQualityFlag.query( '{0}:DMT-ANALYSIS_READY:1'.format(ifo), start, end) # Define the start and stop time of analysis ready analysis_ready_start = analysis_ready.active.extent()[0] analysis_ready_end = analysis_ready.active.extent()[1] # Query for vetos found during this time. vdf = DataQualityDict.from_veto_definer_file( cp.get('segfind', 'veto-file'), analysis_ready_start, analysis_ready_end) # Populate only for analysis ready segments. vdf.populate(segments=analysis_ready.active) return vdf
def main(args=None): """Run the online Guardian node visualization tool """ parser = create_parser() args = parser.parse_args(args=args) # parse command line options ifo = args.ifo if not args.ifo: parser.error('--ifo must be given if not obvious from the host') start = getattr(args, 'gpsstart') end = getattr(args, 'gpsend') duration = int(ceil(end) - floor(start)) categories = args.categories.split(',') for i, c in enumerate(categories): try: categories[i] = int(c) except (TypeError, ValueError): pass vetofile = getattr(args, 'veto-definer-file') vetofile = (urlparse(vetofile).netloc or os.path.abspath(vetofile)) args.metric = args.metric or DEFAULT_METRICS # -- setup -------------------------------------- tag = '%d-%d' % (start.seconds, end.seconds) outdir = os.path.abspath(os.path.join(args.output_directory, tag)) mkdir(outdir) os.chdir(outdir) mkdir('etc', 'segments', 'condor') # -- segment handling --------------------------- os.chdir('segments') ALLSEGMENTS = DataQualityDict() # -- get analysis segments ---------------------- aflags = args.analysis_segments asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo) for i, flag in enumerate(aflags): # use union of segments from a file if os.path.isfile(flag): asegments += DataQualityFlag.read(flag) # or intersection of segments from multiple flags else: new = DataQualityFlag.query(flag, start, end, url=args.segdb) if i: asegments.known &= new.known asegments.active &= new.active else: asegments.known = new.known asegments.active = new.active ALLSEGMENTS[asegments.name] = asegments if os.path.isfile(aflags[0]): asegments.filename = aflags # -- read veto definer and process -------------- if urlparse(vetofile).netloc: tmp = urlopen(vetofile) vetofile = os.path.abspath(os.path.basename(vetofile)) with open(vetofile, 'w') as f: f.write(tmp.read()) LOGGER.info('Downloaded veto definer file') vdf = DataQualityDict.from_veto_definer_file(vetofile, format='ligolw', start=start, end=end, ifo=ifo) LOGGER.info('Read %d flags from veto definer' % len(vdf.keys())) # populate veto definer file from database vdf.populate(source=args.segdb, on_error=args.on_segdb_error) ALLSEGMENTS += vdf # organise flags into categories flags = dict((c, DataQualityDict()) for c in categories) for name, flag in vdf.items(): try: flags[flag.category][name] = flag except KeyError: pass # find the states and segments for each category states, after, oldtitle = (dict(), None, '') for i, category in enumerate(categories): title = isinstance(category, int) and 'Cat %d' % category or category tag = re_cchar.sub('_', str(title).upper()) states[category] = SummaryState( 'After %s' % oldtitle, key=tag, known=after.known, active=after.active, definition=after.name, ) if i else SummaryState( args.analysis_name, key=args.analysis_name, definition=asegments.name, ) try: segs = flags[category].union() except TypeError: # no flags segs = DataQualityFlag() segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag) ALLSEGMENTS[segs.name] = segs after = (after - segs) if i else (asegments - segs) after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag) ALLSEGMENTS[after.name] = after oldtitle = title # write all segments to disk segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' % (ifo, start.seconds, duration)) ALLSEGMENTS.write(segfile) os.chdir(os.pardir) if args.verbose: LOGGER.debug("All segments accessed and written to\n%s" % segfile) # -- job preparation ---------------------------- os.chdir('etc') configs = [] for category in categories: title = (isinstance(category, int) and 'Category %d' % category or category) tab = 'tab-%s' % title config = ConfigParser() # add segment-database configuration add_config_section(config, 'segment-database', url=args.segdb) # add plot configurations pconfig = ConfigParser() pconfig.read(args.config_file) for section in pconfig.sections(): if section.startswith('plot-'): config._sections[section] = pconfig._sections[section].copy() try: plots = pconfig.items('plots-%s' % category, raw=True) except NoSectionError: try: plots = pconfig.items('plots', raw=True) except NoSectionError: plots = [] # add state if args.independent: state = states[categories[0]] else: state = states[category] sname = 'state-%s' % state.key add_config_section(config, sname, key=state.key, name=state.name, definition=state.definition, filename=segfile) # add plugin add_config_section(config, 'plugins', **{'gwvet.tabs': ''}) # define metrics if category == 1: metrics = ['Deadtime'] else: metrics = args.metric # define summary tab if category == 1: tab = configure_veto_tab(config, title, title, state, flags[category].keys(), segfile, metrics, name='Summary', **{'veto-name': title}) else: tab = configure_veto_tab(config, title, title, state, flags[category].keys(), segfile, metrics, name='Summary', **{ 'veto-name': title, 'event-channel': args.event_channel, 'event-generator': args.event_generator, }) if len(categories) == 1: config.set(tab, 'index', '%(gps-start-time)s-%(gps-end-time)s/index.html') for key, value in plots: if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605 config.set(tab, key, '%%(union)s,%s' % value) if '%s-labels' % key not in plots: config.set(tab, '%s-labels' % key, 'Union,%(flags)s') else: config.set(tab, key, value) # now a tab for each flag for flag in flags[category]: if category == 1: tab = configure_veto_tab(config, flag, title, state, [flag], segfile, metrics) else: tab = configure_veto_tab( config, flag, title, state, [flag], segfile, metrics, **{ 'event-channel': args.event_channel, 'event-generator': args.event_generator }) if args.event_file: config.set(tab, 'event-file', args.event_file) for key, value in plots: config.set(tab, key, value) if len(categories) > 1 and category != categories[-1]: with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: config.write(f) configs.append(os.path.abspath(f.name)) # configure summary job if len(categories) > 1: state = states[categories[0]] add_config_section(config, 'state-%s' % state.key, key=state.key, name=state.name, definition=state.definition, filename=segfile) try: plots = pconfig.items('plots', raw=True) except NoSectionError: plots = [] flags = [f for c in categories for f in flags[c].keys()] tab = configure_veto_tab( config, 'Impact of full veto definer file', None, state, flags, segfile, args.metric, shortname='Summary', index='%(gps-start-time)s-%(gps-end-time)s/index.html', **{ 'event-channel': args.event_channel, 'event-generator': args.event_generator, 'veto-name': 'All vetoes' }) if args.event_file: config.set(tab, 'event-file', args.event_file) for key, value in plots: config.set(tab, key, value) with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: config.write(f) configs.append(os.path.abspath(f.name)) os.chdir(os.pardir) if args.verbose: LOGGER.debug("Generated configuration files for each category") # -- condor preparation ------------------------- os.chdir(os.pardir) # get condor variables if getuser() == 'detchar': accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy' else: accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy' gwsumm_args = [ '--gps-start-time', str(start.seconds), '--gps-end-time', str(end.seconds), '--ifo', ifo, '--file-tag', 'gwpy-vet', '--condor-command', 'accounting_group=%s' % accgroup, '--condor-command', 'accounting_group_user=%s' % getuser(), '--on-segdb-error', args.on_segdb_error, '--output-dir', args.output_directory, ] for cf in args.global_config: gwsumm_args.extend(('--global-config', cf)) for cf in configs: gwsumm_args.extend(('--config-file', cf)) if args.verbose: gwsumm_args.append('--verbose') if args.verbose: LOGGER.debug('Generating summary DAG via:\n') LOGGER.debug(' '.join([batch.PROG] + gwsumm_args)) # execute gwsumm in batch mode batch.main(args=gwsumm_args)
new = DataQualityFlag.query(flag, start, end) #try get_segments gwvet.segments ALLSEGMENTS[new.name] = new else: allFlags.remove(flag) #Download VDFs if urlparse(vetofile).netloc: tmp = urlopen(vetofile) vetofile = os.path.abspath(os.path.basename(vetofile)) with open(vetofile, 'w') as f: f.write(tmp.read()) print('Downloaded veto definer file') vdf = DataQualityDict.from_veto_definer_file(vetofile, format='ligolw', start=start, end=end, ifo=ifo) vdf.populate() print('Read %d flags from veto definer' % len(vdf.keys())) categories = [1, 2, 3, 4] vdfCats = dict((c, DataQualityDict()) for c in categories) for name, flag in vdf.iteritems(): try: vdfCats[flag.category][name] = flag except KeyError: pass #generate the triggers needed to run vet if analysis == 'omicron':