Пример #1
0
 def test_populate(self):
     # read veto definer
     start, end = VETO_DEFINER_TEST_SEGMENTS[0]
     vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
                                                  ifo='H1',
                                                  start=start,
                                                  end=end)
     # test query that should fail with 404
     try:
         vdf.populate(url='https://segments.ligo.org')
     except (ImportError, UnboundLocalError) as e:
         self.skipTest(str(e))
     except URLError as e:
         if e.code == 401:  # 401 is uninformative
             self.skipTest(str(e))
         elif e.code == 404:
             pass
         else:
             raise
     else:
         raise AssertionError("URLError not raised")
     # check reduction to warning
     vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
                                                  ifo='H1',
                                                  start=start,
                                                  end=end)
     with pytest.warns(UserWarning):
         vdf.populate(url='https://segments.ligo.org', on_error='warn')
     # check results
     self.assertEqual(len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
     # check use of specific segments
     vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
                                                  ifo='H1')
     vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
Пример #2
0
 def test_populate(self):
     # read veto definer
     start, end = VETO_DEFINER_TEST_SEGMENTS[0]
     vdf = DataQualityDict.from_veto_definer_file(
         VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
     # test query that should fail with 404
     try:
         vdf.populate(url='https://segments.ligo.org')
     except (ImportError, UnboundLocalError) as e:
         self.skipTest(str(e))
     except URLError as e:
         if e.code == 401:  # 401 is uninformative
             self.skipTest(str(e))
         elif e.code == 404:
             pass
         else:
             raise
     else:
         raise AssertionError("URLError not raised")
     # check reduction to warning
     vdf = DataQualityDict.from_veto_definer_file(
         VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
     with pytest.warns(UserWarning):
         vdf.populate(url='https://segments.ligo.org', on_error='warn')
     # check results
     self.assertEqual(
         len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
     # check use of specific segments
     vdf = DataQualityDict.from_veto_definer_file(
         VETO_DEFINER_FILE, ifo='H1')
     vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
Пример #3
0
 def test_from_veto_definer_file(self):
     vdf = DataQualityDict.from_veto_definer_file(self.VETO_DEFINER)
     self.assertNotEqual(len(vdf.keys()), 0)
     # test missing h(t) flag
     self.assertIn('H1:DCH-MISSING_H1_HOFT_C00:1', vdf)
     self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][0],
                       1073779216)
     self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][-1],
                       PosInfinity)
     self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].category, 1)
     # test injections padding
     self.assertEquals(vdf['H1:ODC-INJECTION_CBC:1'].padding, Segment(-8, 8))
     # test download URL
     vdf2 = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE)
     self.assertEqual(len(vdf.keys()), len(vdf2.keys()))
Пример #4
0
 def test_populate(self):
     start, end = VETO_DEFINER_TEST_SEGMENTS[0]
     vdf = DataQualityDict.from_veto_definer_file(
         VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
     vdf.pop("H1:FAKE_CAT5:1", None)
     try:
         vdf.populate(url='https://segments.ligo.org')
     except ImportError as e:
         self.skipTest(str(e))
     else:
         self.assertEqual(
             len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
         vdf = DataQualityDict.from_veto_definer_file(
             VETO_DEFINER_FILE, ifo='H1')
         vdf.pop("H1:FAKE_CAT5:1", None)
         vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS)
Пример #5
0
 def test_from_veto_definer_file(self):
     vdf = DataQualityDict.from_veto_definer_file(self.VETO_DEFINER)
     self.assertNotEqual(len(vdf.keys()), 0)
     # test missing h(t) flag
     self.assertIn('H1:DCH-MISSING_H1_HOFT_C00:1', vdf)
     self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][0],
                       1073779216)
     self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][-1],
                       PosInfinity)
     self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].category, 1)
     # test injections padding
     self.assertEquals(vdf['H1:ODC-INJECTION_CBC:1'].padding,
                       Segment(-8, 8))
     # test download URL
     vdf2 = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE)
     self.assertEqual(len(vdf.keys()), len(vdf2.keys()))
Пример #6
0
def read_veto_definer_file(vetofile, start=None, end=None, ifo=None):
    """Read a veto definer file, downloading it if necessary
    """
    if urlparse(vetofile).netloc:
        tmp = urlopen(vetofile)
        vetofile = os.path.abspath(os.path.basename(vetofile))
        with open(vetofile, 'w') as f:
            f.write(str(tmp.read()))
    return DataQualityDict.from_veto_definer_file(
        vetofile, format='ligolw', start=start, end=end, ifo=ifo)
Пример #7
0
def query_veto_definer_file(ifo, start, end, cp):
    # Obtain segments that are analysis ready
    analysis_ready = DataQualityFlag.query(
        '{0}:DMT-ANALYSIS_READY:1'.format(ifo), start, end)
    # Define the start and stop time of analysis ready
    analysis_ready_start = analysis_ready.active.extent()[0]
    analysis_ready_end = analysis_ready.active.extent()[1]
    # Query for vetos found during this time.
    vdf = DataQualityDict.from_veto_definer_file(
        cp.get('segfind', 'veto-file'), analysis_ready_start,
        analysis_ready_end)
    # Populate only for analysis ready segments.
    vdf.populate(segments=analysis_ready.active)

    return vdf
Пример #8
0
def main(args=None):
    """Run the online Guardian node visualization tool
    """
    parser = create_parser()
    args = parser.parse_args(args=args)

    # parse command line options
    ifo = args.ifo
    if not args.ifo:
        parser.error('--ifo must be given if not obvious from the host')
    start = getattr(args, 'gpsstart')
    end = getattr(args, 'gpsend')
    duration = int(ceil(end) - floor(start))
    categories = args.categories.split(',')
    for i, c in enumerate(categories):
        try:
            categories[i] = int(c)
        except (TypeError, ValueError):
            pass
    vetofile = getattr(args, 'veto-definer-file')
    vetofile = (urlparse(vetofile).netloc or os.path.abspath(vetofile))
    args.metric = args.metric or DEFAULT_METRICS

    # -- setup --------------------------------------

    tag = '%d-%d' % (start.seconds, end.seconds)
    outdir = os.path.abspath(os.path.join(args.output_directory, tag))
    mkdir(outdir)
    os.chdir(outdir)
    mkdir('etc', 'segments', 'condor')

    # -- segment handling ---------------------------

    os.chdir('segments')
    ALLSEGMENTS = DataQualityDict()

    # -- get analysis segments ----------------------

    aflags = args.analysis_segments
    asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo)
    for i, flag in enumerate(aflags):
        # use union of segments from a file
        if os.path.isfile(flag):
            asegments += DataQualityFlag.read(flag)
        # or intersection of segments from multiple flags
        else:
            new = DataQualityFlag.query(flag, start, end, url=args.segdb)
            if i:
                asegments.known &= new.known
                asegments.active &= new.active
            else:
                asegments.known = new.known
                asegments.active = new.active
    ALLSEGMENTS[asegments.name] = asegments

    if os.path.isfile(aflags[0]):
        asegments.filename = aflags

    # -- read veto definer and process --------------

    if urlparse(vetofile).netloc:
        tmp = urlopen(vetofile)
        vetofile = os.path.abspath(os.path.basename(vetofile))
        with open(vetofile, 'w') as f:
            f.write(tmp.read())
        LOGGER.info('Downloaded veto definer file')
    vdf = DataQualityDict.from_veto_definer_file(vetofile,
                                                 format='ligolw',
                                                 start=start,
                                                 end=end,
                                                 ifo=ifo)
    LOGGER.info('Read %d flags from veto definer' % len(vdf.keys()))

    # populate veto definer file from database
    vdf.populate(source=args.segdb, on_error=args.on_segdb_error)
    ALLSEGMENTS += vdf

    # organise flags into categories
    flags = dict((c, DataQualityDict()) for c in categories)
    for name, flag in vdf.items():
        try:
            flags[flag.category][name] = flag
        except KeyError:
            pass

    # find the states and segments for each category
    states, after, oldtitle = (dict(), None, '')
    for i, category in enumerate(categories):
        title = isinstance(category, int) and 'Cat %d' % category or category
        tag = re_cchar.sub('_', str(title).upper())
        states[category] = SummaryState(
            'After %s' % oldtitle,
            key=tag,
            known=after.known,
            active=after.active,
            definition=after.name,
        ) if i else SummaryState(
            args.analysis_name,
            key=args.analysis_name,
            definition=asegments.name,
        )
        try:
            segs = flags[category].union()
        except TypeError:  # no flags
            segs = DataQualityFlag()
        segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag)
        ALLSEGMENTS[segs.name] = segs
        after = (after - segs) if i else (asegments - segs)
        after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag)
        ALLSEGMENTS[after.name] = after
        oldtitle = title

    # write all segments to disk
    segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' %
                              (ifo, start.seconds, duration))
    ALLSEGMENTS.write(segfile)

    os.chdir(os.pardir)

    if args.verbose:
        LOGGER.debug("All segments accessed and written to\n%s" % segfile)

    # -- job preparation ----------------------------

    os.chdir('etc')

    configs = []
    for category in categories:
        title = (isinstance(category, int) and 'Category %d' % category
                 or category)
        tab = 'tab-%s' % title
        config = ConfigParser()

        # add segment-database configuration
        add_config_section(config, 'segment-database', url=args.segdb)

        # add plot configurations
        pconfig = ConfigParser()
        pconfig.read(args.config_file)
        for section in pconfig.sections():
            if section.startswith('plot-'):
                config._sections[section] = pconfig._sections[section].copy()

        try:
            plots = pconfig.items('plots-%s' % category, raw=True)
        except NoSectionError:
            try:
                plots = pconfig.items('plots', raw=True)
            except NoSectionError:
                plots = []

        # add state
        if args.independent:
            state = states[categories[0]]
        else:
            state = states[category]
        sname = 'state-%s' % state.key
        add_config_section(config,
                           sname,
                           key=state.key,
                           name=state.name,
                           definition=state.definition,
                           filename=segfile)

        # add plugin
        add_config_section(config, 'plugins', **{'gwvet.tabs': ''})

        # define metrics
        if category == 1:
            metrics = ['Deadtime']
        else:
            metrics = args.metric

        # define summary tab
        if category == 1:
            tab = configure_veto_tab(config,
                                     title,
                                     title,
                                     state,
                                     flags[category].keys(),
                                     segfile,
                                     metrics,
                                     name='Summary',
                                     **{'veto-name': title})
        else:
            tab = configure_veto_tab(config,
                                     title,
                                     title,
                                     state,
                                     flags[category].keys(),
                                     segfile,
                                     metrics,
                                     name='Summary',
                                     **{
                                         'veto-name': title,
                                         'event-channel': args.event_channel,
                                         'event-generator':
                                         args.event_generator,
                                     })
        if len(categories) == 1:
            config.set(tab, 'index',
                       '%(gps-start-time)s-%(gps-end-time)s/index.html')
        for key, value in plots:
            if re.match('%\(flags\)s (?:plot-)?segments', value):  # noqa: W605
                config.set(tab, key, '%%(union)s,%s' % value)
                if '%s-labels' % key not in plots:
                    config.set(tab, '%s-labels' % key, 'Union,%(flags)s')
            else:
                config.set(tab, key, value)

        # now a tab for each flag
        for flag in flags[category]:
            if category == 1:
                tab = configure_veto_tab(config, flag, title, state, [flag],
                                         segfile, metrics)
            else:
                tab = configure_veto_tab(
                    config, flag, title, state, [flag], segfile, metrics, **{
                        'event-channel': args.event_channel,
                        'event-generator': args.event_generator
                    })
                if args.event_file:
                    config.set(tab, 'event-file', args.event_file)
            for key, value in plots:
                config.set(tab, key, value)

        if len(categories) > 1 and category != categories[-1]:
            with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
                config.write(f)
                configs.append(os.path.abspath(f.name))

    # configure summary job
    if len(categories) > 1:
        state = states[categories[0]]
        add_config_section(config,
                           'state-%s' % state.key,
                           key=state.key,
                           name=state.name,
                           definition=state.definition,
                           filename=segfile)
        try:
            plots = pconfig.items('plots', raw=True)
        except NoSectionError:
            plots = []
        flags = [f for c in categories for f in flags[c].keys()]
        tab = configure_veto_tab(
            config,
            'Impact of full veto definer file',
            None,
            state,
            flags,
            segfile,
            args.metric,
            shortname='Summary',
            index='%(gps-start-time)s-%(gps-end-time)s/index.html',
            **{
                'event-channel': args.event_channel,
                'event-generator': args.event_generator,
                'veto-name': 'All vetoes'
            })
        if args.event_file:
            config.set(tab, 'event-file', args.event_file)
        for key, value in plots:
            config.set(tab, key, value)
        with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
            config.write(f)
            configs.append(os.path.abspath(f.name))

    os.chdir(os.pardir)

    if args.verbose:
        LOGGER.debug("Generated configuration files for each category")

    # -- condor preparation -------------------------

    os.chdir(os.pardir)

    # get condor variables
    if getuser() == 'detchar':
        accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy'
    else:
        accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy'

    gwsumm_args = [
        '--gps-start-time',
        str(start.seconds),
        '--gps-end-time',
        str(end.seconds),
        '--ifo',
        ifo,
        '--file-tag',
        'gwpy-vet',
        '--condor-command',
        'accounting_group=%s' % accgroup,
        '--condor-command',
        'accounting_group_user=%s' % getuser(),
        '--on-segdb-error',
        args.on_segdb_error,
        '--output-dir',
        args.output_directory,
    ]
    for cf in args.global_config:
        gwsumm_args.extend(('--global-config', cf))
    for cf in configs:
        gwsumm_args.extend(('--config-file', cf))
    if args.verbose:
        gwsumm_args.append('--verbose')

    if args.verbose:
        LOGGER.debug('Generating summary DAG via:\n')
        LOGGER.debug(' '.join([batch.PROG] + gwsumm_args))

    # execute gwsumm in batch mode
    batch.main(args=gwsumm_args)
Пример #9
0
        new = DataQualityFlag.query(flag, start,
                                    end)  #try get_segments gwvet.segments
        ALLSEGMENTS[new.name] = new
    else:
        allFlags.remove(flag)

#Download VDFs
if urlparse(vetofile).netloc:
    tmp = urlopen(vetofile)
    vetofile = os.path.abspath(os.path.basename(vetofile))
    with open(vetofile, 'w') as f:
        f.write(tmp.read())
    print('Downloaded veto definer file')
vdf = DataQualityDict.from_veto_definer_file(vetofile,
                                             format='ligolw',
                                             start=start,
                                             end=end,
                                             ifo=ifo)
vdf.populate()
print('Read %d flags from veto definer' % len(vdf.keys()))

categories = [1, 2, 3, 4]
vdfCats = dict((c, DataQualityDict()) for c in categories)
for name, flag in vdf.iteritems():
    try:
        vdfCats[flag.category][name] = flag
    except KeyError:
        pass

#generate the triggers needed to run vet
if analysis == 'omicron':