def test_populate(self):
# read veto definer
start, end = VETO_DEFINER_TEST_SEGMENTS[0]
vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
ifo='H1',
start=start,
end=end)
# test query that should fail with 404
try:
vdf.populate(url='https://segments.ligo.org')
except (ImportError, UnboundLocalError) as e:
self.skipTest(str(e))
except URLError as e:
if e.code == 401: # 401 is uninformative
self.skipTest(str(e))
elif e.code == 404:
pass
else:
raise
else:
raise AssertionError("URLError not raised")
# check reduction to warning
vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
ifo='H1',
start=start,
end=end)
with pytest.warns(UserWarning):
vdf.populate(url='https://segments.ligo.org', on_error='warn')
# check results
self.assertEqual(len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
# check use of specific segments
vdf = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE,
ifo='H1')
vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
def test_populate(self):
# read veto definer
start, end = VETO_DEFINER_TEST_SEGMENTS[0]
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
# test query that should fail with 404
try:
vdf.populate(url='https://segments.ligo.org')
except (ImportError, UnboundLocalError) as e:
self.skipTest(str(e))
except URLError as e:
if e.code == 401: # 401 is uninformative
self.skipTest(str(e))
elif e.code == 404:
pass
else:
raise
else:
raise AssertionError("URLError not raised")
# check reduction to warning
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
with pytest.warns(UserWarning):
vdf.populate(url='https://segments.ligo.org', on_error='warn')
# check results
self.assertEqual(
len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
# check use of specific segments
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1')
vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS, on_error='ignore')
def test_from_veto_definer_file(self):
vdf = DataQualityDict.from_veto_definer_file(self.VETO_DEFINER)
self.assertNotEqual(len(vdf.keys()), 0)
# test missing h(t) flag
self.assertIn('H1:DCH-MISSING_H1_HOFT_C00:1', vdf)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][0],
1073779216)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][-1],
PosInfinity)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].category, 1)
# test injections padding
self.assertEquals(vdf['H1:ODC-INJECTION_CBC:1'].padding, Segment(-8, 8))
# test download URL
vdf2 = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE)
self.assertEqual(len(vdf.keys()), len(vdf2.keys()))
def test_populate(self):
start, end = VETO_DEFINER_TEST_SEGMENTS[0]
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1', start=start, end=end)
vdf.pop("H1:FAKE_CAT5:1", None)
try:
vdf.populate(url='https://segments.ligo.org')
except ImportError as e:
self.skipTest(str(e))
else:
self.assertEqual(
len(vdf['H1:HVT-ER7_A_RND17:1'].active), 36)
vdf = DataQualityDict.from_veto_definer_file(
VETO_DEFINER_FILE, ifo='H1')
vdf.pop("H1:FAKE_CAT5:1", None)
vdf.populate(segments=VETO_DEFINER_TEST_SEGMENTS)
def test_from_veto_definer_file(self):
vdf = DataQualityDict.from_veto_definer_file(self.VETO_DEFINER)
self.assertNotEqual(len(vdf.keys()), 0)
# test missing h(t) flag
self.assertIn('H1:DCH-MISSING_H1_HOFT_C00:1', vdf)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][0],
1073779216)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].known[0][-1],
PosInfinity)
self.assertEquals(vdf['H1:DCH-MISSING_H1_HOFT_C00:1'].category, 1)
# test injections padding
self.assertEquals(vdf['H1:ODC-INJECTION_CBC:1'].padding,
Segment(-8, 8))
# test download URL
vdf2 = DataQualityDict.from_veto_definer_file(VETO_DEFINER_FILE)
self.assertEqual(len(vdf.keys()), len(vdf2.keys()))
def read_veto_definer_file(vetofile, start=None, end=None, ifo=None):
"""Read a veto definer file, downloading it if necessary
"""
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
with open(vetofile, 'w') as f:
f.write(str(tmp.read()))
return DataQualityDict.from_veto_definer_file(
vetofile, format='ligolw', start=start, end=end, ifo=ifo)
def query_veto_definer_file(ifo, start, end, cp):
# Obtain segments that are analysis ready
analysis_ready = DataQualityFlag.query(
'{0}:DMT-ANALYSIS_READY:1'.format(ifo), start, end)
# Define the start and stop time of analysis ready
analysis_ready_start = analysis_ready.active.extent()[0]
analysis_ready_end = analysis_ready.active.extent()[1]
# Query for vetos found during this time.
vdf = DataQualityDict.from_veto_definer_file(
cp.get('segfind', 'veto-file'), analysis_ready_start,
analysis_ready_end)
# Populate only for analysis ready segments.
vdf.populate(segments=analysis_ready.active)
return vdf
def main(args=None):
"""Run the online Guardian node visualization tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# parse command line options
ifo = args.ifo
if not args.ifo:
parser.error('--ifo must be given if not obvious from the host')
start = getattr(args, 'gpsstart')
end = getattr(args, 'gpsend')
duration = int(ceil(end) - floor(start))
categories = args.categories.split(',')
for i, c in enumerate(categories):
try:
categories[i] = int(c)
except (TypeError, ValueError):
pass
vetofile = getattr(args, 'veto-definer-file')
vetofile = (urlparse(vetofile).netloc or os.path.abspath(vetofile))
args.metric = args.metric or DEFAULT_METRICS
# -- setup --------------------------------------
tag = '%d-%d' % (start.seconds, end.seconds)
outdir = os.path.abspath(os.path.join(args.output_directory, tag))
mkdir(outdir)
os.chdir(outdir)
mkdir('etc', 'segments', 'condor')
# -- segment handling ---------------------------
os.chdir('segments')
ALLSEGMENTS = DataQualityDict()
# -- get analysis segments ----------------------
aflags = args.analysis_segments
asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo)
for i, flag in enumerate(aflags):
# use union of segments from a file
if os.path.isfile(flag):
asegments += DataQualityFlag.read(flag)
# or intersection of segments from multiple flags
else:
new = DataQualityFlag.query(flag, start, end, url=args.segdb)
if i:
asegments.known &= new.known
asegments.active &= new.active
else:
asegments.known = new.known
asegments.active = new.active
ALLSEGMENTS[asegments.name] = asegments
if os.path.isfile(aflags[0]):
asegments.filename = aflags
# -- read veto definer and process --------------
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
with open(vetofile, 'w') as f:
f.write(tmp.read())
LOGGER.info('Downloaded veto definer file')
vdf = DataQualityDict.from_veto_definer_file(vetofile,
format='ligolw',
start=start,
end=end,
ifo=ifo)
LOGGER.info('Read %d flags from veto definer' % len(vdf.keys()))
# populate veto definer file from database
vdf.populate(source=args.segdb, on_error=args.on_segdb_error)
ALLSEGMENTS += vdf
# organise flags into categories
flags = dict((c, DataQualityDict()) for c in categories)
for name, flag in vdf.items():
try:
flags[flag.category][name] = flag
except KeyError:
pass
# find the states and segments for each category
states, after, oldtitle = (dict(), None, '')
for i, category in enumerate(categories):
title = isinstance(category, int) and 'Cat %d' % category or category
tag = re_cchar.sub('_', str(title).upper())
states[category] = SummaryState(
'After %s' % oldtitle,
key=tag,
known=after.known,
active=after.active,
definition=after.name,
) if i else SummaryState(
args.analysis_name,
key=args.analysis_name,
definition=asegments.name,
)
try:
segs = flags[category].union()
except TypeError: # no flags
segs = DataQualityFlag()
segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag)
ALLSEGMENTS[segs.name] = segs
after = (after - segs) if i else (asegments - segs)
after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag)
ALLSEGMENTS[after.name] = after
oldtitle = title
# write all segments to disk
segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' %
(ifo, start.seconds, duration))
ALLSEGMENTS.write(segfile)
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("All segments accessed and written to\n%s" % segfile)
# -- job preparation ----------------------------
os.chdir('etc')
configs = []
for category in categories:
title = (isinstance(category, int) and 'Category %d' % category
or category)
tab = 'tab-%s' % title
config = ConfigParser()
# add segment-database configuration
add_config_section(config, 'segment-database', url=args.segdb)
# add plot configurations
pconfig = ConfigParser()
pconfig.read(args.config_file)
for section in pconfig.sections():
if section.startswith('plot-'):
config._sections[section] = pconfig._sections[section].copy()
try:
plots = pconfig.items('plots-%s' % category, raw=True)
except NoSectionError:
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
# add state
if args.independent:
state = states[categories[0]]
else:
state = states[category]
sname = 'state-%s' % state.key
add_config_section(config,
sname,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
# add plugin
add_config_section(config, 'plugins', **{'gwvet.tabs': ''})
# define metrics
if category == 1:
metrics = ['Deadtime']
else:
metrics = args.metric
# define summary tab
if category == 1:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{'veto-name': title})
else:
tab = configure_veto_tab(config,
title,
title,
state,
flags[category].keys(),
segfile,
metrics,
name='Summary',
**{
'veto-name': title,
'event-channel': args.event_channel,
'event-generator':
args.event_generator,
})
if len(categories) == 1:
config.set(tab, 'index',
'%(gps-start-time)s-%(gps-end-time)s/index.html')
for key, value in plots:
if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605
config.set(tab, key, '%%(union)s,%s' % value)
if '%s-labels' % key not in plots:
config.set(tab, '%s-labels' % key, 'Union,%(flags)s')
else:
config.set(tab, key, value)
# now a tab for each flag
for flag in flags[category]:
if category == 1:
tab = configure_veto_tab(config, flag, title, state, [flag],
segfile, metrics)
else:
tab = configure_veto_tab(
config, flag, title, state, [flag], segfile, metrics, **{
'event-channel': args.event_channel,
'event-generator': args.event_generator
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
if len(categories) > 1 and category != categories[-1]:
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
# configure summary job
if len(categories) > 1:
state = states[categories[0]]
add_config_section(config,
'state-%s' % state.key,
key=state.key,
name=state.name,
definition=state.definition,
filename=segfile)
try:
plots = pconfig.items('plots', raw=True)
except NoSectionError:
plots = []
flags = [f for c in categories for f in flags[c].keys()]
tab = configure_veto_tab(
config,
'Impact of full veto definer file',
None,
state,
flags,
segfile,
args.metric,
shortname='Summary',
index='%(gps-start-time)s-%(gps-end-time)s/index.html',
**{
'event-channel': args.event_channel,
'event-generator': args.event_generator,
'veto-name': 'All vetoes'
})
if args.event_file:
config.set(tab, 'event-file', args.event_file)
for key, value in plots:
config.set(tab, key, value)
with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f:
config.write(f)
configs.append(os.path.abspath(f.name))
os.chdir(os.pardir)
if args.verbose:
LOGGER.debug("Generated configuration files for each category")
# -- condor preparation -------------------------
os.chdir(os.pardir)
# get condor variables
if getuser() == 'detchar':
accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy'
else:
accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy'
gwsumm_args = [
'--gps-start-time',
str(start.seconds),
'--gps-end-time',
str(end.seconds),
'--ifo',
ifo,
'--file-tag',
'gwpy-vet',
'--condor-command',
'accounting_group=%s' % accgroup,
'--condor-command',
'accounting_group_user=%s' % getuser(),
'--on-segdb-error',
args.on_segdb_error,
'--output-dir',
args.output_directory,
]
for cf in args.global_config:
gwsumm_args.extend(('--global-config', cf))
for cf in configs:
gwsumm_args.extend(('--config-file', cf))
if args.verbose:
gwsumm_args.append('--verbose')
if args.verbose:
LOGGER.debug('Generating summary DAG via:\n')
LOGGER.debug(' '.join([batch.PROG] + gwsumm_args))
# execute gwsumm in batch mode
batch.main(args=gwsumm_args)
new = DataQualityFlag.query(flag, start,
end) #try get_segments gwvet.segments
ALLSEGMENTS[new.name] = new
else:
allFlags.remove(flag)
#Download VDFs
if urlparse(vetofile).netloc:
tmp = urlopen(vetofile)
vetofile = os.path.abspath(os.path.basename(vetofile))
with open(vetofile, 'w') as f:
f.write(tmp.read())
print('Downloaded veto definer file')
vdf = DataQualityDict.from_veto_definer_file(vetofile,
format='ligolw',
start=start,
end=end,
ifo=ifo)
vdf.populate()
print('Read %d flags from veto definer' % len(vdf.keys()))
categories = [1, 2, 3, 4]
vdfCats = dict((c, DataQualityDict()) for c in categories)
for name, flag in vdf.iteritems():
try:
vdfCats[flag.category][name] = flag
except KeyError:
pass
#generate the triggers needed to run vet
if analysis == 'omicron':