def test_read_write_cache(self):
cache = self.make_cache()[0]
with tempfile.NamedTemporaryFile() as f:
io_cache.write_cache(cache, f)
f.seek(0)
# read from fileobj
c2 = io_cache.read_cache(f)
assert cache == c2
# write with file name
io_cache.write_cache(cache, f.name)
# read from file name
c3 = io_cache.read_cache(f.name)
assert cache == c3
def test_read_write_cache(self):
cache = self.make_cache()[0]
with tempfile.NamedTemporaryFile() as f:
io_cache.write_cache(cache, f)
f.seek(0)
# read from fileobj
c2 = io_cache.read_cache(f)
assert cache == c2
# write with file name
io_cache.write_cache(cache, f.name)
# read from file name
c3 = io_cache.read_cache(f.name)
assert cache == c3
def test_write_lal_cache(tmpdir):
cache = [
"/test/path/X-TEST-0-1.txt",
"/test/path/X-TEST-2-3.txt",
]
target = tmpdir.join("cache.lcf")
utils.write_lal_cache(str(target), cache)
assert read_cache(str(target)) == cache
def test_write_lal_cache(tmpdir):
cache = [
"/test/path/X-TEST-0-1.txt",
"/test/path/X-TEST-2-3.txt",
]
target = tmpdir.join("cache.lcf")
utils.write_lal_cache(str(target), cache)
assert read_cache(str(target)) == cache
def main(args=None):
"""Run the cache_events tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
ifo = args.ifo
start = int(args.gpsstart)
end = int(args.gpsend)
duration = end - start
LOGGER.info("-- Welcome to Hveto --")
LOGGER.info("GPS start time: %d" % start)
LOGGER.info("GPS end time: %d" % end)
LOGGER.info("Interferometer: %s" % ifo)
# -- initialisation -------------------------------
# read configuration
cp = config.HvetoConfigParser(ifo=args.ifo)
cp.read(map(str, args.config_file))
LOGGER.info("Parsed configuration file(s)")
# format output directory
outdir = args.output_directory
outdir.mkdir(parents=True, exist_ok=True)
LOGGER.info("Working directory: {}".format(outdir))
trigdir = outdir / 'triggers'
trigdir.mkdir(parents=True, exist_ok=True)
# get segments
aflag = cp.get('segments', 'analysis-flag')
url = cp.get('segments', 'url')
padding = cp.getfloats('segments', 'padding')
if args.analysis_segments:
segs_ = DataQualityDict.read(args.analysis_segments, gpstype=float)
analysis = segs_[aflag]
span = SegmentList([Segment(start, end)])
analysis.active &= span
analysis.known &= span
analysis.coalesce()
LOGGER.debug("Segments read from disk")
else:
analysis = DataQualityFlag.query(aflag, start, end, url=url)
LOGGER.debug("Segments recovered from %s" % url)
analysis.pad(*padding)
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("Retrieved %d segments for %s with %ss (%.2f%%) livetime" %
(len(analysis.active), aflag, livetime, livetimepc))
snrs = cp.getfloats('hveto', 'snr-thresholds')
minsnr = min(snrs)
# -- utility methods ------------------------------
def create_path(channel):
ifo, name = channel.split(':', 1)
name = name.replace('-', '_')
return trigdir / "{}-{}-{}-{}.h5".format(ifo, name, start, duration)
def read_and_cache_events(channel,
etg,
cache=None,
trigfind_kw={},
**read_kw):
cfile = create_path(channel)
# read existing cached triggers and work out new segments to query
if args.append and cfile.is_file():
previous = DataQualityFlag.read(
str(cfile),
path='segments',
format='hdf5',
).coalesce()
new = analysis - previous
else:
new = analysis.copy()
# get cache of files
if cache is None:
cache = find_trigger_files(channel, etg, new.active, **trigfind_kw)
else:
cache = list(
filter(
lambda e: new.active.intersects_segment(file_segment(e)),
cache,
))
# restrict 'active' segments to when we have data
try:
new.active &= cache_segments(cache)
except IndexError:
new.active = type(new.active)()
# find new triggers
try:
trigs = get_triggers(channel,
etg,
new.active,
cache=cache,
raw=True,
**read_kw)
# catch error and continue
except ValueError as e:
warnings.warn('%s: %s' % (type(e).__name__, str(e)))
else:
path = write_events(channel, trigs, new)
try:
return path, len(trigs)
except TypeError: # None
return
def write_events(channel, tab, segments):
"""Write events to file with a given filename
"""
# get filename
path = create_path(channel)
h5f = h5py.File(str(path), 'a')
# read existing table from file
try:
old = tab.read(h5f["triggers"], format="hdf5")
except KeyError:
pass
else:
tab = vstack(old, tab)
# append event table
tab.write(h5f, path="triggers", append=True, overwrite=True)
# write segments
try:
oldsegs = DataQualityFlag.read(h5f, path="segments", format="hdf5")
except KeyError:
pass
else:
segments = oldsegs + segments
segments.write(h5f, path="segments", append=True, overwrite=True)
# write file to disk
h5f.close()
return path
# -- load channels --------------------------------
# get primary channel name
pchannel = cp.get('primary', 'channel')
# read auxiliary cache
if args.auxiliary_cache is not None:
acache = [e for c in args.auxiliary_cache for e in read_cache(str(c))]
else:
acache = None
# load auxiliary channels
auxetg = cp.get('auxiliary', 'trigger-generator')
auxfreq = cp.getfloats('auxiliary', 'frequency-range')
try:
auxchannels = cp.get('auxiliary', 'channels').strip('\n').split('\n')
except config.configparser.NoOptionError:
auxchannels = find_auxiliary_channels(auxetg,
start,
ifo=args.ifo,
cache=acache)
# load unsafe channels list
_unsafe = cp.get('safety', 'unsafe-channels')
if os.path.isfile(_unsafe): # from file
unsafe = set()
with open(_unsafe, 'rb') as f:
for c in f.read().rstrip('\n').split('\n'):
if c.startswith('%(IFO)s'):
unsafe.add(c.replace('%(IFO)s', ifo))
elif not c.startswith('%s:' % ifo):
unsafe.add('%s:%s' % (ifo, c))
else:
unsafe.add(c)
else: # or from line-seprated list
unsafe = set(_unsafe.strip('\n').split('\n'))
unsafe.add(pchannel)
cp.set('safety', 'unsafe-channels', '\n'.join(sorted(unsafe)))
LOGGER.debug("Read list of %d unsafe channels" % len(unsafe))
# remove duplicates
auxchannels = sorted(set(auxchannels))
LOGGER.debug("Read list of %d auxiliary channels" % len(auxchannels))
# remove unsafe channels
nunsafe = 0
for i in range(len(auxchannels) - 1, -1, -1):
if auxchannels[i] in unsafe:
LOGGER.warning("Auxiliary channel %r identified as unsafe and has "
"been removed" % auxchannels[i])
auxchannels.pop(i)
nunsafe += 1
LOGGER.debug("%d auxiliary channels identified as unsafe" % nunsafe)
naux = len(auxchannels)
LOGGER.info("Identified %d auxiliary channels to process" % naux)
# -- load primary triggers -------------------------
LOGGER.info("Reading events for primary channel...")
# read primary cache
if args.primary_cache is not None:
pcache = [e for c in args.primary_cache for e in read_cache(str(c))]
else:
pcache = None
# get primary params
petg = cp.get('primary', 'trigger-generator')
psnr = cp.getfloat('primary', 'snr-threshold')
pfreq = cp.getfloats('primary', 'frequency-range')
preadkw = cp.getparams('primary', 'read-')
ptrigfindkw = cp.getparams('primary', 'trigfind-')
# load primary triggers
out = read_and_cache_events(pchannel,
petg,
snr=psnr,
frange=pfreq,
cache=pcache,
trigfind_kw=ptrigfindkw,
**preadkw)
try:
e, n = out
except TypeError:
e = None
n = 0
if n:
LOGGER.info("Cached %d new events for %s" % (n, pchannel))
elif args.append and e.is_file():
LOGGER.info("Cached 0 new events for %s" % pchannel)
else:
message = "No events found for %r in %d seconds of livetime" % (
pchannel, livetime)
LOGGER.critical(message)
# write primary to local cache
pname = trigdir / '{}-HVETO_PRIMARY_CACHE-{}-{}.lcf'.format(
ifo,
start,
duration,
)
write_lal_cache(str(pname), [e])
LOGGER.info('Primary cache written to {}'.format(pname))
# -- load auxiliary triggers -----------------------
LOGGER.info("Reading triggers for aux channels...")
counter = multiprocessing.Value('i', 0)
areadkw = cp.getparams('auxiliary', 'read-')
atrigfindkw = cp.getparams('auxiliary', 'trigfind-')
def read_and_write_aux_triggers(channel):
if acache is None:
auxcache = None
else:
ifo, name = channel.split(':')
match = "{}-{}".format(ifo, name.replace('-', '_'))
auxcache = [e for e in acache if Path(e).name.startswith(match)]
out = read_and_cache_events(channel,
auxetg,
cache=auxcache,
snr=minsnr,
frange=auxfreq,
trigfind_kw=atrigfindkw,
**areadkw)
try:
e, n = out
except TypeError:
e = None
n = 0
# log result of load
with counter.get_lock():
counter.value += 1
tag = '[%d/%d]' % (counter.value, naux)
if e is None: # something went wrong
LOGGER.critical(" %s Failed to read events for %s" %
(tag, channel))
else: # either read events or nothing new
LOGGER.debug(" %s Cached %d new events for %s" %
(tag, n, channel))
return e
# map with multiprocessing
if args.nproc > 1:
pool = multiprocessing.Pool(processes=args.nproc)
results = pool.map(read_and_write_aux_triggers, auxchannels)
pool.close()
# map without multiprocessing
else:
results = map(read_and_write_aux_triggers, auxchannels)
acache = [x for x in results if x is not None]
aname = trigdir / '{}-HVETO_AUXILIARY_CACHE-{}-{}.lcf'.format(
ifo,
start,
duration,
)
write_lal_cache(str(aname), acache)
LOGGER.info('Auxiliary cache written to {}'.format(aname))
# -- finish ----------------------------------------
LOGGER.info('Done, you can use these cache files in an hveto analysis by '
'passing the following arguments:\n\n--primary-cache {} '
'--auxiliary-cache {}\n'.format(pname, aname))
def main(args=None):
"""Run the hveto command-line interface
"""
# declare global variables
# this is needed for multiprocessing utilities
global acache, analysis, areadkw, atrigfindkw, auxiliary, auxetg
global auxfreq, counter, livetime, minsnr, naux, pchannel, primary
global rnd, snrs, windows
# parse command-line
parser = create_parser()
args = parser.parse_args(args=args)
ifo = args.ifo
start = int(args.gpsstart)
end = int(args.gpsend)
duration = end - start
# log startup
LOGGER.info("-- Welcome to Hveto --")
LOGGER.info("GPS start time: %d" % start)
LOGGER.info("GPS end time: %d" % end)
LOGGER.info("Interferometer: %s" % ifo)
# -- initialisation -------------------------
# read configuration
cp = config.HvetoConfigParser(ifo=ifo)
cp.read(args.config_file)
LOGGER.info("Parsed configuration file(s)")
# format output directory
outdir = _abs_path(args.output_directory)
if not os.path.isdir(outdir):
os.makedirs(outdir)
os.chdir(outdir)
LOGGER.info("Working directory: %s" % outdir)
segdir = 'segments'
plotdir = 'plots'
trigdir = 'triggers'
omegadir = 'scans'
for d in [segdir, plotdir, trigdir, omegadir]:
if not os.path.isdir(d):
os.makedirs(d)
# prepare html variables
htmlv = {
'title': '%s Hveto | %d-%d' % (ifo, start, end),
'config': None,
'prog': PROG,
'context': ifo.lower(),
}
# get segments
aflag = cp.get('segments', 'analysis-flag')
url = cp.get('segments', 'url')
padding = tuple(cp.getfloats('segments', 'padding'))
if args.analysis_segments:
segs_ = DataQualityDict.read(args.analysis_segments, gpstype=float)
analysis = segs_[aflag]
span = SegmentList([Segment(start, end)])
analysis.active &= span
analysis.known &= span
analysis.coalesce()
LOGGER.debug("Segments read from disk")
else:
analysis = DataQualityFlag.query(aflag, start, end, url=url)
LOGGER.debug("Segments recovered from %s" % url)
if padding != (0, 0):
mindur = padding[0] - padding[1]
analysis.active = type(analysis.active)([s for s in analysis.active if
abs(s) >= mindur])
analysis.pad(*padding, inplace=True)
LOGGER.debug("Padding %s applied" % str(padding))
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("Retrieved %d segments for %s with %ss (%.2f%%) livetime"
% (len(analysis.active), aflag, livetime, livetimepc))
# apply vetoes from veto-definer file
try:
vetofile = cp.get('segments', 'veto-definer-file')
except configparser.NoOptionError:
vetofile = None
else:
try:
categories = cp.getfloats('segments', 'veto-definer-categories')
except configparser.NoOptionError:
categories = None
# read file
vdf = read_veto_definer_file(vetofile, start=start, end=end, ifo=ifo)
LOGGER.debug("Read veto-definer file from %s" % vetofile)
# get vetoes from segdb
vdf.populate(source=url, segments=analysis.active, on_error='warn')
# coalesce flags from chosen categories
vetoes = DataQualityFlag('%s:VDF-VETOES:1' % ifo)
nflags = 0
for flag in vdf:
if not categories or vdf[flag].category in categories:
vetoes += vdf[flag]
nflags += 1
try:
deadtime = int(abs(vetoes.active)) / int(abs(vetoes.known)) * 100
except ZeroDivisionError:
deadtime = 0
LOGGER.debug("Coalesced %ss (%.2f%%) of deadtime from %d veto flags"
% (abs(vetoes.active), deadtime, nflags))
# apply to analysis segments
analysis -= vetoes
LOGGER.debug("Applied vetoes from veto-definer file")
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("%ss (%.2f%%) livetime remaining after vetoes"
% (livetime, livetimepc))
snrs = cp.getfloats('hveto', 'snr-thresholds')
minsnr = min(snrs)
windows = cp.getfloats('hveto', 'time-windows')
# record all segments
segments = DataQualityDict()
segments[analysis.name] = analysis
# -- load channels --------------------------
# get primary channel name
pchannel = cp.get('primary', 'channel')
# read auxiliary cache
if args.auxiliary_cache is not None:
acache = read_cache(args.auxiliary_cache)
else:
acache = None
# load auxiliary channels
auxetg = cp.get('auxiliary', 'trigger-generator')
auxfreq = cp.getfloats('auxiliary', 'frequency-range')
try:
auxchannels = cp.get('auxiliary', 'channels').strip('\n').split('\n')
except config.configparser.NoOptionError:
auxchannels = find_auxiliary_channels(auxetg, (start, end), ifo=ifo,
cache=acache)
cp.set('auxiliary', 'channels', '\n'.join(auxchannels))
LOGGER.debug("Auto-discovered %d "
"auxiliary channels" % len(auxchannels))
else:
auxchannels = sorted(set(auxchannels))
LOGGER.debug("Read list of %d auxiliary channels" % len(auxchannels))
# load unsafe channels list
_unsafe = cp.get('safety', 'unsafe-channels')
if os.path.isfile(_unsafe): # from file
unsafe = set()
with open(_unsafe, 'rb') as f:
for c in f.read().rstrip('\n').split('\n'):
if c.startswith('%(IFO)s'):
unsafe.add(c.replace('%(IFO)s', ifo))
elif not c.startswith('%s:' % ifo):
unsafe.add('%s:%s' % (ifo, c))
else:
unsafe.add(c)
else: # or from line-seprated list
unsafe = set(_unsafe.strip('\n').split('\n'))
unsafe.add(pchannel)
cp.set('safety', 'unsafe-channels', '\n'.join(sorted(unsafe)))
LOGGER.debug("Read list of %d unsafe channels" % len(unsafe))
# remove unsafe channels
nunsafe = 0
for i in range(len(auxchannels) - 1, -1, -1):
if auxchannels[i] in unsafe:
LOGGER.warning("Auxiliary channel %r identified as unsafe and has "
"been removed" % auxchannels[i])
auxchannels.pop(i)
nunsafe += 1
LOGGER.debug("%d auxiliary channels identified as unsafe" % nunsafe)
naux = len(auxchannels)
LOGGER.info("Identified %d auxiliary channels to process" % naux)
# record INI file in output HTML directory
inifile = '%s-HVETO_CONFIGURATION-%d-%d.ini' % (ifo, start, duration)
if os.path.isfile(inifile) and any(
os.path.samefile(inifile, x) for x in args.config_file):
LOGGER.debug("Cannot write INI file to %s, file was given as input")
else:
with open(inifile, 'w') as f:
cp.write(f)
LOGGER.info("Configuration recorded as %s" % inifile)
htmlv['config'] = inifile
# -- load primary triggers ------------------
# read primary cache
if args.primary_cache is not None:
pcache = read_cache(args.primary_cache)
else:
pcache = None
# load primary triggers
petg = cp.get('primary', 'trigger-generator')
psnr = cp.getfloat('primary', 'snr-threshold')
pfreq = cp.getfloats('primary', 'frequency-range')
preadkw = cp.getparams('primary', 'read-')
if pcache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the primary trigger file format')
preadkw['format'] = None
preadkw['path'] = 'triggers'
ptrigfindkw = cp.getparams('primary', 'trigfind-')
primary = get_triggers(pchannel, petg, analysis.active, snr=psnr,
frange=pfreq, cache=pcache, nproc=args.nproc,
trigfind_kwargs=ptrigfindkw, **preadkw)
fcol, scol = primary.dtype.names[1:3]
if len(primary):
LOGGER.info("Read %d events for %s" % (len(primary), pchannel))
else:
message = "No events found for %r in %d seconds of livetime" % (
pchannel, livetime)
LOGGER.critical(message)
# cluster primary triggers
clusterkwargs = cp.getparams('primary', 'cluster-')
if clusterkwargs:
primary = primary.cluster(**clusterkwargs)
LOGGER.info("%d primary events remain after clustering over %s" %
(len(primary), clusterkwargs['rank']))
# -- bail out early -------------------------
# the bail out is done here so that we can at least generate the eventual
# configuration file, mainly for HTML purposes
# no segments
if livetime == 0:
message = ("No active segments found for analysis flag %r in interval "
"[%d, %d)" % (aflag, start, end))
LOGGER.critical(message)
htmlv['context'] = 'info'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# no primary triggers
if len(primary) == 0:
htmlv['context'] = 'danger'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# otherwise write all primary triggers to ASCII
trigfile = os.path.join(
trigdir,
'%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration),
)
primary.write(trigfile, format='ascii', overwrite=True)
# -- load auxiliary triggers ----------------
LOGGER.info("Reading triggers for aux channels...")
counter = multiprocessing.Value('i', 0)
areadkw = cp.getparams('auxiliary', 'read-')
if acache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the auxiliary trigger file format')
areadkw['format'] = None
areadkw['path'] = 'triggers'
atrigfindkw = cp.getparams('auxiliary', 'trigfind-')
# map with multiprocessing
if args.nproc > 1:
pool = multiprocessing.Pool(processes=args.nproc)
results = pool.map(_get_aux_triggers, auxchannels)
pool.close()
# map without multiprocessing
else:
results = map(_get_aux_triggers, auxchannels)
LOGGER.info("All aux events loaded")
auxiliary = dict(x for x in results if x is not None)
auxchannels = sorted(auxiliary.keys())
chanfile = '%s-HVETO_CHANNEL_LIST-%d-%d.txt' % (ifo, start, duration)
with open(chanfile, 'w') as f:
for chan in auxchannels:
print(chan, file=f)
LOGGER.info("Recorded list of valid auxiliary channels in %s" % chanfile)
# -- execute hveto analysis -----------------
minsig = cp.getfloat('hveto', 'minimum-significance')
pevents = [primary]
pvetoed = []
auxfcol, auxscol = auxiliary[auxchannels[0]].dtype.names[1:3]
slabel = plot.get_column_label(scol)
flabel = plot.get_column_label(fcol)
auxslabel = plot.get_column_label(auxscol)
auxflabel = plot.get_column_label(auxfcol)
rounds = []
rnd = core.HvetoRound(1, pchannel, rank=scol)
rnd.segments = analysis.active
while True:
LOGGER.info("-- Processing round %d --" % rnd.n)
# write segments for this round
segfile = os.path.join(
segdir, '%s-HVETO_ANALYSIS_SEGS_ROUND_%d-%d-%d.txt'
% (ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.segments)
# calculate significances for this round
if args.nproc > 1: # multiprocessing
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_find_max_significance, chunks)
pool.close()
winners, sigsets = zip(*results)
# find winner of chunk winners
winner = sorted(winners, key=lambda w: w.significance)[-1]
# flatten sets of significances into one list
newsignificances = sigsets[0]
for subdict in sigsets[1:]:
newsignificances.update(subdict)
else: # single process
winner, newsignificances = core.find_max_significance(
primary, auxiliary, pchannel, snrs, windows, rnd.livetime)
LOGGER.info("Round %d winner: %s" % (rnd.n, winner.name))
# plot significance drop here for the last round
# only now do we actually have the new data to
# calculate significance drop
if rnd.n > 1:
svg = (pngname % 'SIG_DROP').replace('.png', '.svg') # noqa: F821
plot.significance_drop(
svg, oldsignificances, newsignificances, # noqa: F821
title=' | '.join([title, subtitle]), # noqa: F821
bbox_inches='tight')
LOGGER.debug("Figure written to %s" % svg)
svg = FancyPlot(svg, caption=plot.ROUND_CAPTION['SIG_DROP'])
rounds[-1].plots.append(svg)
oldsignificances = newsignificances # noqa: F841
# break out of the loop if the significance is below stopping point
if winner.significance < minsig:
LOGGER.info("Maximum signifiance below stopping point")
LOGGER.debug(" (%.2f < %.2f)" % (winner.significance, minsig))
LOGGER.info("-- Rounds complete! --")
break
# work out the vetoes for this round
allaux = auxiliary[winner.name][
auxiliary[winner.name][auxscol] >= winner.snr]
winner.events = allaux
coincs = allaux[core.find_coincidences(allaux['time'], primary['time'],
dt=winner.window)]
rnd.vetoes = winner.get_segments(allaux['time'])
flag = DataQualityFlag(
'%s:HVT-ROUND_%d:1' % (ifo, rnd.n), active=rnd.vetoes,
known=rnd.segments,
description="winner=%s, window=%s, snr=%s" % (
winner.name, winner.window, winner.snr))
segments[flag.name] = flag
LOGGER.debug("Generated veto segments for round %d" % rnd.n)
# link events before veto for plotting
before = primary
beforeaux = auxiliary[winner.name]
# apply vetoes to primary
primary, vetoed = core.veto(primary, rnd.vetoes)
pevents.append(primary)
pvetoed.append(vetoed)
LOGGER.debug("Applied vetoes to primary")
# record results
rnd.winner = winner
rnd.efficiency = (len(vetoed), len(primary) + len(vetoed))
rnd.use_percentage = (len(coincs), len(winner.events))
if rnd.n > 1:
rnd.cum_efficiency = (
len(vetoed) + rounds[-1].cum_efficiency[0],
rounds[0].efficiency[1])
rnd.cum_deadtime = (
rnd.deadtime[0] + rounds[-1].cum_deadtime[0],
livetime)
else:
rnd.cum_efficiency = rnd.efficiency
rnd.cum_deadtime = rnd.deadtime
# apply vetoes to auxiliary
if args.nproc > 1: # multiprocess
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_veto, chunks)
pool.close()
auxiliary = results[0]
for subdict in results[1:]:
auxiliary.update(subdict)
else: # single process
auxiliary = core.veto_all(auxiliary, rnd.vetoes)
LOGGER.debug("Applied vetoes to auxiliary channels")
# log results
LOGGER.info("""Results for round %d:\n\n
winner : %s
significance : %s
mu : %s
snr : %s
dt : %s
use_percentage : %s
efficiency : %s
deadtime : %s
cum. efficiency : %s
cum. deadtime : %s\n\n""" % (
rnd.n, rnd.winner.name, rnd.winner.significance,
rnd.winner.mu, rnd.winner.snr, rnd.winner.window,
rnd.use_percentage, rnd.efficiency, rnd.deadtime,
rnd.cum_efficiency, rnd.cum_deadtime))
# write segments
segfile = os.path.join(
segdir,
'%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % (
ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.vetoes)
LOGGER.debug("Round %d vetoes written to %s" % (rnd.n, segfile))
rnd.files['VETO_SEGS'] = (segfile,)
# write triggers
trigfile = os.path.join(
trigdir,
'%s-HVETO_%%s_TRIGS_ROUND_%d-%d-%d.txt' % (
ifo, rnd.n, start, duration))
for tag, arr in zip(
['WINNER', 'VETOED', 'RAW'],
[winner.events, vetoed, primary]):
f = trigfile % tag
arr.write(f, format='ascii', overwrite=True)
LOGGER.debug("Round %d %s events written to %s"
% (rnd.n, tag.lower(), f))
rnd.files[tag] = f
# record times to omega scan
if args.omega_scans:
N = len(vetoed)
ind = random.sample(range(0, N), min(args.omega_scans, N))
rnd.scans = vetoed[ind]
LOGGER.debug("Collected %d events to omega scan:\n\n%s\n\n"
% (len(rnd.scans), rnd.scans))
# -- make some plots --
pngname = os.path.join(plotdir, '%s-HVETO_%%s_ROUND_%d-%d-%d.png' % (
ifo, rnd.n, start, duration))
wname = texify(rnd.winner.name)
beforel = 'Before\n[%d]' % len(before)
afterl = 'After\n[%d]' % len(primary)
vetoedl = 'Vetoed\n(primary)\n[%d]' % len(vetoed)
beforeauxl = 'All\n[%d]' % len(beforeaux)
usedl = 'Used\n(aux)\n[%d]' % len(winner.events)
coincl = 'Coinc.\n[%d]' % len(coincs)
title = '%s Hveto round %d' % (ifo, rnd.n)
ptitle = '%s: primary impact' % title
atitle = '%s: auxiliary use' % title
subtitle = 'winner: %s [%d-%d]' % (wname, start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
plot.before_after_histogram(
png, before[scol], primary[scol],
label1=beforel, label2=afterl, xlabel=slabel,
title=ptitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['HISTOGRAM'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(
png, before, vetoed, x='time', y=scol, label1=beforel,
label2=vetoedl, epoch=start, xlim=[start, end], ylabel=slabel,
title=ptitle, subtitle=subtitle, legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'SNR_%s' % fcol.upper()
plot.veto_scatter(
png, before, vetoed, x=fcol, y=scol, label1=beforel,
label2=vetoedl, xlabel=flabel, ylabel=slabel, xlim=pfreq,
title=ptitle, subtitle=subtitle, legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % '%s_TIME' % fcol.upper()
plot.veto_scatter(
png, before, vetoed, x='time', y=fcol, color=scol,
label1=None, label2=None, ylabel=flabel,
clabel=slabel, clim=[3, 100], cmap='YlGnBu',
epoch=start, xlim=[start, end], ylim=pfreq,
title=ptitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['TIME'])
rnd.plots.append(png)
# aux used versus frequency
png = pngname % 'USED_SNR_TIME'
plot.veto_scatter(
png, winner.events, vetoed, x='time', y=[auxscol, scol],
label1=usedl, label2=vetoedl, ylabel=slabel, epoch=start,
xlim=[start, end], title=atitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['USED_SNR_TIME'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'AUX_SNR_TIME'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x='time', y=auxscol,
label1=beforeauxl, label2=(usedl, coincl), epoch=start,
xlim=[start, end], ylabel=auxslabel, title=atitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'AUX_SNR_FREQUENCY'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x=auxfcol, y=auxscol,
label1=beforeauxl, label2=(usedl, coincl), xlabel=auxflabel,
ylabel=auxslabel, title=atitle, subtitle=subtitle,
legend_title="Aux:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_FREQUENCY'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % 'AUX_FREQUENCY_TIME'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x='time', y=auxfcol,
color=auxscol, label1=None, label2=[None, None], ylabel=auxflabel,
clabel=auxslabel, clim=[3, 100], cmap='YlGnBu', epoch=start,
xlim=[start, end], title=atitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_FREQUENCY_TIME'])
rnd.plots.append(png)
# move to the next round
rounds.append(rnd)
rnd = core.HvetoRound(rnd.n + 1, pchannel, rank=scol,
segments=rnd.segments-rnd.vetoes)
# write file with all segments
segfile = os.path.join(
segdir, '%s-HVETO_SEGMENTS-%d-%d.h5' % (ifo, start, duration))
segments.write(segfile, overwrite=True)
LOGGER.debug("Segment summary written to %s" % segfile)
LOGGER.debug("Making summary figures...")
# -- exit early if no rounds above threshold
if not rounds:
message = ("No rounds completed above threshold. Analysis stopped "
"with %s achieving significance of %.2f"
% (winner.name, winner.significance))
LOGGER.critical(message)
message = message.replace(
winner.name, cis_link(winner.name, class_='alert-link'))
message += '<br>[T<sub>win</sub>: %ss, SNR: %s]' % (
winner.window, winner.snr)
htmlv['context'] = 'warning'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# -- plot all rounds impact
pngname = os.path.join(plotdir, '%s-HVETO_%%s_ALL_ROUNDS-%d-%d.png' % (
ifo, start, duration))
plots = []
title = '%s Hveto all rounds' % args.ifo
subtitle = '%d rounds | %d-%d' % (len(rounds), start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
beforel = 'Before analysis [%d events]' % len(pevents[0])
afterl = 'After %d rounds [%d]' % (len(pevents) - 1, len(pevents[-1]))
plot.before_after_histogram(
png, pevents[0][scol], pevents[-1][scol],
label1=beforel, label2=afterl, xlabel=slabel,
title=title, subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['HISTOGRAM'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# efficiency/deadtime curve
png = pngname % 'ROC'
plot.hveto_roc(png, rounds, title=title, subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['ROC'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# frequency versus time
png = pngname % '%s_TIME' % fcol.upper()
labels = [str(r.n) for r in rounds]
legtitle = 'Vetoed at\nround'
plot.veto_scatter(
png, pevents[0], pvetoed,
label1='', label2=labels, title=title,
subtitle=subtitle, ylabel=flabel, x='time', y=fcol,
epoch=start, xlim=[start, end], legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(
png, pevents[0], pvetoed, label1='', label2=labels, title=title,
subtitle=subtitle, ylabel=slabel, x='time', y=scol,
epoch=start, xlim=[start, end], legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['SNR_TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# -- write summary states to ASCII table and JSON
json_ = {
'user': getuser(),
'host': getfqdn(),
'date': str(datetime.datetime.now()),
'configuration': inifile,
'ifo': ifo,
'gpsstart': start,
'gpsend': end,
'call': ' '.join(sys.argv),
'rounds': [],
}
with open('summary-stats.txt', 'w') as f:
# print header
print('#N winner window SNR significance nveto use-percentage '
'efficiency deadtime cumulative-efficiency cumulative-deadtime',
file=f)
for r in rounds:
# extract relevant statistics
results = [
('round', r.n),
('name', r.winner.name),
('window', r.winner.window),
('snr', r.winner.snr),
('significance', r.winner.significance),
('nveto', r.efficiency[0]),
('use-percentage',
r.use_percentage[0] / r.use_percentage[1] * 100.),
('efficiency', r.efficiency[0] / r.efficiency[1] * 100.),
('deadtime', r.deadtime[0] / r.deadtime[1] * 100.),
('cumulative-efficiency',
r.cum_efficiency[0] / r.cum_efficiency[1] * 100.),
('cumulative-deadtime',
r.cum_deadtime[0] / r.cum_deadtime[1] * 100.),
]
# write to ASCII
print(' '.join(map(str, list(zip(*results))[1])), file=f)
# write to JSON
results.append(('files', r.files))
json_['rounds'].append(dict(results))
LOGGER.debug("Summary table written to %s" % f.name)
with open('summary-stats.json', 'w') as f:
json.dump(json_, f, sort_keys=True)
LOGGER.debug("Summary JSON written to %s" % f.name)
# -- generate workflow for omega scans
if args.omega_scans:
omegatimes = list(map(str, sorted(numpy.unique(
[t['time'] for r in rounds for t in r.scans]))))
LOGGER.debug("Collected %d times to omega scan" % len(omegatimes))
newtimes = [t for t in omegatimes if not
os.path.exists(os.path.join(omegadir, str(t)))]
LOGGER.debug("%d scans already complete or in progress, %d remaining"
% (len(omegatimes) - len(newtimes), len(newtimes)))
if len(newtimes) > 0:
LOGGER.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(
ifo=ifo,
ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(
timeout=4,
extra_commands=["request_disk='1G'"],
gps=start)
batch.generate_dag(
newtimes,
flags=flags,
submit=True,
outdir=omegadir,
condor_commands=condorcmds)
LOGGER.info('Launched {} omega scans to condor'.format(
len(newtimes)))
else:
LOGGER.debug('Skipping omega scans')
# -- write HTML and finish
index = html.write_hveto_page(
ifo, start, end, rounds, plots,
winners=[r.winner.name for r in rounds], **htmlv)
LOGGER.debug("HTML written to %s" % index)
LOGGER.debug("Analysis completed in %d seconds" % (time.time() - JOBSTART))
LOGGER.info("-- Hveto complete --")
def main(args=None):
parser = create_parser()
args = parser.parse_args(args=args)
# apply verbosity to logger
args.verbose = max(5 - args.verbose, 0)
logger.setLevel(args.verbose * 10)
# validate command line arguments
if args.ifo is None:
parser.error("Cannot determine IFO prefix from sytem, "
"please pass --ifo on the command line")
if args.executable is None:
parser.error("Cannot find omicron on path, please pass "
"--executable on the command line")
# validate processing options
if all((args.skip_root_merge, args.skip_hdf5_merge, args.skip_ligolw_add,
args.skip_gzip, not args.archive)):
args.skip_postprocessing = True
if args.archive:
argsd = vars(args)
for arg in [
'skip-root-merge', 'skip-hdf5-merge', 'skip-ligolw-add',
'skip-gzip'
]:
if argsd[arg.replace('-', '_')]:
parser.error("Cannot use --%s with --archive" % arg)
# check conflicts
if args.gps is None and args.cache_file is not None:
parser.error("Cannot use --cache-file in 'online' mode, "
"please use --cache-file with --gps")
# extract key variables
ifo = args.ifo
group = args.group
online = args.gps is None
# format file-tag as underscore-delimited upper-case string
filetag = args.file_tag
if filetag:
filetag = re.sub(r'[:_\s-]', '_', filetag).rstrip('_').strip('_')
if const.OMICRON_FILETAG.lower() in filetag.lower():
afiletag = filetag
else:
afiletag = '%s_%s' % (filetag, const.OMICRON_FILETAG.upper())
filetag = '_%s' % filetag
else:
filetag = ''
afiletag = const.OMICRON_FILETAG.upper()
logger.info("--- Welcome to the Omicron processor ---")
# set up containers to keep track of files that we create here
tempfiles = []
keepfiles = []
# check rescue against --dagman-option force
if args.rescue and args.dagman_option.count('force') > 1:
parser.error('--rescue is incompatible with --dagman-option force')
elif args.rescue:
args.dagman_option.pop(0)
logger.info(
"Running in RESCUE mode - the workflow will be "
"re-generated in memory without any files being written", )
# set omicron version for future use
omicronv = utils.get_omicron_version(args.executable)
const.OMICRON_VERSION = str(omicronv)
os.environ.setdefault('OMICRON_VERSION', str(omicronv))
logger.debug('Omicron version: %s' % omicronv)
# -- parse configuration file and get parameters --------------------------
cp = configparser.ConfigParser()
cp.read(args.config_file)
# validate
if not cp.has_section(group):
raise configparser.NoSectionError(group)
# get params
channels = cp.get(group, 'channels').strip('\n').rstrip('\n').split('\n')
try: # allow two-column 'channel samplerate' format
channels, crates = zip(*[c.split(' ', 1) for c in channels])
except ValueError:
crates = []
else:
crates = set(crates)
logger.debug("%d channels read" % len(channels))
for i in range(len(channels) - 1, -1, -1): # remove excluded channels
c = channels[i]
if c in args.exclude_channel:
channels.pop(i)
logger.debug(" removed %r" % c)
logger.debug("%d channels to process" % len(channels))
cp.set(group, 'channels', '\n'.join(channels))
frametype = cp.get(group, 'frametype')
logger.debug("frametype = %s" % frametype)
chunkdur = cp.getint(group, 'chunk-duration')
logger.debug("chunkdur = %s" % chunkdur)
segdur = cp.getint(group, 'segment-duration')
logger.debug("segdur = %s" % segdur)
overlap = cp.getint(group, 'overlap-duration')
logger.debug("overlap = %s" % overlap)
padding = int(overlap / 2)
logger.debug("padding = %s" % padding)
try:
frange = tuple(map(float, cp.get(group, 'frequency-range').split()))
except configparser.NoOptionError as e:
try:
flow = cp.getfloat(group, 'flow')
fhigh = cp.getfloat(group, 'flow')
except configparser.NoOptionError:
raise e
frange = (flow, fhigh)
logger.debug('frequencyrange = [%s, %s)' % tuple(frange))
try:
sampling = cp.getfloat(group, 'sample-frequency')
except configparser.NoOptionError:
if len(crates) == 1:
sampling = float(crates[0])
elif len(crates) > 1:
raise ValueError(
"No sample-frequency parameter given, and multiple "
"sample frequencies parsed from channels list, "
"cannot continue", )
else:
sampling = None
if sampling:
logger.debug('samplingfrequency = %s' % sampling)
# get state channel
try:
statechannel = cp.get(group, 'state-channel')
except configparser.NoOptionError:
statechannel = None
else:
try:
statebits = list(
map(
float,
cp.get(group, 'state-bits').split(','),
))
except configparser.NoOptionError:
statebits = [0]
try:
stateft = cp.get(group, 'state-frametype')
except configparser.NoOptionError as e:
e.args = ('%s, this must be specified if state-channel is given' %
str(e), )
raise
# get state flag (if given)
try:
stateflag = cp.get(group, 'state-flag')
except configparser.NoOptionError:
stateflag = None
else:
logger.debug("State flag = %s" % stateflag)
if not statechannel: # map state flag to state channel
try:
statechannel, statebits, stateft = (
segments.STATE_CHANNEL[stateflag])
except KeyError as e:
if online or args.no_segdb: # only raise if channel required
e.args = ('Cannot map state flag %r to channel' %
stateflag, )
raise
else:
pass
if statechannel:
logger.debug("State channel = %s" % statechannel)
logger.debug("State bits = %s" % ', '.join(map(str, statebits)))
logger.debug("State frametype = %s" % stateft)
# parse padding for state segments
if statechannel or stateflag:
try:
statepad = cp.get(group, 'state-padding')
except configparser.NoOptionError:
statepad = (0, 0)
else:
try:
p = int(statepad)
except ValueError:
statepad = tuple(map(float, statepad.split(',', 1)))
else:
statepad = (p, p)
logger.debug("State padding: %s" % str(statepad))
rundir = utils.get_output_path(args)
# convert to omicron parameters format
oconfig = parameters.OmicronParameters.from_channel_list_config(
cp, group, version=omicronv)
# and validate things
oconfig.validate()
# -- set directories ------------------------------------------------------
rundir.mkdir(exist_ok=True, parents=True)
logger.info("Using run directory\n%s" % rundir)
cachedir = rundir / "cache"
condir = rundir / "condor"
logdir = rundir / "logs"
pardir = rundir / "parameters"
trigdir = rundir / "triggers"
for d in [cachedir, condir, logdir, pardir, trigdir]:
d.mkdir(exist_ok=True)
oconfig.set('OUTPUT', 'DIRECTORY', str(trigdir))
# -- check for an existing process ----------------------------------------
dagpath = condir / "{}.dag".format(DAG_TAG)
# check dagman lock file
running = condor.dag_is_running(dagpath)
if running:
msg = "Detected {} already running in {}".format(
dagpath,
rundir,
)
if not args.reattach:
raise RuntimeError(msg)
logger.info("{}, will reattach".format(msg))
else:
args.reattach = False
# check dagman rescue files
nrescue = len(
list(condir.glob("{}.rescue[0-9][0-9][0-9]".format(dagpath.name), )))
if args.rescue and not nrescue:
raise RuntimeError(
"--rescue given but no rescue DAG files found for {}".format(
dagpath, ), )
if nrescue and not args.rescue and "force" not in args.dagman_option:
raise RuntimeError(
"rescue DAGs found for {} but `--rescue` not given and "
"`--dagman-option force` not given, cannot continue".format(
dagpath, ), )
newdag = not args.rescue and not args.reattach
# -- find run segment -----------------------------------------------------
segfile = str(rundir / "segments.txt")
keepfiles.append(segfile)
if newdag and online:
# get limit of available data (allowing for padding)
end = data.get_latest_data_gps(ifo, frametype) - padding
try: # start from where we got to last time
start = segments.get_last_run_segment(segfile)[1]
except IOError: # otherwise start with a sensible amount of data
if args.use_dev_shm: # process one chunk
logger.debug("No online segment record, starting with "
"%s seconds" % chunkdur)
start = end - chunkdur + padding
else: # process the last 4000 seconds (arbitrarily)
logger.debug("No online segment record, starting with "
"4000 seconds")
start = end - 4000
else:
logger.debug("Online segment record recovered")
elif online:
start, end = segments.get_last_run_segment(segfile)
else:
start, end = args.gps
duration = end - start
datastart = start - padding
dataend = end + padding
dataduration = dataend - datastart
logger.info("Processing segment determined as")
logger.info(" %d %d" % (datastart, dataend))
logger.info("Duration = %d seconds" % dataduration)
span = (start, end)
# -- find segments and frame files ----------------------------------------
# minimum allowed duration is one full chunk
minduration = 1 * chunkdur
# validate span is long enough
if dataduration < minduration and online:
logger.info("Segment is too short (%d < %d), please try again later" %
(duration, minduration))
clean_exit(0, tempfiles)
elif dataduration < minduration:
raise ValueError(
"Segment [%d, %d) is too short (%d < %d), please "
"extend the segment, or shorten the timing parameters." %
(start, end, duration, chunkdur - padding * 2), )
# -- find run segments
# get segments from state vector
if (online and statechannel) or (statechannel
and not stateflag) or (statechannel
and args.no_segdb):
logger.info("Finding segments for relevant state...")
if statebits == "guardian": # use guardian
segs = segments.get_guardian_segments(
statechannel,
stateft,
datastart,
dataend,
pad=statepad,
)
else:
segs = segments.get_state_segments(
statechannel,
stateft,
datastart,
dataend,
bits=statebits,
pad=statepad,
)
# get segments from segment database
elif stateflag:
logger.info("Querying segments for relevant state...")
segs = segments.query_state_segments(stateflag,
datastart,
dataend,
pad=statepad)
# get segments from frame availability
else:
segs = segments.get_frame_segments(ifo, frametype, datastart, dataend)
# print frame segments recovered
if len(segs):
logger.info("State/frame segments recovered as")
for seg in segs:
logger.info(" %d %d [%d]" % (seg[0], seg[1], abs(seg)))
logger.info("Duration = %d seconds" % abs(segs))
# if running online, we want to avoid processing up to the extent of
# available data, so that the next run doesn't get left with a segment that
# is too short to process
# There are a few reasons this might be
# - the interferometer loses lock a short time after the end of this run
# - a restart/other problem means that a frame is missing a short time
# after the end of this run
# so, work out whether we need to truncate:
try:
lastseg = segs[-1]
except IndexError:
truncate = False
else:
truncate = online and newdag and lastseg[1] == dataend
# if final segment is shorter than two chunks, remove it entirely
# so that it gets processed next time (when it will either a closed
# segment, or long enough to process safely)
if truncate and abs(lastseg) < chunkdur * 2:
logger.info(
"The final segment is too short, but ends at the limit of "
"available data, presumably this is an active segment. It "
"will be removed so that it can be processed properly later", )
segs = type(segs)(segs[:-1])
dataend = lastseg[0]
# otherwise, we remove the final chunk (so that the next run has at
# least that on which to operate), then truncate to an integer number
# of chunks (so that # PSD estimation operates on a consistent amount
# of data)
elif truncate:
logger.info("The final segment touches the limit of available data, "
"the end chunk will be removed to guarantee that the next "
"online run has enough data over which to operate")
t, e = lastseg
e -= chunkdur + padding # remove one chunk
# now truncate to an integer number of chunks
step = chunkdur
while t + chunkdur <= e:
t += step
step = chunkdur - overlap
segs[-1] = type(segs[-1])(lastseg[0], t)
dataend = segs[-1][1]
logger.info("This analysis will now run to %d" % dataend)
# recalculate the processing segment
dataspan = type(segs)([segments.Segment(datastart, dataend)])
# -- find the frames
# find frames under /dev/shm (which creates a cache of temporary files)
if args.cache_file:
cache = read_cache(str(args.cache_file))
# only cache if we have state segments
elif args.use_dev_shm and len(segs):
cache = data.find_frames(ifo,
frametype,
datastart,
dataend,
on_gaps='warn',
tmpdir=cachedir)
# remove cached files at end of process
tempfiles.extend(filter(lambda p: str(cachedir) in p, cache))
# find frames using datafind
else:
cache = data.find_frames(ifo,
frametype,
datastart,
dataend,
on_gaps='warn')
# if not frames for an online run, panic
if not online and len(cache) == 0:
raise RuntimeError("No frames found for %s-%s" % (ifo[0], frametype))
# work out the segments of data available
try:
cachesegs = (segments.cache_segments(cache) & dataspan).coalesce()
except TypeError: # empty cache
cachesegs = type(dataspan)()
alldata = False
else:
try:
alldata = cachesegs[-1][1] >= dataspan[-1][1]
except IndexError: # no data overlapping span
alldata = False
# write cache of frames (only if creating a new DAG)
cachefile = cachedir / "frames.lcf"
keepfiles.append(cachefile)
if newdag:
data.write_cache(cache, cachefile)
oconfig.set('DATA', 'FFL', str(cachefile))
logger.info("Cache of %d frames written to\n%s" % (len(cache), cachefile))
# restrict analysis to available data (and warn about missing data)
if segs - cachesegs:
logger.warning("Not all state times are available in frames")
segs = (cachesegs & segs).coalesce()
# apply minimum duration requirement
segs = type(segs)(s for s in segs if abs(s) >= segdur)
# if all of the data are available, but no analysable segments were found
# (i.e. IFO not in right state for all times), record segments.txt
if newdag and len(segs) == 0 and online and alldata:
logger.info(
"No analysable segments found, but up-to-date data are "
"available. A segments.txt file will be written so we don't "
"have to search these data again", )
segments.write_segments(cachesegs, segfile)
logger.info("Segments written to\n%s" % segfile)
clean_exit(0, tempfiles)
# otherwise not all data are available, so
elif len(segs) == 0 and online:
logger.info("No analysable segments found, please try again later")
clean_exit(0, tempfiles)
elif len(segs) == 0:
raise RuntimeError("No analysable segments found")
# and calculate trigger output segments
trigsegs = type(segs)(type(s)(*s) for s in segs).contract(padding)
# display segments
logger.info("Final data segments selected as")
for seg in segs:
logger.info(" %d %d " % seg + "[%d]" % abs(seg))
logger.info("Duration = %d seconds" % abs(segs))
span = type(trigsegs)([trigsegs.extent()])
logger.info("This will output triggers for")
for seg in trigsegs:
logger.info(" %d %d " % seg + "[%d]" % abs(seg))
logger.info("Duration = %d seconds" % abs(trigsegs))
# -- config omicron config directory --------------------------------------
tempfiles.append(utils.astropy_config_path(rundir))
# -- make parameters files then generate the DAG --------------------------
fileformats = oconfig.output_formats()
# generate a 'master' parameters.txt file for archival purposes
if not newdag: # if not writing new dag, dump parameters.txt files to /tmp
pardir = gettempdir()
parfile, jobfiles = oconfig.write_distributed(
pardir, nchannels=args.max_channels_per_job)
logger.debug("Created master parameters file\n%s" % parfile)
if newdag:
keepfiles.append(parfile)
# create dag
dag = pipeline.CondorDAG(str(logdir / "{}.log".format(DAG_TAG)))
dag.set_dag_file(str(dagpath.with_suffix("")))
# set up condor commands for all jobs
condorcmds = {
'accounting_group': args.condor_accounting_group,
'accounting_group_user': args.condor_accounting_group_user
}
for cmd_ in args.condor_command:
key, value = cmd_.split('=', 1)
condorcmds[key.rstrip().lower()] = value.strip()
# create omicron job
reqmem = condorcmds.pop('request_memory', 1000)
ojob = condor.OmicronProcessJob(args.universe,
args.executable,
subdir=condir,
logdir=logdir,
**condorcmds)
ojob.add_condor_cmd('request_memory', reqmem)
ojob.add_condor_cmd('+OmicronProcess', '"%s"' % group)
# create post-processing job
ppjob = condor.OmicronProcessJob(args.universe,
find_executable('bash'),
subdir=condir,
logdir=logdir,
tag='post-processing',
**condorcmds)
ppjob.add_condor_cmd('+OmicronPostProcess', '"%s"' % group)
ppjob.add_short_opt('e', '')
ppnodes = []
rootmerge = find_executable('omicron-root-merge')
hdf5merge = find_executable('omicron-hdf5-merge')
ligolw_add = find_executable('ligolw_add')
gzip = find_executable('gzip')
# create node to remove files
rmjob = condor.OmicronProcessJob(args.universe,
str(condir / "post-process-rm.sh"),
subdir=condir,
logdir=logdir,
tag='post-processing-rm',
**condorcmds)
rm = find_executable('rm')
rmfiles = []
rmjob.add_condor_cmd('+OmicronPostProcess', '"%s"' % group)
if args.archive:
archivejob = condor.OmicronProcessJob(args.universe,
str(condir / "archive.sh"),
subdir=condir,
logdir=logdir,
tag='archive',
**condorcmds)
archivejob.add_condor_cmd('+OmicronPostProcess', '"%s"' % group)
archivefiles = {}
# loop over data segments
for s, e in segs:
# build trigger segment
ts = s + padding
te = e - padding
td = te - ts
# distribute segment across multiple nodes
nodesegs = oconfig.distribute_segment(s,
e,
nperjob=args.max_chunks_per_job)
omicronfiles = {}
# build node for each parameter file
for i, pf in enumerate(jobfiles):
chanlist = jobfiles[pf]
nodes = []
# loop over distributed segments
for subseg in nodesegs:
if not args.skip_omicron:
# work out files for this job
nodefiles = oconfig.output_files(*subseg)
# build node
node = pipeline.CondorDAGNode(ojob)
node.set_category('omicron')
node.set_retry(args.condor_retry)
node.add_var_arg(str(subseg[0]))
node.add_var_arg(str(subseg[1]))
node.add_file_arg(pf)
for chan in chanlist:
for form, flist in nodefiles[chan].items():
# record file as output from this node
for f in flist:
node._CondorDAGNode__output_files.append(f)
# record file as output for this channel
try:
omicronfiles[chan][form].extend(flist)
except KeyError:
try:
omicronfiles[chan][form] = flist
except KeyError:
omicronfiles[chan] = {form: flist}
dag.add_node(node)
nodes.append(node)
# post-process (one post-processing job per channel
# per data segment)
if not args.skip_postprocessing:
script = condir / "post-process-{}-{}-{}.sh".format(i, s, e)
ppnode = pipeline.CondorDAGNode(ppjob)
ppnode.add_var_arg(str(script))
operations = []
# build post-processing nodes for each channel
for c in chanlist:
operations.append('\n# %s' % c)
chandir = trigdir / c
# work out filenames for coalesced files
archpath = Path(
io.get_archive_filename(
c,
ts,
td,
filetag=afiletag,
ext='root',
))
mergepath = chandir / archpath.name
target = str(archpath.parent)
# add ROOT operations
if 'root' in fileformats:
rootfiles = ' '.join(omicronfiles[c]['root'])
for f in omicronfiles[c]['root']:
ppnode._CondorDAGNode__input_files.append(f)
if args.skip_root_merge or (len(
omicronfiles[c]['root']) == 1):
root = rootfiles
else:
root = str(mergepath)
operations.append('%s %s %s --strict' %
(rootmerge, rootfiles, root))
rmfiles.append(rootfiles)
ppnode._CondorDAGNode__output_files.append(root)
if args.archive:
try:
archivefiles[target].append(root)
except KeyError:
archivefiles[target] = [root]
rmfiles.append(root)
# add HDF5 operations
if 'hdf5' in fileformats:
hdf5files = ' '.join(omicronfiles[c]['hdf5'])
for f in omicronfiles[c]['hdf5']:
ppnode._CondorDAGNode__input_files.append(f)
if args.skip_hdf5_merge or (len(
omicronfiles[c]['hdf5']) == 1):
hdf5 = hdf5files
else:
hdf5 = str(mergepath.with_suffix(".h5"))
operations.append(
'{cmd} {infiles} {outfile}'.format(
cmd=hdf5merge,
infiles=hdf5files,
outfile=hdf5,
), )
rmfiles.append(hdf5files)
ppnode._CondorDAGNode__output_files.append(hdf5)
if args.archive:
try:
archivefiles[target].append(hdf5)
except KeyError:
archivefiles[target] = [hdf5]
rmfiles.append(hdf5)
# add LIGO_LW operations
if 'xml' in fileformats:
xmlfiles = ' '.join(omicronfiles[c]['xml'])
for f in omicronfiles[c]['xml']:
ppnode._CondorDAGNode__input_files.append(f)
if (args.skip_ligolw_add
or len(omicronfiles[c]['xml']) == 1):
xml = xmlfiles
else:
xml = str(mergepath.with_suffix(".xml"))
operations.append(
'%s %s --ilwdchar-compat --output %s' %
(ligolw_add, xmlfiles, xml), )
rmfiles.append(xmlfiles)
ppnode._CondorDAGNode__output_files.append(xml)
if not args.skip_gzip:
operations.append(
'%s --force --stdout %s > %s.gz' %
(gzip, xml, xml))
rmfiles.append(xml)
xml = str(mergepath.with_suffix(".xml.gz"))
ppnode._CondorDAGNode__output_files.append(xml)
if args.archive:
try:
archivefiles[target].append(xml)
except KeyError:
archivefiles[target] = [xml]
rmfiles.append(xml)
# add ASCII operations
if 'txt' in fileformats:
txtfiles = ' '.join(omicronfiles[c]['txt'])
for f in omicronfiles[c]['txt']:
ppnode._CondorDAGNode__input_files.append(f)
if args.archive:
try:
archivefiles[target].append(txtfiles)
except KeyError:
archivefiles[target] = [txtfiles]
rmfiles.append(txtfiles)
ppnode.set_category('postprocessing')
ppnode.set_retry(str(args.condor_retry))
if not args.skip_omicron:
for node in nodes:
ppnode.add_parent(node)
dag.add_node(ppnode)
ppnodes.append(ppnode)
tempfiles.append(script)
# write post-processing file
if not args.rescue:
with script.open("w") as f:
# add header
print('#!/bin/bash -e\n#', file=f)
print("# omicron-process post-processing", file=f)
print(
'#\n# File created by\n# {}\n#'.format(
' '.join(sys.argv), ),
file=f,
)
print("# Group: %s" % group, file=f)
print("# Segment: [%d, %d)" % (s, e), file=f)
print("# Channels:\n#", file=f)
for c in chanlist:
print('# %s' % c, file=f)
# add post-processing operations
print('\n'.join(operations), file=f)
if newdag:
script.chmod(0o755)
# set 'strict' option for Omicron
# this is done after the nodes are written so that 'strict' is last in
# the call
ojob.add_arg('strict')
# do all archiving last, once all post-processing has completed
if args.archive:
archivenode = pipeline.CondorDAGNode(archivejob)
acache = {fmt: list() for fmt in fileformats}
if newdag:
# write shell script to seed archive
with open(archivejob.get_executable(), 'w') as f:
print('#!/bin/bash -e\n', file=f)
for gpsdir, filelist in archivefiles.items():
for fn in filelist:
archivenode._CondorDAGNode__input_files.append(fn)
# write 'mv' op to script
print("mkdir -p %s" % gpsdir, file=f)
print("cp %s %s" % (' '.join(filelist), gpsdir), file=f)
# record archived files in caches
filenames = [
str(Path(gpsdir) / x.name)
for x in map(Path, filelist)
]
for fn in filenames:
archivenode._CondorDAGNode__output_files.append(fn)
for fmt, extensions in {
'xml': ('.xml.gz', '.xml'),
'root': '.root',
'hdf5': '.h5',
'txt': '.txt',
}.items():
try:
acache[fmt].extend(
filter(lambda x: x.endswith(extensions),
filenames))
except KeyError: # file format not used
continue
os.chmod(archivejob.get_executable(), 0o755)
# write caches to disk
for fmt, fcache in acache.items():
cachefile = cachedir / "omicron-{0}.lcf".format(fmt)
data.write_cache(fcache, cachefile)
logger.debug("{0} cache written to {1}".format(fmt, cachefile))
# add node to DAG
for node in ppnodes:
archivenode.add_parent(node)
archivenode.set_retry(args.condor_retry)
archivenode.set_category('archive')
dag.add_node(archivenode)
tempfiles.append(archivejob.get_executable())
# add rm job right at the end
rmnode = pipeline.CondorDAGNode(rmjob)
rmscript = rmjob.get_executable()
with open(rmscript, 'w') as f:
print('#!/bin/bash -e\n#', file=f)
print("# omicron-process post-processing-rm", file=f)
print('#\n# File created by\n# %s\n#' % ' '.join(sys.argv), file=f)
print("# Group: %s" % group, file=f)
print("# Segment: [%d, %d)" % (s, e), file=f)
print("# Channels:\n#", file=f)
for c in channels:
print('# %s' % c, file=f)
print('', file=f)
for rmset in rmfiles:
print('%s -f %s' % (rm, rmset), file=f)
if newdag:
os.chmod(rmscript, 0o755)
tempfiles.append(rmscript)
rmnode.set_category('postprocessing')
if args.archive: # run this after archiving
rmnode.add_parent(archivenode)
else: # or just after post-processing if not archiving
for node in ppnodes:
rmnode.add_parent(node)
dag.add_node(rmnode)
# print DAG to file
dagfile = Path(dag.get_dag_file()).resolve(strict=False)
if args.rescue:
logger.info(
"In --rescue mode, this DAG has been reproduced in memory "
"for safety, but will not be written to disk, the file is:", )
elif newdag:
dag.write_sub_files()
dag.write_dag()
dag.write_script()
with open(dagfile, 'a') as f:
print("DOT", dagfile.with_suffix(".dot"), file=f)
logger.info("Dag with %d nodes written to" % len(dag.get_nodes()))
print(dagfile)
if args.no_submit:
if newdag:
segments.write_segments(span, segfile)
logger.info("Segments written to\n%s" % segfile)
sys.exit(0)
# -- submit the DAG and babysit -------------------------------------------
# submit DAG
if args.rescue:
logger.info("--- Submitting rescue DAG to condor ----")
elif args.reattach:
logger.info("--- Reattaching to existing DAG --------")
else:
logger.info("--- Submitting DAG to condor -----------")
for i in range(args.submit_rescue_dag + 1):
if args.reattach: # find ID of existing DAG
dagid = int(
condor.find_job(Owner=getuser(),
OmicronDAGMan=group)['ClusterId'])
logger.info("Found existing condor ID = %d" % dagid)
else: # or submit DAG
dagmanargs = set()
if online:
dagmanopts = {'-append': '+OmicronDAGMan=\"%s\"' % group}
else:
dagmanopts = {}
for x in args.dagman_option:
x = '-%s' % x
try:
key, val = x.split('=', 1)
except ValueError:
dagmanargs.add(x)
else:
dagmanopts[key] = val
dagid = condor.submit_dag(
str(dagfile),
*list(dagmanargs),
**dagmanopts,
)
logger.info("Condor ID = %d" % dagid)
# write segments now -- this means that online processing will
# _always_ move on even if the workflow fails
if i == 0:
segments.write_segments(span, segfile)
logger.info("Segments written to\n%s" % segfile)
if 'force' in args.dagman_option:
args.dagman_option.pop(args.dagman_option.index('force'))
# monitor the dag
logger.debug("----------------------------------------")
logger.info("Monitoring DAG:")
check_call([
"pycondor",
"monitor",
"--time",
"5",
"--length",
"36",
str(dagfile),
])
print()
logger.debug("----------------------------------------")
sleep(5)
try:
stat = condor.get_dag_status(dagid)
except OSError as exc: # query failed
logger.warning(str(exc))
stat = {}
# log exitcode
if "exitcode" not in stat:
logger.warning("DAG has exited, status unknown")
break
if not stat["exitcode"]:
logger.info("DAG has exited with status {}".format(
stat.get("exitcode", "unknown"), ))
break
logger.critical(
"DAG has exited with status {}".format(stat['exitcode']), )
# handle failure
if i == args.submit_rescue_dag:
raise RuntimeError("DAG has failed to complete %d times" %
(args.submit_rescue_dag + 1))
else:
rescue = condor.find_rescue_dag(str(dagfile))
logger.warning("Rescue DAG %s was generated" % rescue)
# mark output and error files of condor nodes that passed to be deleted
try:
for node, files in condor.get_out_err_files(dagid, exitcode=0).items():
tempfiles.extend(files)
except RuntimeError:
pass
# archive files
stub = '%d-%d' % (start, end)
for f in map(Path, ["{}.dagman.out".format(dagfile)] + keepfiles):
archive = logdir / "{0[0]}.{1}.{0[1]}".format(
f.name.split(".", 1),
stub,
)
if str(f) == str(segfile):
shutil.copyfile(f, archive)
else:
f.rename(archive)
logger.debug("Archived path\n{} --> {}".format(f, archive))
# clean up temporary files
tempfiles.extend(trigdir.glob("ffconvert.*.ffl"))
clean_tempfiles(tempfiles)
# and exit
logger.info("--- Processing complete ----------------")