def _update_defaults_from_channels(self):
c1, c2 = self.channels
self.pargs.setdefault('xlabel', texify(str(c1)))
self.pargs.setdefault('ylabel', texify(str(c2)))
if hasattr(c1, 'amplitude_range'):
self.pargs.setdefault('xlim', c1.amplitude_range)
if hasattr(c2, 'amplitude_range'):
self.pargs.setdefault('ylim', c2.amplitude_range)
def init_plot(self,
data=[],
FigureClass=Plot,
geometry=(1, 1),
projection='rectilinear',
sharex=True,
sharey=True,
**kwargs):
"""Initialise the Figure and Axes objects for this `DataPlot`.
"""
# update plot defaults using channel data
self._update_defaults_from_channels()
# strip figure and axes params from pargs
for key in NON_PLOT_PARAMS:
try:
kwargs.setdefault(key, self.pargs.pop(key))
except KeyError:
continue
# escape text for TeX
if key in ('title', 'xlabel', 'ylabel'):
kwargs[key] = texify(kwargs[key])
# create figure
self.plot = FigureClass(*data,
geometry=geometry,
projection=projection,
sharex=sharex,
sharey=sharey,
**kwargs)
return self.plot
def draw(self):
"""Read in all necessary data, and generate the figure.
"""
plot = self.init_plot()
ax = plot.gca()
# work out labels
labels = self.pargs.pop('labels', self.channels)
if isinstance(labels, str):
labels = labels.split(',')
labels = [str(s).strip('\n ') for s in labels]
# add data
for label, channel in zip(labels, self.channels):
label = texify(label)
if self.state and not self.all_data:
valid = self.state.active
else:
valid = SegmentList([self.span])
data = get_timeseries(channel, valid, query=False)
# handle no timeseries
if not len(data):
ax.plot([0], [0], visible=False, label=label)
continue
# plot time-series
color = None
for ts in data:
# double-check log scales
if self.logy:
ts.value[ts.value == 0] = 1e-100
if color is None:
line = ax.plot(ts, label=label)[0]
color = line.get_color()
else:
ax.plot(ts, color=color, label=None)
# allow channel data to set parameters
if hasattr(data[0].channel, 'amplitude_range'):
self.pargs.setdefault('ylim', data[0].channel.amplitude_range)
# add horizontal lines to add
for yval in self.pargs['hline']:
try:
yval = float(yval)
except ValueError:
continue
else:
ax.plot([self.start, self.end], [yval, yval],
linestyle='--',
color='red')
# customise plot
legendargs = self.parse_legend_kwargs()
self.apply_parameters(ax, **self.pargs)
if len(self.channels) > 1:
ax.legend(**legendargs)
# finalise
self.add_state_segments(ax)
return self.finalize()
def get_channel_groups(self):
"""Find and group (mean, min, max) sets of channels for plotting.
Returns
-------
groups : `list` of `tuple`
list of (channelname, channellist) tuples giving core channel
name and an ordered list of channels. Ordering in preference
of 'rms', 'mean', 'min', 'max'.
Notes
-----
This method used to return an `OrderedDict`, but was changed to
return a `list` of `tuple` to enable plotting a channel multiple
times on a plot, for whatever reason.
"""
all_ = self.channels
out = []
for c in all_:
if c.ifo == 'G1' and re.search(r'-(av|min|max)\Z', c.name):
name = texify(c.name.rsplit('-', 1)[0])
else:
name = texify(c.name.rsplit('.', 1)[0])
if ' ' in c.name:
out.append((texify(c.name), [c]))
else:
try:
id_ = list(zip(*out))[0].index(name)
except (IndexError, ValueError):
out.append((name, [c]))
else:
out[id_][1].append(c)
order = ['rms', 'mean', 'av', 'min', 'max']
for channel, clist in out:
clist.sort(key=lambda c: c.name.split('.')[-1] in order and order.
index(c.name.split('.')[-1]) + 1 or 10)
return out
def __init__(self,
channels,
start,
end,
state=None,
outdir='.',
etg=None,
**kwargs):
if (len(channels) == 1) and ('title' not in kwargs):
kwargs['title'] = texify('%s (%s)' % (str(channels[0]), etg))
super(TriggerDataPlot, self).__init__(channels,
start,
end,
state=state,
outdir=outdir,
**kwargs)
self.etg = etg
self.columns = [self.pargs.pop(c) for c in ('x', 'y', 'color')]
def draw(self, outputfile=None):
"""Get data and generate the figure.
"""
# get histogram parameters
plot = self.init_plot()
ax = plot.gca()
if self.state:
self.pargs.setdefault(
'suptitle', '[%s-%s, state: %s]' %
(self.span[0], self.span[1], texify(str(self.state))))
suptitle = self.pargs.pop('suptitle', None)
if suptitle:
plot.suptitle(suptitle, y=0.993, va='top')
# get data
data = []
for channel in self.channels:
if self.state and not self.all_data:
valid = self.state.active
else:
valid = SegmentList([self.span])
data.append(
get_timeseries(channel, valid,
query=False).join(gap='ignore', pad=numpy.nan))
if len(data) == 1:
data.append(data[0])
# histogram
hist_kwargs = self.parse_hist_kwargs()
h, xedges, yedges = numpy.histogram2d(data[0], data[1], **hist_kwargs)
h = numpy.ma.masked_where(h == 0, h)
x, y = numpy.meshgrid(xedges, yedges, copy=False, sparse=True)
# plot
pcmesh_kwargs = self.parse_pcmesh_kwargs()
ax.pcolormesh(x, y, h.T, **pcmesh_kwargs)
# customise plot
self.apply_parameters(ax, **self.pargs)
return self.finalize(outputfile=outputfile)
def main(args=None):
"""Run the hveto command-line interface
"""
# declare global variables
# this is needed for multiprocessing utilities
global acache, analysis, areadkw, atrigfindkw, auxiliary, auxetg
global auxfreq, counter, livetime, minsnr, naux, pchannel, primary
global rnd, snrs, windows
# parse command-line
parser = create_parser()
args = parser.parse_args(args=args)
ifo = args.ifo
start = int(args.gpsstart)
end = int(args.gpsend)
duration = end - start
# log startup
LOGGER.info("-- Welcome to Hveto --")
LOGGER.info("GPS start time: %d" % start)
LOGGER.info("GPS end time: %d" % end)
LOGGER.info("Interferometer: %s" % ifo)
# -- initialisation -------------------------
# read configuration
cp = config.HvetoConfigParser(ifo=ifo)
cp.read(args.config_file)
LOGGER.info("Parsed configuration file(s)")
# format output directory
outdir = _abs_path(args.output_directory)
if not os.path.isdir(outdir):
os.makedirs(outdir)
os.chdir(outdir)
LOGGER.info("Working directory: %s" % outdir)
segdir = 'segments'
plotdir = 'plots'
trigdir = 'triggers'
omegadir = 'scans'
for d in [segdir, plotdir, trigdir, omegadir]:
if not os.path.isdir(d):
os.makedirs(d)
# prepare html variables
htmlv = {
'title': '%s Hveto | %d-%d' % (ifo, start, end),
'config': None,
'prog': PROG,
'context': ifo.lower(),
}
# get segments
aflag = cp.get('segments', 'analysis-flag')
url = cp.get('segments', 'url')
padding = tuple(cp.getfloats('segments', 'padding'))
if args.analysis_segments:
segs_ = DataQualityDict.read(args.analysis_segments, gpstype=float)
analysis = segs_[aflag]
span = SegmentList([Segment(start, end)])
analysis.active &= span
analysis.known &= span
analysis.coalesce()
LOGGER.debug("Segments read from disk")
else:
analysis = DataQualityFlag.query(aflag, start, end, url=url)
LOGGER.debug("Segments recovered from %s" % url)
if padding != (0, 0):
mindur = padding[0] - padding[1]
analysis.active = type(analysis.active)([s for s in analysis.active if
abs(s) >= mindur])
analysis.pad(*padding, inplace=True)
LOGGER.debug("Padding %s applied" % str(padding))
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("Retrieved %d segments for %s with %ss (%.2f%%) livetime"
% (len(analysis.active), aflag, livetime, livetimepc))
# apply vetoes from veto-definer file
try:
vetofile = cp.get('segments', 'veto-definer-file')
except configparser.NoOptionError:
vetofile = None
else:
try:
categories = cp.getfloats('segments', 'veto-definer-categories')
except configparser.NoOptionError:
categories = None
# read file
vdf = read_veto_definer_file(vetofile, start=start, end=end, ifo=ifo)
LOGGER.debug("Read veto-definer file from %s" % vetofile)
# get vetoes from segdb
vdf.populate(source=url, segments=analysis.active, on_error='warn')
# coalesce flags from chosen categories
vetoes = DataQualityFlag('%s:VDF-VETOES:1' % ifo)
nflags = 0
for flag in vdf:
if not categories or vdf[flag].category in categories:
vetoes += vdf[flag]
nflags += 1
try:
deadtime = int(abs(vetoes.active)) / int(abs(vetoes.known)) * 100
except ZeroDivisionError:
deadtime = 0
LOGGER.debug("Coalesced %ss (%.2f%%) of deadtime from %d veto flags"
% (abs(vetoes.active), deadtime, nflags))
# apply to analysis segments
analysis -= vetoes
LOGGER.debug("Applied vetoes from veto-definer file")
livetime = int(abs(analysis.active))
livetimepc = livetime / duration * 100.
LOGGER.info("%ss (%.2f%%) livetime remaining after vetoes"
% (livetime, livetimepc))
snrs = cp.getfloats('hveto', 'snr-thresholds')
minsnr = min(snrs)
windows = cp.getfloats('hveto', 'time-windows')
# record all segments
segments = DataQualityDict()
segments[analysis.name] = analysis
# -- load channels --------------------------
# get primary channel name
pchannel = cp.get('primary', 'channel')
# read auxiliary cache
if args.auxiliary_cache is not None:
acache = read_cache(args.auxiliary_cache)
else:
acache = None
# load auxiliary channels
auxetg = cp.get('auxiliary', 'trigger-generator')
auxfreq = cp.getfloats('auxiliary', 'frequency-range')
try:
auxchannels = cp.get('auxiliary', 'channels').strip('\n').split('\n')
except config.configparser.NoOptionError:
auxchannels = find_auxiliary_channels(auxetg, (start, end), ifo=ifo,
cache=acache)
cp.set('auxiliary', 'channels', '\n'.join(auxchannels))
LOGGER.debug("Auto-discovered %d "
"auxiliary channels" % len(auxchannels))
else:
auxchannels = sorted(set(auxchannels))
LOGGER.debug("Read list of %d auxiliary channels" % len(auxchannels))
# load unsafe channels list
_unsafe = cp.get('safety', 'unsafe-channels')
if os.path.isfile(_unsafe): # from file
unsafe = set()
with open(_unsafe, 'rb') as f:
for c in f.read().rstrip('\n').split('\n'):
if c.startswith('%(IFO)s'):
unsafe.add(c.replace('%(IFO)s', ifo))
elif not c.startswith('%s:' % ifo):
unsafe.add('%s:%s' % (ifo, c))
else:
unsafe.add(c)
else: # or from line-seprated list
unsafe = set(_unsafe.strip('\n').split('\n'))
unsafe.add(pchannel)
cp.set('safety', 'unsafe-channels', '\n'.join(sorted(unsafe)))
LOGGER.debug("Read list of %d unsafe channels" % len(unsafe))
# remove unsafe channels
nunsafe = 0
for i in range(len(auxchannels) - 1, -1, -1):
if auxchannels[i] in unsafe:
LOGGER.warning("Auxiliary channel %r identified as unsafe and has "
"been removed" % auxchannels[i])
auxchannels.pop(i)
nunsafe += 1
LOGGER.debug("%d auxiliary channels identified as unsafe" % nunsafe)
naux = len(auxchannels)
LOGGER.info("Identified %d auxiliary channels to process" % naux)
# record INI file in output HTML directory
inifile = '%s-HVETO_CONFIGURATION-%d-%d.ini' % (ifo, start, duration)
if os.path.isfile(inifile) and any(
os.path.samefile(inifile, x) for x in args.config_file):
LOGGER.debug("Cannot write INI file to %s, file was given as input")
else:
with open(inifile, 'w') as f:
cp.write(f)
LOGGER.info("Configuration recorded as %s" % inifile)
htmlv['config'] = inifile
# -- load primary triggers ------------------
# read primary cache
if args.primary_cache is not None:
pcache = read_cache(args.primary_cache)
else:
pcache = None
# load primary triggers
petg = cp.get('primary', 'trigger-generator')
psnr = cp.getfloat('primary', 'snr-threshold')
pfreq = cp.getfloats('primary', 'frequency-range')
preadkw = cp.getparams('primary', 'read-')
if pcache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the primary trigger file format')
preadkw['format'] = None
preadkw['path'] = 'triggers'
ptrigfindkw = cp.getparams('primary', 'trigfind-')
primary = get_triggers(pchannel, petg, analysis.active, snr=psnr,
frange=pfreq, cache=pcache, nproc=args.nproc,
trigfind_kwargs=ptrigfindkw, **preadkw)
fcol, scol = primary.dtype.names[1:3]
if len(primary):
LOGGER.info("Read %d events for %s" % (len(primary), pchannel))
else:
message = "No events found for %r in %d seconds of livetime" % (
pchannel, livetime)
LOGGER.critical(message)
# cluster primary triggers
clusterkwargs = cp.getparams('primary', 'cluster-')
if clusterkwargs:
primary = primary.cluster(**clusterkwargs)
LOGGER.info("%d primary events remain after clustering over %s" %
(len(primary), clusterkwargs['rank']))
# -- bail out early -------------------------
# the bail out is done here so that we can at least generate the eventual
# configuration file, mainly for HTML purposes
# no segments
if livetime == 0:
message = ("No active segments found for analysis flag %r in interval "
"[%d, %d)" % (aflag, start, end))
LOGGER.critical(message)
htmlv['context'] = 'info'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# no primary triggers
if len(primary) == 0:
htmlv['context'] = 'danger'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# otherwise write all primary triggers to ASCII
trigfile = os.path.join(
trigdir,
'%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration),
)
primary.write(trigfile, format='ascii', overwrite=True)
# -- load auxiliary triggers ----------------
LOGGER.info("Reading triggers for aux channels...")
counter = multiprocessing.Value('i', 0)
areadkw = cp.getparams('auxiliary', 'read-')
if acache is not None: # auto-detect the file format
LOGGER.debug('Unsetting the auxiliary trigger file format')
areadkw['format'] = None
areadkw['path'] = 'triggers'
atrigfindkw = cp.getparams('auxiliary', 'trigfind-')
# map with multiprocessing
if args.nproc > 1:
pool = multiprocessing.Pool(processes=args.nproc)
results = pool.map(_get_aux_triggers, auxchannels)
pool.close()
# map without multiprocessing
else:
results = map(_get_aux_triggers, auxchannels)
LOGGER.info("All aux events loaded")
auxiliary = dict(x for x in results if x is not None)
auxchannels = sorted(auxiliary.keys())
chanfile = '%s-HVETO_CHANNEL_LIST-%d-%d.txt' % (ifo, start, duration)
with open(chanfile, 'w') as f:
for chan in auxchannels:
print(chan, file=f)
LOGGER.info("Recorded list of valid auxiliary channels in %s" % chanfile)
# -- execute hveto analysis -----------------
minsig = cp.getfloat('hveto', 'minimum-significance')
pevents = [primary]
pvetoed = []
auxfcol, auxscol = auxiliary[auxchannels[0]].dtype.names[1:3]
slabel = plot.get_column_label(scol)
flabel = plot.get_column_label(fcol)
auxslabel = plot.get_column_label(auxscol)
auxflabel = plot.get_column_label(auxfcol)
rounds = []
rnd = core.HvetoRound(1, pchannel, rank=scol)
rnd.segments = analysis.active
while True:
LOGGER.info("-- Processing round %d --" % rnd.n)
# write segments for this round
segfile = os.path.join(
segdir, '%s-HVETO_ANALYSIS_SEGS_ROUND_%d-%d-%d.txt'
% (ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.segments)
# calculate significances for this round
if args.nproc > 1: # multiprocessing
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_find_max_significance, chunks)
pool.close()
winners, sigsets = zip(*results)
# find winner of chunk winners
winner = sorted(winners, key=lambda w: w.significance)[-1]
# flatten sets of significances into one list
newsignificances = sigsets[0]
for subdict in sigsets[1:]:
newsignificances.update(subdict)
else: # single process
winner, newsignificances = core.find_max_significance(
primary, auxiliary, pchannel, snrs, windows, rnd.livetime)
LOGGER.info("Round %d winner: %s" % (rnd.n, winner.name))
# plot significance drop here for the last round
# only now do we actually have the new data to
# calculate significance drop
if rnd.n > 1:
svg = (pngname % 'SIG_DROP').replace('.png', '.svg') # noqa: F821
plot.significance_drop(
svg, oldsignificances, newsignificances, # noqa: F821
title=' | '.join([title, subtitle]), # noqa: F821
bbox_inches='tight')
LOGGER.debug("Figure written to %s" % svg)
svg = FancyPlot(svg, caption=plot.ROUND_CAPTION['SIG_DROP'])
rounds[-1].plots.append(svg)
oldsignificances = newsignificances # noqa: F841
# break out of the loop if the significance is below stopping point
if winner.significance < minsig:
LOGGER.info("Maximum signifiance below stopping point")
LOGGER.debug(" (%.2f < %.2f)" % (winner.significance, minsig))
LOGGER.info("-- Rounds complete! --")
break
# work out the vetoes for this round
allaux = auxiliary[winner.name][
auxiliary[winner.name][auxscol] >= winner.snr]
winner.events = allaux
coincs = allaux[core.find_coincidences(allaux['time'], primary['time'],
dt=winner.window)]
rnd.vetoes = winner.get_segments(allaux['time'])
flag = DataQualityFlag(
'%s:HVT-ROUND_%d:1' % (ifo, rnd.n), active=rnd.vetoes,
known=rnd.segments,
description="winner=%s, window=%s, snr=%s" % (
winner.name, winner.window, winner.snr))
segments[flag.name] = flag
LOGGER.debug("Generated veto segments for round %d" % rnd.n)
# link events before veto for plotting
before = primary
beforeaux = auxiliary[winner.name]
# apply vetoes to primary
primary, vetoed = core.veto(primary, rnd.vetoes)
pevents.append(primary)
pvetoed.append(vetoed)
LOGGER.debug("Applied vetoes to primary")
# record results
rnd.winner = winner
rnd.efficiency = (len(vetoed), len(primary) + len(vetoed))
rnd.use_percentage = (len(coincs), len(winner.events))
if rnd.n > 1:
rnd.cum_efficiency = (
len(vetoed) + rounds[-1].cum_efficiency[0],
rounds[0].efficiency[1])
rnd.cum_deadtime = (
rnd.deadtime[0] + rounds[-1].cum_deadtime[0],
livetime)
else:
rnd.cum_efficiency = rnd.efficiency
rnd.cum_deadtime = rnd.deadtime
# apply vetoes to auxiliary
if args.nproc > 1: # multiprocess
# separate channel list into chunks and process each chunk
pool = multiprocessing.Pool(
processes=min(args.nproc, len(auxiliary.keys())))
chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc)
results = pool.map(_veto, chunks)
pool.close()
auxiliary = results[0]
for subdict in results[1:]:
auxiliary.update(subdict)
else: # single process
auxiliary = core.veto_all(auxiliary, rnd.vetoes)
LOGGER.debug("Applied vetoes to auxiliary channels")
# log results
LOGGER.info("""Results for round %d:\n\n
winner : %s
significance : %s
mu : %s
snr : %s
dt : %s
use_percentage : %s
efficiency : %s
deadtime : %s
cum. efficiency : %s
cum. deadtime : %s\n\n""" % (
rnd.n, rnd.winner.name, rnd.winner.significance,
rnd.winner.mu, rnd.winner.snr, rnd.winner.window,
rnd.use_percentage, rnd.efficiency, rnd.deadtime,
rnd.cum_efficiency, rnd.cum_deadtime))
# write segments
segfile = os.path.join(
segdir,
'%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % (
ifo, rnd.n, start, duration))
write_ascii_segments(segfile, rnd.vetoes)
LOGGER.debug("Round %d vetoes written to %s" % (rnd.n, segfile))
rnd.files['VETO_SEGS'] = (segfile,)
# write triggers
trigfile = os.path.join(
trigdir,
'%s-HVETO_%%s_TRIGS_ROUND_%d-%d-%d.txt' % (
ifo, rnd.n, start, duration))
for tag, arr in zip(
['WINNER', 'VETOED', 'RAW'],
[winner.events, vetoed, primary]):
f = trigfile % tag
arr.write(f, format='ascii', overwrite=True)
LOGGER.debug("Round %d %s events written to %s"
% (rnd.n, tag.lower(), f))
rnd.files[tag] = f
# record times to omega scan
if args.omega_scans:
N = len(vetoed)
ind = random.sample(range(0, N), min(args.omega_scans, N))
rnd.scans = vetoed[ind]
LOGGER.debug("Collected %d events to omega scan:\n\n%s\n\n"
% (len(rnd.scans), rnd.scans))
# -- make some plots --
pngname = os.path.join(plotdir, '%s-HVETO_%%s_ROUND_%d-%d-%d.png' % (
ifo, rnd.n, start, duration))
wname = texify(rnd.winner.name)
beforel = 'Before\n[%d]' % len(before)
afterl = 'After\n[%d]' % len(primary)
vetoedl = 'Vetoed\n(primary)\n[%d]' % len(vetoed)
beforeauxl = 'All\n[%d]' % len(beforeaux)
usedl = 'Used\n(aux)\n[%d]' % len(winner.events)
coincl = 'Coinc.\n[%d]' % len(coincs)
title = '%s Hveto round %d' % (ifo, rnd.n)
ptitle = '%s: primary impact' % title
atitle = '%s: auxiliary use' % title
subtitle = 'winner: %s [%d-%d]' % (wname, start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
plot.before_after_histogram(
png, before[scol], primary[scol],
label1=beforel, label2=afterl, xlabel=slabel,
title=ptitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['HISTOGRAM'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(
png, before, vetoed, x='time', y=scol, label1=beforel,
label2=vetoedl, epoch=start, xlim=[start, end], ylabel=slabel,
title=ptitle, subtitle=subtitle, legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'SNR_%s' % fcol.upper()
plot.veto_scatter(
png, before, vetoed, x=fcol, y=scol, label1=beforel,
label2=vetoedl, xlabel=flabel, ylabel=slabel, xlim=pfreq,
title=ptitle, subtitle=subtitle, legend_title="Primary:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % '%s_TIME' % fcol.upper()
plot.veto_scatter(
png, before, vetoed, x='time', y=fcol, color=scol,
label1=None, label2=None, ylabel=flabel,
clabel=slabel, clim=[3, 100], cmap='YlGnBu',
epoch=start, xlim=[start, end], ylim=pfreq,
title=ptitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['TIME'])
rnd.plots.append(png)
# aux used versus frequency
png = pngname % 'USED_SNR_TIME'
plot.veto_scatter(
png, winner.events, vetoed, x='time', y=[auxscol, scol],
label1=usedl, label2=vetoedl, ylabel=slabel, epoch=start,
xlim=[start, end], title=atitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['USED_SNR_TIME'])
rnd.plots.append(png)
# snr versus time
png = pngname % 'AUX_SNR_TIME'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x='time', y=auxscol,
label1=beforeauxl, label2=(usedl, coincl), epoch=start,
xlim=[start, end], ylabel=auxslabel, title=atitle,
subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_TIME'])
rnd.plots.append(png)
# snr versus frequency
png = pngname % 'AUX_SNR_FREQUENCY'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x=auxfcol, y=auxscol,
label1=beforeauxl, label2=(usedl, coincl), xlabel=auxflabel,
ylabel=auxslabel, title=atitle, subtitle=subtitle,
legend_title="Aux:")
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_FREQUENCY'])
rnd.plots.append(png)
# frequency versus time coloured by SNR
png = pngname % 'AUX_FREQUENCY_TIME'
plot.veto_scatter(
png, beforeaux, (winner.events, coincs), x='time', y=auxfcol,
color=auxscol, label1=None, label2=[None, None], ylabel=auxflabel,
clabel=auxslabel, clim=[3, 100], cmap='YlGnBu', epoch=start,
xlim=[start, end], title=atitle, subtitle=subtitle)
LOGGER.debug("Figure written to %s" % png)
png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_FREQUENCY_TIME'])
rnd.plots.append(png)
# move to the next round
rounds.append(rnd)
rnd = core.HvetoRound(rnd.n + 1, pchannel, rank=scol,
segments=rnd.segments-rnd.vetoes)
# write file with all segments
segfile = os.path.join(
segdir, '%s-HVETO_SEGMENTS-%d-%d.h5' % (ifo, start, duration))
segments.write(segfile, overwrite=True)
LOGGER.debug("Segment summary written to %s" % segfile)
LOGGER.debug("Making summary figures...")
# -- exit early if no rounds above threshold
if not rounds:
message = ("No rounds completed above threshold. Analysis stopped "
"with %s achieving significance of %.2f"
% (winner.name, winner.significance))
LOGGER.critical(message)
message = message.replace(
winner.name, cis_link(winner.name, class_='alert-link'))
message += '<br>[T<sub>win</sub>: %ss, SNR: %s]' % (
winner.window, winner.snr)
htmlv['context'] = 'warning'
index = html.write_null_page(ifo, start, end, message, **htmlv)
LOGGER.info("HTML report written to %s" % index)
sys.exit(0)
# -- plot all rounds impact
pngname = os.path.join(plotdir, '%s-HVETO_%%s_ALL_ROUNDS-%d-%d.png' % (
ifo, start, duration))
plots = []
title = '%s Hveto all rounds' % args.ifo
subtitle = '%d rounds | %d-%d' % (len(rounds), start, end)
# before/after histogram
png = pngname % 'HISTOGRAM'
beforel = 'Before analysis [%d events]' % len(pevents[0])
afterl = 'After %d rounds [%d]' % (len(pevents) - 1, len(pevents[-1]))
plot.before_after_histogram(
png, pevents[0][scol], pevents[-1][scol],
label1=beforel, label2=afterl, xlabel=slabel,
title=title, subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['HISTOGRAM'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# efficiency/deadtime curve
png = pngname % 'ROC'
plot.hveto_roc(png, rounds, title=title, subtitle=subtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['ROC'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# frequency versus time
png = pngname % '%s_TIME' % fcol.upper()
labels = [str(r.n) for r in rounds]
legtitle = 'Vetoed at\nround'
plot.veto_scatter(
png, pevents[0], pvetoed,
label1='', label2=labels, title=title,
subtitle=subtitle, ylabel=flabel, x='time', y=fcol,
epoch=start, xlim=[start, end], legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# snr versus time
png = pngname % 'SNR_TIME'
plot.veto_scatter(
png, pevents[0], pvetoed, label1='', label2=labels, title=title,
subtitle=subtitle, ylabel=slabel, x='time', y=scol,
epoch=start, xlim=[start, end], legend_title=legtitle)
png = FancyPlot(png, caption=plot.HEADER_CAPTION['SNR_TIME'])
plots.append(png)
LOGGER.debug("Figure written to %s" % png)
# -- write summary states to ASCII table and JSON
json_ = {
'user': getuser(),
'host': getfqdn(),
'date': str(datetime.datetime.now()),
'configuration': inifile,
'ifo': ifo,
'gpsstart': start,
'gpsend': end,
'call': ' '.join(sys.argv),
'rounds': [],
}
with open('summary-stats.txt', 'w') as f:
# print header
print('#N winner window SNR significance nveto use-percentage '
'efficiency deadtime cumulative-efficiency cumulative-deadtime',
file=f)
for r in rounds:
# extract relevant statistics
results = [
('round', r.n),
('name', r.winner.name),
('window', r.winner.window),
('snr', r.winner.snr),
('significance', r.winner.significance),
('nveto', r.efficiency[0]),
('use-percentage',
r.use_percentage[0] / r.use_percentage[1] * 100.),
('efficiency', r.efficiency[0] / r.efficiency[1] * 100.),
('deadtime', r.deadtime[0] / r.deadtime[1] * 100.),
('cumulative-efficiency',
r.cum_efficiency[0] / r.cum_efficiency[1] * 100.),
('cumulative-deadtime',
r.cum_deadtime[0] / r.cum_deadtime[1] * 100.),
]
# write to ASCII
print(' '.join(map(str, list(zip(*results))[1])), file=f)
# write to JSON
results.append(('files', r.files))
json_['rounds'].append(dict(results))
LOGGER.debug("Summary table written to %s" % f.name)
with open('summary-stats.json', 'w') as f:
json.dump(json_, f, sort_keys=True)
LOGGER.debug("Summary JSON written to %s" % f.name)
# -- generate workflow for omega scans
if args.omega_scans:
omegatimes = list(map(str, sorted(numpy.unique(
[t['time'] for r in rounds for t in r.scans]))))
LOGGER.debug("Collected %d times to omega scan" % len(omegatimes))
newtimes = [t for t in omegatimes if not
os.path.exists(os.path.join(omegadir, str(t)))]
LOGGER.debug("%d scans already complete or in progress, %d remaining"
% (len(omegatimes) - len(newtimes), len(newtimes)))
if len(newtimes) > 0:
LOGGER.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(
ifo=ifo,
ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(
timeout=4,
extra_commands=["request_disk='1G'"],
gps=start)
batch.generate_dag(
newtimes,
flags=flags,
submit=True,
outdir=omegadir,
condor_commands=condorcmds)
LOGGER.info('Launched {} omega scans to condor'.format(
len(newtimes)))
else:
LOGGER.debug('Skipping omega scans')
# -- write HTML and finish
index = html.write_hveto_page(
ifo, start, end, rounds, plots,
winners=[r.winner.name for r in rounds], **htmlv)
LOGGER.debug("HTML written to %s" % index)
LOGGER.debug("Analysis completed in %d seconds" % (time.time() - JOBSTART))
LOGGER.info("-- Hveto complete --")
def from_ini(cls, config, section, plotdir='plots', **kwargs):
"""Define a new `GuardianTab`.
"""
new = super(DataTab, cls).from_ini(config, section, **kwargs)
new.plots = []
new.plotdir = plotdir
new.ifo = config.get(section, 'IFO')
# record node and states
new.node = config.get(section, 'node')
new.grdstates = OrderedDict()
plot = []
for key, name in config.nditems(section):
try:
key = int(key)
except ValueError:
continue
else:
if name[0] == '*':
plot.append(False)
else:
plot.append(True)
new.grdstates[int(key)] = name.strip('*')
try:
new.transstates = list(
map(int,
config.get(section, 'transitions').split(',')))
except NoOptionError:
new.transstates = list(new.grdstates)
# -- build plots ------------------------
new.set_layout([1, 2])
grdidxs = dict((state, idx) for idx, state in new.grdstates.items())
new.segmenttag = '%s:%s %%s' % (new.ifo, new.node)
pstates = [
l for i, l in enumerate(list(new.grdstates.values())[::-1])
if plot[-i - 1]
]
flags = [new.segmenttag % name for name in pstates]
labels = ['[%d] %s' % (grdidxs[state], state) for state in pstates]
# define colours
cmap = get_cmap('brg')(Normalize(-1, 1)(numpy.linspace(
-1, 1, len(pstates))))[::-1]
th = len(flags) > 8 and (new.span[1] - new.span[0]) / 200. or 0
for state in new.states:
# get common plot tag prefix
tagprefix = 'GRD_%s' % re.sub(r'[-\s]', '_', new.node.upper())
if state.name.lower() != ALLSTATE: # include state name if not All
tagprefix = '%s_%s' % (state.name, tagprefix)
# segment plot
new.plots.append(
get_plot('guardian')(flags,
new.span[0],
new.span[1],
state=state,
labels=labels,
outdir=plotdir,
known={
'hatch': 'x',
'alpha': 0.1,
'facecolor': 'none'
},
tag='%s_SEGMENTS' % tagprefix,
title='%s Guardian %s state' %
(new.ifo, texify(new.node)),
zorder=2))
# pie
new.plots.append(
get_plot('segment-pie')(
flags,
new.span[0],
new.span[1],
state=state,
labels=pstates,
colors=cmap,
tag='%s_SEGMENT_PIE' % tagprefix,
startangle=180,
counterclock=False,
wedge_linewidth=0.01,
outdir=plotdir,
title='%s Guardian %s state' % (new.ifo, texify(new.node)),
legend_fontsize=16,
legend_sorted=True,
legend_threshold=th,
))
# bar
new.plots.append(
get_plot('segment-bar')(
flags,
new.span[0],
new.span[1],
state=state,
labels=pstates,
sorted=True,
tag='%s_SEGMENT_BAR' % tagprefix,
outdir=plotdir,
title='%s Guardian %s state' % (new.ifo, texify(new.node)),
))
return new
def draw(self):
from ..tabs.guardian import (REQUESTSTUB, NOMINALSTUB,
re_guardian_index)
plot = self.init_plot()
ax = plot.gca()
# get labels
flags = [texify(str(f)) for f in self.flags]
labels = self.pargs.pop('labels', self.pargs.pop('label', flags))
ax.set_insetlabels(self.pargs.pop('insetlabels', True))
if isinstance(labels, str):
labels = labels.split(',')
labels = [re_quote.sub('', str(s).strip('\n ')) for s in labels]
# parse plotting arguments
legendargs = self.parse_legend_kwargs()
plotargs = self.parse_plot_kwargs()[0]
plotargs.pop('label')
height = plotargs.pop('height', .8)
activecolor = plotargs.pop('facecolor')
nominalcolor = self.pargs.pop('nominalcolor')
requestcolor = self.pargs.pop('requestcolor')
actargs = plotargs.copy()
plotargs.update({
'facecolor': nominalcolor,
'edgecolor': 'none',
'known': {
'alpha': 0.1,
'facecolor': 'lightgray'
},
})
reqargs = plotargs.copy()
reqargs.update({
'facecolor': requestcolor,
'known': None,
'height': height,
})
actargs.update({
'facecolor': activecolor,
'known': None,
})
if self.state and not self.all_data:
valid = self.state.active
else:
valid = SegmentList([self.span])
# plot segments
for y, (flag, label) in enumerate(list(zip(self.flags, labels))[::-1]):
inreq = str(flag) + REQUESTSTUB
nominal = str(flag) + NOMINALSTUB
segs = get_segments([flag, inreq, nominal],
validity=valid,
query=False)
# format label
if self.fileformat != 'svg':
try:
idx, label = re_guardian_index.match(label).groups()
except AttributeError:
pass
else:
x = float(self.span[0]) - (float(abs(self.span)) * 0.005)
ax.text(x,
y,
'[%s]' % idx,
ha='right',
va='center',
fontsize=12)
# plot segments
ax.plot(segs[nominal], label=label, y=y, height=1., **plotargs)
ax.plot(segs[inreq],
label=label,
y=y,
collection='ignore',
**reqargs)
ax.plot(segs[flag],
label=label,
y=y,
collection='ignore',
height=.6,
**actargs)
# make custom legend
seg = Segment(self.start - 10, self.start - 9)
v = plotargs.pop('known', None)
n = SegmentRectangle(seg,
0,
facecolor=nominalcolor,
edgecolor=nominalcolor)
a = SegmentRectangle(seg,
0,
facecolor=requestcolor,
edgecolor=requestcolor)
b = SegmentRectangle(seg,
0,
facecolor=activecolor,
edgecolor=actargs['edgecolor'])
handles = [n, a, b]
labels = ['Nominal', 'Request', 'Active']
if v:
v.pop('collection', None)
v['edgecolor'] = v.get('edgecolor') or tint(v['facecolor'], .5)
handles.insert(0, SegmentRectangle(seg, 0, **v))
labels.insert(0, 'Alive')
ax.legend(handles, labels, title='Node state', **legendargs)
# customise plot
for key, val in self.pargs.items():
try:
getattr(ax, 'set_%s' % key)(val)
except AttributeError:
setattr(ax, key, val)
if 'ylim' not in self.pargs:
ax.set_ylim(-0.5, len(self.flags) - 0.5)
epoch = ax.get_epoch()
# add node MODE along the bottom
sax = None
seg_kw = {'y': 0, 'edgecolor': 'none', 'label': 'Mode'}
legentry = OrderedDict()
colors = iter(('#0066ff', '#8000bf', 'hotpink', 'yellow'))
for ctag, mstate in [
# bit listing for channels (Nones are ignored)
('MODE', ('EXEC', 'MANAGED', 'MANUAL')),
('OP', (None, 'PAUSE', None)),
]:
# get data
data = get_timeseries('{0.ifo}:GRD-{0.node}_{1}'.format(
self, ctag),
valid,
query=False).join(gap='pad', pad=-1)
for i, m in filter(lambda x: x[1] is not None, enumerate(mstate)):
x = (data == i).to_dqflag()
fc = next(colors)
if sax is None:
sax = plot.add_segments_bar(x.active,
facecolor=fc,
**seg_kw)
else:
sax.plot_segmentlist(x.active, facecolor=fc, **seg_kw)
legentry[m.title()] = SegmentRectangle(seg,
0,
facecolor=fc,
edgecolor=fc)
seg_kw.update({'collection': 'ignore', 'label': None})
# add OK segments along the bottom
ok = get_segments(flag.split(' ', 1)[0] + ' OK',
validity=valid,
query=False)
seg_kw.pop('edgecolor', None)
sax.plot_segmentlist(ok.active,
facecolor=activecolor,
edgecolor=actargs['edgecolor'],
**seg_kw)
legentry['`OK\''] = SegmentRectangle(seg,
0,
facecolor=activecolor,
edgecolor=actargs['edgecolor'])
# make custom legend
legendargs.update({
'bbox_to_anchor': (1., 0.),
'loc': 'lower left',
})
sax.legend(list(legentry.values()),
list(legentry),
title='Node mode',
**legendargs)
sax.tick_params(axis='y', which='major', labelsize=12)
for ax_ in (ax, sax):
ax_.set_epoch(epoch)
return self.finalize()
def init_plots(self, plotdir=os.curdir):
"""Configure the default list of plots for this tab
This method configures a veto-trigger glitchgram, histograms of
before/after SNR and frequency/template duration,
before and after glitchgrams, and a segment plot.
This method is a mess, and should be re-written in a better way.
"""
namestr = self.title.split('/')[0]
before = get_channel(str(self.channel))
for state in self.states:
# -- configure segment plot
segargs = {
'state': state,
'known': {
'alpha': 0.1,
'facecolor': 'lightgray'
},
'color': 'red',
}
if len(self.flags) == 1:
sp = get_plot('segments')(self.flags,
self.start,
self.end,
outdir=plotdir,
labels=self.labels,
title='Veto segments: %s' %
(texify(namestr)),
**segargs)
else:
sp = get_plot('segments')(
[self.metaflag] + self.flags,
self.start,
self.end,
labels=([self.intersection and 'Intersection' or 'Union'] +
self.labels),
outdir=plotdir,
title='Veto segments: %s' % texify(namestr),
**segargs)
self.plots.append(sp)
if self.channel:
self.set_layout([
2,
])
after = get_channel(
veto_tag(before, self.metaflag, mode='after'))
vetoed = get_channel(
veto_tag(before, self.metaflag, mode='vetoed'))
# -- configure trigger plots
params = etg.get_etg_parameters(self.etg, IFO=before.ifo)
glitchgramargs = {
'etg':
self.etg,
'x':
params['time'],
'y':
params['frequency'],
'yscale':
params.get('frequency-scale', 'log'),
'ylabel':
params.get('frequency-label',
get_column_label(params['frequency'])),
'ylim':
params.get('ylim', [10, 5000]),
'edgecolor':
'none',
'legend-scatterpoints':
1,
'legend-borderaxespad':
0,
'legend-bbox_to_anchor': (1, 1),
'legend-loc':
'upper left',
'legend-frameon':
False,
}
glitchgramargs.update(self.glitchgramargs)
# plot before/after glitchgram
self.plots.append(
get_plot('triggers')([after, vetoed],
self.start,
self.end,
state=state,
title='Impact of %s (%s)' %
(texify(namestr), self.etg),
outdir=plotdir,
labels=['After', 'Vetoed'],
**glitchgramargs))
# plot histograms
if params['det'] != params['snr']:
statistics = ['snr', 'det']
xlims = [(5, 16384), (6, 15), (8, 8192)]
else:
statistics = ['snr']
xlims = [(5, 16384), (8, 8192)]
self.layout.append(len(statistics) + 1)
for column, xlim in zip(statistics + ['frequency'], xlims):
self.plots.append(
get_plot('trigger-histogram')(
[before, after],
self.start,
self.end,
state=state,
column=params[column],
etg=self.etg,
outdir=plotdir,
filterstr=self.filterstr,
title='Impact of %s (%s)' %
(texify(namestr), self.etg),
labels=['Before', 'After'],
xlim=xlim,
xlabel=params.get('%s-label' % column,
get_column_label(
params[column])),
color=['#ffa07a', '#1f77b4'],
alpha=1,
xscale=params.get('%s-scale' % column, 'log'),
yscale='log',
histtype='stepfilled',
weights=1 / float(abs(self.span)),
bins=100,
ylim=(1 / float(abs(self.span)) * 0.5, 1),
**{
'legend-borderaxespad': 0,
'legend-bbox_to_anchor': (1., 1.),
'legend-loc': 'upper left',
'legend-frameon': False,
}))
# plot triggers before and after
for stat in statistics:
column = params[stat]
glitchgramargs.update({
'color':
column,
'clim':
params.get('%s-limits' % stat, [3, 100]),
'logcolor':
params.get('%s-log' % stat, True),
'colorlabel':
params.get('%s-label' % stat,
get_column_label(column)),
})
self.plots.append(
get_plot('triggers')([before],
self.start,
self.end,
state=state,
outdir=plotdir,
filterstr=self.filterstr,
**glitchgramargs))
self.plots.append(
get_plot('triggers')(
[after],
self.start,
self.end,
state=state,
title='%s after %s (%s)' % (texify(
str(self.channel)), texify(namestr), self.etg),
outdir=plotdir,
filterstr=self.filterstr,
**glitchgramargs))
self.layout.append(2)
else:
self.set_layout([
1,
])
def get_column_string(column):
r"""
Format the string columnName (e.g. xml table column) into latex format for
an axis label.
Parameters
----------
column : `str`
string to format
Examples
--------
>>> get_column_string('snr')
'SNR'
>>> get_column_string('bank_chisq_dof')
r'Bank $\chi^2$ DOF'
"""
acro = ['snr', 'ra', 'dof', 'id', 'ms', 'far']
greek = [
'alpha', 'beta', 'gamma', 'delta', 'epsilon', 'zeta', 'eta', 'theta',
'iota', 'kappa', 'lamda', 'mu', 'nu', 'xi', 'omicron', 'pi', 'rho',
'sigma', 'tau', 'upsilon', 'phi', 'chi', 'psi', 'omega'
]
unit = ['ns']
sub = ['flow', 'fhigh', 'hrss', 'mtotal', 'mchirp']
tex = rcParams['text.usetex']
words = []
for word in re.split(r'\s', column):
if word.isupper():
words.append(word)
else:
words.extend(re.split(r'_', word))
# parse words
for i, word in enumerate(words):
# get acronym in lower case
if word in acro:
words[i] = word.upper()
# get numerical unit
elif word in unit:
words[i] = '$(%s)$' % word
# get character with subscript text
elif word in sub and tex:
words[i] = r'%s$_{\mbox{\small %s}}$' % (word[0], word[1:])
# get greek word
elif word in greek and tex:
words[i] = r'$\%s$' % word
# get starting with greek word
elif re.match(r'(%s)' % '|'.join(greek), word) and tex:
if word[-1].isdigit():
words[i] = r'$\%s_{%s}$' '' % tuple(
re.findall(r"[a-zA-Z]+|\d+", word))
elif word.endswith('sq'):
words[i] = r'$\%s^2$' % word.rstrip('sq')
# get everything else
else:
if word.isupper():
words[i] = word
else:
words[i] = word.title()
# escape underscore
words[i] = texify(words[i])
return ' '.join(words)
def draw(self):
# get columns
xcolumn, ycolumn, ccolumn = self.columns
# initialise figure
plot = self.init_plot()
ax = plot.gca()
ax.grid(True, which='both')
# work out labels
labels = self.pargs.pop('labels', self.channels)
if isinstance(labels, str):
labels = labels.split(',')
labels = [str(s).strip('\n ') for s in labels]
# get colouring params
cmap = self.pargs.pop('cmap')
clim = self.pargs.pop('clim', self.pargs.pop('colorlim', None))
cnorm = 'log' if self.pargs.pop('logcolor', False) else None
clabel = self.pargs.pop('colorlabel', None)
no_loudest = self.pargs.pop('no-loudest', False) is not False
loudest_by = self.pargs.pop('loudest-by', None)
# get plot arguments
plotargs = []
for i in range(len(self.channels)):
plotargs.append(dict())
# get plot arguments
for key in [
'vmin', 'vmax', 'edgecolor', 'facecolor', 'cmap', 's',
'marker', 'rasterized'
]:
try:
val = self.pargs.pop(key)
except KeyError:
continue
if key == 'facecolor' and len(self.channels) > 1 and val is None:
val = color_cycle()
if key == 'marker' and len(self.channels) > 1 and val is None:
val = marker_cycle()
elif isinstance(val, (list, tuple, cycle)):
val = cycle(val)
else:
val = cycle([val] * len(self.channels))
for i in range(len(self.channels)):
plotargs[i][key] = next(val)
# add data
valid = SegmentList([self.span])
if self.state and not self.all_data:
valid &= self.state.active
ntrigs = 0
for channel, label, pargs in zip(self.channels, labels, plotargs):
try:
channel = get_channel(channel)
except ValueError:
pass
if '#' in str(channel) or '@' in str(channel):
key = '%s,%s' % (str(channel), self.state and str(self.state)
or 'All')
else:
key = str(channel)
table = get_triggers(key, self.etg, valid, query=False)
if self.filterstr is not None:
table = table.filter(self.filterstr)
ntrigs += len(table)
# access channel parameters for limits
for c, column in zip(('x', 'y', 'c'), (xcolumn, ycolumn, ccolumn)):
if not column:
continue
# hack for SnglBurst frequency nonsense
if column in ['peak_frequency', 'central_freq']:
column = 'frequency'
# set x and y in plotargs
param = '%s_range' % column
lim = '%slim' % c
if (getattr(channel, param, None) is not None
and c in ('x', 'y')):
self.pargs.setdefault(lim, getattr(channel, param))
if isinstance(self.pargs[lim], Quantity):
self.pargs[lim] = self.pargs[lim].value
# set clim separately
elif hasattr(channel, param):
if not clim:
clim = getattr(channel, param)
ax.scatter(table[xcolumn],
table[ycolumn],
c=table[ccolumn] if ccolumn else None,
label=label,
**pargs)
# customise plot
legendargs = self.parse_legend_kwargs(markerscale=3)
if len(self.channels) == 1:
self.pargs.setdefault(
'title', texify('%s (%s)' % (str(self.channels[0]), self.etg)))
for axis in ('x', 'y'): # prevent zeros on log scale
scale = getattr(ax, 'get_{0}scale'.format(axis))()
lim = getattr(ax, 'get_{0}lim'.format(axis))()
if scale == 'log' and lim[0] <= 0 and not ntrigs:
getattr(ax, 'set_{0}lim'.format(axis))(1, 10)
self.apply_parameters(ax, **self.pargs)
# correct log-scale empty axes
if any(map(isinf, ax.get_ylim())):
ax.set_ylim(0.1, 10)
# add colorbar
if ccolumn:
if not ntrigs:
ax.scatter([1], [1], c=[1], visible=False)
ax.colorbar(cmap=cmap, clim=clim, norm=cnorm, label=clabel)
if len(self.channels) == 1 and len(table) and not no_loudest:
columns = [
x for x in (loudest_by or ccolumn or ycolumn, xcolumn, ycolumn,
ccolumn) if x is not None
]
self.add_loudest_event(ax, table, *columns, fontsize='large')
if len(self.channels) > 1:
ax.legend(**legendargs)
# add state segments
if isinstance(ax.xaxis.get_transform(), GPSTransform):
self.add_state_segments(ax)
self.add_future_shade()
# finalise
return self.finalize()
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs):
"""Plot the signifiance drop for each channel
"""
channels = sorted(old.keys())
if show_channel_names is None:
show_channel_names = len(channels) <= 50
plot = Plot(figsize=(20, 5))
plot.subplots_adjust(left=.07, right=.93)
ax = plot.gca()
if show_channel_names:
plot.subplots_adjust(bottom=.4)
winner = sorted(old.items(), key=lambda x: x[1])[-1][0]
for i, c in enumerate(channels):
if c == winner:
color = 'orange'
elif old[c] > new[c]:
color = 'dodgerblue'
else:
color = 'red'
ax.plot([i, i], [old[c], new[c]],
color=color,
linestyle='-',
marker='o',
markeredgecolor='k',
markeredgewidth=.5,
markersize=10,
label=c,
zorder=old[c])
ax.set_xlim(-1, len(channels))
ax.set_ybound(lower=0)
# set xticks to show channel names
if show_channel_names:
ax.set_xticks(range(len(channels)))
ax.set_xticklabels([texify(c) for c in channels])
for i, t in enumerate(ax.get_xticklabels()):
t.set_rotation(270)
t.set_verticalalignment('top')
t.set_horizontalalignment('center')
t.set_fontsize(8)
# or just show systems of channels
else:
plot.canvas.draw()
systems = {}
for i, c in enumerate(channels):
sys = c.split(':', 1)[1].split('-')[0].split('_')[0]
try:
systems[sys][1] += 1
except KeyError:
systems[sys] = [i, 1]
systems = sorted(systems.items(), key=lambda x: x[1][0])
labels, counts = zip(*systems)
xticks, xmticks = zip(*[(a, a + b / 2.) for (a, b) in counts])
# show ticks at the edge of each group
ax.set_xticks(xticks, minor=False)
ax.set_xticklabels([], minor=False)
# show label in the centre of each group
ax.set_xticks(xmticks, minor=True)
for t in ax.set_xticklabels(labels, minor=True):
t.set_rotation(270)
kwargs.setdefault('ylabel', 'Significance')
# create interactivity
if outfile.endswith('.svg'):
_finalize_plot(plot,
ax,
outfile.replace('.svg', '.png'),
close=False,
**kwargs)
tooltips = []
ylim = ax.get_ylim()
yoffset = (ylim[1] - ylim[0]) * 0.061
bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'}
xthresh = len(channels) / 10.
for i, l in enumerate(ax.lines):
x = l.get_xdata()[1]
if x < xthresh:
ha = 'left'
elif x > (len(channels) - xthresh):
ha = 'right'
else:
ha = 'center'
y = l.get_ydata()[0] + yoffset
c = l.get_label()
tooltips.append(
ax.annotate(texify(c), (x, y),
ha=ha,
zorder=ylim[1],
bbox=bbox))
l.set_gid('line-%d' % i)
tooltips[-1].set_gid('tooltip-%d' % i)
f = BytesIO()
plot.savefig(f, format='svg')
tree, xmlid = etree.XMLID(f.getvalue())
tree.set('onload', 'init(evt)')
for i in range(len(tooltips)):
try:
e = xmlid['tooltip-%d' % i]
except KeyError:
warnings.warn("Failed to recover tooltip %d" % i)
continue
e.set('visibility', 'hidden')
for i, l in enumerate(ax.lines):
e = xmlid['line-%d' % i]
e.set('onmouseover', 'ShowTooltip(this)')
e.set('onmouseout', 'HideTooltip(this)')
tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT))
etree.ElementTree(tree).write(outfile)
plot.close()
else:
_finalize_plot(plot, ax, outfile, **kwargs)
def format_label(label):
label = str(label).strip('\n ')
label = re_quote.sub('', label)
return texify(label)
def _process_channel(input_):
"""Handle individual channels for multiprocessing
"""
if USETEX:
gwplot.configure_mpl_tex()
p4 = (.1, .1, .9, .95)
chan = input_[1][0]
ts = input_[1][1]
lassocoef = nonzerocoef[chan]
zeroed = lassocoef == 0
if zeroed:
plot4 = None
plot5 = None
plot6 = None
pcorr = None
else:
plot4 = None
plot5 = None
plot6 = None
if trend_type == 'minute':
pcorr = numpy.corrcoef(ts.value, primaryts.value)[0, 1]
else:
pcorr = 0.0
if abs(lassocoef) < threshold:
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info(
"Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(chan)))
sys.stdout.flush()
return (chan, lassocoef, plot4, plot5, plot6, ts)
# create time series subplots
fig = Plot(figsize=(12, 8))
fig.subplots_adjust(*p4)
ax1 = fig.add_subplot(2, 1, 1, xscale='auto-gps', epoch=start)
ax1.plot(primaryts,
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax1.set_xlabel(None)
ax2 = fig.add_subplot(2, 1, 2, sharex=ax1, xlim=xlim)
ax2.plot(ts, label=texify(chan), linewidth=line_size_aux)
if range_is_primary:
ax1.set_ylabel('Sensitive range [Mpc]')
else:
ax1.set_ylabel('Primary channel units')
ax2.set_ylabel('Channel units')
for ax in fig.axes:
ax.legend(loc='best')
channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1)
plot4 = gwplot.save_figure(fig,
'%s_TRENDS-%s.png' % (channelstub, gpsstub),
bbox_inches='tight')
# create scaled, sign-corrected, and overlayed timeseries
tsscaled = scale(ts.value)
if lassocoef < 0:
tsscaled = numpy.negative(tsscaled)
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p1)
ax = fig.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax.plot(times,
_descaler(tsscaled),
label=texify(chan),
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.legend(loc='best')
plot5 = gwplot.save_figure(fig,
'%s_COMPARISON-%s.png' %
(channelstub, gpsstub),
bbox_inches='tight')
# scatter plot
tsCopy = ts.value.reshape(-1, 1)
primarytsCopy = primaryts.value.reshape(-1, 1)
primaryReg = linear_model.LinearRegression()
primaryReg.fit(tsCopy, primarytsCopy)
primaryFit = primaryReg.predict(tsCopy)
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p1)
ax = fig.gca()
ax.set_xlabel(texify(chan) + ' [Channel units]')
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary channel units')
y_min = min(primaryts.value)
y_max = max(primaryts.value)
y_range = y_max - y_min
ax.set_ylim(y_min - (y_range * 0.1), y_max + (y_range * 0.1))
ax.text(.9,
.1,
'r = ' + str('{0:.2}'.format(pcorr)),
verticalalignment='bottom',
horizontalalignment='right',
transform=ax.transAxes,
color='black',
size=20,
bbox=dict(boxstyle='square',
facecolor='white',
alpha=.75,
edgecolor='black'))
ax.scatter(ts.value, primaryts.value, color='red')
ax.plot(ts.value, primaryFit, color='black')
ax.autoscale_view(tight=False, scalex=True, scaley=True)
plot6 = gwplot.save_figure(fig,
'%s_SCATTER-%s.png' %
(channelstub, gpsstub),
bbox_inches='tight')
# increment counter and print status
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info("Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(chan)))
sys.stdout.flush()
return (chan, lassocoef, plot4, plot5, plot6, ts)
def _draw(self):
"""Load all data, and generate this `SpectrumDataPlot`
"""
plot = self.init_plot()
ax = plot.gca()
ax.grid(b=True, axis='both', which='both')
if self.state:
self.pargs.setdefault(
'suptitle', '[%s-%s, state: %s]' %
(self.span[0], self.span[1], texify(str(self.state))))
suptitle = self.pargs.pop('suptitle', None)
if suptitle:
plot.suptitle(suptitle, y=0.993, va='top')
# get spectrum format: 'amplitude' or 'power'
sdform = self.pargs.pop('format')
if sdform == 'rayleigh':
method = 'rayleigh'
else:
method = None
use_percentiles = str(
self.pargs.pop('no-percentiles')).lower() == 'false'
# parse plotting arguments
plotargs = self.parse_plot_kwargs()
legendargs = self.parse_legend_kwargs()
use_legend = False
# get reference arguments
refs = self.parse_references()
# add data
if self.type == 'coherence-spectrum':
iterator = list(
zip(self.channels[0::2], self.channels[1::2], plotargs))
else:
iterator = list(zip(self.channels, plotargs))
for chantuple in iterator:
channel = chantuple[0]
channel2 = chantuple[1]
pargs = chantuple[-1]
if self.state and not self.all_data:
valid = self.state
else:
valid = SegmentList([self.span])
if self.type == 'coherence-spectrum':
data = get_coherence_spectrum(
[str(channel), str(channel2)], valid, query=False)
elif self.type == 'range-spectrum':
data = get_range_spectrum(str(channel), valid, query=False)
elif self.type == 'cumulative-range-spectrum':
data = get_range_spectrum(str(channel),
valid,
query=False,
which='mean')
if str(data.unit) == 'Mpc':
data = (data**3).cumsum()**(1 / 3.)
else:
data = (data**2).cumsum()**(1 / 2.)
try:
data = (100 * data / data[-1], )
except IndexError:
data = tuple()
else:
try:
data = get_spectrum(str(channel),
valid,
query=False,
format=sdform,
method=method)
except ValueError as exc:
# math op failed beacuse one of the datasets is empty
if ('could not be broadcast' in str(exc)
and '(0,)' in str(exc)):
data = []
else:
raise
# undo demodulation
data = list(data)
for i, spec in enumerate(data):
data[i] = undo_demodulation(spec, channel,
self.pargs.get('xlim', None))
# anticipate log problems
if self.logx:
data = [s[1:] for s in data]
if self.logy:
for sp in data:
sp.value[sp.value == 0] = 1e-100
if 'label' in pargs:
use_legend = True
if data and self.type == 'cumulative-range-spectrum':
pargs_reverse = pargs.copy()
pargs_reverse.pop('label', None)
pargs_reverse['linestyle'] = 'dashed'
# plot cumulative spectrum and its reverse
ax.plot(data[0], **pargs)
ax.plot(100 - data[0], **pargs_reverse)
elif data and use_percentiles:
_, minline, maxline, _ = ax.plot_mmm(*data, **pargs)
# make min, max lines lighter:
minline.set_alpha(pargs.get('alpha', .1) * 2)
maxline.set_alpha(pargs.get('alpha', .1) * 2)
elif data:
ax.plot(data[0], **pargs)
# display references
for source, refparams in refs.items():
refspec = io.read_frequencyseries(source)
refparams.setdefault('zorder', -len(refs) + 1)
if 'filter' in refparams:
refspec = refspec.filter(*refparams.pop('filter'))
if 'scale' in refparams:
refspec *= refparams.pop('scale', 1)
if 'label' in refparams:
use_legend = True
ax.plot(refspec, **refparams)
# customise
self.add_hvlines()
self.apply_parameters(ax, **self.pargs)
if use_legend or len(self.channels) > 1 or ax.legend_ is not None:
ax.legend(**legendargs)
return self.finalize()
def draw(self, outputfile=None):
"""Get data and generate the figure.
"""
# get plot and axes
plot = self.init_plot()
axes = plot.axes
if self.state:
self.pargs.setdefault(
'suptitle', '[%s-%s, state: %s]' %
(self.span[0], self.span[1], texify(str(self.state))))
suptitle = self.pargs.pop('suptitle', None)
if suptitle:
plot.suptitle(suptitle, y=0.993, va='top')
# extract histogram arguments
histargs = self.parse_plot_kwargs()
# get data
data = []
for channel in self.channels:
if self.state and not self.all_data:
valid = self.state.active
else:
valid = SegmentList([self.span])
data.append(
get_timeseries(channel, valid,
query=False).join(gap='ignore', pad=numpy.nan))
# plot
for ax, arr, pargs in zip(cycle(axes), data, histargs):
# set range if not given
if pargs.get('range') is None:
pargs['range'] = self._get_range(
data,
# use range from first dataset if already calculated
range=histargs[0].get('range'),
# use xlim if manually set (user or INI)
xlim=None if ax.get_autoscalex_on() else ax.get_xlim(),
)
# plot histogram
_, _, patches = ax.hist(arr, **pargs)
# update edge color of histogram to be tinted version of face
if pargs.get('histtype', None) == 'stepfilled':
for p in patches:
if not p.get_edgecolor()[3]:
p.set_edgecolor(tint(p.get_facecolor(), .7))
# customise plot
legendargs = self.parse_legend_kwargs()
for i, ax in enumerate(axes):
for key, val in self.pargs.items():
if key == 'title' and i > 0:
continue
if key == 'xlabel' and i < (len(axes) - 1):
continue
if key == 'ylabel' and ((len(axes) % 2 and i != len(axes) // 2)
or (len(axes) % 2 == 0 and i > 0)):
continue
try:
getattr(ax, 'set_%s' % key)(val)
except AttributeError:
setattr(ax, key, val)
if len(self.channels) > 1:
ax.legend(**legendargs)
if len(axes) % 2 == 0 and axes[0].get_ylabel():
label = axes[0].yaxis.label
ax = axes[int(len(axes) // 2) - 1]
ax.set_ylabel(label.get_text())
ax.yaxis.label.set_position((0, -.2 / len(axes)))
if len(axes) != 2:
label.set_text('')
# add extra axes and finalise
return self.finalize(outputfile=outputfile)
def draw(self, outputfile=None):
"""Read in all necessary data, and generate the figure.
"""
plot = self.init_plot()
ax = plot.gca()
plotargs = self.parse_plot_kwargs()
legendargs = self.parse_legend_kwargs()
# add data
channels, groups = list(zip(*self.get_channel_groups()))
for clist, pargs in list(zip(groups, plotargs)):
# get data
valid = self._get_data_segments(clist[0])
data = [get_timeseries(c, valid, query=False) for c in clist]
if len(clist) > 1:
data = [tsl.join(gap='pad', pad=numpy.nan) for tsl in data]
flatdata = [ts for tsl in data for ts in tsl]
# validate parameters
for ts in flatdata:
# double-check empty
if ts.x0 is None:
ts.epoch = self.start
# double-check log scales
if self.logy:
ts.value[ts.value == 0] = 1e-100
# set label
try:
label = pargs.pop('label')
except KeyError:
try:
label = texify(flatdata[0].name)
except IndexError:
label = clist[0]
else:
if self.fileformat == 'svg' and not label.startswith(
texify(str(flatdata[0].channel)).split('.')[0]):
label += ' [%s]' % (texify(str(flatdata[0].channel)))
# plot groups or single TimeSeries
if len(clist) > 1:
data[1].name = None # force no labels for shades
data[2].name = None
ax.plot_mmm(*data, label=label, **pargs)
elif len(flatdata) == 0:
ax.plot(data[0].EntryClass([],
epoch=self.start,
unit='s',
name=label),
label=label,
**pargs)
else:
for ts in data[0]:
line, = ax.plot(ts, label=label, **pargs)
label = None
pargs['color'] = line.get_color()
# customise plot
self.add_hvlines()
self.apply_parameters(ax, **self.pargs)
# add legend
if ax.get_legend_handles_labels()[0]:
ax.legend(**legendargs)
self.add_state_segments(ax)
self.add_future_shade()
return self.finalize(outputfile=outputfile)
def from_ini(cls, config, section, plotdir='plots', **kwargs):
new = super(ParentTab, cls).from_ini(config, section, **kwargs)
# add information
new.plots = []
new.channel = config.get(section, 'channel')
new.ifo = config.get(section, 'IFO')
new.modes = OrderedDict((int(idx), name)
for (idx, name) in config.nditems(section)
if idx.isdigit())
# -----------
# build plots
new.set_layout([2, 2])
new.segmenttag = '%s:%%s' % (new.channel)
tag = new.channel.split(':', 1)[-1].replace('-', '_')
groups = type(new.modes)((idx, name.strip('*'))
for (idx, name) in new.modes.items()
if int(idx) % 10 == 0)
pstates = OrderedDict((idx, name) for (idx, name) in new.modes.items()
if not name.startswith('*'))
# parse colors
colors = dict((int(key[6:]), eval(color))
for (key, color) in config.nditems(section)
if re.match(r'color-\d+', key))
for flag in pstates:
group = int(flag // 10 * 10)
if flag not in colors:
colors[flag] = colors.get(group, None)
# plot segments
for flags, ptag in zip([groups, pstates], ['overview', 'details']):
segcolors = [colors.get(flag, None) for flag in flags]
if not any(segcolors):
segcolors = None
for state in new.states:
new.plots.append(
get_plot('segments')(
[new.segmenttag % idx for idx in flags],
new.span[0],
new.span[1],
labels=list(flags.values()),
state=state,
outdir=plotdir,
pid='%s_SEGMENTS_%s' % (tag, ptag.upper()),
active=segcolors,
known={
'alpha': 0.1,
'facecolor': 'lightgray'
},
title='%s operating mode %s' % (new.ifo, ptag)))
# plot pie charts
try:
explode = list(
map(float,
config.get(section, 'pie-explode').split(',')))
except NoOptionError:
explode = None
ptag = 'overview'
piecolors = [colors.get(flag, None) for flag in groups]
if not any(piecolors):
piecolors = None
for state in new.states:
labels = list(groups.values())
new.plots.append(
get_plot('segment-pie')(
[new.segmenttag % idx for idx in groups],
new.span[0],
new.span[1],
state=state,
labels=labels,
outdir=plotdir,
pid='%s_PIE_%s' % (tag, ptag.upper()),
colors=piecolors,
explode=explode,
title='%s operating mode %s' % (new.ifo, ptag)))
shortlabels = [
'\n'.join('{0:.9}.'.format(w) if len(w) > 9 else w
for w in l.split()) for l in labels
]
new.plots.append(
get_plot('duty')(
[new.segmenttag % idx for idx in groups],
new.span[0],
new.span[1],
state=state,
labels=shortlabels,
outdir=plotdir,
pid='%s_SEGMENT_BAR_%s' % (tag, ptag.upper()),
colors=piecolors,
stacked=True,
ylim=[0, 100],
ylabel=texify('Percentage [%] of available time'),
legend_loc='upper left',
legend_bbox_to_anchor=(1., 1),
legend_fontsize=12,
legend_borderaxespad=0,
legend_frameon=False,
legend_handlelength=1.0,
legend_handletextpad=.5,
title='%s operating mode %s' % (new.ifo, ptag)))
return new
def main(args=None):
"""Run the lasso command-line interface
"""
# declare global variables
# this is needed for multiprocessing utilities
global auxdata, cluster_threshold, cmap, colors, counter, gpsstub
global line_size_aux, line_size_primary, max_correlated_channels
global nonzerocoef, nonzerodata, p1, primary, primary_mean, primary_std
global primaryts, range_is_primary, re_delim, start, target, times
global threshold, trend_type, xlim
parser = create_parser()
args = parser.parse_args(args=args)
# get run params
start = int(args.gpsstart)
end = int(args.gpsend)
pad = args.filter_padding
# set pertinent global variables
cluster_threshold = args.cluster_coefficient
line_size_aux = args.line_size_aux
line_size_primary = args.line_size_primary
threshold = args.threshold
trend_type = args.trend_type
# let's go
LOGGER.info('{} Lasso correlations {}-{}'.format(args.ifo, start, end))
# get primary channel frametype
primary = args.primary_channel.format(ifo=args.ifo)
range_is_primary = 'EFFECTIVE_RANGE_MPC' in args.primary_channel
if args.primary_cache is not None:
LOGGER.info("Using custom primary cache file")
elif args.primary_frametype is None:
try:
args.primary_frametype = DEFAULT_FRAMETYPE[
args.primary_channel.split(':')[1]].format(ifo=args.ifo)
except KeyError as exc:
raise type(exc)("Could not determine primary channel's frametype, "
"please specify with --primary-frametype")
# create output directory
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
os.chdir(args.output_dir)
# multiprocessing for plots
nprocplot = (args.nproc_plot or args.nproc) if USETEX else 1
# bandpass primary
if args.band_pass:
try:
flower, fupper = args.band_pass
except TypeError:
flower, fupper = None
LOGGER.info("-- Loading primary channel data")
bandts = get_data(primary,
start - pad,
end + pad,
verbose='Reading primary:'.rjust(30),
frametype=args.primary_frametype,
source=args.primary_cache,
nproc=args.nproc)
if flower < 0 or fupper >= float((bandts.sample_rate / 2.).value):
raise ValueError(
"bandpass frequency is out of range for this "
"channel, band (Hz): {0}, sample rate: {1}".format(
args.band_pass, bandts.sample_rate))
# get darm BLRMS
LOGGER.debug("-- Filtering data")
if trend_type == 'minute':
stride = 60
else:
stride = 1
if flower:
darmbl = (bandts.highpass(
flower / 2., fstop=flower / 4., filtfilt=False,
ftype='butter').notch(60, filtfilt=False).bandpass(
flower,
fupper,
fstop=[flower / 2., fupper * 1.5],
filtfilt=False,
ftype='butter').crop(start, end))
darmblrms = darmbl.rms(stride)
darmblrms.name = '%s %s-%s Hz BLRMS' % (primary, flower, fupper)
else:
darmbl = bandts.notch(60).crop(start, end)
darmblrms = darmbl.rms(stride)
darmblrms.name = '%s RMS' % primary
primaryts = darmblrms
bandts_asd = bandts.asd(4, 2, method='median')
darmbl_asd = darmbl.asd(4, 2, method='median')
spectrum_plots = gwplot.make_spectrum_plots(start, end, flower, fupper,
args.primary_channel,
bandts_asd, darmbl_asd)
spectrum_plot_zoomed_out = spectrum_plots[0]
spectrum_plot_zoomed_in = spectrum_plots[1]
else:
# load primary channel data
LOGGER.info("-- Loading primary channel data")
primaryts = get_data(primary,
start,
end,
frametype=args.primary_frametype,
source=args.primary_cache,
verbose='Reading:'.rjust(30),
nproc=args.nproc).crop(start, end)
if args.remove_outliers:
LOGGER.debug("-- Removing outliers above %f sigma" %
args.remove_outliers)
gwlasso.remove_outliers(primaryts, args.remove_outliers)
elif args.remove_outliers_pf:
LOGGER.debug("-- Removing outliers in the bottom {} percent "
"of data".format(args.remove_outliers_pf))
gwlasso.remove_outliers(primaryts,
args.remove_outliers_pf,
method='pf')
start = int(primaryts.span()[0])
end = int(primaryts.span()[1])
primary_mean = numpy.mean(primaryts.value)
primary_std = numpy.std(primaryts.value)
# get aux data
LOGGER.info("-- Loading auxiliary channel data")
if args.channel_file is None:
host, port = io_nds2.host_resolution_order(args.ifo)[0]
channels = ChannelList.query_nds2('*.mean',
host=host,
port=port,
type='m-trend')
else:
with open(args.channel_file, 'r') as f:
channels = [name.rstrip('\n') for name in f]
nchan = len(channels)
LOGGER.debug("Identified %d channels" % nchan)
if trend_type == 'minute':
frametype = '%s_M' % args.ifo # for minute trends
else:
frametype = '%s_T' % args.ifo # for second trends
# read aux channels
auxdata = get_data(channels,
start,
end,
verbose='Reading:'.rjust(30),
frametype=frametype,
nproc=args.nproc,
pad=0).crop(start, end)
# -- removes flat data to be re-introdused later
LOGGER.info('-- Pre-processing auxiliary channel data')
auxdata = gwlasso.remove_flat(auxdata)
flatable = Table(data=(list(set(channels) - set(auxdata.keys())), ),
names=('Channels', ))
LOGGER.debug('Removed {0} channels with flat data'.format(len(flatable)))
LOGGER.debug('{0} channels remaining'.format(len(auxdata)))
# -- remove bad data
LOGGER.info("Removing any channels with bad data...")
nbefore = len(auxdata)
auxdata = gwlasso.remove_bad(auxdata)
nafter = len(auxdata)
LOGGER.debug('Removed {0} channels with bad data'.format(nbefore - nafter))
LOGGER.debug('{0} channels remaining'.format(nafter))
data = numpy.array([scale(ts.value) for ts in auxdata.values()]).T
# -- perform lasso regression -------------------
# create model
LOGGER.info('-- Fitting data to target')
target = scale(primaryts.value)
model = gwlasso.fit(data, target, alpha=args.alpha)
LOGGER.info('Alpha: {}'.format(model.alpha))
# restructure results for convenience
allresults = Table(data=(list(auxdata.keys()), model.coef_,
numpy.abs(model.coef_)),
names=('Channel', 'Lasso coefficient', 'rank'))
allresults.sort('rank')
allresults.reverse()
useful = allresults['rank'] > 0
allresults.remove_column('rank')
results = allresults[useful] # non-zero coefficient
zeroed = allresults[numpy.invert(useful)] # zero coefficient
# extract data for useful channels
nonzerodata = {name: auxdata[name] for name in results['Channel']}
nonzerocoef = {name: coeff for name, coeff in results.as_array()}
# print results
LOGGER.info('Found {} channels with |Lasso coefficient| >= {}:\n\n'.format(
len(results), threshold))
print(results)
print('\n\n')
# convert to pandas
set_option('max_colwidth', -1)
df = results.to_pandas()
df.index += 1
# write results to files
gpsstub = '%d-%d' % (start, end - start)
resultsfile = '%s-LASSO_RESULTS-%s.csv' % (args.ifo, gpsstub)
results.write(resultsfile, format='csv', overwrite=True)
zerofile = '%s-ZERO_COEFFICIENT_CHANNELS-%s.csv' % (args.ifo, gpsstub)
zeroed.write(zerofile, format='csv', overwrite=True)
flatfile = '%s-FLAT_CHANNELS-%s.csv' % (args.ifo, gpsstub)
flatable.write(flatfile, format='csv', overwrite=True)
# -- generate lasso plots
modelFit = model.predict(data)
re_delim = re.compile(r'[:_-]')
p1 = (.1, .15, .9, .9) # global plot defaults for plot1, lasso model
times = primaryts.times.value
xlim = primaryts.span
cmap = get_cmap('tab20')
colors = [cmap(i) for i in numpy.linspace(0, 1, len(nonzerodata) + 1)]
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax.plot(times,
_descaler(modelFit),
label='Lasso model',
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
ax.set_title('Lasso Model of Range')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Lasso Model of Primary Channel')
ax.legend(loc='best')
plot1 = gwplot.save_figure(plot,
'%s-LASSO_MODEL-%s.png' % (args.ifo, gpsstub),
bbox_inches='tight')
# summed contributions
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
summed = 0
for i, name in enumerate(results['Channel']):
summed += scale(nonzerodata[name].value) * nonzerocoef[name]
if i:
label = 'Channels 1-{0}'.format(i + 1)
else:
label = 'Channel 1'
ax.plot(times,
_descaler(summed),
label=label,
color=colors[i],
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Summations of Channel Contributions to Model')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot2 = gwplot.save_figure(plot,
'%s-LASSO_CHANNEL_SUMMATION-%s.png' %
(args.ifo, gpsstub),
bbox_inches='tight')
# individual contributions
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
for i, name in enumerate(results['Channel']):
this = _descaler(scale(nonzerodata[name].value) * nonzerocoef[name])
if i:
label = 'Channels 1-{0}'.format(i + 1)
else:
label = 'Channel 1'
ax.plot(times,
this,
label=texify(name),
color=colors[i],
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Individual Channel Contributions to Model')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot3 = gwplot.save_figure(plot,
'%s-LASSO_CHANNEL_CONTRIBUTIONS-%s.png' %
(args.ifo, gpsstub),
bbox_inches='tight')
# -- process aux channels, making plots
LOGGER.info("-- Processing channels")
counter = multiprocessing.Value('i', 0)
# process channels
pool = multiprocessing.Pool(nprocplot)
results = pool.map(_process_channel, enumerate(list(nonzerodata.items())))
results = sorted(results, key=lambda x: abs(x[1]), reverse=True)
# generate clustered time series plots
counter = multiprocessing.Value('i', 0)
max_correlated_channels = 20
if args.no_cluster is False:
LOGGER.info("-- Generating clusters")
pool = multiprocessing.Pool(nprocplot)
clusters = pool.map(_generate_cluster, enumerate(results))
channelsfile = '%s-CHANNELS-%s.csv' % (args.ifo, gpsstub)
numpy.savetxt(channelsfile, channels, delimiter=',', fmt='%s')
# write html
trange = '%d-%d' % (start, end)
title = '%s Lasso Correlation: %s' % (args.ifo, trange)
if args.band_pass:
links = [trange
] + [(s, '#%s' % s.lower())
for s in ['Parameters', 'Spectra', 'Model', 'Results']]
else:
links = [trange] + [(s, '#%s' % s.lower())
for s in ['Parameters', 'Model', 'Results']]
(brand, class_) = htmlio.get_brand(args.ifo, 'Lasso', start)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(title='%s Lasso | %s' %
(args.ifo, trange),
navbar=navbar)
page.h1(title, class_='pb-2 mt-3 mb-2 border-bottom')
# -- summary table
content = [
('Primary channel', markup.oneliner.code(primary)),
('Primary frametype', markup.oneliner.code(args.primary_frametype)
or '-'),
('Primary cache file', markup.oneliner.code(args.primary_cache)
or '-'), ('Outlier threshold', '%s sigma' % args.remove_outliers),
('Lasso coefficient threshold', str(threshold)),
('Cluster coefficient threshold', str(args.cluster_coefficient)),
('Non-zero coefficients', str(numpy.count_nonzero(model.coef_))),
('α (model)', '%.4f' % model.alpha)
]
if args.band_pass:
content.insert(
2, ('Primary bandpass', '{0}-{1} Hz'.format(flower, fupper)))
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(content, start=start, end=end))
page.div.close() # col-md-9 col-sm-12
# -- download button
files = [('%s analyzed channels (CSV)' % nchan, channelsfile),
('%s flat channels (CSV)' % len(flatable), flatfile),
('%s zeroed channels (CSV)' % len(zeroed), zerofile)]
page.div(class_='col-md-3 col-sm-12')
page.add(
htmlio.download_btn(files,
label='Channel information',
btnclass='btn btn-%s dropdown-toggle' %
args.ifo.lower()))
page.div.close() # col-md-3 col-sm-12
page.div.close() # rowa
# -- command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
if args.band_pass:
page.h2('Primary channel spectra', class_='mt-4', id_='spectra')
page.div(class_='card border-light card-body shadow-sm')
page.div(class_='row')
page.div(class_='col-md-6')
spectra_img1 = htmlio.FancyPlot(spectrum_plot_zoomed_out)
page.add(htmlio.fancybox_img(spectra_img1))
page.div.close() # col-md-6
page.div(class_='col-md-6')
spectra_img2 = htmlio.FancyPlot(spectrum_plot_zoomed_in)
page.add(htmlio.fancybox_img(spectra_img2))
page.div.close() # col-md-6
page.div.close() # row
page.div.close() # card border-light card-body shadow-sm
# -- model information
page.h2('Model information', class_='mt-4', id_='model')
page.div(class_='card card-%s card-body shadow-sm' % args.ifo.lower())
page.div(class_='row')
page.div(class_='col-md-8 offset-md-2', id_='results-table')
page.p('Below are the top {} mean minute-trend channels, ranked by '
'Lasso correlation with the primary.'.format(df.shape[0]))
page.add(
df.to_html(classes=('table', 'table-sm', 'table-hover'),
formatters={
'Lasso coefficient': lambda x: "%.4f" % x,
'Channel':
lambda x: str(htmlio.cis_link(x.split('.')[0])),
'__index__': lambda x: str(x)
},
escape=False,
border=0).replace(' style="text-align: right;"', ''))
page.div.close() # col-md-10 offset-md-1
page.div.close() # row
page.div(class_='row', id_='primary-lasso')
page.div(class_='col-md-8 offset-md-2')
img1 = htmlio.FancyPlot(plot1)
page.add(htmlio.fancybox_img(img1)) # primary lasso plot
page.div.close() # col-md-8 offset-md-2
page.div.close() # primary-lasso
page.div(class_='row', id_='channel-summation')
img2 = htmlio.FancyPlot(plot2)
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img2))
page.div.close() # col-md-8 offset-md-2
page.div.close() # channel-summation
page.div(class_='row', id_='channels-and-primary')
img3 = htmlio.FancyPlot(plot3)
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img3))
page.div.close() # col-md-8 offset-md-2
page.div.close() # channels-and-primary
page.div.close() # card card-<ifo> card-body shadow-sm
# -- results
page.h2('Top channels', class_='mt-4', id_='results')
page.div(id_='results')
# for each aux channel, create information container and put plots in it
for i, (ch, lassocoef, plot4, plot5, plot6, ts) in enumerate(results):
# set container color/context based on lasso coefficient
if lassocoef == 0:
break
elif abs(lassocoef) < threshold:
h = '%s [lasso coefficient = %.4f] (Below threshold)' % (ch,
lassocoef)
else:
h = '%s [lasso coefficient = %.4f]' % (ch, lassocoef)
if ((lassocoef is None) or (lassocoef == 0)
or (abs(lassocoef) < threshold)):
card = 'card border-light mb-1 shadow-sm'
card_header = 'card-header bg-light'
elif abs(lassocoef) >= .5:
card = 'card border-danger mb-1 shadow-sm'
card_header = 'card-header text-white bg-danger'
elif abs(lassocoef) >= .2:
card = 'card border-warning mb-1 shadow-sm'
card_header = 'card-header text-white bg-warning'
else:
card = 'card border-info mb-1 shadow-sm'
card_header = 'card-header text-white bg-info'
page.div(class_=card)
# heading
page.div(class_=card_header)
page.a(h,
class_='collapsed card-link cis-link',
href='#channel%d' % i,
**{'data-toggle': 'collapse'})
page.div.close() # card-header
# body
page.div(id_='channel%d' % i,
class_='collapse',
**{'data-parent': '#results'})
page.div(class_='card-body')
if lassocoef is None:
page.p('The amplitude data for this channel is flat (does not '
'change) within the chosen time period.')
elif abs(lassocoef) < threshold:
page.p('Lasso coefficient below the threshold of %g.' %
(threshold))
else:
for image in [plot4, plot5, plot6]:
img = htmlio.FancyPlot(image)
page.div(class_='row')
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img))
page.div.close() # col-md-8 offset-md-2
page.div.close() # row
page.add('<hr class="row-divider">')
if args.no_cluster is False:
if clusters[i][0] is None:
page.p("<font size='3'><br />No channels were highly "
"correlated with this channel.</font>")
else:
page.div(class_='row', id_='clusters')
page.div(class_='col-md-12')
cimg = htmlio.FancyPlot(clusters[i][0])
page.add(htmlio.fancybox_img(cimg))
page.div.close() # col-md-12
page.div.close() # clusters
if clusters[i][1] is not None:
corr_link = markup.oneliner.a(
'Export {} channels (CSV)'.format(
max_correlated_channels),
href=clusters[i][1],
download=clusters[i][1],
)
page.button(
corr_link,
class_='btn btn-%s' % args.ifo.lower(),
)
page.div.close() # card-body
page.div.close() # collapse
page.div.close() # card
page.div.close() # results
htmlio.close_page(page, 'index.html') # save and close
LOGGER.info("-- Process Completed")
def draw(self, outputfile=None):
"""Generate the figure for this plot
"""
plot = self.init_plot()
ax = plot.axes[0]
# get plotting arguments
cumulative = self.pargs.pop('cumulative', False)
plotargs = self.parse_plot_kwargs()
legendargs = self.parse_legend_kwargs()
# set ylabel
if cumulative:
self.pargs.setdefault(
'ylabel',
'Cumulative time-volume [Mpc$^3$ kyr]',
)
else:
self.pargs.setdefault(
'ylabel',
'Time-volume [Mpc$^3$ kyr]',
)
# get data
allsegs, allranges = ([], [])
for channel, flag, pargs in zip(self.channels, self.flags, plotargs):
pad = 0
if self.state and not self.all_data:
valid = self.state.active
pad = numpy.nan
elif channel.sample_rate.value:
valid = SegmentList(
[self.span.protract(1 / channel.sample_rate.value)])
else:
valid = SegmentList([self.span])
data = get_timeseries(channel, valid, query=False).join(gap='pad',
pad=pad)
if not data.unit or data.unit.to_string() in ['', 'undef']:
data.override_unit('Mpc')
segments = get_segments(flag, valid, query=False)
timevolume = self.calculate_time_volume(segments.active, data)
if cumulative:
ax.plot(timevolume.cumsum(), **pargs)
else:
ax.plot(timevolume, **pargs)
allsegs.append(segments.active)
allranges.append(data)
# estimate combined time-volume
if self.all_data and len(self.channels) > 1:
pargs = plotargs[-1]
pargs['color'] = '#000000'
pargs['label'] = 'Combined'
pargs['linestyle'] = '--'
combined_timevolume = self.combined_time_volume(allsegs, allranges)
if cumulative:
ax.plot(combined_timevolume.cumsum(), **pargs)
else:
ax.plot(combined_timevolume, **pargs)
# add horizontal lines to add
for yval in self.pargs.get('hline', []):
try:
yval = float(yval)
except ValueError:
continue
else:
ax.plot([self.start, self.end], [yval, yval],
linestyle='--',
color='red')
# customise plot
self.apply_parameters(ax, **self.pargs)
if (len(self.channels) > 1 or plotargs[0].get('label', None)
in [texify(str(self.channels[0])), None]):
ax.legend(**legendargs)
# add extra axes and finalise
self.add_state_segments(ax)
self.add_future_shade()
if ax.get_yscale() == 'log':
ax.yaxis.set_major_locator(LogLocator())
else:
ax.yaxis.set_major_locator(MaxNLocator(8))
ticks = ax.get_yticks()
ax.yaxis.set_ticklabels(ticks)
return self.finalize(outputfile=outputfile)
def _draw(self):
"""Load all data, and generate this `SpectrumDataPlot`
"""
plot = self.init_plot()
ax = plot.gca()
if self.state:
self.pargs.setdefault(
'suptitle', '[%s-%s, state: %s]' %
(self.span[0], self.span[1], texify(str(self.state))))
suptitle = self.pargs.pop('suptitle', None)
if suptitle:
plot.suptitle(suptitle, y=0.993, va='top')
# parse plotting arguments
cmap = self.pargs.pop('cmap', None)
varargs = self.parse_variance_kwargs()
plotargs = self.parse_plot_kwargs()[0]
# get reference arguments
refs = self.parse_references()
# calculate spectral variance and plot
# pad data request to over-fill plots (no gaps at the end)
if self.state and not self.all_data:
valid = self.state.active
else:
valid = SegmentList([self.span])
livetime = float(abs(valid))
if livetime:
plotargs.setdefault('vmin', 1 / livetime)
plotargs.setdefault('vmax', 1.)
plotargs.pop('label')
specgram = get_spectrogram(self.channels[0],
valid,
query=False,
format='asd').join(gap='ignore')
if specgram.size:
asd = specgram.median(axis=0)
asd.name = None
variance = specgram.variance(**varargs)
# normalize the variance
variance /= livetime / specgram.dt.value
# undo demodulation
variance = undo_demodulation(variance, self.channels[0],
ax.get_xlim())
# plot
ax.plot(asd, color='grey', linewidth=0.3)
ax.imshow(variance, cmap=cmap, **plotargs)
# display references
for source, refparams in refs.items():
refspec = io.read_frequencyseries(source)
if 'filter' in refparams:
refspec = refspec.filter(*refparams.pop('filter'))
if 'scale' in refparams:
refspec *= refparams.pop('scale', 1)
ax.plot(refspec, **refparams)
# customise
self.add_hvlines()
self.apply_parameters(ax, **self.pargs)
ax.grid(b=True, axis='both', which='both')
return self.finalize()
def _draw(self):
"""Load all data, and generate this `SpectrumDataPlot`
"""
plot = self.init_plot()
ax = plot.gca()
ax.grid(b=True, axis='both', which='both')
if self.state:
self.pargs.setdefault(
'suptitle', '[%s-%s, state: %s]' %
(self.span[0], self.span[1], texify(str(self.state))))
suptitle = self.pargs.pop('suptitle', None)
if suptitle:
plot.suptitle(suptitle, y=0.993, va='top')
# get spectrum format: 'amplitude' or 'power'
sdform = self.pargs.pop('format')
# parse plotting arguments
plotargs = self.parse_plot_kwargs()
legendargs = self.parse_legend_kwargs()
# add data
sumdata = []
for i, (channel, pargs) in enumerate(zip(self.channels, plotargs)):
if self.state and not self.all_data:
valid = self.state
else:
valid = SegmentList([self.span])
data = get_spectrum(str(channel),
valid,
query=False,
format=sdform,
method=None)[0]
if i and data.size:
sumdata.append(data)
else:
darmdata = data
# anticipate log problems
if self.logx:
data = data[1:]
if self.logy:
data.value[data.value == 0] = 1e-100
pargs.setdefault('zorder', -i)
ax.plot(data, **pargs)
# assert all noise terms have the same resolution
if any([x.dx != sumdata[0].dx for x in sumdata]):
raise RuntimeError("Noise components have different resolutions, "
"cannot construct sum of noises")
# reshape noises if required
n = max(x.size for x in sumdata)
for i, d in enumerate(sumdata):
if d.size < n:
sumdata[i] = numpy.resize(numpy.require(d, requirements=['O']),
(n, ))
# plot sum of noises
sumargs = self.parse_sum_params()
sum_ = numpy.zeros_like(darmdata**2, shape=(n, ))
for d in sumdata:
d.override_unit(darmdata.unit) # enforce consistent units
sum_ += d**2
ax.plot(sum_**(1 / 2.), zorder=1, **sumargs)
ax.lines.insert(1, ax.lines.pop(-1))
# plot residual of noises
if not self.pargs.pop('no-residual', False):
resargs = self.parse_residual_params()
try:
residual = (darmdata**2 - sum_)**(1 / 2.)
except ValueError:
if not darmdata.size: # if no data, just copy nothing
residual = darmdata
else: # other error
raise
ax.plot(residual, zorder=-1000, **resargs)
ax.lines.insert(1, ax.lines.pop(-1))
# finalize
self.apply_parameters(ax, **self.pargs)
ax.legend(**legendargs)
return self.finalize()
def _generate_cluster(input_):
"""Generate cluster data for use below
"""
if USETEX:
gwplot.configure_mpl_tex()
currentchan = input_[1][0]
currentts = input_[1][5]
current = input_[0]
p7 = (.135, .15, .95, .9)
plot7 = None
plot7_list = None
if current < len(nonzerodata):
cluster = []
for i, otheritem in enumerate(list(auxdata.items())):
chan_, ts_ = otheritem
if chan_ != currentchan:
pcorr = numpy.corrcoef(currentts.value, ts_.value)[0, 1]
if abs(pcorr) >= cluster_threshold:
stub = re_delim.sub('_', chan_).replace('_', '-', 1)
cluster.append([i, ts_, pcorr, chan_, stub])
if cluster:
# write cluster table to file
cluster = sorted(cluster, key=lambda x: abs(x[2]), reverse=True)
clustertab = Table(data=list(zip(*cluster))[2:4],
names=('Pearson Coefficient', 'Channel'))
plot7_list = '%s_CLUSTER_LIST-%s.csv' % (re_delim.sub(
'_', str(currentchan)).replace('_', '-', 1), gpsstub)
clustertab.write(plot7_list, format='csv', overwrite=True)
ncluster = min(len(cluster), max_correlated_channels)
colors2 = [cmap(i) for i in numpy.linspace(0, 1, ncluster + 1)]
# plot
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p7)
ax = fig.gca(xscale='auto-gps')
ax.plot(times,
scale(currentts.value) * numpy.sign(input_[1][1]),
label=texify(currentchan),
linewidth=line_size_aux,
color=colors[0])
for i in range(0, ncluster):
this = cluster[i]
ax.plot(
times,
scale(this[1].value) * numpy.sign(input_[1][1]) *
numpy.sign(this[2]),
color=colors2[i + 1],
linewidth=line_size_aux,
label=('{0}, r = {1:.2}'.format(texify(cluster[i][3]),
cluster[i][2])),
)
ax.margins(x=0)
ax.set_ylabel('Scaled amplitude [arbitrary units]')
ax.set_title('Highly Correlated Channels')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot7 = gwplot.save_figure(
fig,
'%s_CLUSTER-%s.png' % (re_delim.sub(
'_', str(currentchan)).replace('_', '-', 1), gpsstub),
bbox_inches='tight')
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info(
"Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(currentchan)))
sys.stdout.flush()
return (plot7, plot7_list)
def _draw(self):
"""Load all data, and generate this `SpectrumDataPlot`
"""
plot = self.init_plot()
ax = plot.gca()
ax.grid(b=True, axis='both', which='both')
if self.state:
self.pargs.setdefault(
'suptitle', '[%s-%s, state: %s]' %
(self.span[0], self.span[1], texify(str(self.state))))
suptitle = self.pargs.pop('suptitle', None)
if suptitle:
plot.suptitle(suptitle, y=0.993, va='top')
# get spectrum format: 'amplitude' or 'power'
sdform = self.pargs.pop('format')
# parse plotting arguments
plotargs = self.parse_plot_kwargs()[0]
# add data
sumdata = []
for i, channel in enumerate(self.channels):
if self.state and not self.all_data:
valid = self.state
else:
valid = SegmentList([self.span])
data = get_spectrum(str(channel),
valid,
query=False,
format=sdform,
method=None)[0]
if i and data.size:
sumdata.append(data)
else:
target = data
if target.size:
# assert all noise terms have the same resolution
if any([x.dx != target.dx for x in sumdata]):
raise RuntimeError("Noise components have different "
"resolutions, cannot construct "
"sum of noises")
# reshape noises if required
n = target.size
for i, d in enumerate(sumdata):
if d.size < n:
sumdata[i] = numpy.resize(
numpy.require(d, requirements=['O']), (n, ))
# calculate sum of noises
sum_ = numpy.zeros_like(sumdata[0]**2, shape=(n, ))
for d in sumdata:
d.override_unit(sumdata[0].unit) # enforce consistent units
sum_ += d**2
sum_ **= (1 / 2.)
relative = sum_ / target
else: # no data, so just use anything as a proxy
relative = target
# plot ratio of h(t) to sum of noises
ax.plot(relative, **plotargs)
# finalize plot
self.apply_parameters(ax, **self.pargs)
return self.finalize()
def draw(self):
# data span
start = self.gpstime - self.duration / 2.
end = self.gpstime + self.duration / 2.
# get data
if self.use_nds:
data = TimeSeriesDict.fetch(self.chanlist, start, end)
else:
from glue.datafind import GWDataFindHTTPConnection
conn = GWDataFindHTTPConnection()
cache = conn.find_frame_urls(self.ifo[0],
'%s_R' % self.ifo,
self.start,
self.end,
urltype='file')
if len(cache) == 0:
data = {}
else:
data = TimeSeriesDict.read(cache,
self.chanlist,
start=start,
end=end)
# make plot
plot, axes = subplots(nrows=self.geometry[0],
ncols=self.geometry[1],
sharex=True,
subplot_kw={'xscale': 'auto-gps'},
FigureClass=Plot,
figsize=[12, 6])
axes[0, 0].set_xlim(start, end)
for channel, ax in zip(self.chanlist, axes.flat):
ax.set_epoch(self.gpstime)
# plot data
try:
ax.plot(data[channel])
except KeyError:
ax.text(self.gpstime,
0.5,
"No data",
va='center',
ha='center',
transform=ax.transData)
# plot trip indicator
ax.axvline(self.gpstime,
linewidth=0.5,
linestyle='--',
color='red')
ax.set_xlabel('')
ax.set_ylabel('')
ax.set_title(texify(channel.name), fontsize=10)
ax.xaxis.set_minor_locator(NullLocator())
for tick in ax.yaxis.get_major_ticks():
tick.label.set_fontsize(10)
for tick in ax.xaxis.get_major_ticks():
tick.label.set_fontsize(16)
plot.text(0.5,
0.02,
'Time [seconds] from trip (%s)' % self.gpstime,
ha='center',
va='bottom',
fontsize=24)
plot.text(0.01,
0.5,
'Amplitude %s' % self.unit,
ha='left',
va='center',
rotation='vertical',
fontsize=24)
plot.suptitle('%s %s %s watchdog trip: %s' %
(self.ifo, self.chamber, self.sensor, self.gpstime),
fontsize=24)
plot.save(self.outputfile)
plot.close()
return self.outputfile
