def main_compare_gif_gotic2():
start = tconvert('Oct 15 2018 00:00:00')
end = tconvert('Oct 21 2018 00:00:00')
# gif data
pfx = '/Users/miyo/Dropbox/KagraData/gif/'
segments = GifData.findfiles(start, end, 'CALC_STRAIN', prefix=pfx)
allfiles = [path for files in segments for path in files]
strain = TimeSeries.read(source=allfiles,
name='CALC_STRAIN',
format='gif',
pad=numpy.nan,
nproc=2)
strain = strain.detrend('linear')
# gotic data
source = '201805010000_201811010000.gotic'
gifx = KagraGoticStrain.read(source, start=start, end=end).x
gifx = gifx.detrend('linear')
gifx = gifx * 0.9
# plot
plot = Plot(gifx, strain, xscale='auto-gps')
plot.legend()
plot.subplots_adjust(right=.86)
plot.savefig('result.png')
plot.close()
def spectral_comparison(gps,
qspecgram,
fringe,
output,
thresh=15,
multipliers=(1, 2, 4, 8),
colormap='viridis',
figsize=[12, 8]):
"""Compare a high-resolution spectrogram with projected fringe frequencies
Parameters
----------
gps : `float`
reference GPS time (in seconds) to serve as the origin
qspecgram : `~gwpy.spectrogram.Spectrogram`
an interpolated high-resolution spectrogram
fringe : `~gwpy.timeseries.TimeSeries`
projected fringe frequencies (in Hz)
output : `str`
name of the output file
thresh : `float`, optional
frequency threshold (Hz) for scattering fringes, default: 15
multipliers : `tuple`, optional
collection of fringe harmonic numbers to plot, can be given in
any order, default: `(1, 2, 4, 8)`
colormap : `str`, optional
matplotlib colormap to use, default: viridis
figsize : `tuple`, optional
size (width x height) of the final figure, default: `(12, 8)`
"""
plot = Plot(figsize=figsize)
# format spectral plot
ax1 = plot.add_subplot(211)
ax1.set_title('{0} at {1:.2f} with $Q$ of {2:.1f}'.format(
qspecgram.name, gps, qspecgram.q))
_format_spectrogram(ax1, qspecgram, colormap=colormap)
ax1.set_xlabel(None)
# format fringe frequency plot
ax2 = plot.add_subplot(212, sharex=ax1)
ax2.set_title('{} scattering fringes'.format(fringe.name))
_format_timeseries(ax2,
gps,
fringe,
multipliers=multipliers,
thresh=thresh)
# format timeseries axes
ax2.set_ylim([-1, 60])
ax2.set_ylabel('Projected Frequency [Hz]')
# save plot and close
plot.savefig(output, bbox_inches='tight')
plot.close()
def spectral_comparison(gps, qspecgram, fringe, output, thresh=15,
multipliers=(1, 2, 4, 8), colormap='viridis',
figsize=[12, 8]):
"""Compare a high-resolution spectrogram with projected fringe frequencies
Parameters
----------
gps : `float`
reference GPS time (in seconds) to serve as the origin
qspecgram : `~gwpy.spectrogram.Spectrogram`
an interpolated high-resolution spectrogram
fringe : `~gwpy.timeseries.TimeSeries`
projected fringe frequencies (in Hz)
output : `str`
name of the output file
thresh : `float`, optional
frequency threshold (Hz) for scattering fringes, default: 15
multipliers : `tuple`, optional
collection of fringe harmonic numbers to plot, can be given in
any order, default: `(1, 2, 4, 8)`
colormap : `str`, optional
matplotlib colormap to use, default: viridis
figsize : `tuple`, optional
size (width x height) of the final figure, default: `(12, 8)`
"""
plot = Plot(figsize=figsize)
# format spectral plot
ax1 = plot.add_subplot(211)
ax1.set_title('{0} at {1:.2f} with $Q$ of {2:.1f}'.format(
qspecgram.name, gps, qspecgram.q))
_format_spectrogram(ax1, qspecgram, colormap=colormap)
ax1.set_xlabel(None)
# format fringe frequency plot
ax2 = plot.add_subplot(212, sharex=ax1)
ax2.set_title('{} scattering fringes'.format(fringe.name))
_format_timeseries(ax2, gps, fringe, multipliers=multipliers,
thresh=thresh)
# format timeseries axes
ax2.set_ylim([-1, 60])
ax2.set_ylabel('Projected Frequency [Hz]')
# save plot and close
plot.savefig(output, bbox_inches='tight')
plot.close()
def plot_timeseries(*data, **kwargs):
title = kwargs.pop('title', None)
ylim = kwargs.pop('ylim', None)
fname = kwargs.pop('fname', 'TimeSeries.png')
plot = Plot(figsize=(15, 10))
ax0 = plot.gca()
ax0.plot(*data)
ax0.legend([text.to_string(_data.name) for _data in data], fontsize=20)
ax0.set_xscale('auto-gps')
ax0.set_ylabel(text.to_string(data[0].unit))
plot.add_state_segments(segments_daq_iy0_ok, label='IY0 DAQ State')
plot.add_state_segments(segments_daq_ix1_ok, label='IX1 DAQ State')
plot.suptitle(title, fontsize=40)
if ylim:
ax0.set_ylim(ylim[0], ylim[1])
plot.savefig(fname)
plot.close()
def spectral_overlay(gps,
qspecgram,
fringe,
output,
multipliers=(1, 2, 4, 8),
figsize=[12, 4]):
"""Overlay scattering fringe projections on top of a high-resolution
spectrogram
Parameters
----------
gps : `float`
reference GPS time (in seconds) to serve as the origin
qspecgram : `~gwpy.spectrogram.Spectrogram`
an interpolated high-resolution spectrogram
fringe : `~gwpy.timeseries.TimeSeries`
projected fringe frequencies (in Hz)
output : `str`
name of the output file
multipliers : `tuple`, optional
collection of fringe harmonic numbers to plot, can be given in
any order, default: `(1, 2, 4, 8)`
figsize : `tuple`, optional
size (width x height) of the final figure, default: `(12, 4)`
"""
plot = Plot(figsize=figsize)
ax = plot.gca()
# format spectrogram plot
ax.set_title('Fringes: {0}, Spectrogram: {1}'.format(
fringe.name, qspecgram.name))
_format_spectrogram(ax, qspecgram, colormap='binary')
# overlay fringe frequencies
_format_timeseries(ax, gps, fringe, multipliers=multipliers, linewidth=1.5)
ax.set_ylim([
qspecgram.f0.to('Hz').value,
qspecgram.frequencies.max().to('Hz').value
])
# save plot and close
plot.savefig(output, bbox_inches='tight')
plot.close()
def spectral_overlay(gps, qspecgram, fringe, output,
multipliers=(1, 2, 4, 8), figsize=[12, 4]):
"""Overlay scattering fringe projections on top of a high-resolution
spectrogram
Parameters
----------
gps : `float`
reference GPS time (in seconds) to serve as the origin
qspecgram : `~gwpy.spectrogram.Spectrogram`
an interpolated high-resolution spectrogram
fringe : `~gwpy.timeseries.TimeSeries`
projected fringe frequencies (in Hz)
output : `str`
name of the output file
multipliers : `tuple`, optional
collection of fringe harmonic numbers to plot, can be given in
any order, default: `(1, 2, 4, 8)`
figsize : `tuple`, optional
size (width x height) of the final figure, default: `(12, 4)`
"""
plot = Plot(figsize=figsize)
ax = plot.gca()
# format spectrogram plot
ax.set_title('Fringes: {0}, Spectrogram: {1}'.format(
fringe.name, qspecgram.name))
_format_spectrogram(ax, qspecgram, colormap='binary')
# overlay fringe frequencies
_format_timeseries(ax, gps, fringe, multipliers=multipliers,
linewidth=1.5)
ax.set_ylim([qspecgram.f0.to('Hz').value,
qspecgram.frequencies.max().to('Hz').value])
# save plot and close
plot.savefig(output, bbox_inches='tight')
plot.close()
figsize=[12, 6],
separate=True,
sharex=True,
color='gwpy:ligo-hanford')
ax1, ax2 = plot.axes
ax1.set_title('LIGO-Hanford strain data around GW150914')
ax1.text(1.0, 1.01, 'Unfiltered data', transform=ax1.transAxes, ha='right')
ax1.set_ylabel('Amplitude [strain]', y=-0.2)
ax2.set_ylabel('')
ax2.text(1.0,
1.01,
'50-250\,Hz bandpass, notches at 60, 120, 180 Hz',
transform=ax2.transAxes,
ha='right')
plot.show()
plot.close() # hide
# We see now a spike around 16 seconds into the data, so let's zoom into
# that time (and prettify):
plot = hfilt.plot(color='gwpy:ligo-hanford')
ax = plot.gca()
ax.set_title('LIGO-Hanford strain data around GW150914')
ax.set_ylabel('Amplitude [strain]')
ax.set_xlim(1126259462, 1126259462.6)
ax.set_xscale('seconds', epoch=1126259462)
plot.show()
plot.close() # hide
# Congratulations, you have succesfully filtered LIGO data to uncover the
# first ever directly-detected gravitational wave signal, GW150914!
ax.loglog(adcnoise, 'r--', label='ADC Noise', linewidth=1, alpha=0.7)
ax.loglog(ampnoise, 'g--', label='Amp Noise', linewidth=1, alpha=0.7)
ax.loglog(aanoise, 'b--', label='AA Noise', linewidth=1, alpha=0.7)
ax.loglog(exv0, 'k-', label='EXV')
ax.loglog(ixv1, label='IXV1')
#ax.loglog(ixv2,label='IXV2')
#ax.loglog(d12,label='IXV1-IXV2')
#ax.loglog(strain,label='StrainMeter')
#ImportError: No module named gwpy.plot
ax.legend(fontsize=8, loc='lower left')
ax.set_title(
'Seismometer, {dname}'.format(dname=dataname.replace('_', '')),
fontsize=16)
plot.savefig('./{dname}/ASD_{dname}_X.png'.format(dname=dataname))
plot.close()
print('save ./{dname}/ASD_{dname}_X.png'.format(dname=dataname))
#
f = np.logspace(-3, 2, 1e4)
w = 2.0 * np.pi * f
L = 3000.0 # m
c_p = 5500.0 # m/sec
cdmr_p = lambda w, c: np.sqrt((1.0 + np.cos(L * w / c)) /
(1.0 - np.cos(L * w / c)))
cdmr_ps = lambda w, c: np.sqrt((1.0 + jv(0, 2 * L * w / c)) /
(1.0 - jv(0, 2 * L * w / c)))
if True:
from gwpy.plot import Plot
import matplotlib.pyplot as plt
# Finally, we can :meth:`~TimeSeries.plot` the original and filtered data,
# adding some code to prettify the figure:
from gwpy.plot import Plot
plot = Plot(hdata, hfilt, figsize=[12, 6], separate=True, sharex=True,
color='gwpy:ligo-hanford')
ax1, ax2 = plot.axes
ax1.set_title('LIGO-Hanford strain data around GW150914')
ax1.text(1.0, 1.01, 'Unfiltered data', transform=ax1.transAxes, ha='right')
ax1.set_ylabel('Amplitude [strain]', y=-0.2)
ax2.set_ylabel('')
ax2.text(1.0, 1.01, '50-250\,Hz bandpass, notches at 60, 120, 180 Hz',
transform=ax2.transAxes, ha='right')
plot.show()
plot.close() # hide
# We see now a spike around 16 seconds into the data, so let's zoom into
# that time (and prettify):
plot = hfilt.plot(color='gwpy:ligo-hanford')
ax = plot.gca()
ax.set_title('LIGO-Hanford strain data around GW150914')
ax.set_ylabel('Amplitude [strain]')
ax.set_xlim(1126259462, 1126259462.6)
ax.set_xscale('seconds', epoch=1126259462)
plot.show()
plot.close() # hide
# Congratulations, you have succesfully filtered LIGO data to uncover the
# first ever directly-detected gravitational wave signal, GW150914!
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs):
"""Plot the signifiance drop for each channel
"""
channels = sorted(old.keys())
if show_channel_names is None:
show_channel_names = len(channels) <= 50
plot = Plot(figsize=(18, 6))
plot.subplots_adjust(left=.07, right=.93)
ax = plot.gca()
if show_channel_names:
plot.subplots_adjust(bottom=.4)
winner = sorted(old.items(), key=lambda x: x[1])[-1][0]
for i, c in enumerate(channels):
if c == winner:
color = 'orange'
elif old[c] > new[c]:
color = 'dodgerblue'
else:
color = 'red'
ax.plot([i, i], [old[c], new[c]], color=color, linestyle='-',
marker='o', markeredgecolor='k', markeredgewidth=.5,
markersize=10, label=c, zorder=old[c])
ax.set_xlim(-1, len(channels))
ax.set_ybound(lower=0)
# set xticks to show channel names
if show_channel_names:
ax.set_xticks(range(len(channels)))
ax.set_xticklabels([c.replace('_','\_') for c in channels])
for i, t in enumerate(ax.get_xticklabels()):
t.set_rotation(270)
t.set_verticalalignment('top')
t.set_horizontalalignment('center')
t.set_fontsize(8)
# or just show systems of channels
else:
plot.canvas.draw()
systems = {}
for i, c in enumerate(channels):
sys = c.split(':', 1)[1].split('-')[0].split('_')[0]
try:
systems[sys][1] += 1
except KeyError:
systems[sys] = [i, 1]
systems = sorted(systems.items(), key=lambda x: x[1][0])
labels, counts = zip(*systems)
xticks, xmticks = zip(*[(a, a+b/2.) for (a, b) in counts])
# show ticks at the edge of each group
ax.set_xticks(xticks, minor=False)
ax.set_xticklabels([], minor=False)
# show label in the centre of each group
ax.set_xticks(xmticks, minor=True)
for t in ax.set_xticklabels(labels, minor=True):
t.set_rotation(270)
kwargs.setdefault('ylabel', 'Significance')
# create interactivity
if outfile.endswith('.svg'):
_finalize_plot(plot, ax, outfile.replace('.svg', '.png'),
close=False, **kwargs)
tooltips = []
ylim = ax.get_ylim()
yoffset = (ylim[1] - ylim[0]) * 0.061
bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'}
xthresh = len(channels) / 10.
for i, l in enumerate(ax.lines):
x = l.get_xdata()[1]
if x < xthresh:
ha = 'left'
elif x > (len(channels) - xthresh):
ha ='right'
else:
ha = 'center'
y = l.get_ydata()[0] + yoffset
c = l.get_label()
tooltips.append(ax.annotate(c.replace('_', r'\_'), (x, y),
ha=ha, zorder=ylim[1], bbox=bbox))
l.set_gid('line-%d' % i)
tooltips[-1].set_gid('tooltip-%d' % i)
f = BytesIO()
plot.savefig(f, format='svg')
tree, xmlid = etree.XMLID(f.getvalue())
tree.set('onload', 'init(evt)')
for i in range(len(tooltips)):
try:
e = xmlid['tooltip-%d' % i]
except KeyError:
warnings.warn("Failed to recover tooltip %d" % i)
continue
e.set('visibility', 'hidden')
for i, l in enumerate(ax.lines):
e = xmlid['line-%d' % i]
e.set('onmouseover', 'ShowTooltip(this)')
e.set('onmouseout', 'HideTooltip(this)')
tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT))
etree.ElementTree(tree).write(outfile)
plot.close()
else:
_finalize_plot(plot, ax, outfile, **kwargs)
def process_channel(input_,):
chan, ts = input_
flat = ts.value.min() == ts.value.max()
if flat:
corr1 = None
corr2 = None
corr1s = None
corr2s = None
plot1 = None
plot2 = None
plot3 = None
else:
corr1 = numpy.corrcoef(ts.value, darmblrms.value)[0, 1]
corr1s = spearmanr(ts.value, darmblrms.value)[0]
if args.trend_type == 'minute':
corr2 = numpy.corrcoef(ts.value, rangets.value)[0, 1]
corr2s = spearmanr(ts.value, rangets.value)[0]
else:
corr2 = 0.0
corr2s = 0.0
# if all corralations are below threshold it does not plot
if((abs(corr1) < args.threshold)
and (abs(corr1s) < args.threshold)
and (abs(corr2) < args.threshold)
and (abs(corr2s) < args.threshold)):
plot1 = None
plot2 = None
plot3 = None
return (chan, corr1, corr2, plot1,
plot2, plot3, corr1s, corr2s)
plot = Plot(darmblrms, ts, rangets,
xscale="auto-gps", separate=True,
figsize=(12, 12))
plot.subplots_adjust(*p1)
plot.axes[0].set_ylabel('$h(t)$ BLRMS [strain]')
plot.axes[1].set_ylabel('Channel units')
plot.axes[2].set_ylabel('Sensitive range [Mpc]')
for ax in plot.axes:
ax.legend(loc='best')
ax.set_xlim(start, end)
ax.set_epoch(start)
channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1)
plot1 = '%s_TRENDS-%s.png' % (channelstub, gpsstub)
try:
plot.save(plot1)
except (IOError, IndexError):
plot.save(plot1)
except RuntimeError as e:
if 'latex' in str(e).lower():
plot.save(plot1)
else:
raise
plot.close()
# plot auto-scaled verions
tsscaled = ts.detrend()
tsrms = numpy.sqrt(sum(tsscaled**2.0)/len(tsscaled))
if args.trend_type == 'minute':
tsscaled *= (rangerms / tsrms)
if corr1 > 0:
tsscaled *= -1
else:
tsscaled *= (darmrms / tsrms)
if corr1 < 0:
tsscaled *= -1
plot = Plot(darmscaled, rangescaled, tsscaled,
xscale="auto-gps", figsize=[12, 6])
plot.subplots_adjust(*p2)
ax = plot.gca()
ax.set_xlim(start, end)
ax.set_epoch(start)
ax.set_ylabel('Scaled amplitude [arbitrary units]')
ax.legend(loc='best')
plot2 = '%s_COMPARISON-%s.png' % (channelstub, gpsstub)
try:
plot.save(plot2)
except (IOError, IndexError):
plot.save(plot2)
except RuntimeError as e:
if 'latex' in str(e).lower():
plot.save(plot2)
else:
raise
plot.close()
# plot scatter plots
rangeColor = 'red'
darmblrmsColor = 'blue'
tsCopy = ts.reshape(-1, 1)
rangetsCopy = rangets.reshape(-1, 1)
darmblrmsCopy = darmblrms.reshape(-1, 1)
darmblrmsReg = linear_model.LinearRegression()
darmblrmsReg.fit(tsCopy, darmblrmsCopy)
darmblrmsFit = darmblrmsReg.predict(tsCopy)
rangeReg = linear_model.LinearRegression()
rangeReg.fit(tsCopy, rangetsCopy)
rangeFit = rangeReg.predict(tsCopy)
fig = Plot(figsize=(12, 6))
fig.subplots_adjust(*p2)
ax = fig.add_subplot(121)
ax.set_xlabel('Channel units')
ax.set_ylabel('Sensitive range [Mpc]')
yrange = abs(max(darmblrms.value) - min(darmblrms.value))
upperLim = max(darmblrms.value) + .1 * yrange
lowerLim = min(darmblrms.value) - .1 * yrange
ax.set_ylim(lowerLim, upperLim)
ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr1)),
verticalalignment='bottom', horizontalalignment='right',
transform=ax.transAxes, color='black', size=20,
bbox=dict(boxstyle='square', facecolor='white', alpha=.75,
edgecolor='black'))
fig.add_scatter(ts, darmblrms, color=darmblrmsColor)
fig.add_line(ts, darmblrmsFit, color='black')
ax = fig.add_subplot(122)
ax.set_xlabel('Channel units')
ax.set_ylabel('$h(t)$ BLRMS [strain]')
ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr2)),
verticalalignment='bottom', horizontalalignment='right',
transform=ax.transAxes, color='black', size=20,
bbox=dict(boxstyle='square', facecolor='white', alpha=.75,
edgecolor='black'))
fig.add_scatter(ts, rangets, color=rangeColor)
fig.add_line(ts, rangeFit, color='black')
plot3 = '%s_SCATTER-%s.png' % (channelstub, gpsstub)
try:
fig.save(plot3)
except (IOError, IndexError):
fig.save(plot3)
except RuntimeError as e:
if 'latex' in str(e).lower():
fig.save(plot3)
else:
raise
plt.close(fig)
# increment counter and print status
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / nchan
LOGGER.debug("Completed [%d/%d] %3d%% %-50s"
% (counter.value, nchan, pc, '(%s)' % str(chan)))
sys.stdout.flush()
return chan, corr1, corr2, plot1, plot2, plot3, corr1s, corr2s
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs):
"""Plot the signifiance drop for each channel
"""
channels = sorted(old.keys())
if show_channel_names is None:
show_channel_names = len(channels) <= 50
plot = Plot(figsize=(20, 5))
plot.subplots_adjust(left=.07, right=.93)
ax = plot.gca()
if show_channel_names:
plot.subplots_adjust(bottom=.4)
winner = sorted(old.items(), key=lambda x: x[1])[-1][0]
for i, c in enumerate(channels):
if c == winner:
color = 'orange'
elif old[c] > new[c]:
color = 'dodgerblue'
else:
color = 'red'
ax.plot([i, i], [old[c], new[c]],
color=color,
linestyle='-',
marker='o',
markeredgecolor='k',
markeredgewidth=.5,
markersize=10,
label=c,
zorder=old[c])
ax.set_xlim(-1, len(channels))
ax.set_ybound(lower=0)
# set xticks to show channel names
if show_channel_names:
ax.set_xticks(range(len(channels)))
ax.set_xticklabels([texify(c) for c in channels])
for i, t in enumerate(ax.get_xticklabels()):
t.set_rotation(270)
t.set_verticalalignment('top')
t.set_horizontalalignment('center')
t.set_fontsize(8)
# or just show systems of channels
else:
plot.canvas.draw()
systems = {}
for i, c in enumerate(channels):
sys = c.split(':', 1)[1].split('-')[0].split('_')[0]
try:
systems[sys][1] += 1
except KeyError:
systems[sys] = [i, 1]
systems = sorted(systems.items(), key=lambda x: x[1][0])
labels, counts = zip(*systems)
xticks, xmticks = zip(*[(a, a + b / 2.) for (a, b) in counts])
# show ticks at the edge of each group
ax.set_xticks(xticks, minor=False)
ax.set_xticklabels([], minor=False)
# show label in the centre of each group
ax.set_xticks(xmticks, minor=True)
for t in ax.set_xticklabels(labels, minor=True):
t.set_rotation(270)
kwargs.setdefault('ylabel', 'Significance')
# create interactivity
if outfile.endswith('.svg'):
_finalize_plot(plot,
ax,
outfile.replace('.svg', '.png'),
close=False,
**kwargs)
tooltips = []
ylim = ax.get_ylim()
yoffset = (ylim[1] - ylim[0]) * 0.061
bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'}
xthresh = len(channels) / 10.
for i, l in enumerate(ax.lines):
x = l.get_xdata()[1]
if x < xthresh:
ha = 'left'
elif x > (len(channels) - xthresh):
ha = 'right'
else:
ha = 'center'
y = l.get_ydata()[0] + yoffset
c = l.get_label()
tooltips.append(
ax.annotate(texify(c), (x, y),
ha=ha,
zorder=ylim[1],
bbox=bbox))
l.set_gid('line-%d' % i)
tooltips[-1].set_gid('tooltip-%d' % i)
f = BytesIO()
plot.savefig(f, format='svg')
tree, xmlid = etree.XMLID(f.getvalue())
tree.set('onload', 'init(evt)')
for i in range(len(tooltips)):
try:
e = xmlid['tooltip-%d' % i]
except KeyError:
warnings.warn("Failed to recover tooltip %d" % i)
continue
e.set('visibility', 'hidden')
for i, l in enumerate(ax.lines):
e = xmlid['line-%d' % i]
e.set('onmouseover', 'ShowTooltip(this)')
e.set('onmouseout', 'HideTooltip(this)')
tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT))
etree.ElementTree(tree).write(outfile)
plot.close()
else:
_finalize_plot(plot, ax, outfile, **kwargs)
def main(args=None):
use('agg')
rcParams.update({
'figure.subplot.bottom': 0.15,
'figure.subplot.left': 0.1,
'figure.subplot.right': 0.83,
'figure.subplot.top': 0.93,
'figure.subplot.hspace': 0.25,
'axes.labelsize': 20,
'grid.color': 'gray',
})
grid = GridSpec(2, 1)
logger = log.Logger('omicron-status')
try:
omicronversion = str(get_omicron_version())
except KeyError:
omicronversion = 'Unknown'
logger.warning("Omicron version unknown")
else:
logger.info("Found omicron version: %s" % omicronversion)
parser = create_parser()
args = parser.parse_args(args=args)
if args.ifo is None:
parser.error("Cannot determine IFO prefix from sytem, "
"please pass --ifo on the command line")
group = args.group
logger.info("Checking status for %r group" % group)
archive = args.archive_directory
proddir = args.production_directory.with_name(
args.production_directory.name.format(group=args.group), )
outdir = args.output_directory
outdir.mkdir(exist_ok=True, parents=True)
tag = args.latency_archive_tag.format(group=args.group)
filetypes = ['h5', 'xml.gz', 'root']
logger.debug("Set output directory to %s" % outdir)
logger.debug(
"Will process the following filetypes: {}".format(
", ".join(filetypes), ), )
# -- parse configuration file and get parameters --------------------------
cp = configparser.ConfigParser()
ok = cp.read(args.config_file)
if args.config_file not in ok:
raise IOError(
"Failed to read configuration file %r" % args.config_file, )
logger.info("Configuration read")
# validate
if not cp.has_section(group):
raise configparser.NoSectionError(group)
# get parameters
obs = args.ifo[0]
frametype = cp.get(group, 'frametype')
padding = cp.getint(group, 'overlap-duration') / 2.
mingap = cp.getint(group, 'chunk-duration')
channels = args.channel
if not channels:
channels = [
c.split()[0]
for c in cp.get(group, 'channels').strip('\n').split('\n')
]
channels.sort()
logger.debug("Found %d channels" % len(channels))
start = args.gps_start_time
end = args.gps_end_time
if end == NOW:
end -= padding
if args.state_flag:
stateflag = args.state_flag
statepad = tuple(map(float, args.state_pad.split(',')))
else:
try:
stateflag = cp.get(group, 'state-flag')
except configparser.NoOptionError:
stateflag = None
else:
try:
statepad = tuple(
map(
float,
cp.get(group, 'state-padding').split(','),
))
except configparser.NoOptionError:
statepad = (0, 0)
if stateflag:
logger.debug("Parsed state flag: %r" % stateflag)
logger.debug("Parsed state padding: %s" % repr(statepad))
logger.info("Processing %d-%d" % (start, end))
# -- define nagios JSON printer -------------------------------------------
def print_nagios_json(code, message, outfile, tag='status', **extras):
out = {
'created_gps':
NOW,
'status_intervals': [
{
'start_sec': 0,
'end_sec': args.unknown,
'num_status': code,
'txt_status': message
},
{
'start_sec': args.unknown,
'num_status': 3,
'txt_status': 'Omicron %s check is not running' % tag
},
],
'author': {
'name': 'Duncan Macleod',
'email': '*****@*****.**',
},
'omicron': {
'version': omicronversion,
'group': group,
'channels': ' '.join(channels),
'frametype': frametype,
'state-flag': stateflag,
},
'pyomicron': {
'version': __version__,
},
}
out.update(extras)
with open(outfile, 'w') as f:
f.write(json.dumps(out))
logger.debug("nagios info written to %s" % outfile)
# -- get condor status ------------------------------------------------
if not args.skip_condor:
# connect to scheduler
try:
schedd = htcondor.Schedd()
except RuntimeError as e:
logger.warning("Caught %s: %s" % (type(e).__name__, e))
logger.info("Failed to connect to HTCondor scheduler, cannot "
"determine condor status for %s" % group)
schedd = None
if not args.skip_condor and schedd:
logger.info("-- Checking condor status --")
# get DAG status
jsonfp = outdir / "nagios-condor-{}.json".format(group)
okstates = ['Running', 'Idle', 'Completed']
try:
# check manager status
qstr = 'OmicronManager == "{}" && Owner == "{}"'.format(
group,
args.user,
)
try:
jobs = schedd.query(qstr, ['JobStatus'])
except IOError as e:
warnings.warn("Caught IOError: %s [retrying...]" % str(e))
sleep(2)
jobs = schedd.query(qstr, ['JobStatus'])
logger.debug(
"Found {} jobs for query {!r}".format(len(jobs), qstr), )
if len(jobs) > 1:
raise RuntimeError(
"Multiple OmicronManager jobs found for %r" % group)
elif len(jobs) == 0:
raise RuntimeError(
"No OmicronManager job found for %r" % group, )
status = condor.JOB_STATUS[jobs[0]['JobStatus']]
if status not in okstates:
raise RuntimeError("OmicronManager status for %r: %r" %
(group, status))
logger.debug("Manager status is %r" % status)
# check node status
jobs = schedd.query(
'OmicronProcess == "{}" && Owner == "{}"'.format(
group,
args.user,
),
['JobStatus', 'ClusterId'],
)
logger.debug(
"Found {} jobs for query {!r}".format(len(jobs), qstr), )
for job in jobs:
status = condor.JOB_STATUS[job['JobStatus']]
if status not in okstates:
raise RuntimeError("Omicron node %s (%r) is %r" %
(job['ClusterId'], group, status))
except RuntimeError as e:
print_nagios_json(2, str(e), jsonfp, tag='condor')
logger.warning("Failed to determine condor status: %r" % str(e))
except IOError as e:
logger.warning("Caught %s: %s" % (type(e).__name__, e))
logger.info("Failed to connect to HTCondor scheduler, cannot "
"determine condor status for %s" % group)
else:
print_nagios_json(
0,
"Condor processing for %r is OK" % group,
jsonfp,
tag='condor',
)
logger.info("Condor processing is OK")
if not args.skip_job_duration:
# get job duration history
plot = Plot(figsize=[12, 3])
plot.subplots_adjust(bottom=.22, top=.87)
ax = plot.gca(xscale="auto-gps")
times, jobdur = condor.get_job_duration_history_shell('OmicronProcess',
group,
maxjobs=5000)
logger.debug("Recovered duration history for %d omicron.exe jobs" %
len(times))
line = ax.plot([0], [1], label='Omicron.exe')[0]
ax.plot(times,
jobdur,
linestyle=' ',
marker='.',
color=line.get_color())
times, jobdur = condor.get_job_duration_history_shell(
'OmicronPostProcess', group, maxjobs=5000)
logger.debug("Recovered duration history for %d post-processing jobs" %
len(times))
line = ax.plot([0], [1], label='Post-processing')[0]
ax.plot(times,
jobdur,
linestyle=' ',
marker='.',
color=line.get_color())
ax.legend(loc='upper left',
borderaxespad=0,
bbox_to_anchor=(1.01, 1),
handlelength=1)
ax.set_xlim(args.gps_start_time, args.gps_end_time)
ax.set_epoch(ax.get_xlim()[1])
ax.set_yscale('log')
ax.set_title('Omicron job durations for %r' % group)
ax.set_ylabel('Job duration [seconds]')
ax.xaxis.labelpad = 5
png = str(outdir / "nagios-condor-{}.png".format(group))
plot.save(png)
plot.close()
logger.debug("Saved condor plot to %s" % png)
if args.skip_file_checks:
sys.exit(0)
# -- get file latency and archive completeness ----------------------------
logger.info("-- Checking file archive --")
# get frame segments
segs = segments.get_frame_segments(obs, frametype, start, end)
# get state segments
if stateflag is not None:
segs &= segments.query_state_segments(
stateflag,
start,
end,
pad=statepad,
)
try:
end = segs[-1][1]
except IndexError:
pass
# apply inwards padding to generate resolvable segments
for i in range(len(segs) - 1, -1, -1):
# if segment is shorter than padding, ignore it completely
if abs(segs[i]) <= padding * 2:
del segs[i]
# otherwise apply padding to generate trigger segment
else:
segs[i] = segs[i].contract(padding)
logger.debug("Found %d seconds of analysable time" % abs(segs))
# load archive latency
latencyfile = outdir / "nagios-latency-{}.h5".format(tag)
times = dict((c, dict((ft, None) for ft in filetypes)) for c in channels)
ldata = dict((c, dict((ft, None) for ft in filetypes)) for c in channels)
try:
with h5py.File(latencyfile, 'r') as h5file:
for c in channels:
for ft in filetypes:
try:
times[c][ft] = h5file[c]['time'][ft][:]
ldata[c][ft] = h5file[c]['latency'][ft][:]
except KeyError:
times[c][ft] = numpy.ndarray((0, ))
ldata[c][ft] = numpy.ndarray((0, ))
except OSError as exc: # file not found, or is corrupt
warnings.warn("failed to load latency data from {}: {}".format(
latencyfile,
str(exc),
))
for c in channels:
for ft in filetypes:
if not times[c].get(ft):
times[c][ft] = numpy.ndarray((0, ))
ldata[c][ft] = numpy.ndarray((0, ))
else:
logger.debug("Parsed latency data from %s" % latencyfile)
# load acknowledged gaps
acksegfile = str(outdir / "acknowledged-gaps-{}.txt".format(tag))
try:
acknowledged = SegmentList.read(acksegfile,
gpstype=float,
format="segwizard")
except IOError: # no file
acknowledged = SegmentList()
else:
logger.debug(
"Read %d segments from %s" % (len(acknowledged), acksegfile), )
acknowledged.coalesce()
# build legend for segments
leg = OrderedDict()
leg['Analysable'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightgray',
edgecolor='gray',
)
leg['Available'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightgreen',
edgecolor='green',
)
leg['Missing'] = SegmentRectangle(
[0, 1],
0,
facecolor='red',
edgecolor='darkred',
)
leg['Unresolvable'] = SegmentRectangle(
[0, 1],
0,
facecolor='magenta',
edgecolor='purple',
)
leg['Overlapping'] = SegmentRectangle(
[0, 1],
0,
facecolor='yellow',
edgecolor='orange',
)
leg['Pending'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightskyblue',
edgecolor='blue',
)
leg['Acknowledged'] = SegmentRectangle(
[0, 1],
0,
facecolor='sandybrown',
edgecolor='brown',
)
logger.debug("Checking archive for each channel...")
# find files
latency = {}
gaps = {}
overlap = {}
pending = {}
plots = {}
for c in channels:
# create data storate
latency[c] = {}
gaps[c] = {}
overlap[c] = {}
pending[c] = {}
# create figure
plot = Plot(figsize=[12, 5])
lax = plot.add_subplot(grid[0, 0], xscale="auto-gps")
sax = plot.add_subplot(grid[1, 0], sharex=lax, projection='segments')
colors = ['lightblue', 'dodgerblue', 'black']
for y, ft in enumerate(filetypes):
# find files
cache = io.find_omicron_files(c, start, end, archive, ext=ft)
cpend = sieve_cache(io.find_pending_files(c, proddir, ext=ft),
segment=Segment(start, end))
# get available segments
avail = segments.cache_segments(cache)
found = avail & segs
pending[c][ft] = segments.cache_segments(cpend) & segs
# remove gaps at the end that represent latency
try:
latency[c][ft] = abs(segs & type(
segs)([type(segs[0])(found[-1][1], segs[-1][1])])) / 3600.
except IndexError:
latency[c][ft] = 0
processed = segs
else:
processed = segs & type(segs)(
[type(segs[0])(start, found[-1][1])])
gaps[c][ft] = type(found)()
lost = type(found)()
for s in processed - found:
if abs(s) < mingap and s in list(segs):
lost.append(s)
else:
gaps[c][ft].append(s)
# remove acknowledged gaps
ack = gaps[c][ft] & acknowledged
gaps[c][ft] -= acknowledged
# print warnings
if abs(gaps[c][ft]):
warnings.warn("Gaps found in %s files for %s:\n%s" %
(c, ft, gaps[c][ft]))
overlap[c][ft] = segments.cache_overlaps(cache)
if abs(overlap[c][ft]):
warnings.warn("Overlap found in %s files for %s:\n%s" %
(c, ft, overlap[c][ft]))
# append archive
times[c][ft] = numpy.concatenate((times[c][ft][-99999:], [NOW]))
ldata[c][ft] = numpy.concatenate(
(ldata[c][ft][-99999:], [latency[c][ft]]))
# plot
line = lax.plot(
times[c][ft],
ldata[c][ft],
label=ft,
color=colors[y],
)[0]
lax.plot(times[c][ft],
ldata[c][ft],
marker='.',
linestyle=' ',
color=line.get_color())
sax.plot_segmentlist(segs,
y=y,
label=ft,
alpha=.5,
facecolor=leg['Analysable'].get_facecolor(),
edgecolor=leg['Analysable'].get_edgecolor())
sax.plot_segmentlist(pending[c][ft],
y=y,
facecolor=leg['Pending'].get_facecolor(),
edgecolor=leg['Pending'].get_edgecolor())
sax.plot_segmentlist(avail,
y=y,
label=ft,
alpha=.2,
height=.1,
facecolor=leg['Available'].get_facecolor(),
edgecolor=leg['Available'].get_edgecolor())
sax.plot_segmentlist(found,
y=y,
label=ft,
alpha=.5,
facecolor=leg['Available'].get_facecolor(),
edgecolor=leg['Available'].get_edgecolor())
sax.plot_segmentlist(lost,
y=y,
facecolor=leg['Unresolvable'].get_facecolor(),
edgecolor=leg['Unresolvable'].get_edgecolor())
sax.plot_segmentlist(gaps[c][ft],
y=y,
facecolor=leg['Missing'].get_facecolor(),
edgecolor=leg['Missing'].get_edgecolor())
sax.plot_segmentlist(overlap[c][ft],
y=y,
facecolor=leg['Overlapping'].get_facecolor(),
edgecolor=leg['Overlapping'].get_edgecolor())
sax.plot_segmentlist(ack,
y=y,
facecolor=leg['Acknowledged'].get_facecolor(),
edgecolor=leg['Acknowledged'].get_edgecolor())
# finalise plot
lax.axhline(args.warning / 3600.,
color=(1.0, 0.7, 0.0),
linestyle='--',
linewidth=2,
label='Warning',
zorder=-1)
lax.axhline(args.error / 3600.,
color='red',
linestyle='--',
linewidth=2,
label='Critical',
zorder=-1)
lax.set_title('Omicron status: {}'.format(c))
lax.set_ylim(0, args.error / 1800.)
lax.set_ylabel('Latency [hours]')
lax.legend(loc='upper left',
bbox_to_anchor=(1.01, 1),
borderaxespad=0,
handlelength=2,
fontsize=12.4)
lax.set_xlabel(' ')
for ax in plot.axes:
ax.set_xlim(args.gps_start_time, args.gps_end_time)
ax.set_epoch(ax.get_xlim()[1])
sax.xaxis.labelpad = 5
sax.set_ylim(-.5, len(filetypes) - .5)
sax.legend(leg.values(),
leg.keys(),
handlelength=1,
fontsize=12.4,
loc='lower left',
bbox_to_anchor=(1.01, 0),
borderaxespad=0)
plots[c] = png = outdir / "nagios-latency-{}.png".format(
c.replace(':', '-'), )
plot.save(png)
plot.close()
logger.debug(" %s" % c)
# update latency and write archive
h5file = h5py.File(latencyfile, 'w')
for c in channels:
g = h5file.create_group(c)
for name, d in zip(['time', 'latency'], [times[c], ldata[c]]):
g2 = g.create_group(name)
for ft in filetypes:
g2.create_dataset(ft, data=d[ft], compression='gzip')
h5file.close()
logger.debug("Stored latency data as HDF in %s" % latencyfile)
# write nagios output for files
status = []
for segset, tag in zip([gaps, overlap], ['gaps', 'overlap']):
chans = [(c, segset[c]) for c in segset
if abs(reduce(operator.or_, segset[c].values()))]
jsonfp = outdir / "nagios-{}-{}.json".format(tag, group)
status.append((tag, jsonfp))
if chans:
gapstr = '\n'.join('%s: %s' % c for c in chans)
code = 1
message = ("%s found in Omicron files for group %r\n%s" %
(tag.title(), group, gapstr))
else:
code = 0
message = ("No %s found in Omicron files for group %r" %
(tag, group))
print_nagios_json(code, message, jsonfp, tag=tag, **{tag: dict(chans)})
# write group JSON
jsonfp = outdir / "nagios-latency-{}.json".format(group)
status.append(('latency', jsonfp))
code = 0
message = 'No channels have high latency for group %r' % group
ldict = dict((c, max(latency[c].values())) for c in latency)
for x, dt in zip([2, 1], [args.error, args.warning]):
dh = dt / 3600.
chans = [c for c in ldict if ldict[c] >= dh]
if chans:
code = x
message = (
"%d channels found with high latency (above %s seconds)" %
(len(chans), dt))
break
print_nagios_json(code, message, jsonfp, tag='latency', latency=ldict)
# auto-detect 'standard' JSON files
for tag, name in zip(
['condor', 'omicron-online'],
['condor', 'processing'],
):
f = outdir / "nagios-{}-{}.json".format(tag, group)
if f.is_file():
status.insert(0, (name, f))
# write HTML summary
if args.html:
page = markup.page()
page.init(
title="%s Omicron Online status" % group,
css=[
('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/'
'bootstrap.min.css'),
('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/'
'jquery.fancybox.min.css'),
],
script=[
'//code.jquery.com/jquery-1.11.2.min.js',
('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/'
'bootstrap.min.js'),
('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/'
'jquery.fancybox.min.js'),
],
)
page.div(class_='container')
# write header
page.div(class_='page-header')
page.h1('Omicron Online status: %s' % group)
page.div.close() # page-header
# write summary
page.div(id_='json')
page.h2("Processing status")
for tag, f in status:
jf = f.name
page.a("%s status" % tag.title(),
href=jf,
role='button',
target="_blank",
id_="nagios-%s" % tag,
class_='btn btn-default json-status')
page.p(style="padding-top: 5px;")
page.small(
"Hover over button for explanation, click to open JSON file", )
page.p.close()
page.div.close() # id=json
# show plots
page.div(id_='plots')
page.h2("Channel details")
page.div(class_='row')
for channel in sorted(channels):
png = plots[channel].name
page.div(class_="col-sm-6 col-md-4")
page.div(class_="panel panel-default")
page.div(class_='panel-heading')
page.h3(channel, class_='panel-title', style="font-size: 14px;")
page.div.close() # panel-heading
page.div(class_='panel-body')
page.a(href=png,
target="_blank",
class_="fancybox",
rel="channel-status-img")
page.img(src=png, class_='img-responsive')
page.a.close()
page.div.close() # panel-body
page.div.close() # panel
page.div.close() # col
page.div.close() # row
page.div.close() # id=plots
# dump parameters
page.div(id_="parameters")
page.h2("Parameters")
for key, val in cp.items(group):
page.p()
page.strong("%s:" % key)
page.add(val)
page.p.close()
page.div.close() # id=parameters
# finish and close
page.div.close() # container
page.script("""
function setStatus(data, id) {
var txt = data.status_intervals[0].txt_status.split("\\n")[0];
$("#"+id).attr("title", txt);
var stat = data.status_intervals[0].num_status;
if (stat == 0) {
$("#"+id).addClass("btn-success"); }
else if (stat == 1) {
$("#"+id).addClass("btn-warning"); }
else if (stat == 2){
$("#"+id).addClass("btn-danger"); }
}
$(document).ready(function() {
$(".json-status").each(function() {
var jsonf = $(this).attr("href");
var id = $(this).attr("id");
$.getJSON(jsonf, function(data) { setStatus(data, id); });
});
$(".fancybox").fancybox({nextEffect: 'none', prevEffect: 'none'});
});""",
type="text/javascript")
with (outdir / "index.html").open("w") as f:
f.write(str(page))
logger.debug("HTML summary written to %s" % f.name)
def main(args=None):
"""Run the primary scattering command-line tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# set up logger
logger = cli.logger(
name=PROG.split('python -m ').pop(),
level='DEBUG' if args.verbose else 'INFO',
)
# useful variables
fthresh = (
int(args.frequency_threshold) if args.frequency_threshold.is_integer()
else args.frequency_threshold)
multiplier = args.multiplier_for_threshold
tstr = str(fthresh).replace('.', '_')
gpsstr = '%s-%s' % (int(args.gpsstart), int(args.gpsend - args.gpsstart))
args.optic = args.optic or list(OPTIC_MOTION_CHANNELS.keys())
# go to working directory
indir = os.getcwd()
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
os.chdir(args.output_dir)
# set up output files
summfile = '{}-SCATTERING_SUMMARY-{}.csv'.format(
args.ifo, gpsstr)
segfile = '{}-SCATTERING_SEGMENTS_{}_HZ-{}.h5'.format(
args.ifo, tstr, gpsstr)
# log start of process
logger.info('{} Scattering {}-{}'.format(
args.ifo, int(args.gpsstart), int(args.gpsend)))
# -- get state segments -----------
span = Segment(args.gpsstart, args.gpsend)
# get segments
if args.state_flag is not None:
state = DataQualityFlag.query(
args.state_flag, int(args.gpsstart), int(args.gpsend),
url=DEFAULT_SEGMENT_SERVER,
).coalesce()
statea = []
padding = args.segment_start_pad + args.segment_end_pad
for i, seg in enumerate(state.active):
if abs(seg) > padding:
statea.append(Segment(
seg[0] + args.segment_start_pad,
seg[1] - args.segment_end_pad,
))
else:
logger.debug(
"Segment length {} shorter than padding length {}, "
"skipping segment {}-{}".format(abs(seg), padding, *seg),
)
statea = SegmentList(statea)
logger.debug("Downloaded %d segments for %s"
% (len(statea), args.state_flag))
else:
statea = SegmentList([span])
livetime = float(abs(statea))
logger.debug("Processing %.2f s of livetime" % livetime)
# -- load h(t) --------------------
args.main_channel = args.main_channel.format(IFO=args.ifo)
logger.debug("Loading Omicron triggers for %s" % args.main_channel)
if args.gpsstart >= 1230336018: # Jan 1 2019
ext = "h5"
names = ["time", "frequency", "snr"]
read_kw = {
"columns": names,
"selection": [
"{0} < frequency < {1}".format(
args.fmin, multiplier * fthresh),
("time", in_segmentlist, statea),
],
"format": "hdf5",
"path": "triggers",
}
else:
ext = "xml.gz"
names = ['peak', 'peak_frequency', 'snr']
read_kw = {
"columns": names,
"selection": [
"{0} < peak_frequency < {1}".format(
args.fmin, multiplier * fthresh),
('peak', in_segmentlist, statea),
],
"format": 'ligolw',
"tablename": "sngl_burst",
}
fullcache = []
for seg in statea:
cache = gwtrigfind.find_trigger_files(
args.main_channel, 'omicron', seg[0], seg[1], ext=ext,
)
if len(cache) == 0:
warnings.warn(
"No Omicron triggers found for %s in segment [%d .. %d)"
% (args.main_channel, seg[0], seg[1]),
)
continue
fullcache.extend(cache)
# read triggers
if fullcache:
trigs = EventTable.read(fullcache, nproc=args.nproc, **read_kw)
else: # no files (no livetime?)
trigs = EventTable(names=names)
highsnrtrigs = trigs[trigs['snr'] >= 8]
logger.debug("%d read" % len(trigs))
# -- prepare HTML -----------------
links = [
'%d-%d' % (int(args.gpsstart), int(args.gpsend)),
('Parameters', '#parameters'),
('Segments', (
('State flag', '#state-flag'),
('Optical sensors', '#osems'),
('Transmons', '#transmons'),
)),
]
if args.omega_scans:
links.append(('Scans', '#omega-scans'))
(brand, class_) = htmlio.get_brand(args.ifo, 'Scattering', args.gpsstart)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(
title='%s Scattering | %d-%d' % (
args.ifo, int(args.gpsstart), int(args.gpsend)),
navbar=navbar)
page.div(class_='pb-2 mt-3 mb-2 border-bottom')
page.h1('%s Scattering: %d-%d'
% (args.ifo, int(args.gpsstart), int(args.gpsend)))
page.div.close() # pb-2 mt-3 mb-2 border-bottom
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(
start=int(args.gpsstart), end=int(args.gpsend), flag=args.state_flag))
page.div.close() # col-md-9 col-sm-12
# link to summary files
page.div(class_='col-md-3 col-sm-12')
page.add(htmlio.download_btn(
[('Segments (HDF)', segfile),
('Triggers (CSV)', summfile)],
btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(),
))
page.div.close() # col-md-3 col-sm-12
page.div.close() # row
# command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
# section header
page.h2('Segments', class_='mt-4', id_='segments')
if statea: # contextual information
paper = markup.oneliner.a(
'Accadia et al. (2010)', target='_blank', class_='alert-link',
href='http://iopscience.iop.org/article/10.1088/0264-9381/27'
'/19/194011')
msg = (
"Segments marked \"optical sensors\" below show evidence of beam "
"scattering between {0} and {1} Hz based on the velocity of optic "
"motion, with fringe frequencies projected using equation (3) of "
"{2}. Segments marked \"transmons\" are based on whitened, "
"band-limited RMS trends of transmon sensors. In both cases, "
"yellow panels denote weak evidence for scattering, while red "
"panels denote strong evidence."
).format(args.fmin, multiplier * fthresh, str(paper))
page.add(htmlio.alert(msg, context=args.ifo.lower()))
else: # null segments
page.add(htmlio.alert('No active analysis segments were found',
context='warning', dismiss=False))
# record state segments
if args.state_flag is not None:
page.h3('State flag', class_='mt-3', id_='state-flag')
page.div(id_='accordion1')
page.add(htmlio.write_flag_html(
state, span, 'state', parent='accordion1', context='success',
plotdir='', facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen',
known={'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4}))
page.div.close()
# -- find scattering evidence -----
# read data for OSEMs and transmons
osems = ['%s:%s' % (args.ifo, c) for optic in args.optic for
c in OPTIC_MOTION_CHANNELS[optic]]
transmons = ['%s:%s' % (args.ifo, c) for c in TRANSMON_CHANNELS]
allchannels = osems + transmons
logger.info("Reading all timeseries data")
alldata = []
n = len(statea)
for i, seg in enumerate(statea):
msg = "{0}/{1} {2}:".rjust(30).format(
str(i + 1).rjust(len(str(n))),
n,
str(seg),
) if args.verbose else False
alldata.append(
get_data(allchannels, seg[0], seg[1],
frametype=args.frametype.format(IFO=args.ifo),
verbose=msg, nproc=args.nproc).resample(128))
try: # ensure that only available channels are analyzed
osems = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(osems))
transmons = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(transmons))
except IndexError:
osems = []
transmons = []
# initialize scattering segments
scatter_segments = DataQualityDict()
actives = SegmentList()
# scattering based on OSEM velocity
if statea:
page.h3('Optical sensors (OSEMs)', class_='mt-3', id_='osems')
page.div(id_='osems-group')
logger.info('Searching for scatter based on OSEM velocity')
for i, channel in enumerate(sorted(osems)):
logger.info("-- Processing %s --" % channel)
chanstr = re.sub('[:-]', '_', channel).replace('_', '-', 1)
optic = channel.split('-')[1].split('_')[0]
flag = '%s:DCH-%s_SCATTERING_GE_%s_HZ:1' % (args.ifo, optic, tstr)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering above {0} Hz from {1} in "
"{2}".format(fthresh, optic, channel),
)
# set up plot(s)
plot = Plot(figsize=[12, 12])
axes = {}
axes['position'] = plot.add_subplot(
411, xscale='auto-gps', xlabel='')
axes['fringef'] = plot.add_subplot(
412, sharex=axes['position'], xlabel='')
axes['triggers'] = plot.add_subplot(
413, sharex=axes['position'], xlabel='')
axes['segments'] = plot.add_subplot(
414, projection='segments', sharex=axes['position'])
plot.subplots_adjust(bottom=.07, top=.95)
fringecolors = [None] * len(FREQUENCY_MULTIPLIERS)
histdata = dict((x, numpy.ndarray((0,))) for
x in FREQUENCY_MULTIPLIERS)
linecolor = None
# loop over state segments and find scattering fringes
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
ts = alldata[j][channel]
# get raw data and plot
line = axes['position'].plot(ts, color=linecolor)[0]
linecolor = line.get_color()
# get fringe frequency and plot
fringef = get_fringe_frequency(ts, multiplier=1)
for k, m in list(enumerate(FREQUENCY_MULTIPLIERS))[::-1]:
fm = fringef * m
line = axes['fringef'].plot(
fm, color=fringecolors[k],
label=(j == 0 and r'$f\times%d$' % m or None))[0]
fringecolors[k] = line.get_color()
histdata[m] = numpy.resize(
histdata[m], (histdata[m].size + fm.size,))
histdata[m][-fm.size:] = fm.value
# get segments and plot
scatter = get_segments(
fringef * multiplier,
fthresh,
name=flag,
pad=args.segment_padding
)
axes['segments'].plot(
scatter, facecolor='red', edgecolor='darkred',
known={'alpha': 0.6, 'facecolor': 'lightgray',
'edgecolor': 'gray', 'height': 0.4},
height=0.8, y=0, label=' ',
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# finalize plot
logger.debug("Plotting")
name = texify(channel)
axes['position'].set_title("Scattering evidence in %s" % name)
axes['position'].set_xlabel('')
axes['position'].set_ylabel(r'Position [$\mu$m]')
axes['position'].text(
0.01, 0.95, 'Optic position',
transform=axes['position'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['fringef'].plot(
span, [fthresh, fthresh], 'k--')
axes['fringef'].set_xlabel('')
axes['fringef'].set_ylabel(r'Frequency [Hz]')
axes['fringef'].yaxis.tick_right()
axes['fringef'].yaxis.set_label_position("right")
axes['fringef'].set_ylim(0, multiplier * fthresh)
axes['fringef'].text(
0.01, 0.95, 'Calculated fringe frequency',
transform=axes['fringef'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
handles, labels = axes['fringef'].get_legend_handles_labels()
axes['fringef'].legend(handles[::-1], labels[::-1], loc='upper right',
borderaxespad=0, bbox_to_anchor=(-0.01, 1.),
handlelength=1)
axes['triggers'].scatter(
trigs[names[0]],
trigs[names[1]],
c=trigs[names[2]],
edgecolor='none',
)
name = texify(args.main_channel)
axes['triggers'].text(
0.01, 0.95,
'%s event triggers (Omicron)' % name,
transform=axes['triggers'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['triggers'].set_ylabel('Frequency [Hz]')
axes['triggers'].set_ylim(args.fmin, multiplier * fthresh)
axes['triggers'].colorbar(cmap='YlGnBu', clim=(3, 100), norm='log',
label='Signal-to-noise ratio')
axes['segments'].set_ylim(-.55, .55)
axes['segments'].text(
0.01, 0.95,
r'Time segments with $f\times%d > %.2f$ Hz' % (
multiplier, fthresh),
transform=axes['segments'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
for ax in axes.values():
ax.set_epoch(int(args.gpsstart))
ax.set_xlim(*span)
png = '%s_SCATTERING_%s_HZ-%s.png' % (chanstr, tstr, gpsstr)
try:
plot.save(png)
except OverflowError as e:
warnings.warn(str(e))
plot.axes[1].set_ylim(0, multiplier * fthresh)
plot.refresh()
plot.save(png)
plot.close()
logger.debug("%s written." % png)
# make histogram
histogram = Plot(figsize=[12, 6])
ax = histogram.gca()
hrange = (0, multiplier * fthresh)
for m, color in list(zip(histdata, fringecolors))[::-1]:
if histdata[m].size:
ax.hist(
histdata[m], facecolor=color, alpha=.6, range=hrange,
bins=50, histtype='stepfilled', label=r'$f\times%d$' % m,
cumulative=-1, weights=ts.dx.value, bottom=1e-100,
log=True)
else:
ax.plot(histdata[m], color=color, label=r'$f\times%d$' % m)
ax.set_yscale('log')
ax.set_ylim(.01, float(livetime))
ax.set_ylabel('Time with fringe above frequency [s]')
ax.set_xlim(*hrange)
ax.set_xlabel('Frequency [Hz]')
ax.set_title(axes['position'].get_title())
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles[::-1], labels[::-1], loc='upper right')
hpng = '%s_SCATTERING_HISTOGRAM-%s.png' % (chanstr, gpsstr)
histogram.save(hpng)
histogram.close()
logger.debug("%s written." % hpng)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif ((deadtime != 0 and effdt < 2) or
(histdata[multiplier].size and
histdata[multiplier].max() >=
fthresh/2.)):
context = 'warning'
else:
continue
page.div(class_='card border-%s mb-1 shadow-sm' % context)
page.div(class_='card-header text-white bg-%s' % context)
page.a(channel, class_='collapsed card-link cis-link',
href='#osem%s' % i, **{'data-toggle': 'collapse'})
page.div.close() # card-header
page.div(id_='osem%s' % i, class_='collapse',
**{'data-parent': '#osems-group'})
page.div(class_='card-body')
page.div(class_='row')
img = htmlio.FancyPlot(
png, caption=SCATTER_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(img))
page.div.close() # col-md-10 offset-md-1
himg = htmlio.FancyPlot(
hpng, caption=HIST_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(himg))
page.div.close() # col-md-10 offset-md-1
page.div.close() # row
segs = StringIO()
if deadtime:
page.p("%d segments were found predicting a scattering fringe "
"above %.2f Hz." % (
len(scatter_segments[channel].active),
fthresh))
page.table(class_='table table-sm table-hover')
page.tbody()
page.tr()
page.th('Deadtime')
page.td('%.2f/%d seconds' % (deadtime, livetime))
page.td('%.2f%%' % deadtimepc)
page.tr.close()
page.tr()
page.th('Efficiency<br><small>(SNR≥8 and '
'%.2f Hz</sub><f<sub>peak</sub><%.2f Hz)</small>'
% (args.fmin, multiplier * fthresh))
page.td('%d/%d events' % (efficiency, len(highsnrtrigs)))
page.td('%.2f%%' % efficiencypc)
page.tr.close()
page.tr()
page.th('Efficiency/Deadtime')
page.td()
page.td('%.2f' % effdt)
page.tr.close()
page.tbody.close()
page.table.close()
scatter_segments[channel].active.write(segs, format='segwizard',
coltype=float)
page.pre(segs.getvalue())
else:
page.p("No segments were found with scattering above %.2f Hz."
% fthresh)
page.div.close() # card-body
page.div.close() # collapse
page.div.close() # card
if statea: # close accordion
page.div.close() # osems-group
# scattering based on transmon BLRMS
if statea:
page.h3('Transmons', class_='mt-3', id_='transmons')
page.div(id_='transmons-group')
logger.info('Searching for scatter based on band-limited RMS of transmons')
for i, channel in enumerate(sorted(transmons)):
logger.info("-- Processing %s --" % channel)
optic = channel.split('-')[1][:6]
flag = '%s:DCH-%s_SCATTERING_BLRMS:1' % (args.ifo, optic)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering from whitened, band-limited "
"RMS trends of {0}".format(channel),
)
# loop over state segments and compute BLRMS
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
wblrms = get_blrms(
alldata[j][channel],
flow=args.bandpass_flow,
fhigh=args.bandpass_fhigh,
)
scatter = get_segments(
wblrms,
numpy.mean(wblrms) + args.sigma * numpy.std(wblrms),
name=flag,
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
highsnrtrigs = trigs[trigs['snr'] <= 200]
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif deadtime != 0 and effdt < 2:
context = 'warning'
else:
continue
page.add(htmlio.write_flag_html(
scatter_segments[channel], span, i, parent='transmons-group',
title=channel, context=context, plotdir=''))
if statea: # close accordion
page.div.close() # transmons-group
actives = actives.coalesce() # merge contiguous segments
if statea and not actives:
page.add(htmlio.alert(
'No evidence of scattering found in the channels analyzed',
context=args.ifo.lower(), dismiss=False))
# identify triggers during active segments
logger.debug('Writing a summary CSV record')
ind = [i for i, trigtime in enumerate(highsnrtrigs[names[0]])
if trigtime in actives]
gps = highsnrtrigs[names[0]][ind]
freq = highsnrtrigs[names[1]][ind]
snr = highsnrtrigs[names[2]][ind]
segs = [y for x in gps for y in actives if x in y]
table = EventTable(
[gps, freq, snr, [seg[0] for seg in segs], [seg[1] for seg in segs]],
names=('trigger_time', 'trigger_frequency', 'trigger_snr',
'segment_start', 'segment_end'))
logger.info('The following {} triggers fell within active scattering '
'segments:\n\n'.format(len(table)))
print(table)
print('\n\n')
table.write(summfile, overwrite=True)
# -- launch omega scans -----------
nscans = min(args.omega_scans, len(table))
if nscans > 0:
# launch scans
scandir = 'scans'
ind = random.sample(range(0, len(table)), nscans)
omegatimes = [str(t) for t in table['trigger_time'][ind]]
logger.debug('Collected {} event times to omega scan: {}'.format(
nscans, ', '.join(omegatimes)))
logger.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(
ifo=args.ifo, ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(timeout=4, gps=args.gpsstart)
batch.generate_dag(omegatimes, flags=flags, submit=True,
outdir=scandir, condor_commands=condorcmds)
logger.info('Launched {} omega scans to condor'.format(nscans))
# render HTML
page.h2('Omega scans', class_='mt-4', id_='omega-scans')
msg = (
'The following event times correspond to significant Omicron '
'triggers that occur during the scattering segments found above. '
'To compare these against fringe frequency projections, please '
'use the "simple scattering" module:',
markup.oneliner.pre(
'$ python -m gwdetchar.scattering.simple --help',
),
)
page.add(htmlio.alert(msg, context=args.ifo.lower()))
page.add(htmlio.scaffold_omega_scans(
omegatimes, args.main_channel, scandir=scandir))
elif args.omega_scans:
logger.info('No events found during active scattering segments')
# -- finalize ---------------------
# write segments
scatter_segments.write(segfile, path="segments", overwrite=True)
logger.debug("%s written" % segfile)
# write HTML
htmlio.close_page(page, 'index.html')
logger.info("-- index.html written, all done --")
# return to original directory
os.chdir(indir)
