def plot_spectra(clusters,
channel,
unit='cts',
xlog=True,
legend=None,
xlim=None,
**kwargs):
from glob import glob
from gwpy.frequencyseries import FrequencySeries
from gwpy.plot import Plot
title = channel
psds = {}
for cluster in clusters:
for filename in glob('*.hdf5'):
try:
psds[cluster] = FrequencySeries.read(filename,
f'{cluster}-{channel}')
print(f'found in {filename}')
break
except KeyError:
continue
else:
raise KeyError(f'Could not find Nº{cluster}')
if legend is None:
legend = clusters
# plot the group in one figure.
plt = Plot(*(psds[cluster] for cluster in psds),
separate=False,
sharex=True,
zorder=1,
**kwargs)
if xlim is not None:
plt.gca().set_xlim(xlim)
plt.gca().set_ylim((1e-48, 1e-37))
# modify the figure as a whole.
# plt.add_segments_bar(dq, label='')
# plt.gca().set_color_cycle(['red', 'green', 'blue', 'yellow'])
if xlog:
plt.gca().set_xscale('log')
plt.gca().set_yscale('log')
plt.gca().set_ylabel(f'Power Spectral Density [{unit}^2/Hz]')
plt.suptitle(title)
plt.legend(legend, prop={'size': 15})
# save to png.
plt.save(f'{title}.png')
def test_save_legend(tmpdir):
base = str(tmpdir)
fig = Plot()
ax = fig.gca()
ax.plot(SERIES, label=SERIES.name)
tsplot = plot.save_legend(ax, os.path.join(base, 'test.png'))
assert tsplot == os.path.join(base, 'test.png')
shutil.rmtree(base, ignore_errors=True)
def test_apply_parameters(self, plot):
fig = Plot()
ax = fig.gca()
plot.apply_parameters(ax, **{
'xlim': (10, 20),
'no-blah': 'anything',
'grid': False,
})
assert ax.get_xlim() == (10, 20)
def test_apply_parameters(self, plot):
fig = Plot()
ax = fig.gca()
plot.apply_parameters(ax, **{
'xlim': (10, 20),
'no-blah': 'anything',
'grid': False,
})
assert ax.get_xlim() == (10, 20)
def hveto_roc(outfile,
rounds,
figsize=[9, 6],
constants=[1, 5, 10, 20],
**kwargs):
efficiency = []
deadtime = []
for r in rounds:
try:
efficiency.append(r.cum_efficiency[0] / r.cum_efficiency[1])
except ZeroDivisionError:
efficiency.append(0.)
try:
deadtime.append(r.cum_deadtime[0] / r.cum_deadtime[1])
except ZeroDivisionError:
deadtime.append(0.)
plot = Plot(figsize=figsize)
ax = plot.gca()
ax.plot(deadtime, efficiency, marker='o', linestyle='-')
try:
xbound = 10**floor(log10(deadtime[0]))
except ValueError:
xbound = 1e-4
try:
ybound = 10**floor(log10(efficiency[0]))
except ValueError:
ybound = 1e-4
bound = min(xbound, ybound)
axargs = {
'xlabel': 'Fractional deadtime',
'ylabel': 'Fractional efficiency',
'xscale': 'log',
'yscale': 'log',
'xlim': (bound, 1.),
'ylim': (bound, 1.),
}
axargs.update(kwargs)
# draw some eff/dt contours
if len(constants):
for i, c in enumerate(constants):
g = 1 - ((i + 1) / len(constants) * .5)
x = axargs['xlim']
y = [a * c for a in x]
ax.plot(x, y, linestyle='--', color=(g, g, g), label=str(c))
ax.legend(title='Eff/dt:',
borderaxespad=0,
bbox_to_anchor=(1.01, 1),
handlelength=1,
handletextpad=.5,
loc='upper left')
# save and close
_finalize_plot(plot, ax, outfile, **axargs)
def plot_timeseries(*data, **kwargs):
title = kwargs.pop('title', None)
ylim = kwargs.pop('ylim', None)
fname = kwargs.pop('fname', 'TimeSeries.png')
plot = Plot(figsize=(15, 10))
ax0 = plot.gca()
ax0.plot(*data)
ax0.legend([text.to_string(_data.name) for _data in data], fontsize=20)
ax0.set_xscale('auto-gps')
ax0.set_ylabel(text.to_string(data[0].unit))
plot.add_state_segments(segments_daq_iy0_ok, label='IY0 DAQ State')
plot.add_state_segments(segments_daq_ix1_ok, label='IX1 DAQ State')
plot.suptitle(title, fontsize=40)
if ylim:
ax0.set_ylim(ylim[0], ylim[1])
plot.savefig(fname)
plot.close()
def before_after_histogram(outfile,
x,
y,
label1='Before',
label2='After',
bins=100,
histtype='stepfilled',
range=None,
figsize=[9, 6],
**kwargs):
"""Plot a histogram of SNR for two event distributions
"""
# format axis arguments
axargs = {
'xscale': 'log',
'xlabel': 'Loudness',
'yscale': 'log',
'ylabel': 'Number of events',
}
axargs.update(kwargs)
# create figure
plot = Plot(figsize=figsize)
ax = plot.gca()
# make histogram
if range is None:
range = min(map(numpy.min, (x, y))), max(map(numpy.max, (x, y)))
axargs.setdefault('xlim', range)
histargs = {
'range': range,
'histtype': histtype,
'bins': bins,
'linewidth': 2,
'logbins': axargs['xscale'] == 'log',
'alpha': .8,
}
ax.hist(x, label=label1, facecolor='red', edgecolor='darkred', **histargs)
ax.hist(y,
label=label2,
facecolor='dodgerblue',
edgecolor='blue',
**histargs)
# add legend
ax.legend(loc='upper right')
# format axes
axargs.setdefault('ylim', (.5, ax.yaxis.get_data_interval()[1] * 1.05))
_finalize_plot(plot, ax, outfile, **axargs)
def spectral_overlay(gps,
qspecgram,
fringe,
output,
multipliers=(1, 2, 4, 8),
figsize=[12, 4]):
"""Overlay scattering fringe projections on top of a high-resolution
spectrogram
Parameters
----------
gps : `float`
reference GPS time (in seconds) to serve as the origin
qspecgram : `~gwpy.spectrogram.Spectrogram`
an interpolated high-resolution spectrogram
fringe : `~gwpy.timeseries.TimeSeries`
projected fringe frequencies (in Hz)
output : `str`
name of the output file
multipliers : `tuple`, optional
collection of fringe harmonic numbers to plot, can be given in
any order, default: `(1, 2, 4, 8)`
figsize : `tuple`, optional
size (width x height) of the final figure, default: `(12, 4)`
"""
plot = Plot(figsize=figsize)
ax = plot.gca()
# format spectrogram plot
ax.set_title('Fringes: {0}, Spectrogram: {1}'.format(
fringe.name, qspecgram.name))
_format_spectrogram(ax, qspecgram, colormap='binary')
# overlay fringe frequencies
_format_timeseries(ax, gps, fringe, multipliers=multipliers, linewidth=1.5)
ax.set_ylim([
qspecgram.f0.to('Hz').value,
qspecgram.frequencies.max().to('Hz').value
])
# save plot and close
plot.savefig(output, bbox_inches='tight')
plot.close()
def hveto_roc(outfile, rounds, figsize=[9, 6], constants=[1, 5, 10, 20],
**kwargs):
efficiency = []
deadtime = []
for r in rounds:
try:
efficiency.append(r.cum_efficiency[0] / r.cum_efficiency[1])
except ZeroDivisionError:
efficiency.append(0.)
try:
deadtime.append(r.cum_deadtime[0] / r.cum_deadtime[1])
except ZeroDivisionError:
deadtime.append(0.)
plot = Plot(figsize=figsize)
ax = plot.gca()
ax.plot(deadtime, efficiency, marker='o', linestyle='-')
try:
xbound = 10 ** floor(log10(deadtime[0]))
except ValueError:
xbound = 1e-4
try:
ybound = 10 ** floor(log10(efficiency[0]))
except ValueError:
ybound = 1e-4
bound = min(xbound, ybound)
axargs = {
'xlabel': 'Fractional deadtime',
'ylabel': 'Fractional efficiency',
'xscale': 'log',
'yscale': 'log',
'xlim': (bound, 1.),
'ylim': (bound, 1.),
}
axargs.update(kwargs)
# draw some eff/dt contours
if len(constants):
for i, c in enumerate(constants):
g = 1 - ((i+1)/len(constants) * .5)
x = axargs['xlim']
y = [a * c for a in x]
ax.plot(x, y, linestyle='--', color=(g, g, g), label=str(c))
ax.legend(title='Eff/dt:', borderaxespad=0, bbox_to_anchor=(1.01, 1),
handlelength=1, handletextpad=.5, loc='upper left')
# save and close
_finalize_plot(plot, ax, outfile, **axargs)
def spectral_overlay(gps, qspecgram, fringe, output,
multipliers=(1, 2, 4, 8), figsize=[12, 4]):
"""Overlay scattering fringe projections on top of a high-resolution
spectrogram
Parameters
----------
gps : `float`
reference GPS time (in seconds) to serve as the origin
qspecgram : `~gwpy.spectrogram.Spectrogram`
an interpolated high-resolution spectrogram
fringe : `~gwpy.timeseries.TimeSeries`
projected fringe frequencies (in Hz)
output : `str`
name of the output file
multipliers : `tuple`, optional
collection of fringe harmonic numbers to plot, can be given in
any order, default: `(1, 2, 4, 8)`
figsize : `tuple`, optional
size (width x height) of the final figure, default: `(12, 4)`
"""
plot = Plot(figsize=figsize)
ax = plot.gca()
# format spectrogram plot
ax.set_title('Fringes: {0}, Spectrogram: {1}'.format(
fringe.name, qspecgram.name))
_format_spectrogram(ax, qspecgram, colormap='binary')
# overlay fringe frequencies
_format_timeseries(ax, gps, fringe, multipliers=multipliers,
linewidth=1.5)
ax.set_ylim([qspecgram.f0.to('Hz').value,
qspecgram.frequencies.max().to('Hz').value])
# save plot and close
plot.savefig(output, bbox_inches='tight')
plot.close()
def plot_asd(data,replot=False,fftlength=2**7,show=False,**kwargs):
if isinstance(data,TimeSeries):
chname = data.name
psd_specgram = data.spectrogram2(fftlength=fftlength,
overlap=fftlength/2.0,
window='hanning')
elif isinstance(data,Spectrogram):
chname = data.name
specgram = data
pngfname = to_pngfname(chname,ftype='ASD',**kwargs)
if not replot and os.path.exists(pngfname):
print('Skip plot {0}'.format(pngfname))
return None
median = specgram.percentile(50)
low = specgram.percentile(5)
high = specgram.percentile(95)
median = vel2vel(median)
low = vel2vel(low)
high = vel2vel(high)
_f, _selfnoise = trillium.selfnoise(trillium='120QA',psd='ASD',unit='velo')
_selfnoise = _selfnoise*1e6
plot = Plot()
ax = plot.gca(xscale='log', xlim=(1e-3, 3e2), xlabel='Frequency [Hz]',
#yscale='log', ylim=(1e-11, 3e-6),
yscale='log', ylim=(1e-5, 3e-0),
ylabel=r'Velocity [m/sec/\rtHz]')
ax.plot(_f,_selfnoise,'-',linewidth=1,color='gray')
ax.plot_mmm(median, low, high, color='gwpy:ligo-livingston')
ax.set_title(chname.replace('_',' '),fontsize=16)
ax.legend(labels=['Selfnoise','Measurement'])
plot.savefig(pngfname)
print('plot in ',pngfname)
return plot
def before_after_histogram(
outfile, x, y, label1='Before', label2='After',
bins=100, histtype='stepfilled', range=None, figsize=[9, 6], **kwargs):
"""Plot a histogram of SNR for two event distributions
"""
# format axis arguments
axargs = {
'xscale': 'log',
'xlabel': 'Loudness',
'yscale': 'log',
'ylabel': 'Number of events',
}
axargs.update(kwargs)
# create figure
plot = Plot(figsize=figsize)
ax = plot.gca()
# make histogram
if range is None:
range = min(map(numpy.min, (x, y))), max(map(numpy.max, (x, y)))
axargs.setdefault('xlim', range)
histargs = {
'range': range,
'histtype': histtype,
'bins': bins,
'linewidth': 2,
'logbins': axargs['xscale'] == 'log',
'alpha': .8,
}
ax.hist(x, label=label1, facecolor='red', edgecolor='darkred',
**histargs)
ax.hist(y, label=label2, facecolor='dodgerblue', edgecolor='blue',
**histargs)
# add legend
ax.legend(loc='upper right')
# format axes
axargs.setdefault('ylim', (.5, ax.yaxis.get_data_interval()[1] * 1.05))
_finalize_plot(plot, ax, outfile, **axargs)
from gwpy.timeseries import TimeSeries
from gwpy.plot import Plot
h1 = TimeSeries.fetch_open_data('H1', 1126259457, 1126259467)
h1b = h1.bandpass(50, 250).notch(60).notch(120)
l1 = TimeSeries.fetch_open_data('L1', 1126259457, 1126259467)
l1b = l1.bandpass(50, 250).notch(60).notch(120)
plot = Plot(figsize=(12, 4.8))
ax = plot.gca(xscale='auto-gps')
ax.plot(h1b, color='gwpy:ligo-hanford', label='LIGO-Hanford')
ax.plot(l1b, color='gwpy:ligo-livingston', label='LIGO-Livingston')
ax.set_epoch(1126259462.427)
ax.set_xlim(1126259462, 1126259462.6)
ax.set_ylim(-1e-21, 1e-21)
ax.set_ylabel('Strain noise')
ax.legend()
plot.show()
def _process_channel(input_):
"""Handle individual channels for multiprocessing
"""
if USETEX:
gwplot.configure_mpl_tex()
p4 = (.1, .1, .9, .95)
chan = input_[1][0]
ts = input_[1][1]
lassocoef = nonzerocoef[chan]
zeroed = lassocoef == 0
if zeroed:
plot4 = None
plot5 = None
plot6 = None
pcorr = None
else:
plot4 = None
plot5 = None
plot6 = None
if trend_type == 'minute':
pcorr = numpy.corrcoef(ts.value, primaryts.value)[0, 1]
else:
pcorr = 0.0
if abs(lassocoef) < threshold:
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info(
"Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(chan)))
sys.stdout.flush()
return (chan, lassocoef, plot4, plot5, plot6, ts)
# create time series subplots
fig = Plot(figsize=(12, 8))
fig.subplots_adjust(*p4)
ax1 = fig.add_subplot(2, 1, 1, xscale='auto-gps', epoch=start)
ax1.plot(primaryts,
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax1.set_xlabel(None)
ax2 = fig.add_subplot(2, 1, 2, sharex=ax1, xlim=xlim)
ax2.plot(ts, label=texify(chan), linewidth=line_size_aux)
if range_is_primary:
ax1.set_ylabel('Sensitive range [Mpc]')
else:
ax1.set_ylabel('Primary channel units')
ax2.set_ylabel('Channel units')
for ax in fig.axes:
ax.legend(loc='best')
channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1)
plot4 = gwplot.save_figure(fig,
'%s_TRENDS-%s.png' % (channelstub, gpsstub),
bbox_inches='tight')
# create scaled, sign-corrected, and overlayed timeseries
tsscaled = scale(ts.value)
if lassocoef < 0:
tsscaled = numpy.negative(tsscaled)
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p1)
ax = fig.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax.plot(times,
_descaler(tsscaled),
label=texify(chan),
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.legend(loc='best')
plot5 = gwplot.save_figure(fig,
'%s_COMPARISON-%s.png' %
(channelstub, gpsstub),
bbox_inches='tight')
# scatter plot
tsCopy = ts.value.reshape(-1, 1)
primarytsCopy = primaryts.value.reshape(-1, 1)
primaryReg = linear_model.LinearRegression()
primaryReg.fit(tsCopy, primarytsCopy)
primaryFit = primaryReg.predict(tsCopy)
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p1)
ax = fig.gca()
ax.set_xlabel(texify(chan) + ' [Channel units]')
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary channel units')
y_min = min(primaryts.value)
y_max = max(primaryts.value)
y_range = y_max - y_min
ax.set_ylim(y_min - (y_range * 0.1), y_max + (y_range * 0.1))
ax.text(.9,
.1,
'r = ' + str('{0:.2}'.format(pcorr)),
verticalalignment='bottom',
horizontalalignment='right',
transform=ax.transAxes,
color='black',
size=20,
bbox=dict(boxstyle='square',
facecolor='white',
alpha=.75,
edgecolor='black'))
ax.scatter(ts.value, primaryts.value, color='red')
ax.plot(ts.value, primaryFit, color='black')
ax.autoscale_view(tight=False, scalex=True, scaley=True)
plot6 = gwplot.save_figure(fig,
'%s_SCATTER-%s.png' %
(channelstub, gpsstub),
bbox_inches='tight')
# increment counter and print status
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info("Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(chan)))
sys.stdout.flush()
return (chan, lassocoef, plot4, plot5, plot6, ts)
def main(args=None):
use('agg')
rcParams.update({
'figure.subplot.bottom': 0.15,
'figure.subplot.left': 0.1,
'figure.subplot.right': 0.83,
'figure.subplot.top': 0.93,
'figure.subplot.hspace': 0.25,
'axes.labelsize': 20,
'grid.color': 'gray',
})
grid = GridSpec(2, 1)
logger = log.Logger('omicron-status')
try:
omicronversion = str(get_omicron_version())
except KeyError:
omicronversion = 'Unknown'
logger.warning("Omicron version unknown")
else:
logger.info("Found omicron version: %s" % omicronversion)
parser = create_parser()
args = parser.parse_args(args=args)
if args.ifo is None:
parser.error("Cannot determine IFO prefix from sytem, "
"please pass --ifo on the command line")
group = args.group
logger.info("Checking status for %r group" % group)
archive = args.archive_directory
proddir = args.production_directory.with_name(
args.production_directory.name.format(group=args.group), )
outdir = args.output_directory
outdir.mkdir(exist_ok=True, parents=True)
tag = args.latency_archive_tag.format(group=args.group)
filetypes = ['h5', 'xml.gz', 'root']
logger.debug("Set output directory to %s" % outdir)
logger.debug(
"Will process the following filetypes: {}".format(
", ".join(filetypes), ), )
# -- parse configuration file and get parameters --------------------------
cp = configparser.ConfigParser()
ok = cp.read(args.config_file)
if args.config_file not in ok:
raise IOError(
"Failed to read configuration file %r" % args.config_file, )
logger.info("Configuration read")
# validate
if not cp.has_section(group):
raise configparser.NoSectionError(group)
# get parameters
obs = args.ifo[0]
frametype = cp.get(group, 'frametype')
padding = cp.getint(group, 'overlap-duration') / 2.
mingap = cp.getint(group, 'chunk-duration')
channels = args.channel
if not channels:
channels = [
c.split()[0]
for c in cp.get(group, 'channels').strip('\n').split('\n')
]
channels.sort()
logger.debug("Found %d channels" % len(channels))
start = args.gps_start_time
end = args.gps_end_time
if end == NOW:
end -= padding
if args.state_flag:
stateflag = args.state_flag
statepad = tuple(map(float, args.state_pad.split(',')))
else:
try:
stateflag = cp.get(group, 'state-flag')
except configparser.NoOptionError:
stateflag = None
else:
try:
statepad = tuple(
map(
float,
cp.get(group, 'state-padding').split(','),
))
except configparser.NoOptionError:
statepad = (0, 0)
if stateflag:
logger.debug("Parsed state flag: %r" % stateflag)
logger.debug("Parsed state padding: %s" % repr(statepad))
logger.info("Processing %d-%d" % (start, end))
# -- define nagios JSON printer -------------------------------------------
def print_nagios_json(code, message, outfile, tag='status', **extras):
out = {
'created_gps':
NOW,
'status_intervals': [
{
'start_sec': 0,
'end_sec': args.unknown,
'num_status': code,
'txt_status': message
},
{
'start_sec': args.unknown,
'num_status': 3,
'txt_status': 'Omicron %s check is not running' % tag
},
],
'author': {
'name': 'Duncan Macleod',
'email': '*****@*****.**',
},
'omicron': {
'version': omicronversion,
'group': group,
'channels': ' '.join(channels),
'frametype': frametype,
'state-flag': stateflag,
},
'pyomicron': {
'version': __version__,
},
}
out.update(extras)
with open(outfile, 'w') as f:
f.write(json.dumps(out))
logger.debug("nagios info written to %s" % outfile)
# -- get condor status ------------------------------------------------
if not args.skip_condor:
# connect to scheduler
try:
schedd = htcondor.Schedd()
except RuntimeError as e:
logger.warning("Caught %s: %s" % (type(e).__name__, e))
logger.info("Failed to connect to HTCondor scheduler, cannot "
"determine condor status for %s" % group)
schedd = None
if not args.skip_condor and schedd:
logger.info("-- Checking condor status --")
# get DAG status
jsonfp = outdir / "nagios-condor-{}.json".format(group)
okstates = ['Running', 'Idle', 'Completed']
try:
# check manager status
qstr = 'OmicronManager == "{}" && Owner == "{}"'.format(
group,
args.user,
)
try:
jobs = schedd.query(qstr, ['JobStatus'])
except IOError as e:
warnings.warn("Caught IOError: %s [retrying...]" % str(e))
sleep(2)
jobs = schedd.query(qstr, ['JobStatus'])
logger.debug(
"Found {} jobs for query {!r}".format(len(jobs), qstr), )
if len(jobs) > 1:
raise RuntimeError(
"Multiple OmicronManager jobs found for %r" % group)
elif len(jobs) == 0:
raise RuntimeError(
"No OmicronManager job found for %r" % group, )
status = condor.JOB_STATUS[jobs[0]['JobStatus']]
if status not in okstates:
raise RuntimeError("OmicronManager status for %r: %r" %
(group, status))
logger.debug("Manager status is %r" % status)
# check node status
jobs = schedd.query(
'OmicronProcess == "{}" && Owner == "{}"'.format(
group,
args.user,
),
['JobStatus', 'ClusterId'],
)
logger.debug(
"Found {} jobs for query {!r}".format(len(jobs), qstr), )
for job in jobs:
status = condor.JOB_STATUS[job['JobStatus']]
if status not in okstates:
raise RuntimeError("Omicron node %s (%r) is %r" %
(job['ClusterId'], group, status))
except RuntimeError as e:
print_nagios_json(2, str(e), jsonfp, tag='condor')
logger.warning("Failed to determine condor status: %r" % str(e))
except IOError as e:
logger.warning("Caught %s: %s" % (type(e).__name__, e))
logger.info("Failed to connect to HTCondor scheduler, cannot "
"determine condor status for %s" % group)
else:
print_nagios_json(
0,
"Condor processing for %r is OK" % group,
jsonfp,
tag='condor',
)
logger.info("Condor processing is OK")
if not args.skip_job_duration:
# get job duration history
plot = Plot(figsize=[12, 3])
plot.subplots_adjust(bottom=.22, top=.87)
ax = plot.gca(xscale="auto-gps")
times, jobdur = condor.get_job_duration_history_shell('OmicronProcess',
group,
maxjobs=5000)
logger.debug("Recovered duration history for %d omicron.exe jobs" %
len(times))
line = ax.plot([0], [1], label='Omicron.exe')[0]
ax.plot(times,
jobdur,
linestyle=' ',
marker='.',
color=line.get_color())
times, jobdur = condor.get_job_duration_history_shell(
'OmicronPostProcess', group, maxjobs=5000)
logger.debug("Recovered duration history for %d post-processing jobs" %
len(times))
line = ax.plot([0], [1], label='Post-processing')[0]
ax.plot(times,
jobdur,
linestyle=' ',
marker='.',
color=line.get_color())
ax.legend(loc='upper left',
borderaxespad=0,
bbox_to_anchor=(1.01, 1),
handlelength=1)
ax.set_xlim(args.gps_start_time, args.gps_end_time)
ax.set_epoch(ax.get_xlim()[1])
ax.set_yscale('log')
ax.set_title('Omicron job durations for %r' % group)
ax.set_ylabel('Job duration [seconds]')
ax.xaxis.labelpad = 5
png = str(outdir / "nagios-condor-{}.png".format(group))
plot.save(png)
plot.close()
logger.debug("Saved condor plot to %s" % png)
if args.skip_file_checks:
sys.exit(0)
# -- get file latency and archive completeness ----------------------------
logger.info("-- Checking file archive --")
# get frame segments
segs = segments.get_frame_segments(obs, frametype, start, end)
# get state segments
if stateflag is not None:
segs &= segments.query_state_segments(
stateflag,
start,
end,
pad=statepad,
)
try:
end = segs[-1][1]
except IndexError:
pass
# apply inwards padding to generate resolvable segments
for i in range(len(segs) - 1, -1, -1):
# if segment is shorter than padding, ignore it completely
if abs(segs[i]) <= padding * 2:
del segs[i]
# otherwise apply padding to generate trigger segment
else:
segs[i] = segs[i].contract(padding)
logger.debug("Found %d seconds of analysable time" % abs(segs))
# load archive latency
latencyfile = outdir / "nagios-latency-{}.h5".format(tag)
times = dict((c, dict((ft, None) for ft in filetypes)) for c in channels)
ldata = dict((c, dict((ft, None) for ft in filetypes)) for c in channels)
try:
with h5py.File(latencyfile, 'r') as h5file:
for c in channels:
for ft in filetypes:
try:
times[c][ft] = h5file[c]['time'][ft][:]
ldata[c][ft] = h5file[c]['latency'][ft][:]
except KeyError:
times[c][ft] = numpy.ndarray((0, ))
ldata[c][ft] = numpy.ndarray((0, ))
except OSError as exc: # file not found, or is corrupt
warnings.warn("failed to load latency data from {}: {}".format(
latencyfile,
str(exc),
))
for c in channels:
for ft in filetypes:
if not times[c].get(ft):
times[c][ft] = numpy.ndarray((0, ))
ldata[c][ft] = numpy.ndarray((0, ))
else:
logger.debug("Parsed latency data from %s" % latencyfile)
# load acknowledged gaps
acksegfile = str(outdir / "acknowledged-gaps-{}.txt".format(tag))
try:
acknowledged = SegmentList.read(acksegfile,
gpstype=float,
format="segwizard")
except IOError: # no file
acknowledged = SegmentList()
else:
logger.debug(
"Read %d segments from %s" % (len(acknowledged), acksegfile), )
acknowledged.coalesce()
# build legend for segments
leg = OrderedDict()
leg['Analysable'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightgray',
edgecolor='gray',
)
leg['Available'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightgreen',
edgecolor='green',
)
leg['Missing'] = SegmentRectangle(
[0, 1],
0,
facecolor='red',
edgecolor='darkred',
)
leg['Unresolvable'] = SegmentRectangle(
[0, 1],
0,
facecolor='magenta',
edgecolor='purple',
)
leg['Overlapping'] = SegmentRectangle(
[0, 1],
0,
facecolor='yellow',
edgecolor='orange',
)
leg['Pending'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightskyblue',
edgecolor='blue',
)
leg['Acknowledged'] = SegmentRectangle(
[0, 1],
0,
facecolor='sandybrown',
edgecolor='brown',
)
logger.debug("Checking archive for each channel...")
# find files
latency = {}
gaps = {}
overlap = {}
pending = {}
plots = {}
for c in channels:
# create data storate
latency[c] = {}
gaps[c] = {}
overlap[c] = {}
pending[c] = {}
# create figure
plot = Plot(figsize=[12, 5])
lax = plot.add_subplot(grid[0, 0], xscale="auto-gps")
sax = plot.add_subplot(grid[1, 0], sharex=lax, projection='segments')
colors = ['lightblue', 'dodgerblue', 'black']
for y, ft in enumerate(filetypes):
# find files
cache = io.find_omicron_files(c, start, end, archive, ext=ft)
cpend = sieve_cache(io.find_pending_files(c, proddir, ext=ft),
segment=Segment(start, end))
# get available segments
avail = segments.cache_segments(cache)
found = avail & segs
pending[c][ft] = segments.cache_segments(cpend) & segs
# remove gaps at the end that represent latency
try:
latency[c][ft] = abs(segs & type(
segs)([type(segs[0])(found[-1][1], segs[-1][1])])) / 3600.
except IndexError:
latency[c][ft] = 0
processed = segs
else:
processed = segs & type(segs)(
[type(segs[0])(start, found[-1][1])])
gaps[c][ft] = type(found)()
lost = type(found)()
for s in processed - found:
if abs(s) < mingap and s in list(segs):
lost.append(s)
else:
gaps[c][ft].append(s)
# remove acknowledged gaps
ack = gaps[c][ft] & acknowledged
gaps[c][ft] -= acknowledged
# print warnings
if abs(gaps[c][ft]):
warnings.warn("Gaps found in %s files for %s:\n%s" %
(c, ft, gaps[c][ft]))
overlap[c][ft] = segments.cache_overlaps(cache)
if abs(overlap[c][ft]):
warnings.warn("Overlap found in %s files for %s:\n%s" %
(c, ft, overlap[c][ft]))
# append archive
times[c][ft] = numpy.concatenate((times[c][ft][-99999:], [NOW]))
ldata[c][ft] = numpy.concatenate(
(ldata[c][ft][-99999:], [latency[c][ft]]))
# plot
line = lax.plot(
times[c][ft],
ldata[c][ft],
label=ft,
color=colors[y],
)[0]
lax.plot(times[c][ft],
ldata[c][ft],
marker='.',
linestyle=' ',
color=line.get_color())
sax.plot_segmentlist(segs,
y=y,
label=ft,
alpha=.5,
facecolor=leg['Analysable'].get_facecolor(),
edgecolor=leg['Analysable'].get_edgecolor())
sax.plot_segmentlist(pending[c][ft],
y=y,
facecolor=leg['Pending'].get_facecolor(),
edgecolor=leg['Pending'].get_edgecolor())
sax.plot_segmentlist(avail,
y=y,
label=ft,
alpha=.2,
height=.1,
facecolor=leg['Available'].get_facecolor(),
edgecolor=leg['Available'].get_edgecolor())
sax.plot_segmentlist(found,
y=y,
label=ft,
alpha=.5,
facecolor=leg['Available'].get_facecolor(),
edgecolor=leg['Available'].get_edgecolor())
sax.plot_segmentlist(lost,
y=y,
facecolor=leg['Unresolvable'].get_facecolor(),
edgecolor=leg['Unresolvable'].get_edgecolor())
sax.plot_segmentlist(gaps[c][ft],
y=y,
facecolor=leg['Missing'].get_facecolor(),
edgecolor=leg['Missing'].get_edgecolor())
sax.plot_segmentlist(overlap[c][ft],
y=y,
facecolor=leg['Overlapping'].get_facecolor(),
edgecolor=leg['Overlapping'].get_edgecolor())
sax.plot_segmentlist(ack,
y=y,
facecolor=leg['Acknowledged'].get_facecolor(),
edgecolor=leg['Acknowledged'].get_edgecolor())
# finalise plot
lax.axhline(args.warning / 3600.,
color=(1.0, 0.7, 0.0),
linestyle='--',
linewidth=2,
label='Warning',
zorder=-1)
lax.axhline(args.error / 3600.,
color='red',
linestyle='--',
linewidth=2,
label='Critical',
zorder=-1)
lax.set_title('Omicron status: {}'.format(c))
lax.set_ylim(0, args.error / 1800.)
lax.set_ylabel('Latency [hours]')
lax.legend(loc='upper left',
bbox_to_anchor=(1.01, 1),
borderaxespad=0,
handlelength=2,
fontsize=12.4)
lax.set_xlabel(' ')
for ax in plot.axes:
ax.set_xlim(args.gps_start_time, args.gps_end_time)
ax.set_epoch(ax.get_xlim()[1])
sax.xaxis.labelpad = 5
sax.set_ylim(-.5, len(filetypes) - .5)
sax.legend(leg.values(),
leg.keys(),
handlelength=1,
fontsize=12.4,
loc='lower left',
bbox_to_anchor=(1.01, 0),
borderaxespad=0)
plots[c] = png = outdir / "nagios-latency-{}.png".format(
c.replace(':', '-'), )
plot.save(png)
plot.close()
logger.debug(" %s" % c)
# update latency and write archive
h5file = h5py.File(latencyfile, 'w')
for c in channels:
g = h5file.create_group(c)
for name, d in zip(['time', 'latency'], [times[c], ldata[c]]):
g2 = g.create_group(name)
for ft in filetypes:
g2.create_dataset(ft, data=d[ft], compression='gzip')
h5file.close()
logger.debug("Stored latency data as HDF in %s" % latencyfile)
# write nagios output for files
status = []
for segset, tag in zip([gaps, overlap], ['gaps', 'overlap']):
chans = [(c, segset[c]) for c in segset
if abs(reduce(operator.or_, segset[c].values()))]
jsonfp = outdir / "nagios-{}-{}.json".format(tag, group)
status.append((tag, jsonfp))
if chans:
gapstr = '\n'.join('%s: %s' % c for c in chans)
code = 1
message = ("%s found in Omicron files for group %r\n%s" %
(tag.title(), group, gapstr))
else:
code = 0
message = ("No %s found in Omicron files for group %r" %
(tag, group))
print_nagios_json(code, message, jsonfp, tag=tag, **{tag: dict(chans)})
# write group JSON
jsonfp = outdir / "nagios-latency-{}.json".format(group)
status.append(('latency', jsonfp))
code = 0
message = 'No channels have high latency for group %r' % group
ldict = dict((c, max(latency[c].values())) for c in latency)
for x, dt in zip([2, 1], [args.error, args.warning]):
dh = dt / 3600.
chans = [c for c in ldict if ldict[c] >= dh]
if chans:
code = x
message = (
"%d channels found with high latency (above %s seconds)" %
(len(chans), dt))
break
print_nagios_json(code, message, jsonfp, tag='latency', latency=ldict)
# auto-detect 'standard' JSON files
for tag, name in zip(
['condor', 'omicron-online'],
['condor', 'processing'],
):
f = outdir / "nagios-{}-{}.json".format(tag, group)
if f.is_file():
status.insert(0, (name, f))
# write HTML summary
if args.html:
page = markup.page()
page.init(
title="%s Omicron Online status" % group,
css=[
('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/'
'bootstrap.min.css'),
('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/'
'jquery.fancybox.min.css'),
],
script=[
'//code.jquery.com/jquery-1.11.2.min.js',
('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/'
'bootstrap.min.js'),
('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/'
'jquery.fancybox.min.js'),
],
)
page.div(class_='container')
# write header
page.div(class_='page-header')
page.h1('Omicron Online status: %s' % group)
page.div.close() # page-header
# write summary
page.div(id_='json')
page.h2("Processing status")
for tag, f in status:
jf = f.name
page.a("%s status" % tag.title(),
href=jf,
role='button',
target="_blank",
id_="nagios-%s" % tag,
class_='btn btn-default json-status')
page.p(style="padding-top: 5px;")
page.small(
"Hover over button for explanation, click to open JSON file", )
page.p.close()
page.div.close() # id=json
# show plots
page.div(id_='plots')
page.h2("Channel details")
page.div(class_='row')
for channel in sorted(channels):
png = plots[channel].name
page.div(class_="col-sm-6 col-md-4")
page.div(class_="panel panel-default")
page.div(class_='panel-heading')
page.h3(channel, class_='panel-title', style="font-size: 14px;")
page.div.close() # panel-heading
page.div(class_='panel-body')
page.a(href=png,
target="_blank",
class_="fancybox",
rel="channel-status-img")
page.img(src=png, class_='img-responsive')
page.a.close()
page.div.close() # panel-body
page.div.close() # panel
page.div.close() # col
page.div.close() # row
page.div.close() # id=plots
# dump parameters
page.div(id_="parameters")
page.h2("Parameters")
for key, val in cp.items(group):
page.p()
page.strong("%s:" % key)
page.add(val)
page.p.close()
page.div.close() # id=parameters
# finish and close
page.div.close() # container
page.script("""
function setStatus(data, id) {
var txt = data.status_intervals[0].txt_status.split("\\n")[0];
$("#"+id).attr("title", txt);
var stat = data.status_intervals[0].num_status;
if (stat == 0) {
$("#"+id).addClass("btn-success"); }
else if (stat == 1) {
$("#"+id).addClass("btn-warning"); }
else if (stat == 2){
$("#"+id).addClass("btn-danger"); }
}
$(document).ready(function() {
$(".json-status").each(function() {
var jsonf = $(this).attr("href");
var id = $(this).attr("id");
$.getJSON(jsonf, function(data) { setStatus(data, id); });
});
$(".fancybox").fancybox({nextEffect: 'none', prevEffect: 'none'});
});""",
type="text/javascript")
with (outdir / "index.html").open("w") as f:
f.write(str(page))
logger.debug("HTML summary written to %s" % f.name)
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(xscale='log',
xlim=(10, 1500),
xlabel='Frequency [Hz]',
yscale='log',
ylim=(3e-24, 2e-20),
ylabel=r'Strain noise [1/$\sqrt{\mathrm{Hz}}$]')
ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford')
ax.set_title('LIGO-Hanford strain noise variation around GW170817',
fontsize=16)
plot.show()
# Calc Spectrum with percentile
pdp = ppol.spectrogram2(fftlength=2, overlap=1, window='hanning')**(1 / 2.)
pdp_m = pdp.percentile(50)
pdp_l = pdp.percentile(5)
pdp_h = pdp.percentile(95)
pds = spol.spectrogram2(fftlength=2, overlap=1, window='hanning')**(1 / 2.)
pds_m = pds.percentile(50)
pds_l = pds.percentile(5)
pds_h = pds.percentile(95)
# Plot asd graph
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(
xscale='log',
xlim=(1, 50000),
xlabel='Frequency [Hz]',
yscale='log', #ylim=(3e-24, 2e-20),
ylabel=r'Voltage [V/\rtHz]')
ax.plot_mmm(pdp_m,
pdp_l,
pdp_h,
color='gwpy:ligo-hanford',
label='P-polarized signal')
ax.plot_mmm(pds_m,
pds_l,
pds_h,
color='gwpy:ligo-livingston',
label='S-polarized signal')
ax.set_title('PD voltage', fontsize=16)
ax.legend()
plot.savefig('asd.png')
nproc=2,
window='hanning')**(1 / 2.)
print('fft done')
# percentile
median = sg.percentile(50)
low = sg.percentile(5)
high = sg.percentile(95)
print('percentile done')
# plot TimeSeries
plot = data.plot()
plot.savefig('./img_timeseries.png')
plot.close()
# plot Spectrum
plot = Plot()
ax = plot.gca(
xscale='log',
xlim=(1e-3, 200),
xlabel='Frequency [Hz]',
yscale='log', # ylim=(3e-24, 2e-20),
ylabel=r'Ground Velocity [um/sec/\rtHz]')
ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford')
ax.set_title('No title', fontsize=16)
plot.savefig('./img_asd.png')
# write data
low.write('data1_exv_x_5pct.hdf5', format='hdf5', overwrite=True)
median.write('data1_exv_x_50pct.hdf5', format='hdf5', overwrite=True)
high.write('data1_exv_x_95pct.hdf5', format='hdf5', overwrite=True)
# Then we can download a simulation of the GW150914 signal from GWOSC:
from astropy.utils.data import get_readable_fileobj
url = ("https://www.gw-openscience.org/s/events/GW150914/P150914/"
"fig2-unfiltered-waveform-H.txt")
with get_readable_fileobj(url) as f:
signal = TimeSeries.read(f, format='txt')
signal.t0 = .5 # make sure this intersects with noise time samples
# Note, since this simulation cuts off before a certain time, it is
# important to taper its ends to zero to avoid ringing artifacts.
# We can accomplish this using the
# :meth:`~gwpy.timeseries.TimeSeries.taper` method.
signal = signal.taper()
# Since the time samples overlap, we can inject this into our noise data
# using :meth:`~gwpy.types.series.Series.inject`:
data = noise.inject(signal)
# Finally, we can visualize the full process in the time domain:
from gwpy.plot import Plot
plot = Plot(noise, signal, data, separate=True, sharex=True, sharey=True)
plot.gca().set_epoch(0)
plot.show()
# We can clearly see that the loud GW150914-like signal has been layered
# on top of Gaussian noise with the correct amplitude and phase evolution.
def zoom(ts: [TimeSeries], t, dt=0.5 * u.s):
print(t)
plot2 = Plot(ts)
ax = plot2.gca()
ax.set_xlim(t - dt, t + dt)
plot2.show()
def _generate_cluster(input_):
"""Generate cluster data for use below
"""
if USETEX:
gwplot.configure_mpl_tex()
currentchan = input_[1][0]
currentts = input_[1][5]
current = input_[0]
p7 = (.135, .15, .95, .9)
plot7 = None
plot7_list = None
if current < len(nonzerodata):
cluster = []
for i, otheritem in enumerate(list(auxdata.items())):
chan_, ts_ = otheritem
if chan_ != currentchan:
pcorr = numpy.corrcoef(currentts.value, ts_.value)[0, 1]
if abs(pcorr) >= cluster_threshold:
stub = re_delim.sub('_', chan_).replace('_', '-', 1)
cluster.append([i, ts_, pcorr, chan_, stub])
if cluster:
# write cluster table to file
cluster = sorted(cluster, key=lambda x: abs(x[2]), reverse=True)
clustertab = Table(data=list(zip(*cluster))[2:4],
names=('Pearson Coefficient', 'Channel'))
plot7_list = '%s_CLUSTER_LIST-%s.csv' % (re_delim.sub(
'_', str(currentchan)).replace('_', '-', 1), gpsstub)
clustertab.write(plot7_list, format='csv', overwrite=True)
ncluster = min(len(cluster), max_correlated_channels)
colors2 = [cmap(i) for i in numpy.linspace(0, 1, ncluster + 1)]
# plot
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p7)
ax = fig.gca(xscale='auto-gps')
ax.plot(times,
scale(currentts.value) * numpy.sign(input_[1][1]),
label=texify(currentchan),
linewidth=line_size_aux,
color=colors[0])
for i in range(0, ncluster):
this = cluster[i]
ax.plot(
times,
scale(this[1].value) * numpy.sign(input_[1][1]) *
numpy.sign(this[2]),
color=colors2[i + 1],
linewidth=line_size_aux,
label=('{0}, r = {1:.2}'.format(texify(cluster[i][3]),
cluster[i][2])),
)
ax.margins(x=0)
ax.set_ylabel('Scaled amplitude [arbitrary units]')
ax.set_title('Highly Correlated Channels')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot7 = gwplot.save_figure(
fig,
'%s_CLUSTER-%s.png' % (re_delim.sub(
'_', str(currentchan)).replace('_', '-', 1), gpsstub),
bbox_inches='tight')
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info(
"Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(currentchan)))
sys.stdout.flush()
return (plot7, plot7_list)
def representative_spectra(channels,
start,
stop,
rate,
label='kmeans-labels',
filename=DEFAULT_FILENAME,
prefix='.',
downloader=TimeSeriesDict.get,
cluster_numbers=None,
groups=None,
**kwargs):
"""
Make representative spectra for each cluster based on the median psd for minutes in that cluster.
Downloads only the raw minutes in the cluster to save.
"""
if groups is None:
groups = channels
# read the labels from the save file.
labels = TimeSeries.read(filename,
label,
start=to_gps(start),
end=to_gps(stop))
logger.info(f'Read labels {start} to {stop} from {filename}')
if cluster_numbers is None:
clusters = list(range(max(labels.value) + 1))
cluster_counts = list(
len(labels.value[labels.value == c]) for c in clusters)
largest_cluster = cluster_counts.index(max(cluster_counts))
clusters.remove(largest_cluster)
logger.info(
f'Largest cluster found to be Nº{largest_cluster} ({100 * max(cluster_counts) // len(labels.value)}%). Doing {clusters}.'
)
cluster_counts.remove(max(cluster_counts))
else:
clusters = cluster_numbers
cluster_counts = list(
len(labels.value[labels.value == c]) for c in clusters)
t, v, d = labels.times, labels.value, diff(labels.value)
pairs = list(
zip([t[0]] + list(t[:-1][d != 0]),
list(t[1:][d != 0]) + [t[-1]]))
values = list(v[:-1][d != 0]) + [v[-1]]
assert len(pairs) == len(values) # need to include start-| and |-end
# l|r l|r l|r l|r
# l,r l,r l,r l,r
# l r,l r,l r,l r # zip(start + l[1:], r[:-1] + stop)
print(pairs)
for pair in pairs:
print(int(pair[1].value) - int(pair[0].value))
print(values)
# use h5py to make a mutable object pointing to a file on disk.
save_file, filename = path2h5file(
get_path(f'spectra-cache {start}', 'hdf5', prefix=prefix))
logger.debug(f'Initiated hdf5 stream to {filename}')
logger.info(f'Patching {filename}...')
for i, (dl_start, end) in enumerate(pairs):
if values[i] in clusters:
if not data_exists(channels, to_gps(end).seconds, save_file):
logger.debug(
f'Downloading Nº{values[i]} from {dl_start} to {end}...')
try:
dl = downloader(channels,
start=to_gps(dl_start) - LIGOTimeGPS(60),
end=to_gps(end) + LIGOTimeGPS(seconds=1))
out = TimeSeriesDict()
for n in dl:
out[n] = dl[n].resample(**better_aa_opts(dl[n], rate))
write_to_disk(out, to_gps(dl_start).seconds, save_file)
except RuntimeError: # Cannot find all relevant data on any known server
logger.warning(
f"SKIPPING Nº{values[i]} from {dl_start} to {end} !!")
logger.info('Reading data...')
data = TimeSeriesDict.read(save_file, channels)
logger.info('Starting PSD generation...')
f = data[channels[0]].crop(
start=to_gps(data[channels[0]].times[-1]) - LIGOTimeGPS(60),
end=to_gps(data[channels[0]].times[-1])).psd().frequencies
d = (to_gps(labels.times[-1]).seconds - to_gps(labels.times[1]).seconds)
for i, cluster in enumerate(clusters):
try:
psds = {
channel: FrequencySeries.read(filename, f'{cluster}-{channel}')
for channel in channels
}
logger.info(f'Loaded Nº{cluster}.')
except KeyError:
logger.info(
f'Doing Nº{cluster} ({100 * cluster_counts[i] / len(labels.value):.2f}% of data)...'
)
with Progress(f'psd Nº{cluster} ({i + 1}/{len(clusters)})',
len(channels) * d) as progress:
psds = {
channel: FrequencySeries(median(stack([
progress(data[channel].crop,
pc * d + (to_gps(time).seconds -
to_gps(labels.times[1]).seconds),
start=to_gps(time) - LIGOTimeGPS(60),
end=to_gps(time)).psd().value
for c, time in zip(labels.value, labels.times)
if c == cluster
]),
axis=0),
frequencies=f,
name=f'{cluster}-{channel}')
for pc, channel in enumerate(channels)
}
for name in psds.keys():
psds[name].write(filename, **writing_opts)
# plotting is slow, so show a nice progress bar.
logger.debug('Initiating plotting routine...')
with Progress('plotting', len(groups)) as progress:
for p, (group, lbls, title) in enumerate(groups):
# plot the group in one figure.
plt = Plot(*(psds[channel] for channel in group),
separate=False,
sharex=True,
zorder=1,
**kwargs)
# plt.gca().set_xlim((30,60))
# modify the figure as a whole.
# plt.add_segments_bar(dq, label='')
plt.gca().set_xscale('log')
plt.gca().set_yscale('log')
plt.suptitle(title)
plt.legend(lbls)
# save to png.
progress(
plt.save, p,
get_path(f'{cluster}-{title}',
'png',
prefix=f'{prefix}/{cluster}'))
# a number of ASDs, using the :meth:`~gwpy.timeseries.TimeSeries.spectrogram2`
# method:
sg = hoft.spectrogram2(fftlength=4, overlap=2, window='hanning')**(1 / 2.)
# From this we can trivially extract the median, 5th and 95th percentiles:
median = sg.percentile(50)
low = sg.percentile(5)
high = sg.percentile(95)
# Finally, we can make plot, using :meth:`~gwpy.plot.Axes.plot_mmm` to
# display the 5th and 95th percentiles as a shaded region around the median:
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(xscale='log',
xlim=(10, 1500),
xlabel='Frequency [Hz]',
yscale='log',
ylim=(3e-24, 2e-20),
ylabel=r'Strain noise [1/\rtHz]')
ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford')
ax.set_title('LIGO-Hanford strain noise variation around GW170817',
fontsize=16)
plot.show()
# Now we can see that the ASD varies by factors of a few across most of the
# frequency band, with notable exceptions, e.g. around the 60-Hz power line
# harmonics (60 Hz, 120 Hz, 180 Hz, ...) where the noise is very stable.
def veto_scatter(
outfile, a, b, label1='All', label2='Vetoed', x='time', y='snr',
color=None, clim=None, clabel=None, cmap=None, clog=True,
figsize=[9, 6],**kwargs):
"""Plot an x-y scatter of all/vetoed events
"""
# format axis arguments
axargs = {
'yscale': 'log',
'ylabel': 'Loudness',
}
axargs['xscale'] = 'auto-gps' if x == 'time' else 'log'
if isinstance(y, (list, tuple)):
ya = y[0]
yb = y[1]
else:
ya = yb = y
axargs.update(kwargs)
# create figure
plot = Plot(figsize=figsize)
ax = plot.gca()
# add data
if color is None:
ax.scatter(a[x], a[ya], color='black', marker='o', label=label1, s=40)
else:
colorargs = {'edgecolor': 'none'}
if clim:
colorargs['vmin'] = clim[0]
colorargs['vmax'] = clim[1]
if clog:
colorargs['norm'] = LogNorm(vmin=clim[0], vmax=clim[1])
a = a.copy()
a.sort(color)
m = ax.scatter(a[x], a[ya], c=a[color], label=label1, **colorargs)
# add colorbar
ax.colorbar(mappable=m, cmap=cmap, label=clabel)
if isinstance(b, (list, tuple)) and len(b) == 2:
# aux channel used/coinc (probably)
colors = [{'color': c} for c in (
'#ffd200', # yellow
'#d62728', # red
)]
elif isinstance(b, (list, tuple)):
colors = list(rcParams['axes.prop_cycle'])
else:
b = [b]
label2 = [label2]
colors = [{'color': '#d62728'}]
for i, data in enumerate(b):
# setting the color here looks complicated, but is just a fancy
# way of looping through the color cycle when scattering, but using
# red if we only have one other data set
ax.scatter(data[x], data[yb], marker='+', linewidth=1.5,
label=label2[i], s=40, **colors[i % len(colors)])
# add legend
if ax.get_legend_handles_labels()[0]:
legargs = {
'loc': 'upper left',
'bbox_to_anchor': (1.01, 1),
'borderaxespad': 0,
'numpoints': 1,
'scatterpoints': 1,
'handlelength': 1,
'handletextpad': .5
}
legargs.update(dict(
(x[7:], axargs.pop(x)) for x in list(axargs.keys()) if
x.startswith('legend_')
))
ax.legend(**legargs)
# finalize
for axis in ['x', 'y']:
# get data limits
lim = list(getattr(ax, '%saxis' % axis).get_data_interval())
# use given ybound
lim[0] = axargs.get('%sbound' % axis, lim[0])
# scale out for visual
lim[0] *= 0.95
lim[1] *= 1.05
# handle logs
if axargs.get("%sscale" % axis, "linear") == "log" and lim[0] <= 0.:
lim[0] = None
axargs.setdefault('%slim' % axis, lim)
_finalize_plot(plot, ax, outfile, **axargs)
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs):
"""Plot the signifiance drop for each channel
"""
channels = sorted(old.keys())
if show_channel_names is None:
show_channel_names = len(channels) <= 50
plot = Plot(figsize=(20, 5))
plot.subplots_adjust(left=.07, right=.93)
ax = plot.gca()
if show_channel_names:
plot.subplots_adjust(bottom=.4)
winner = sorted(old.items(), key=lambda x: x[1])[-1][0]
for i, c in enumerate(channels):
if c == winner:
color = 'orange'
elif old[c] > new[c]:
color = 'dodgerblue'
else:
color = 'red'
ax.plot([i, i], [old[c], new[c]],
color=color,
linestyle='-',
marker='o',
markeredgecolor='k',
markeredgewidth=.5,
markersize=10,
label=c,
zorder=old[c])
ax.set_xlim(-1, len(channels))
ax.set_ybound(lower=0)
# set xticks to show channel names
if show_channel_names:
ax.set_xticks(range(len(channels)))
ax.set_xticklabels([texify(c) for c in channels])
for i, t in enumerate(ax.get_xticklabels()):
t.set_rotation(270)
t.set_verticalalignment('top')
t.set_horizontalalignment('center')
t.set_fontsize(8)
# or just show systems of channels
else:
plot.canvas.draw()
systems = {}
for i, c in enumerate(channels):
sys = c.split(':', 1)[1].split('-')[0].split('_')[0]
try:
systems[sys][1] += 1
except KeyError:
systems[sys] = [i, 1]
systems = sorted(systems.items(), key=lambda x: x[1][0])
labels, counts = zip(*systems)
xticks, xmticks = zip(*[(a, a + b / 2.) for (a, b) in counts])
# show ticks at the edge of each group
ax.set_xticks(xticks, minor=False)
ax.set_xticklabels([], minor=False)
# show label in the centre of each group
ax.set_xticks(xmticks, minor=True)
for t in ax.set_xticklabels(labels, minor=True):
t.set_rotation(270)
kwargs.setdefault('ylabel', 'Significance')
# create interactivity
if outfile.endswith('.svg'):
_finalize_plot(plot,
ax,
outfile.replace('.svg', '.png'),
close=False,
**kwargs)
tooltips = []
ylim = ax.get_ylim()
yoffset = (ylim[1] - ylim[0]) * 0.061
bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'}
xthresh = len(channels) / 10.
for i, l in enumerate(ax.lines):
x = l.get_xdata()[1]
if x < xthresh:
ha = 'left'
elif x > (len(channels) - xthresh):
ha = 'right'
else:
ha = 'center'
y = l.get_ydata()[0] + yoffset
c = l.get_label()
tooltips.append(
ax.annotate(texify(c), (x, y),
ha=ha,
zorder=ylim[1],
bbox=bbox))
l.set_gid('line-%d' % i)
tooltips[-1].set_gid('tooltip-%d' % i)
f = BytesIO()
plot.savefig(f, format='svg')
tree, xmlid = etree.XMLID(f.getvalue())
tree.set('onload', 'init(evt)')
for i in range(len(tooltips)):
try:
e = xmlid['tooltip-%d' % i]
except KeyError:
warnings.warn("Failed to recover tooltip %d" % i)
continue
e.set('visibility', 'hidden')
for i, l in enumerate(ax.lines):
e = xmlid['line-%d' % i]
e.set('onmouseover', 'ShowTooltip(this)')
e.set('onmouseout', 'HideTooltip(this)')
tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT))
etree.ElementTree(tree).write(outfile)
plot.close()
else:
_finalize_plot(plot, ax, outfile, **kwargs)
def veto_scatter(outfile,
a,
b,
label1='All',
label2='Vetoed',
x='time',
y='snr',
color=None,
clim=None,
clabel=None,
cmap=None,
clog=True,
figsize=[9, 6],
**kwargs):
"""Plot an x-y scatter of all/vetoed events
"""
# format axis arguments
axargs = {
'yscale': 'log',
'ylabel': 'Loudness',
}
axargs['xscale'] = 'auto-gps' if x == 'time' else 'log'
if isinstance(y, (list, tuple)):
ya = y[0]
yb = y[1]
else:
ya = yb = y
axargs.update(kwargs)
# create figure
plot = Plot(figsize=figsize)
ax = plot.gca()
# add data
if color is None:
ax.scatter(a[x], a[ya], color='black', marker='o', label=label1, s=40)
else:
colorargs = {'edgecolor': 'none'}
if clim:
colorargs['vmin'] = clim[0]
colorargs['vmax'] = clim[1]
if clog:
colorargs['norm'] = LogNorm(vmin=clim[0], vmax=clim[1])
a = a.copy()
a.sort(color)
m = ax.scatter(a[x], a[ya], c=a[color], label=label1, **colorargs)
# add colorbar
ax.colorbar(mappable=m, cmap=cmap, label=clabel)
if isinstance(b, (list, tuple)) and len(b) == 2:
# aux channel used/coinc (probably)
colors = [
{
'color': c
} for c in (
'#ffd200', # yellow
'#d62728', # red
)
]
elif isinstance(b, (list, tuple)):
colors = list(rcParams['axes.prop_cycle'])
else:
b = [b]
label2 = [label2]
colors = [{'color': '#d62728'}]
for i, data in enumerate(b):
# setting the color here looks complicated, but is just a fancy
# way of looping through the color cycle when scattering, but using
# red if we only have one other data set
ax.scatter(data[x],
data[yb],
marker='+',
linewidth=1.5,
label=label2[i],
s=40,
**colors[i % len(colors)])
# add legend
if ax.get_legend_handles_labels()[0]:
legargs = {
'loc': 'upper left',
'bbox_to_anchor': (1.01, 1),
'borderaxespad': 0,
'numpoints': 1,
'scatterpoints': 1,
'handlelength': 1,
'handletextpad': .5
}
legargs.update(
dict((x[7:], axargs.pop(x)) for x in list(axargs.keys())
if x.startswith('legend_')))
ax.legend(**legargs)
# finalize
for axis in ['x', 'y']:
# get data limits
lim = list(getattr(ax, '%saxis' % axis).get_data_interval())
# use given ybound
lim[0] = axargs.get('%sbound' % axis, lim[0])
# scale out for visual
lim[0] *= 0.95
lim[1] *= 1.05
# handle logs
if axargs.get("%sscale" % axis, "linear") == "log" and lim[0] <= 0.:
lim[0] = None
axargs.setdefault('%slim' % axis, lim)
_finalize_plot(plot, ax, outfile, **axargs)
def cluster_plotter(channels,
start,
stop,
prefix='.',
label='kmeans-labels',
groups=None,
filename=DEFAULT_FILENAME,
dqflag='L1:DMT-ANALYSIS_READY:1',
xscale=None,
unit=None,
progressbar=True,
**kwargs):
"""
Plots data with clusters labeled by color in the working directory, or a relative path given by prefix.
Requires a .hdf5 file produced with a clustering function defined in this module to be in the working directory.
**kwargs are forwarded to TimeSeries.plot().
:param prefix: relative path to output images.
:param label: name attribute of labels TimeSeries saved in filename.
:param groups: groups of channels to plot in the same figure. See the example.
:param dqflag: data quality flag for segments bar.
:param xscale: gps x-axis scale to use.
:param unit: override y-axis unit.
:param progressbar: show progress bar.
>>> from gwpy.time import tconvert, from_gps
>>> from datetime import timedelta
>>> from cluster import cluster_plotter
>>>
>>> channels = [f'L1:ISI-GND_STS_ETMX_Z_BLRMS_1_3.mean,m-trend', 'L1:ISI-GND_STS_ETMY_Z_BLRMS_1_3.mean,m-trend']
>>> groups = [[channels, ('ETMX', 'ETMY'), 'L1:ISI-GND_STS_BLRMS_1_3 Z-axis']] # plot on the same figure.
>>>
>>> stop = from_gps(60 * (int(tconvert('now')) // 60)) # gets nearest minute to now
>>> start = stop - timedelta(days=1) # cluster the past day
>>> cluster_plotter(channels, start, stop, filename='my_kmeans.hdf5', groups=groups)
"""
# some defaults.
if not kwargs:
kwargs['color'] = 'k'
kwargs['alpha'] = 0.3
if groups is None:
groups = channels
# read the data from the save file.
data = TimeSeriesDict.read(filename,
channels + [label],
start=to_gps(start),
end=to_gps(stop))
logger.info(f'Read {start} to {stop} from {filename}')
# get segments for the duration specified. Note that this may require doing `ligo-proxy-init -p`.
logger.debug(f'Getting segments for {dqflag} from {start} to {stop}...')
dq = DataQualityFlag.query(dqflag, to_gps(start), to_gps(stop))
logger.info(f'Got segments for {dqflag} from {start} to {stop}.')
# plotting is slow, so show a nice progress bar.
logger.debug('Initiating plotting routine...')
with Progress('plotting', len(channels),
quiet=not progressbar) as progress:
for p, (group, labels, title) in enumerate(groups):
# plot the group in one figure.
plt = Plot(*(data[channel] for channel in group),
separate=True,
sharex=True,
zorder=1,
**kwargs)
# modify the axes one by one.
axes = plt.get_axes()
for i, ax in enumerate(axes):
# namely, add a colored overlay that indicates clustering labels.
ax.scatter(data[group[i]].times,
data[group[i]].value,
c=[colors[j] for j in data[label]],
edgecolor='',
s=4,
zorder=2)
ax.set_ylabel(
f'{labels[i]} {data[group[i]].unit if unit is None else unit}'
)
setp(ax.get_xticklabels(), visible=False)
# modify the figure as a whole.
plt.add_segments_bar(dq, label='')
if xscale is not None:
plt.gca().set_xscale(xscale)
plt.suptitle(title)
# save to png.
progress(plt.save, p, get_path(title, 'png', prefix=prefix))
logger.info(f'Completed plotting for {start} to {stop} from {filename}')
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs):
"""Plot the signifiance drop for each channel
"""
channels = sorted(old.keys())
if show_channel_names is None:
show_channel_names = len(channels) <= 50
plot = Plot(figsize=(18, 6))
plot.subplots_adjust(left=.07, right=.93)
ax = plot.gca()
if show_channel_names:
plot.subplots_adjust(bottom=.4)
winner = sorted(old.items(), key=lambda x: x[1])[-1][0]
for i, c in enumerate(channels):
if c == winner:
color = 'orange'
elif old[c] > new[c]:
color = 'dodgerblue'
else:
color = 'red'
ax.plot([i, i], [old[c], new[c]], color=color, linestyle='-',
marker='o', markeredgecolor='k', markeredgewidth=.5,
markersize=10, label=c, zorder=old[c])
ax.set_xlim(-1, len(channels))
ax.set_ybound(lower=0)
# set xticks to show channel names
if show_channel_names:
ax.set_xticks(range(len(channels)))
ax.set_xticklabels([c.replace('_','\_') for c in channels])
for i, t in enumerate(ax.get_xticklabels()):
t.set_rotation(270)
t.set_verticalalignment('top')
t.set_horizontalalignment('center')
t.set_fontsize(8)
# or just show systems of channels
else:
plot.canvas.draw()
systems = {}
for i, c in enumerate(channels):
sys = c.split(':', 1)[1].split('-')[0].split('_')[0]
try:
systems[sys][1] += 1
except KeyError:
systems[sys] = [i, 1]
systems = sorted(systems.items(), key=lambda x: x[1][0])
labels, counts = zip(*systems)
xticks, xmticks = zip(*[(a, a+b/2.) for (a, b) in counts])
# show ticks at the edge of each group
ax.set_xticks(xticks, minor=False)
ax.set_xticklabels([], minor=False)
# show label in the centre of each group
ax.set_xticks(xmticks, minor=True)
for t in ax.set_xticklabels(labels, minor=True):
t.set_rotation(270)
kwargs.setdefault('ylabel', 'Significance')
# create interactivity
if outfile.endswith('.svg'):
_finalize_plot(plot, ax, outfile.replace('.svg', '.png'),
close=False, **kwargs)
tooltips = []
ylim = ax.get_ylim()
yoffset = (ylim[1] - ylim[0]) * 0.061
bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'}
xthresh = len(channels) / 10.
for i, l in enumerate(ax.lines):
x = l.get_xdata()[1]
if x < xthresh:
ha = 'left'
elif x > (len(channels) - xthresh):
ha ='right'
else:
ha = 'center'
y = l.get_ydata()[0] + yoffset
c = l.get_label()
tooltips.append(ax.annotate(c.replace('_', r'\_'), (x, y),
ha=ha, zorder=ylim[1], bbox=bbox))
l.set_gid('line-%d' % i)
tooltips[-1].set_gid('tooltip-%d' % i)
f = BytesIO()
plot.savefig(f, format='svg')
tree, xmlid = etree.XMLID(f.getvalue())
tree.set('onload', 'init(evt)')
for i in range(len(tooltips)):
try:
e = xmlid['tooltip-%d' % i]
except KeyError:
warnings.warn("Failed to recover tooltip %d" % i)
continue
e.set('visibility', 'hidden')
for i, l in enumerate(ax.lines):
e = xmlid['line-%d' % i]
e.set('onmouseover', 'ShowTooltip(this)')
e.set('onmouseout', 'HideTooltip(this)')
tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT))
etree.ElementTree(tree).write(outfile)
plot.close()
else:
_finalize_plot(plot, ax, outfile, **kwargs)
path='hoge')
strain.write('./fs_gif.hdf5', format='hdf5', overwrite=True, path='hoge')
# -----------------------------------------------
# Main
# -----------------------------------------------
# -----------------------------------------------
# IXV1 DIFF12
# -----------------------------------------------
if comparison_ixv1_diff12:
print('Comparison_IXV1_DIFF12')
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(xscale='log',
xlim=(1e-3, 100),
yscale='log',
ylim=(1e-7, 1e2))
ax.plot(ixv1, color='k', label='IXV1')
ax.plot(diff12, color='r', label=r'(IXV1 - IXV2)/$\sqrt{2}$')
ax.plot(tr120_selfnoise,
color='k',
label='Selfnoise',
linestyle='--',
zorder=0)
ax.set_xlabel('Frequency [Hz]', fontsize=15)
ax.set_ylabel(r'Displacement [um/$\sqrt{\mathrm{Hz}}$]', fontsize=15)
ax.set_title('Trillium120QA Noise Investigation', fontsize=20)
ax.text(110,
1e-7,
'START : {0}'.format(start),
rotation=90,
_aanoise = 7e-8*u.V*np.ones(len(freq)) # [V/rtHz]
_adcnoise = FrequencySeries(_adcnoise, frequencies=freq)
_adcnoise2 = FrequencySeries(_adcnoise2, frequencies=freq)
_ampnoise = FrequencySeries(_ampnoise, frequencies=freq)
_aanoise = FrequencySeries(_aanoise, frequencies=freq)
adcnoise = v2vel_120(_adcnoise)/amp
ampnoise = v2vel_120(_ampnoise)/amp #
aanoise = v2vel_120(_aanoise)/amp
selfnoise_120q = tr120q.selfnoise(unit=unit)
selfnoise_240 = tr240.selfnoise(unit=unit)
#
# Plot
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(xscale='log', xlim=(1e-3, 10), xlabel='Frequency [Hz]',
yscale='log', ylim=(1e-12, 1e-4),
ylabel=r'Velocity [{0}/\rtHz]'.format(unit))
#ax.loglog(x1500,label='x1500 (ref. GIF data)')
ax.loglog(selfnoise_120q,'k--',label='Self Noise 120Q',linewidth=1,alpha=0.7)
#ax.loglog(selfnoise_240,'m--',label='Self Noise 240',linewidth=1,alpha=0.7)
#ax.loglog(adcnoise,'r--',label='ADC Noise',linewidth=1,alpha=0.7)
#ax.loglog(ampnoise,'g--',label='Amp Noise',linewidth=1,alpha=0.7)
#ax.loglog(aanoise,'b--',label='AA Noise',linewidth=1,alpha=0.7)
ax.loglog(exv,'k-',label='EXV')
ax.loglog(ixv,label='IXV1')
ax.set_xlim(1e-2,10)
#ax.loglog(ixv2,label='IXV2')
#ax.loglog(d12,label='IXV1-IXV2')
#ax.loglog(strain,label='StrainMeter')
ax.legend(fontsize=8,loc='lower left')
ax.set_title('Seismometer, {dname}'.format(dname=dataname.replace('_','')),
def main(args=None):
"""Run the lasso command-line interface
"""
# declare global variables
# this is needed for multiprocessing utilities
global auxdata, cluster_threshold, cmap, colors, counter, gpsstub
global line_size_aux, line_size_primary, max_correlated_channels
global nonzerocoef, nonzerodata, p1, primary, primary_mean, primary_std
global primaryts, range_is_primary, re_delim, start, target, times
global threshold, trend_type, xlim
parser = create_parser()
args = parser.parse_args(args=args)
# get run params
start = int(args.gpsstart)
end = int(args.gpsend)
pad = args.filter_padding
# set pertinent global variables
cluster_threshold = args.cluster_coefficient
line_size_aux = args.line_size_aux
line_size_primary = args.line_size_primary
threshold = args.threshold
trend_type = args.trend_type
# let's go
LOGGER.info('{} Lasso correlations {}-{}'.format(args.ifo, start, end))
# get primary channel frametype
primary = args.primary_channel.format(ifo=args.ifo)
range_is_primary = 'EFFECTIVE_RANGE_MPC' in args.primary_channel
if args.primary_cache is not None:
LOGGER.info("Using custom primary cache file")
elif args.primary_frametype is None:
try:
args.primary_frametype = DEFAULT_FRAMETYPE[
args.primary_channel.split(':')[1]].format(ifo=args.ifo)
except KeyError as exc:
raise type(exc)("Could not determine primary channel's frametype, "
"please specify with --primary-frametype")
# create output directory
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
os.chdir(args.output_dir)
# multiprocessing for plots
nprocplot = (args.nproc_plot or args.nproc) if USETEX else 1
# bandpass primary
if args.band_pass:
try:
flower, fupper = args.band_pass
except TypeError:
flower, fupper = None
LOGGER.info("-- Loading primary channel data")
bandts = get_data(primary,
start - pad,
end + pad,
verbose='Reading primary:'.rjust(30),
frametype=args.primary_frametype,
source=args.primary_cache,
nproc=args.nproc)
if flower < 0 or fupper >= float((bandts.sample_rate / 2.).value):
raise ValueError(
"bandpass frequency is out of range for this "
"channel, band (Hz): {0}, sample rate: {1}".format(
args.band_pass, bandts.sample_rate))
# get darm BLRMS
LOGGER.debug("-- Filtering data")
if trend_type == 'minute':
stride = 60
else:
stride = 1
if flower:
darmbl = (bandts.highpass(
flower / 2., fstop=flower / 4., filtfilt=False,
ftype='butter').notch(60, filtfilt=False).bandpass(
flower,
fupper,
fstop=[flower / 2., fupper * 1.5],
filtfilt=False,
ftype='butter').crop(start, end))
darmblrms = darmbl.rms(stride)
darmblrms.name = '%s %s-%s Hz BLRMS' % (primary, flower, fupper)
else:
darmbl = bandts.notch(60).crop(start, end)
darmblrms = darmbl.rms(stride)
darmblrms.name = '%s RMS' % primary
primaryts = darmblrms
bandts_asd = bandts.asd(4, 2, method='median')
darmbl_asd = darmbl.asd(4, 2, method='median')
spectrum_plots = gwplot.make_spectrum_plots(start, end, flower, fupper,
args.primary_channel,
bandts_asd, darmbl_asd)
spectrum_plot_zoomed_out = spectrum_plots[0]
spectrum_plot_zoomed_in = spectrum_plots[1]
else:
# load primary channel data
LOGGER.info("-- Loading primary channel data")
primaryts = get_data(primary,
start,
end,
frametype=args.primary_frametype,
source=args.primary_cache,
verbose='Reading:'.rjust(30),
nproc=args.nproc).crop(start, end)
if args.remove_outliers:
LOGGER.debug("-- Removing outliers above %f sigma" %
args.remove_outliers)
gwlasso.remove_outliers(primaryts, args.remove_outliers)
elif args.remove_outliers_pf:
LOGGER.debug("-- Removing outliers in the bottom {} percent "
"of data".format(args.remove_outliers_pf))
gwlasso.remove_outliers(primaryts,
args.remove_outliers_pf,
method='pf')
start = int(primaryts.span()[0])
end = int(primaryts.span()[1])
primary_mean = numpy.mean(primaryts.value)
primary_std = numpy.std(primaryts.value)
# get aux data
LOGGER.info("-- Loading auxiliary channel data")
if args.channel_file is None:
host, port = io_nds2.host_resolution_order(args.ifo)[0]
channels = ChannelList.query_nds2('*.mean',
host=host,
port=port,
type='m-trend')
else:
with open(args.channel_file, 'r') as f:
channels = [name.rstrip('\n') for name in f]
nchan = len(channels)
LOGGER.debug("Identified %d channels" % nchan)
if trend_type == 'minute':
frametype = '%s_M' % args.ifo # for minute trends
else:
frametype = '%s_T' % args.ifo # for second trends
# read aux channels
auxdata = get_data(channels,
start,
end,
verbose='Reading:'.rjust(30),
frametype=frametype,
nproc=args.nproc,
pad=0).crop(start, end)
# -- removes flat data to be re-introdused later
LOGGER.info('-- Pre-processing auxiliary channel data')
auxdata = gwlasso.remove_flat(auxdata)
flatable = Table(data=(list(set(channels) - set(auxdata.keys())), ),
names=('Channels', ))
LOGGER.debug('Removed {0} channels with flat data'.format(len(flatable)))
LOGGER.debug('{0} channels remaining'.format(len(auxdata)))
# -- remove bad data
LOGGER.info("Removing any channels with bad data...")
nbefore = len(auxdata)
auxdata = gwlasso.remove_bad(auxdata)
nafter = len(auxdata)
LOGGER.debug('Removed {0} channels with bad data'.format(nbefore - nafter))
LOGGER.debug('{0} channels remaining'.format(nafter))
data = numpy.array([scale(ts.value) for ts in auxdata.values()]).T
# -- perform lasso regression -------------------
# create model
LOGGER.info('-- Fitting data to target')
target = scale(primaryts.value)
model = gwlasso.fit(data, target, alpha=args.alpha)
LOGGER.info('Alpha: {}'.format(model.alpha))
# restructure results for convenience
allresults = Table(data=(list(auxdata.keys()), model.coef_,
numpy.abs(model.coef_)),
names=('Channel', 'Lasso coefficient', 'rank'))
allresults.sort('rank')
allresults.reverse()
useful = allresults['rank'] > 0
allresults.remove_column('rank')
results = allresults[useful] # non-zero coefficient
zeroed = allresults[numpy.invert(useful)] # zero coefficient
# extract data for useful channels
nonzerodata = {name: auxdata[name] for name in results['Channel']}
nonzerocoef = {name: coeff for name, coeff in results.as_array()}
# print results
LOGGER.info('Found {} channels with |Lasso coefficient| >= {}:\n\n'.format(
len(results), threshold))
print(results)
print('\n\n')
# convert to pandas
set_option('max_colwidth', -1)
df = results.to_pandas()
df.index += 1
# write results to files
gpsstub = '%d-%d' % (start, end - start)
resultsfile = '%s-LASSO_RESULTS-%s.csv' % (args.ifo, gpsstub)
results.write(resultsfile, format='csv', overwrite=True)
zerofile = '%s-ZERO_COEFFICIENT_CHANNELS-%s.csv' % (args.ifo, gpsstub)
zeroed.write(zerofile, format='csv', overwrite=True)
flatfile = '%s-FLAT_CHANNELS-%s.csv' % (args.ifo, gpsstub)
flatable.write(flatfile, format='csv', overwrite=True)
# -- generate lasso plots
modelFit = model.predict(data)
re_delim = re.compile(r'[:_-]')
p1 = (.1, .15, .9, .9) # global plot defaults for plot1, lasso model
times = primaryts.times.value
xlim = primaryts.span
cmap = get_cmap('tab20')
colors = [cmap(i) for i in numpy.linspace(0, 1, len(nonzerodata) + 1)]
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax.plot(times,
_descaler(modelFit),
label='Lasso model',
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
ax.set_title('Lasso Model of Range')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Lasso Model of Primary Channel')
ax.legend(loc='best')
plot1 = gwplot.save_figure(plot,
'%s-LASSO_MODEL-%s.png' % (args.ifo, gpsstub),
bbox_inches='tight')
# summed contributions
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
summed = 0
for i, name in enumerate(results['Channel']):
summed += scale(nonzerodata[name].value) * nonzerocoef[name]
if i:
label = 'Channels 1-{0}'.format(i + 1)
else:
label = 'Channel 1'
ax.plot(times,
_descaler(summed),
label=label,
color=colors[i],
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Summations of Channel Contributions to Model')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot2 = gwplot.save_figure(plot,
'%s-LASSO_CHANNEL_SUMMATION-%s.png' %
(args.ifo, gpsstub),
bbox_inches='tight')
# individual contributions
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
for i, name in enumerate(results['Channel']):
this = _descaler(scale(nonzerodata[name].value) * nonzerocoef[name])
if i:
label = 'Channels 1-{0}'.format(i + 1)
else:
label = 'Channel 1'
ax.plot(times,
this,
label=texify(name),
color=colors[i],
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Individual Channel Contributions to Model')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot3 = gwplot.save_figure(plot,
'%s-LASSO_CHANNEL_CONTRIBUTIONS-%s.png' %
(args.ifo, gpsstub),
bbox_inches='tight')
# -- process aux channels, making plots
LOGGER.info("-- Processing channels")
counter = multiprocessing.Value('i', 0)
# process channels
pool = multiprocessing.Pool(nprocplot)
results = pool.map(_process_channel, enumerate(list(nonzerodata.items())))
results = sorted(results, key=lambda x: abs(x[1]), reverse=True)
# generate clustered time series plots
counter = multiprocessing.Value('i', 0)
max_correlated_channels = 20
if args.no_cluster is False:
LOGGER.info("-- Generating clusters")
pool = multiprocessing.Pool(nprocplot)
clusters = pool.map(_generate_cluster, enumerate(results))
channelsfile = '%s-CHANNELS-%s.csv' % (args.ifo, gpsstub)
numpy.savetxt(channelsfile, channels, delimiter=',', fmt='%s')
# write html
trange = '%d-%d' % (start, end)
title = '%s Lasso Correlation: %s' % (args.ifo, trange)
if args.band_pass:
links = [trange
] + [(s, '#%s' % s.lower())
for s in ['Parameters', 'Spectra', 'Model', 'Results']]
else:
links = [trange] + [(s, '#%s' % s.lower())
for s in ['Parameters', 'Model', 'Results']]
(brand, class_) = htmlio.get_brand(args.ifo, 'Lasso', start)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(title='%s Lasso | %s' %
(args.ifo, trange),
navbar=navbar)
page.h1(title, class_='pb-2 mt-3 mb-2 border-bottom')
# -- summary table
content = [
('Primary channel', markup.oneliner.code(primary)),
('Primary frametype', markup.oneliner.code(args.primary_frametype)
or '-'),
('Primary cache file', markup.oneliner.code(args.primary_cache)
or '-'), ('Outlier threshold', '%s sigma' % args.remove_outliers),
('Lasso coefficient threshold', str(threshold)),
('Cluster coefficient threshold', str(args.cluster_coefficient)),
('Non-zero coefficients', str(numpy.count_nonzero(model.coef_))),
('α (model)', '%.4f' % model.alpha)
]
if args.band_pass:
content.insert(
2, ('Primary bandpass', '{0}-{1} Hz'.format(flower, fupper)))
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(content, start=start, end=end))
page.div.close() # col-md-9 col-sm-12
# -- download button
files = [('%s analyzed channels (CSV)' % nchan, channelsfile),
('%s flat channels (CSV)' % len(flatable), flatfile),
('%s zeroed channels (CSV)' % len(zeroed), zerofile)]
page.div(class_='col-md-3 col-sm-12')
page.add(
htmlio.download_btn(files,
label='Channel information',
btnclass='btn btn-%s dropdown-toggle' %
args.ifo.lower()))
page.div.close() # col-md-3 col-sm-12
page.div.close() # rowa
# -- command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
if args.band_pass:
page.h2('Primary channel spectra', class_='mt-4', id_='spectra')
page.div(class_='card border-light card-body shadow-sm')
page.div(class_='row')
page.div(class_='col-md-6')
spectra_img1 = htmlio.FancyPlot(spectrum_plot_zoomed_out)
page.add(htmlio.fancybox_img(spectra_img1))
page.div.close() # col-md-6
page.div(class_='col-md-6')
spectra_img2 = htmlio.FancyPlot(spectrum_plot_zoomed_in)
page.add(htmlio.fancybox_img(spectra_img2))
page.div.close() # col-md-6
page.div.close() # row
page.div.close() # card border-light card-body shadow-sm
# -- model information
page.h2('Model information', class_='mt-4', id_='model')
page.div(class_='card card-%s card-body shadow-sm' % args.ifo.lower())
page.div(class_='row')
page.div(class_='col-md-8 offset-md-2', id_='results-table')
page.p('Below are the top {} mean minute-trend channels, ranked by '
'Lasso correlation with the primary.'.format(df.shape[0]))
page.add(
df.to_html(classes=('table', 'table-sm', 'table-hover'),
formatters={
'Lasso coefficient': lambda x: "%.4f" % x,
'Channel':
lambda x: str(htmlio.cis_link(x.split('.')[0])),
'__index__': lambda x: str(x)
},
escape=False,
border=0).replace(' style="text-align: right;"', ''))
page.div.close() # col-md-10 offset-md-1
page.div.close() # row
page.div(class_='row', id_='primary-lasso')
page.div(class_='col-md-8 offset-md-2')
img1 = htmlio.FancyPlot(plot1)
page.add(htmlio.fancybox_img(img1)) # primary lasso plot
page.div.close() # col-md-8 offset-md-2
page.div.close() # primary-lasso
page.div(class_='row', id_='channel-summation')
img2 = htmlio.FancyPlot(plot2)
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img2))
page.div.close() # col-md-8 offset-md-2
page.div.close() # channel-summation
page.div(class_='row', id_='channels-and-primary')
img3 = htmlio.FancyPlot(plot3)
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img3))
page.div.close() # col-md-8 offset-md-2
page.div.close() # channels-and-primary
page.div.close() # card card-<ifo> card-body shadow-sm
# -- results
page.h2('Top channels', class_='mt-4', id_='results')
page.div(id_='results')
# for each aux channel, create information container and put plots in it
for i, (ch, lassocoef, plot4, plot5, plot6, ts) in enumerate(results):
# set container color/context based on lasso coefficient
if lassocoef == 0:
break
elif abs(lassocoef) < threshold:
h = '%s [lasso coefficient = %.4f] (Below threshold)' % (ch,
lassocoef)
else:
h = '%s [lasso coefficient = %.4f]' % (ch, lassocoef)
if ((lassocoef is None) or (lassocoef == 0)
or (abs(lassocoef) < threshold)):
card = 'card border-light mb-1 shadow-sm'
card_header = 'card-header bg-light'
elif abs(lassocoef) >= .5:
card = 'card border-danger mb-1 shadow-sm'
card_header = 'card-header text-white bg-danger'
elif abs(lassocoef) >= .2:
card = 'card border-warning mb-1 shadow-sm'
card_header = 'card-header text-white bg-warning'
else:
card = 'card border-info mb-1 shadow-sm'
card_header = 'card-header text-white bg-info'
page.div(class_=card)
# heading
page.div(class_=card_header)
page.a(h,
class_='collapsed card-link cis-link',
href='#channel%d' % i,
**{'data-toggle': 'collapse'})
page.div.close() # card-header
# body
page.div(id_='channel%d' % i,
class_='collapse',
**{'data-parent': '#results'})
page.div(class_='card-body')
if lassocoef is None:
page.p('The amplitude data for this channel is flat (does not '
'change) within the chosen time period.')
elif abs(lassocoef) < threshold:
page.p('Lasso coefficient below the threshold of %g.' %
(threshold))
else:
for image in [plot4, plot5, plot6]:
img = htmlio.FancyPlot(image)
page.div(class_='row')
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img))
page.div.close() # col-md-8 offset-md-2
page.div.close() # row
page.add('<hr class="row-divider">')
if args.no_cluster is False:
if clusters[i][0] is None:
page.p("<font size='3'><br />No channels were highly "
"correlated with this channel.</font>")
else:
page.div(class_='row', id_='clusters')
page.div(class_='col-md-12')
cimg = htmlio.FancyPlot(clusters[i][0])
page.add(htmlio.fancybox_img(cimg))
page.div.close() # col-md-12
page.div.close() # clusters
if clusters[i][1] is not None:
corr_link = markup.oneliner.a(
'Export {} channels (CSV)'.format(
max_correlated_channels),
href=clusters[i][1],
download=clusters[i][1],
)
page.button(
corr_link,
class_='btn btn-%s' % args.ifo.lower(),
)
page.div.close() # card-body
page.div.close() # collapse
page.div.close() # card
page.div.close() # results
htmlio.close_page(page, 'index.html') # save and close
LOGGER.info("-- Process Completed")
# Then we can download a simulation of the GW150914 signal from LOSC:
from astropy.utils.data import get_readable_fileobj
source = 'https://losc.ligo.org/s/events/GW150914/P150914/'
url = '%s/fig2-unfiltered-waveform-H.txt' % source
with get_readable_fileobj(url) as f:
signal = TimeSeries.read(f, format='txt')
signal.t0 = .5 # make sure this intersects with noise time samples
# Note, since this simulation cuts off before a certain time, it is
# important to taper its ends to zero to avoid ringing artifacts.
# We can accomplish this using the
# :meth:`~gwpy.timeseries.TimeSeries.taper` method.
signal = signal.taper()
# Since the time samples overlap, we can inject this into our noise data
# using :meth:`~gwpy.types.series.Series.inject`:
data = noise.inject(signal)
# Finally, we can visualize the full process in the time domain:
from gwpy.plot import Plot
plot = Plot(noise, signal, data, separate=True, sharex=True, sharey=True)
plot.gca().set_epoch(0)
plot.show()
# We can clearly see that the loud GW150914-like signal has been layered
# on top of Gaussian noise with the correct amplitude and phase evolution.
def whiten(data, ffttime, window, low_f, high_f, notch, rate):
"""
This function whitens the data and band-pass it in the range [low_f, high_f].
Parameters
----------
data: numpy array
The signal to whiten as numpy array
ffttime: int
Portion of the strain to compute the psd
window: str
Type of function for the windowing
low_f: int
Lower bound of the band-pass filter
high_f: int
Upper bound of the band-pass filter
notch: list
Frequencies of the notch filters. Depends on the detector
rate: int
Resampling rate. Represents the sampling frequency
Returns
-------
whitened: numpy array
The whitened and band-passed numpy array
"""
# Band-pass filter in [35, 250]
bp = bandpass(float(low_f), float(high_f), data.sample_rate)
#Notches for the 1st three harminics of the 60 Hz AC
notches = [filter_design.notch(line, data.sample_rate) for line in notch]
#Concatenate both filters
zpk = filter_design.concatenate_zpks(bp, *notches)
#Whiten and band-pass filter
white = data.whiten(ffttime, int(ffttime / 2),
window='hann') #whiten the data
white_down = white.filter(zpk, filtfilt=True).resample(
rate=rate, window='hann') #downsample to 2048Hz
whitened = np.array(white_down)
#Plot version with and without notches
plot = Plot(figsize=(15, 6))
ax = plot.gca()
ax.plot(white_down, label='Downsampled', alpha=0.7)
ax.plot(white.filter(zpk, filtfilt=True),
label='Not downsampled',
alpha=0.7)
ax.set_xscale('auto-gps')
ax.set_ylabel('Frequency [Hz]')
ax.set_title(
'LIGO-Livingston strain data whitened, band-passed in range [' +
str(low_f) + '' + str(high_f) + '] $Hz$')
plot.legend()
plt.savefig('/home/melissa.lopez/Anomaly_Detection/Algorithm/dummy.png')
plt.close()
return whitened
color='gwpy:ligo-hanford')
ax1, ax2 = plot.axes
ax1.set_title('LIGO-Hanford strain data around GW150914')
ax1.text(1.0, 1.01, 'Unfiltered data', transform=ax1.transAxes, ha='right')
ax1.set_ylabel('Amplitude [strain]', y=-0.2)
ax2.set_ylabel('')
ax2.text(1.0, 1.01, '50-250\,Hz bandpass, notches at 60, 120, 180 Hz',
transform=ax2.transAxes, ha='right')
plot.show()
plot.close() # hide
# We see now a spike around 16 seconds into the data, so let's zoom into
# that time (and prettify):
plot = hfilt.plot(color='gwpy:ligo-hanford')
ax = plot.gca()
ax.set_title('LIGO-Hanford strain data around GW150914')
ax.set_ylabel('Amplitude [strain]')
ax.set_xlim(1126259462, 1126259462.6)
ax.set_xscale('seconds', epoch=1126259462)
plot.show()
plot.close() # hide
# Congratulations, you have succesfully filtered LIGO data to uncover the
# first ever directly-detected gravitational wave signal, GW150914!
# But wait, what about LIGO-Livingston?
# We can easily add that to our figure by following the same procedure.
#
# First, we load the new data
ldata = TimeSeries.fetch_open_data('L1', 1126259446, 1126259478)
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(xscale='log', xlim=(10, 1500), xlabel='Frequency [Hz]',
yscale='log', ylim=(3e-24, 2e-20),
ylabel=r'Strain noise [1/\rtHz]')
ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford')
ax.set_title('LIGO-Hanford strain noise variation around GW170817',
fontsize=16)
plot.show()
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(xscale='log', xlim=(10, 1500), xlabel='Frequency [Hz]',
yscale='log', ylim=(3e-24, 2e-20),
ylabel=r'Strain noise [1/$\sqrt{\mathrm{Hz}}$]')
ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford')
ax.set_title('LIGO-Hanford strain noise variation around GW170817',
fontsize=16)
plot.show()
def process_channel(input_,):
chan, ts = input_
flat = ts.value.min() == ts.value.max()
if flat:
corr1 = None
corr2 = None
corr1s = None
corr2s = None
plot1 = None
plot2 = None
plot3 = None
else:
corr1 = numpy.corrcoef(ts.value, darmblrms.value)[0, 1]
corr1s = spearmanr(ts.value, darmblrms.value)[0]
if args.trend_type == 'minute':
corr2 = numpy.corrcoef(ts.value, rangets.value)[0, 1]
corr2s = spearmanr(ts.value, rangets.value)[0]
else:
corr2 = 0.0
corr2s = 0.0
# if all corralations are below threshold it does not plot
if((abs(corr1) < args.threshold)
and (abs(corr1s) < args.threshold)
and (abs(corr2) < args.threshold)
and (abs(corr2s) < args.threshold)):
plot1 = None
plot2 = None
plot3 = None
return (chan, corr1, corr2, plot1,
plot2, plot3, corr1s, corr2s)
plot = Plot(darmblrms, ts, rangets,
xscale="auto-gps", separate=True,
figsize=(12, 12))
plot.subplots_adjust(*p1)
plot.axes[0].set_ylabel('$h(t)$ BLRMS [strain]')
plot.axes[1].set_ylabel('Channel units')
plot.axes[2].set_ylabel('Sensitive range [Mpc]')
for ax in plot.axes:
ax.legend(loc='best')
ax.set_xlim(start, end)
ax.set_epoch(start)
channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1)
plot1 = '%s_TRENDS-%s.png' % (channelstub, gpsstub)
try:
plot.save(plot1)
except (IOError, IndexError):
plot.save(plot1)
except RuntimeError as e:
if 'latex' in str(e).lower():
plot.save(plot1)
else:
raise
plot.close()
# plot auto-scaled verions
tsscaled = ts.detrend()
tsrms = numpy.sqrt(sum(tsscaled**2.0)/len(tsscaled))
if args.trend_type == 'minute':
tsscaled *= (rangerms / tsrms)
if corr1 > 0:
tsscaled *= -1
else:
tsscaled *= (darmrms / tsrms)
if corr1 < 0:
tsscaled *= -1
plot = Plot(darmscaled, rangescaled, tsscaled,
xscale="auto-gps", figsize=[12, 6])
plot.subplots_adjust(*p2)
ax = plot.gca()
ax.set_xlim(start, end)
ax.set_epoch(start)
ax.set_ylabel('Scaled amplitude [arbitrary units]')
ax.legend(loc='best')
plot2 = '%s_COMPARISON-%s.png' % (channelstub, gpsstub)
try:
plot.save(plot2)
except (IOError, IndexError):
plot.save(plot2)
except RuntimeError as e:
if 'latex' in str(e).lower():
plot.save(plot2)
else:
raise
plot.close()
# plot scatter plots
rangeColor = 'red'
darmblrmsColor = 'blue'
tsCopy = ts.reshape(-1, 1)
rangetsCopy = rangets.reshape(-1, 1)
darmblrmsCopy = darmblrms.reshape(-1, 1)
darmblrmsReg = linear_model.LinearRegression()
darmblrmsReg.fit(tsCopy, darmblrmsCopy)
darmblrmsFit = darmblrmsReg.predict(tsCopy)
rangeReg = linear_model.LinearRegression()
rangeReg.fit(tsCopy, rangetsCopy)
rangeFit = rangeReg.predict(tsCopy)
fig = Plot(figsize=(12, 6))
fig.subplots_adjust(*p2)
ax = fig.add_subplot(121)
ax.set_xlabel('Channel units')
ax.set_ylabel('Sensitive range [Mpc]')
yrange = abs(max(darmblrms.value) - min(darmblrms.value))
upperLim = max(darmblrms.value) + .1 * yrange
lowerLim = min(darmblrms.value) - .1 * yrange
ax.set_ylim(lowerLim, upperLim)
ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr1)),
verticalalignment='bottom', horizontalalignment='right',
transform=ax.transAxes, color='black', size=20,
bbox=dict(boxstyle='square', facecolor='white', alpha=.75,
edgecolor='black'))
fig.add_scatter(ts, darmblrms, color=darmblrmsColor)
fig.add_line(ts, darmblrmsFit, color='black')
ax = fig.add_subplot(122)
ax.set_xlabel('Channel units')
ax.set_ylabel('$h(t)$ BLRMS [strain]')
ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr2)),
verticalalignment='bottom', horizontalalignment='right',
transform=ax.transAxes, color='black', size=20,
bbox=dict(boxstyle='square', facecolor='white', alpha=.75,
edgecolor='black'))
fig.add_scatter(ts, rangets, color=rangeColor)
fig.add_line(ts, rangeFit, color='black')
plot3 = '%s_SCATTER-%s.png' % (channelstub, gpsstub)
try:
fig.save(plot3)
except (IOError, IndexError):
fig.save(plot3)
except RuntimeError as e:
if 'latex' in str(e).lower():
fig.save(plot3)
else:
raise
plt.close(fig)
# increment counter and print status
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / nchan
LOGGER.debug("Completed [%d/%d] %3d%% %-50s"
% (counter.value, nchan, pc, '(%s)' % str(chan)))
sys.stdout.flush()
return chan, corr1, corr2, plot1, plot2, plot3, corr1s, corr2s
tr240 = Trillium('240')
v2vel = tr120q.v2vel
v2vel = tr240.v2vel
adcnoise = v2vel(_adcnoise) / amp
ampnoise = v2vel(_ampnoise) / amp #
aanoise = v2vel(_aanoise) / amp
selfnoise_120q = tr120q.selfnoise()
selfnoise_240 = tr240.selfnoise()
# Plot
if True:
from gwpy.plot import Plot
plot = Plot()
ax = plot.gca(xscale='log',
xlim=(1e-3, 10),
xlabel='Frequency [Hz]',
yscale='log',
ylim=(1e-12, 1e-4),
ylabel=r'Velocity [m/sec/\rtHz]')
#ax.loglog(x1500,label='x1500 (ref. GIF data)')
ax.loglog(selfnoise_120q,
'k--',
label='Self Noise 120Q',
linewidth=1,
alpha=0.7)
ax.loglog(selfnoise_240,
'm--',
label='Self Noise 240',
linewidth=1,
alpha=0.7)
ax.loglog(adcnoise, 'r--', label='ADC Noise', linewidth=1, alpha=0.7)
ax.loglog(ampnoise, 'g--', label='Amp Noise', linewidth=1, alpha=0.7)
# convert with tf
asd1 = vel2vel(asd1)
asd2 = vel2vel(asd2)
asd3 = vel2vel(asd3)
adc = vel2vel(adc)
noise12 = vel2vel(noise12)
noise13 = vel2vel(noise13)
signal12 = vel2vel(signal12)
signal13 = vel2vel(signal13)
print('convert with tf')
# plot psd with noise
plot = Plot()
ax = plot.gca(xscale='log',
xlim=(1e-3, 3e2),
xlabel='Frequency [Hz]',
yscale='log',
ylim=(1e-5, 3e-0),
ylabel=r'Velocity [um/sec/\rtHz]')
ax.plot(_f, _selfnoise, '-', linewidth=1, color='gray')
ax.plot(asd1, label='1', color='black', linewidth=3)
#ax.plot(asd2,label='2',color='red',linewidth=1)
#ax.plot(asd3,label='3',color='blue',linewidth=1)
#ax.plot(adc,label='adc',color='green',linewidth=1)
#ax.plot(noise13,'o',label='Noise13 : IXV*(1-coh13) ',markersize=1)
#ax.plot(noise12,'o',label='Noise12 : IXV*(1-coh12) ',markersize=1)
ax.plot(signal13, label='Signal13 : IXV*coh13 ', markersize=1)
ax.plot(signal12, label='Signal12 : IXV*coh12 ', markersize=1)
#ax.plot(signal_local,label='Local Signal')
ax.legend()
plot.savefig('ASD.png')
print 'plot in ASD.png'
# Now, we can loop through the active segments of ``'H1_DATA'`` and fetch the
# strain `TimeSeries` for each segment, calculating a
# :class:`~gwpy.spectrogram.Spectrogram` for each segment.
from gwpy.timeseries import TimeSeries
spectrograms = []
for start, end in h1segs.active:
h1strain = TimeSeries.fetch_open_data('H1', start, end, verbose=True)
specgram = h1strain.spectrogram(30, fftlength=4)**(1 / 2.)
spectrograms.append(specgram)
# Finally, we can build a :meth:`~gwpy.spectrogram.Spectrogram.plot`:
from gwpy.plot import Plot
plot = Plot(figsize=(12, 6))
ax = plot.gca()
for specgram in spectrograms:
ax.imshow(specgram)
ax.set_xscale('auto-gps', epoch='Sep 16 2010')
ax.set_xlim('Sep 16 2010', 'Sep 17 2010')
ax.set_ylim(40, 2000)
ax.set_yscale('log')
ax.set_ylabel('Frequency [Hz]')
ax.set_title('LIGO-Hanford strain data')
ax.colorbar(cmap='viridis',
norm='log',
clim=(1e-23, 1e-19),
label=r'Strain noise [1/\rtHz]')
plot.add_segments_bar(h1segs)
plot.show()
def main(args=None):
"""Run the primary scattering command-line tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# set up logger
logger = cli.logger(
name=PROG.split('python -m ').pop(),
level='DEBUG' if args.verbose else 'INFO',
)
# useful variables
fthresh = (
int(args.frequency_threshold) if args.frequency_threshold.is_integer()
else args.frequency_threshold)
multiplier = args.multiplier_for_threshold
tstr = str(fthresh).replace('.', '_')
gpsstr = '%s-%s' % (int(args.gpsstart), int(args.gpsend - args.gpsstart))
args.optic = args.optic or list(OPTIC_MOTION_CHANNELS.keys())
# go to working directory
indir = os.getcwd()
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
os.chdir(args.output_dir)
# set up output files
summfile = '{}-SCATTERING_SUMMARY-{}.csv'.format(
args.ifo, gpsstr)
segfile = '{}-SCATTERING_SEGMENTS_{}_HZ-{}.h5'.format(
args.ifo, tstr, gpsstr)
# log start of process
logger.info('{} Scattering {}-{}'.format(
args.ifo, int(args.gpsstart), int(args.gpsend)))
# -- get state segments -----------
span = Segment(args.gpsstart, args.gpsend)
# get segments
if args.state_flag is not None:
state = DataQualityFlag.query(
args.state_flag, int(args.gpsstart), int(args.gpsend),
url=DEFAULT_SEGMENT_SERVER,
).coalesce()
statea = []
padding = args.segment_start_pad + args.segment_end_pad
for i, seg in enumerate(state.active):
if abs(seg) > padding:
statea.append(Segment(
seg[0] + args.segment_start_pad,
seg[1] - args.segment_end_pad,
))
else:
logger.debug(
"Segment length {} shorter than padding length {}, "
"skipping segment {}-{}".format(abs(seg), padding, *seg),
)
statea = SegmentList(statea)
logger.debug("Downloaded %d segments for %s"
% (len(statea), args.state_flag))
else:
statea = SegmentList([span])
livetime = float(abs(statea))
logger.debug("Processing %.2f s of livetime" % livetime)
# -- load h(t) --------------------
args.main_channel = args.main_channel.format(IFO=args.ifo)
logger.debug("Loading Omicron triggers for %s" % args.main_channel)
if args.gpsstart >= 1230336018: # Jan 1 2019
ext = "h5"
names = ["time", "frequency", "snr"]
read_kw = {
"columns": names,
"selection": [
"{0} < frequency < {1}".format(
args.fmin, multiplier * fthresh),
("time", in_segmentlist, statea),
],
"format": "hdf5",
"path": "triggers",
}
else:
ext = "xml.gz"
names = ['peak', 'peak_frequency', 'snr']
read_kw = {
"columns": names,
"selection": [
"{0} < peak_frequency < {1}".format(
args.fmin, multiplier * fthresh),
('peak', in_segmentlist, statea),
],
"format": 'ligolw',
"tablename": "sngl_burst",
}
fullcache = []
for seg in statea:
cache = gwtrigfind.find_trigger_files(
args.main_channel, 'omicron', seg[0], seg[1], ext=ext,
)
if len(cache) == 0:
warnings.warn(
"No Omicron triggers found for %s in segment [%d .. %d)"
% (args.main_channel, seg[0], seg[1]),
)
continue
fullcache.extend(cache)
# read triggers
if fullcache:
trigs = EventTable.read(fullcache, nproc=args.nproc, **read_kw)
else: # no files (no livetime?)
trigs = EventTable(names=names)
highsnrtrigs = trigs[trigs['snr'] >= 8]
logger.debug("%d read" % len(trigs))
# -- prepare HTML -----------------
links = [
'%d-%d' % (int(args.gpsstart), int(args.gpsend)),
('Parameters', '#parameters'),
('Segments', (
('State flag', '#state-flag'),
('Optical sensors', '#osems'),
('Transmons', '#transmons'),
)),
]
if args.omega_scans:
links.append(('Scans', '#omega-scans'))
(brand, class_) = htmlio.get_brand(args.ifo, 'Scattering', args.gpsstart)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(
title='%s Scattering | %d-%d' % (
args.ifo, int(args.gpsstart), int(args.gpsend)),
navbar=navbar)
page.div(class_='pb-2 mt-3 mb-2 border-bottom')
page.h1('%s Scattering: %d-%d'
% (args.ifo, int(args.gpsstart), int(args.gpsend)))
page.div.close() # pb-2 mt-3 mb-2 border-bottom
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(
start=int(args.gpsstart), end=int(args.gpsend), flag=args.state_flag))
page.div.close() # col-md-9 col-sm-12
# link to summary files
page.div(class_='col-md-3 col-sm-12')
page.add(htmlio.download_btn(
[('Segments (HDF)', segfile),
('Triggers (CSV)', summfile)],
btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(),
))
page.div.close() # col-md-3 col-sm-12
page.div.close() # row
# command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
# section header
page.h2('Segments', class_='mt-4', id_='segments')
if statea: # contextual information
paper = markup.oneliner.a(
'Accadia et al. (2010)', target='_blank', class_='alert-link',
href='http://iopscience.iop.org/article/10.1088/0264-9381/27'
'/19/194011')
msg = (
"Segments marked \"optical sensors\" below show evidence of beam "
"scattering between {0} and {1} Hz based on the velocity of optic "
"motion, with fringe frequencies projected using equation (3) of "
"{2}. Segments marked \"transmons\" are based on whitened, "
"band-limited RMS trends of transmon sensors. In both cases, "
"yellow panels denote weak evidence for scattering, while red "
"panels denote strong evidence."
).format(args.fmin, multiplier * fthresh, str(paper))
page.add(htmlio.alert(msg, context=args.ifo.lower()))
else: # null segments
page.add(htmlio.alert('No active analysis segments were found',
context='warning', dismiss=False))
# record state segments
if args.state_flag is not None:
page.h3('State flag', class_='mt-3', id_='state-flag')
page.div(id_='accordion1')
page.add(htmlio.write_flag_html(
state, span, 'state', parent='accordion1', context='success',
plotdir='', facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen',
known={'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4}))
page.div.close()
# -- find scattering evidence -----
# read data for OSEMs and transmons
osems = ['%s:%s' % (args.ifo, c) for optic in args.optic for
c in OPTIC_MOTION_CHANNELS[optic]]
transmons = ['%s:%s' % (args.ifo, c) for c in TRANSMON_CHANNELS]
allchannels = osems + transmons
logger.info("Reading all timeseries data")
alldata = []
n = len(statea)
for i, seg in enumerate(statea):
msg = "{0}/{1} {2}:".rjust(30).format(
str(i + 1).rjust(len(str(n))),
n,
str(seg),
) if args.verbose else False
alldata.append(
get_data(allchannels, seg[0], seg[1],
frametype=args.frametype.format(IFO=args.ifo),
verbose=msg, nproc=args.nproc).resample(128))
try: # ensure that only available channels are analyzed
osems = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(osems))
transmons = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(transmons))
except IndexError:
osems = []
transmons = []
# initialize scattering segments
scatter_segments = DataQualityDict()
actives = SegmentList()
# scattering based on OSEM velocity
if statea:
page.h3('Optical sensors (OSEMs)', class_='mt-3', id_='osems')
page.div(id_='osems-group')
logger.info('Searching for scatter based on OSEM velocity')
for i, channel in enumerate(sorted(osems)):
logger.info("-- Processing %s --" % channel)
chanstr = re.sub('[:-]', '_', channel).replace('_', '-', 1)
optic = channel.split('-')[1].split('_')[0]
flag = '%s:DCH-%s_SCATTERING_GE_%s_HZ:1' % (args.ifo, optic, tstr)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering above {0} Hz from {1} in "
"{2}".format(fthresh, optic, channel),
)
# set up plot(s)
plot = Plot(figsize=[12, 12])
axes = {}
axes['position'] = plot.add_subplot(
411, xscale='auto-gps', xlabel='')
axes['fringef'] = plot.add_subplot(
412, sharex=axes['position'], xlabel='')
axes['triggers'] = plot.add_subplot(
413, sharex=axes['position'], xlabel='')
axes['segments'] = plot.add_subplot(
414, projection='segments', sharex=axes['position'])
plot.subplots_adjust(bottom=.07, top=.95)
fringecolors = [None] * len(FREQUENCY_MULTIPLIERS)
histdata = dict((x, numpy.ndarray((0,))) for
x in FREQUENCY_MULTIPLIERS)
linecolor = None
# loop over state segments and find scattering fringes
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
ts = alldata[j][channel]
# get raw data and plot
line = axes['position'].plot(ts, color=linecolor)[0]
linecolor = line.get_color()
# get fringe frequency and plot
fringef = get_fringe_frequency(ts, multiplier=1)
for k, m in list(enumerate(FREQUENCY_MULTIPLIERS))[::-1]:
fm = fringef * m
line = axes['fringef'].plot(
fm, color=fringecolors[k],
label=(j == 0 and r'$f\times%d$' % m or None))[0]
fringecolors[k] = line.get_color()
histdata[m] = numpy.resize(
histdata[m], (histdata[m].size + fm.size,))
histdata[m][-fm.size:] = fm.value
# get segments and plot
scatter = get_segments(
fringef * multiplier,
fthresh,
name=flag,
pad=args.segment_padding
)
axes['segments'].plot(
scatter, facecolor='red', edgecolor='darkred',
known={'alpha': 0.6, 'facecolor': 'lightgray',
'edgecolor': 'gray', 'height': 0.4},
height=0.8, y=0, label=' ',
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# finalize plot
logger.debug("Plotting")
name = texify(channel)
axes['position'].set_title("Scattering evidence in %s" % name)
axes['position'].set_xlabel('')
axes['position'].set_ylabel(r'Position [$\mu$m]')
axes['position'].text(
0.01, 0.95, 'Optic position',
transform=axes['position'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['fringef'].plot(
span, [fthresh, fthresh], 'k--')
axes['fringef'].set_xlabel('')
axes['fringef'].set_ylabel(r'Frequency [Hz]')
axes['fringef'].yaxis.tick_right()
axes['fringef'].yaxis.set_label_position("right")
axes['fringef'].set_ylim(0, multiplier * fthresh)
axes['fringef'].text(
0.01, 0.95, 'Calculated fringe frequency',
transform=axes['fringef'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
handles, labels = axes['fringef'].get_legend_handles_labels()
axes['fringef'].legend(handles[::-1], labels[::-1], loc='upper right',
borderaxespad=0, bbox_to_anchor=(-0.01, 1.),
handlelength=1)
axes['triggers'].scatter(
trigs[names[0]],
trigs[names[1]],
c=trigs[names[2]],
edgecolor='none',
)
name = texify(args.main_channel)
axes['triggers'].text(
0.01, 0.95,
'%s event triggers (Omicron)' % name,
transform=axes['triggers'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['triggers'].set_ylabel('Frequency [Hz]')
axes['triggers'].set_ylim(args.fmin, multiplier * fthresh)
axes['triggers'].colorbar(cmap='YlGnBu', clim=(3, 100), norm='log',
label='Signal-to-noise ratio')
axes['segments'].set_ylim(-.55, .55)
axes['segments'].text(
0.01, 0.95,
r'Time segments with $f\times%d > %.2f$ Hz' % (
multiplier, fthresh),
transform=axes['segments'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
for ax in axes.values():
ax.set_epoch(int(args.gpsstart))
ax.set_xlim(*span)
png = '%s_SCATTERING_%s_HZ-%s.png' % (chanstr, tstr, gpsstr)
try:
plot.save(png)
except OverflowError as e:
warnings.warn(str(e))
plot.axes[1].set_ylim(0, multiplier * fthresh)
plot.refresh()
plot.save(png)
plot.close()
logger.debug("%s written." % png)
# make histogram
histogram = Plot(figsize=[12, 6])
ax = histogram.gca()
hrange = (0, multiplier * fthresh)
for m, color in list(zip(histdata, fringecolors))[::-1]:
if histdata[m].size:
ax.hist(
histdata[m], facecolor=color, alpha=.6, range=hrange,
bins=50, histtype='stepfilled', label=r'$f\times%d$' % m,
cumulative=-1, weights=ts.dx.value, bottom=1e-100,
log=True)
else:
ax.plot(histdata[m], color=color, label=r'$f\times%d$' % m)
ax.set_yscale('log')
ax.set_ylim(.01, float(livetime))
ax.set_ylabel('Time with fringe above frequency [s]')
ax.set_xlim(*hrange)
ax.set_xlabel('Frequency [Hz]')
ax.set_title(axes['position'].get_title())
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles[::-1], labels[::-1], loc='upper right')
hpng = '%s_SCATTERING_HISTOGRAM-%s.png' % (chanstr, gpsstr)
histogram.save(hpng)
histogram.close()
logger.debug("%s written." % hpng)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif ((deadtime != 0 and effdt < 2) or
(histdata[multiplier].size and
histdata[multiplier].max() >=
fthresh/2.)):
context = 'warning'
else:
continue
page.div(class_='card border-%s mb-1 shadow-sm' % context)
page.div(class_='card-header text-white bg-%s' % context)
page.a(channel, class_='collapsed card-link cis-link',
href='#osem%s' % i, **{'data-toggle': 'collapse'})
page.div.close() # card-header
page.div(id_='osem%s' % i, class_='collapse',
**{'data-parent': '#osems-group'})
page.div(class_='card-body')
page.div(class_='row')
img = htmlio.FancyPlot(
png, caption=SCATTER_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(img))
page.div.close() # col-md-10 offset-md-1
himg = htmlio.FancyPlot(
hpng, caption=HIST_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(himg))
page.div.close() # col-md-10 offset-md-1
page.div.close() # row
segs = StringIO()
if deadtime:
page.p("%d segments were found predicting a scattering fringe "
"above %.2f Hz." % (
len(scatter_segments[channel].active),
fthresh))
page.table(class_='table table-sm table-hover')
page.tbody()
page.tr()
page.th('Deadtime')
page.td('%.2f/%d seconds' % (deadtime, livetime))
page.td('%.2f%%' % deadtimepc)
page.tr.close()
page.tr()
page.th('Efficiency<br><small>(SNR≥8 and '
'%.2f Hz</sub><f<sub>peak</sub><%.2f Hz)</small>'
% (args.fmin, multiplier * fthresh))
page.td('%d/%d events' % (efficiency, len(highsnrtrigs)))
page.td('%.2f%%' % efficiencypc)
page.tr.close()
page.tr()
page.th('Efficiency/Deadtime')
page.td()
page.td('%.2f' % effdt)
page.tr.close()
page.tbody.close()
page.table.close()
scatter_segments[channel].active.write(segs, format='segwizard',
coltype=float)
page.pre(segs.getvalue())
else:
page.p("No segments were found with scattering above %.2f Hz."
% fthresh)
page.div.close() # card-body
page.div.close() # collapse
page.div.close() # card
if statea: # close accordion
page.div.close() # osems-group
# scattering based on transmon BLRMS
if statea:
page.h3('Transmons', class_='mt-3', id_='transmons')
page.div(id_='transmons-group')
logger.info('Searching for scatter based on band-limited RMS of transmons')
for i, channel in enumerate(sorted(transmons)):
logger.info("-- Processing %s --" % channel)
optic = channel.split('-')[1][:6]
flag = '%s:DCH-%s_SCATTERING_BLRMS:1' % (args.ifo, optic)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering from whitened, band-limited "
"RMS trends of {0}".format(channel),
)
# loop over state segments and compute BLRMS
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
wblrms = get_blrms(
alldata[j][channel],
flow=args.bandpass_flow,
fhigh=args.bandpass_fhigh,
)
scatter = get_segments(
wblrms,
numpy.mean(wblrms) + args.sigma * numpy.std(wblrms),
name=flag,
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
highsnrtrigs = trigs[trigs['snr'] <= 200]
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif deadtime != 0 and effdt < 2:
context = 'warning'
else:
continue
page.add(htmlio.write_flag_html(
scatter_segments[channel], span, i, parent='transmons-group',
title=channel, context=context, plotdir=''))
if statea: # close accordion
page.div.close() # transmons-group
actives = actives.coalesce() # merge contiguous segments
if statea and not actives:
page.add(htmlio.alert(
'No evidence of scattering found in the channels analyzed',
context=args.ifo.lower(), dismiss=False))
# identify triggers during active segments
logger.debug('Writing a summary CSV record')
ind = [i for i, trigtime in enumerate(highsnrtrigs[names[0]])
if trigtime in actives]
gps = highsnrtrigs[names[0]][ind]
freq = highsnrtrigs[names[1]][ind]
snr = highsnrtrigs[names[2]][ind]
segs = [y for x in gps for y in actives if x in y]
table = EventTable(
[gps, freq, snr, [seg[0] for seg in segs], [seg[1] for seg in segs]],
names=('trigger_time', 'trigger_frequency', 'trigger_snr',
'segment_start', 'segment_end'))
logger.info('The following {} triggers fell within active scattering '
'segments:\n\n'.format(len(table)))
print(table)
print('\n\n')
table.write(summfile, overwrite=True)
# -- launch omega scans -----------
nscans = min(args.omega_scans, len(table))
if nscans > 0:
# launch scans
scandir = 'scans'
ind = random.sample(range(0, len(table)), nscans)
omegatimes = [str(t) for t in table['trigger_time'][ind]]
logger.debug('Collected {} event times to omega scan: {}'.format(
nscans, ', '.join(omegatimes)))
logger.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(
ifo=args.ifo, ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(timeout=4, gps=args.gpsstart)
batch.generate_dag(omegatimes, flags=flags, submit=True,
outdir=scandir, condor_commands=condorcmds)
logger.info('Launched {} omega scans to condor'.format(nscans))
# render HTML
page.h2('Omega scans', class_='mt-4', id_='omega-scans')
msg = (
'The following event times correspond to significant Omicron '
'triggers that occur during the scattering segments found above. '
'To compare these against fringe frequency projections, please '
'use the "simple scattering" module:',
markup.oneliner.pre(
'$ python -m gwdetchar.scattering.simple --help',
),
)
page.add(htmlio.alert(msg, context=args.ifo.lower()))
page.add(htmlio.scaffold_omega_scans(
omegatimes, args.main_channel, scandir=scandir))
elif args.omega_scans:
logger.info('No events found during active scattering segments')
# -- finalize ---------------------
# write segments
scatter_segments.write(segfile, path="segments", overwrite=True)
logger.debug("%s written" % segfile)
# write HTML
htmlio.close_page(page, 'index.html')
logger.info("-- index.html written, all done --")
# return to original directory
os.chdir(indir)
