def main_compare_gif_gotic2():
start = tconvert('Oct 15 2018 00:00:00')
end = tconvert('Oct 21 2018 00:00:00')
# gif data
pfx = '/Users/miyo/Dropbox/KagraData/gif/'
segments = GifData.findfiles(start, end, 'CALC_STRAIN', prefix=pfx)
allfiles = [path for files in segments for path in files]
strain = TimeSeries.read(source=allfiles,
name='CALC_STRAIN',
format='gif',
pad=numpy.nan,
nproc=2)
strain = strain.detrend('linear')
# gotic data
source = '201805010000_201811010000.gotic'
gifx = KagraGoticStrain.read(source, start=start, end=end).x
gifx = gifx.detrend('linear')
gifx = gifx * 0.9
# plot
plot = Plot(gifx, strain, xscale='auto-gps')
plot.legend()
plot.subplots_adjust(right=.86)
plot.savefig('result.png')
plot.close()
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs):
"""Plot the signifiance drop for each channel
"""
channels = sorted(old.keys())
if show_channel_names is None:
show_channel_names = len(channels) <= 50
plot = Plot(figsize=(18, 6))
plot.subplots_adjust(left=.07, right=.93)
ax = plot.gca()
if show_channel_names:
plot.subplots_adjust(bottom=.4)
winner = sorted(old.items(), key=lambda x: x[1])[-1][0]
for i, c in enumerate(channels):
if c == winner:
color = 'orange'
elif old[c] > new[c]:
color = 'dodgerblue'
else:
color = 'red'
ax.plot([i, i], [old[c], new[c]], color=color, linestyle='-',
marker='o', markeredgecolor='k', markeredgewidth=.5,
markersize=10, label=c, zorder=old[c])
ax.set_xlim(-1, len(channels))
ax.set_ybound(lower=0)
# set xticks to show channel names
if show_channel_names:
ax.set_xticks(range(len(channels)))
ax.set_xticklabels([c.replace('_','\_') for c in channels])
for i, t in enumerate(ax.get_xticklabels()):
t.set_rotation(270)
t.set_verticalalignment('top')
t.set_horizontalalignment('center')
t.set_fontsize(8)
# or just show systems of channels
else:
plot.canvas.draw()
systems = {}
for i, c in enumerate(channels):
sys = c.split(':', 1)[1].split('-')[0].split('_')[0]
try:
systems[sys][1] += 1
except KeyError:
systems[sys] = [i, 1]
systems = sorted(systems.items(), key=lambda x: x[1][0])
labels, counts = zip(*systems)
xticks, xmticks = zip(*[(a, a+b/2.) for (a, b) in counts])
# show ticks at the edge of each group
ax.set_xticks(xticks, minor=False)
ax.set_xticklabels([], minor=False)
# show label in the centre of each group
ax.set_xticks(xmticks, minor=True)
for t in ax.set_xticklabels(labels, minor=True):
t.set_rotation(270)
kwargs.setdefault('ylabel', 'Significance')
# create interactivity
if outfile.endswith('.svg'):
_finalize_plot(plot, ax, outfile.replace('.svg', '.png'),
close=False, **kwargs)
tooltips = []
ylim = ax.get_ylim()
yoffset = (ylim[1] - ylim[0]) * 0.061
bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'}
xthresh = len(channels) / 10.
for i, l in enumerate(ax.lines):
x = l.get_xdata()[1]
if x < xthresh:
ha = 'left'
elif x > (len(channels) - xthresh):
ha ='right'
else:
ha = 'center'
y = l.get_ydata()[0] + yoffset
c = l.get_label()
tooltips.append(ax.annotate(c.replace('_', r'\_'), (x, y),
ha=ha, zorder=ylim[1], bbox=bbox))
l.set_gid('line-%d' % i)
tooltips[-1].set_gid('tooltip-%d' % i)
f = BytesIO()
plot.savefig(f, format='svg')
tree, xmlid = etree.XMLID(f.getvalue())
tree.set('onload', 'init(evt)')
for i in range(len(tooltips)):
try:
e = xmlid['tooltip-%d' % i]
except KeyError:
warnings.warn("Failed to recover tooltip %d" % i)
continue
e.set('visibility', 'hidden')
for i, l in enumerate(ax.lines):
e = xmlid['line-%d' % i]
e.set('onmouseover', 'ShowTooltip(this)')
e.set('onmouseout', 'HideTooltip(this)')
tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT))
etree.ElementTree(tree).write(outfile)
plot.close()
else:
_finalize_plot(plot, ax, outfile, **kwargs)
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs):
"""Plot the signifiance drop for each channel
"""
channels = sorted(old.keys())
if show_channel_names is None:
show_channel_names = len(channels) <= 50
plot = Plot(figsize=(20, 5))
plot.subplots_adjust(left=.07, right=.93)
ax = plot.gca()
if show_channel_names:
plot.subplots_adjust(bottom=.4)
winner = sorted(old.items(), key=lambda x: x[1])[-1][0]
for i, c in enumerate(channels):
if c == winner:
color = 'orange'
elif old[c] > new[c]:
color = 'dodgerblue'
else:
color = 'red'
ax.plot([i, i], [old[c], new[c]],
color=color,
linestyle='-',
marker='o',
markeredgecolor='k',
markeredgewidth=.5,
markersize=10,
label=c,
zorder=old[c])
ax.set_xlim(-1, len(channels))
ax.set_ybound(lower=0)
# set xticks to show channel names
if show_channel_names:
ax.set_xticks(range(len(channels)))
ax.set_xticklabels([texify(c) for c in channels])
for i, t in enumerate(ax.get_xticklabels()):
t.set_rotation(270)
t.set_verticalalignment('top')
t.set_horizontalalignment('center')
t.set_fontsize(8)
# or just show systems of channels
else:
plot.canvas.draw()
systems = {}
for i, c in enumerate(channels):
sys = c.split(':', 1)[1].split('-')[0].split('_')[0]
try:
systems[sys][1] += 1
except KeyError:
systems[sys] = [i, 1]
systems = sorted(systems.items(), key=lambda x: x[1][0])
labels, counts = zip(*systems)
xticks, xmticks = zip(*[(a, a + b / 2.) for (a, b) in counts])
# show ticks at the edge of each group
ax.set_xticks(xticks, minor=False)
ax.set_xticklabels([], minor=False)
# show label in the centre of each group
ax.set_xticks(xmticks, minor=True)
for t in ax.set_xticklabels(labels, minor=True):
t.set_rotation(270)
kwargs.setdefault('ylabel', 'Significance')
# create interactivity
if outfile.endswith('.svg'):
_finalize_plot(plot,
ax,
outfile.replace('.svg', '.png'),
close=False,
**kwargs)
tooltips = []
ylim = ax.get_ylim()
yoffset = (ylim[1] - ylim[0]) * 0.061
bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'}
xthresh = len(channels) / 10.
for i, l in enumerate(ax.lines):
x = l.get_xdata()[1]
if x < xthresh:
ha = 'left'
elif x > (len(channels) - xthresh):
ha = 'right'
else:
ha = 'center'
y = l.get_ydata()[0] + yoffset
c = l.get_label()
tooltips.append(
ax.annotate(texify(c), (x, y),
ha=ha,
zorder=ylim[1],
bbox=bbox))
l.set_gid('line-%d' % i)
tooltips[-1].set_gid('tooltip-%d' % i)
f = BytesIO()
plot.savefig(f, format='svg')
tree, xmlid = etree.XMLID(f.getvalue())
tree.set('onload', 'init(evt)')
for i in range(len(tooltips)):
try:
e = xmlid['tooltip-%d' % i]
except KeyError:
warnings.warn("Failed to recover tooltip %d" % i)
continue
e.set('visibility', 'hidden')
for i, l in enumerate(ax.lines):
e = xmlid['line-%d' % i]
e.set('onmouseover', 'ShowTooltip(this)')
e.set('onmouseout', 'HideTooltip(this)')
tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT))
etree.ElementTree(tree).write(outfile)
plot.close()
else:
_finalize_plot(plot, ax, outfile, **kwargs)
def process_channel(input_,):
chan, ts = input_
flat = ts.value.min() == ts.value.max()
if flat:
corr1 = None
corr2 = None
corr1s = None
corr2s = None
plot1 = None
plot2 = None
plot3 = None
else:
corr1 = numpy.corrcoef(ts.value, darmblrms.value)[0, 1]
corr1s = spearmanr(ts.value, darmblrms.value)[0]
if args.trend_type == 'minute':
corr2 = numpy.corrcoef(ts.value, rangets.value)[0, 1]
corr2s = spearmanr(ts.value, rangets.value)[0]
else:
corr2 = 0.0
corr2s = 0.0
# if all corralations are below threshold it does not plot
if((abs(corr1) < args.threshold)
and (abs(corr1s) < args.threshold)
and (abs(corr2) < args.threshold)
and (abs(corr2s) < args.threshold)):
plot1 = None
plot2 = None
plot3 = None
return (chan, corr1, corr2, plot1,
plot2, plot3, corr1s, corr2s)
plot = Plot(darmblrms, ts, rangets,
xscale="auto-gps", separate=True,
figsize=(12, 12))
plot.subplots_adjust(*p1)
plot.axes[0].set_ylabel('$h(t)$ BLRMS [strain]')
plot.axes[1].set_ylabel('Channel units')
plot.axes[2].set_ylabel('Sensitive range [Mpc]')
for ax in plot.axes:
ax.legend(loc='best')
ax.set_xlim(start, end)
ax.set_epoch(start)
channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1)
plot1 = '%s_TRENDS-%s.png' % (channelstub, gpsstub)
try:
plot.save(plot1)
except (IOError, IndexError):
plot.save(plot1)
except RuntimeError as e:
if 'latex' in str(e).lower():
plot.save(plot1)
else:
raise
plot.close()
# plot auto-scaled verions
tsscaled = ts.detrend()
tsrms = numpy.sqrt(sum(tsscaled**2.0)/len(tsscaled))
if args.trend_type == 'minute':
tsscaled *= (rangerms / tsrms)
if corr1 > 0:
tsscaled *= -1
else:
tsscaled *= (darmrms / tsrms)
if corr1 < 0:
tsscaled *= -1
plot = Plot(darmscaled, rangescaled, tsscaled,
xscale="auto-gps", figsize=[12, 6])
plot.subplots_adjust(*p2)
ax = plot.gca()
ax.set_xlim(start, end)
ax.set_epoch(start)
ax.set_ylabel('Scaled amplitude [arbitrary units]')
ax.legend(loc='best')
plot2 = '%s_COMPARISON-%s.png' % (channelstub, gpsstub)
try:
plot.save(plot2)
except (IOError, IndexError):
plot.save(plot2)
except RuntimeError as e:
if 'latex' in str(e).lower():
plot.save(plot2)
else:
raise
plot.close()
# plot scatter plots
rangeColor = 'red'
darmblrmsColor = 'blue'
tsCopy = ts.reshape(-1, 1)
rangetsCopy = rangets.reshape(-1, 1)
darmblrmsCopy = darmblrms.reshape(-1, 1)
darmblrmsReg = linear_model.LinearRegression()
darmblrmsReg.fit(tsCopy, darmblrmsCopy)
darmblrmsFit = darmblrmsReg.predict(tsCopy)
rangeReg = linear_model.LinearRegression()
rangeReg.fit(tsCopy, rangetsCopy)
rangeFit = rangeReg.predict(tsCopy)
fig = Plot(figsize=(12, 6))
fig.subplots_adjust(*p2)
ax = fig.add_subplot(121)
ax.set_xlabel('Channel units')
ax.set_ylabel('Sensitive range [Mpc]')
yrange = abs(max(darmblrms.value) - min(darmblrms.value))
upperLim = max(darmblrms.value) + .1 * yrange
lowerLim = min(darmblrms.value) - .1 * yrange
ax.set_ylim(lowerLim, upperLim)
ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr1)),
verticalalignment='bottom', horizontalalignment='right',
transform=ax.transAxes, color='black', size=20,
bbox=dict(boxstyle='square', facecolor='white', alpha=.75,
edgecolor='black'))
fig.add_scatter(ts, darmblrms, color=darmblrmsColor)
fig.add_line(ts, darmblrmsFit, color='black')
ax = fig.add_subplot(122)
ax.set_xlabel('Channel units')
ax.set_ylabel('$h(t)$ BLRMS [strain]')
ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr2)),
verticalalignment='bottom', horizontalalignment='right',
transform=ax.transAxes, color='black', size=20,
bbox=dict(boxstyle='square', facecolor='white', alpha=.75,
edgecolor='black'))
fig.add_scatter(ts, rangets, color=rangeColor)
fig.add_line(ts, rangeFit, color='black')
plot3 = '%s_SCATTER-%s.png' % (channelstub, gpsstub)
try:
fig.save(plot3)
except (IOError, IndexError):
fig.save(plot3)
except RuntimeError as e:
if 'latex' in str(e).lower():
fig.save(plot3)
else:
raise
plt.close(fig)
# increment counter and print status
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / nchan
LOGGER.debug("Completed [%d/%d] %3d%% %-50s"
% (counter.value, nchan, pc, '(%s)' % str(chan)))
sys.stdout.flush()
return chan, corr1, corr2, plot1, plot2, plot3, corr1s, corr2s
def main(args=None):
use('agg')
rcParams.update({
'figure.subplot.bottom': 0.15,
'figure.subplot.left': 0.1,
'figure.subplot.right': 0.83,
'figure.subplot.top': 0.93,
'figure.subplot.hspace': 0.25,
'axes.labelsize': 20,
'grid.color': 'gray',
})
grid = GridSpec(2, 1)
logger = log.Logger('omicron-status')
try:
omicronversion = str(get_omicron_version())
except KeyError:
omicronversion = 'Unknown'
logger.warning("Omicron version unknown")
else:
logger.info("Found omicron version: %s" % omicronversion)
parser = create_parser()
args = parser.parse_args(args=args)
if args.ifo is None:
parser.error("Cannot determine IFO prefix from sytem, "
"please pass --ifo on the command line")
group = args.group
logger.info("Checking status for %r group" % group)
archive = args.archive_directory
proddir = args.production_directory.with_name(
args.production_directory.name.format(group=args.group), )
outdir = args.output_directory
outdir.mkdir(exist_ok=True, parents=True)
tag = args.latency_archive_tag.format(group=args.group)
filetypes = ['h5', 'xml.gz', 'root']
logger.debug("Set output directory to %s" % outdir)
logger.debug(
"Will process the following filetypes: {}".format(
", ".join(filetypes), ), )
# -- parse configuration file and get parameters --------------------------
cp = configparser.ConfigParser()
ok = cp.read(args.config_file)
if args.config_file not in ok:
raise IOError(
"Failed to read configuration file %r" % args.config_file, )
logger.info("Configuration read")
# validate
if not cp.has_section(group):
raise configparser.NoSectionError(group)
# get parameters
obs = args.ifo[0]
frametype = cp.get(group, 'frametype')
padding = cp.getint(group, 'overlap-duration') / 2.
mingap = cp.getint(group, 'chunk-duration')
channels = args.channel
if not channels:
channels = [
c.split()[0]
for c in cp.get(group, 'channels').strip('\n').split('\n')
]
channels.sort()
logger.debug("Found %d channels" % len(channels))
start = args.gps_start_time
end = args.gps_end_time
if end == NOW:
end -= padding
if args.state_flag:
stateflag = args.state_flag
statepad = tuple(map(float, args.state_pad.split(',')))
else:
try:
stateflag = cp.get(group, 'state-flag')
except configparser.NoOptionError:
stateflag = None
else:
try:
statepad = tuple(
map(
float,
cp.get(group, 'state-padding').split(','),
))
except configparser.NoOptionError:
statepad = (0, 0)
if stateflag:
logger.debug("Parsed state flag: %r" % stateflag)
logger.debug("Parsed state padding: %s" % repr(statepad))
logger.info("Processing %d-%d" % (start, end))
# -- define nagios JSON printer -------------------------------------------
def print_nagios_json(code, message, outfile, tag='status', **extras):
out = {
'created_gps':
NOW,
'status_intervals': [
{
'start_sec': 0,
'end_sec': args.unknown,
'num_status': code,
'txt_status': message
},
{
'start_sec': args.unknown,
'num_status': 3,
'txt_status': 'Omicron %s check is not running' % tag
},
],
'author': {
'name': 'Duncan Macleod',
'email': '*****@*****.**',
},
'omicron': {
'version': omicronversion,
'group': group,
'channels': ' '.join(channels),
'frametype': frametype,
'state-flag': stateflag,
},
'pyomicron': {
'version': __version__,
},
}
out.update(extras)
with open(outfile, 'w') as f:
f.write(json.dumps(out))
logger.debug("nagios info written to %s" % outfile)
# -- get condor status ------------------------------------------------
if not args.skip_condor:
# connect to scheduler
try:
schedd = htcondor.Schedd()
except RuntimeError as e:
logger.warning("Caught %s: %s" % (type(e).__name__, e))
logger.info("Failed to connect to HTCondor scheduler, cannot "
"determine condor status for %s" % group)
schedd = None
if not args.skip_condor and schedd:
logger.info("-- Checking condor status --")
# get DAG status
jsonfp = outdir / "nagios-condor-{}.json".format(group)
okstates = ['Running', 'Idle', 'Completed']
try:
# check manager status
qstr = 'OmicronManager == "{}" && Owner == "{}"'.format(
group,
args.user,
)
try:
jobs = schedd.query(qstr, ['JobStatus'])
except IOError as e:
warnings.warn("Caught IOError: %s [retrying...]" % str(e))
sleep(2)
jobs = schedd.query(qstr, ['JobStatus'])
logger.debug(
"Found {} jobs for query {!r}".format(len(jobs), qstr), )
if len(jobs) > 1:
raise RuntimeError(
"Multiple OmicronManager jobs found for %r" % group)
elif len(jobs) == 0:
raise RuntimeError(
"No OmicronManager job found for %r" % group, )
status = condor.JOB_STATUS[jobs[0]['JobStatus']]
if status not in okstates:
raise RuntimeError("OmicronManager status for %r: %r" %
(group, status))
logger.debug("Manager status is %r" % status)
# check node status
jobs = schedd.query(
'OmicronProcess == "{}" && Owner == "{}"'.format(
group,
args.user,
),
['JobStatus', 'ClusterId'],
)
logger.debug(
"Found {} jobs for query {!r}".format(len(jobs), qstr), )
for job in jobs:
status = condor.JOB_STATUS[job['JobStatus']]
if status not in okstates:
raise RuntimeError("Omicron node %s (%r) is %r" %
(job['ClusterId'], group, status))
except RuntimeError as e:
print_nagios_json(2, str(e), jsonfp, tag='condor')
logger.warning("Failed to determine condor status: %r" % str(e))
except IOError as e:
logger.warning("Caught %s: %s" % (type(e).__name__, e))
logger.info("Failed to connect to HTCondor scheduler, cannot "
"determine condor status for %s" % group)
else:
print_nagios_json(
0,
"Condor processing for %r is OK" % group,
jsonfp,
tag='condor',
)
logger.info("Condor processing is OK")
if not args.skip_job_duration:
# get job duration history
plot = Plot(figsize=[12, 3])
plot.subplots_adjust(bottom=.22, top=.87)
ax = plot.gca(xscale="auto-gps")
times, jobdur = condor.get_job_duration_history_shell('OmicronProcess',
group,
maxjobs=5000)
logger.debug("Recovered duration history for %d omicron.exe jobs" %
len(times))
line = ax.plot([0], [1], label='Omicron.exe')[0]
ax.plot(times,
jobdur,
linestyle=' ',
marker='.',
color=line.get_color())
times, jobdur = condor.get_job_duration_history_shell(
'OmicronPostProcess', group, maxjobs=5000)
logger.debug("Recovered duration history for %d post-processing jobs" %
len(times))
line = ax.plot([0], [1], label='Post-processing')[0]
ax.plot(times,
jobdur,
linestyle=' ',
marker='.',
color=line.get_color())
ax.legend(loc='upper left',
borderaxespad=0,
bbox_to_anchor=(1.01, 1),
handlelength=1)
ax.set_xlim(args.gps_start_time, args.gps_end_time)
ax.set_epoch(ax.get_xlim()[1])
ax.set_yscale('log')
ax.set_title('Omicron job durations for %r' % group)
ax.set_ylabel('Job duration [seconds]')
ax.xaxis.labelpad = 5
png = str(outdir / "nagios-condor-{}.png".format(group))
plot.save(png)
plot.close()
logger.debug("Saved condor plot to %s" % png)
if args.skip_file_checks:
sys.exit(0)
# -- get file latency and archive completeness ----------------------------
logger.info("-- Checking file archive --")
# get frame segments
segs = segments.get_frame_segments(obs, frametype, start, end)
# get state segments
if stateflag is not None:
segs &= segments.query_state_segments(
stateflag,
start,
end,
pad=statepad,
)
try:
end = segs[-1][1]
except IndexError:
pass
# apply inwards padding to generate resolvable segments
for i in range(len(segs) - 1, -1, -1):
# if segment is shorter than padding, ignore it completely
if abs(segs[i]) <= padding * 2:
del segs[i]
# otherwise apply padding to generate trigger segment
else:
segs[i] = segs[i].contract(padding)
logger.debug("Found %d seconds of analysable time" % abs(segs))
# load archive latency
latencyfile = outdir / "nagios-latency-{}.h5".format(tag)
times = dict((c, dict((ft, None) for ft in filetypes)) for c in channels)
ldata = dict((c, dict((ft, None) for ft in filetypes)) for c in channels)
try:
with h5py.File(latencyfile, 'r') as h5file:
for c in channels:
for ft in filetypes:
try:
times[c][ft] = h5file[c]['time'][ft][:]
ldata[c][ft] = h5file[c]['latency'][ft][:]
except KeyError:
times[c][ft] = numpy.ndarray((0, ))
ldata[c][ft] = numpy.ndarray((0, ))
except OSError as exc: # file not found, or is corrupt
warnings.warn("failed to load latency data from {}: {}".format(
latencyfile,
str(exc),
))
for c in channels:
for ft in filetypes:
if not times[c].get(ft):
times[c][ft] = numpy.ndarray((0, ))
ldata[c][ft] = numpy.ndarray((0, ))
else:
logger.debug("Parsed latency data from %s" % latencyfile)
# load acknowledged gaps
acksegfile = str(outdir / "acknowledged-gaps-{}.txt".format(tag))
try:
acknowledged = SegmentList.read(acksegfile,
gpstype=float,
format="segwizard")
except IOError: # no file
acknowledged = SegmentList()
else:
logger.debug(
"Read %d segments from %s" % (len(acknowledged), acksegfile), )
acknowledged.coalesce()
# build legend for segments
leg = OrderedDict()
leg['Analysable'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightgray',
edgecolor='gray',
)
leg['Available'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightgreen',
edgecolor='green',
)
leg['Missing'] = SegmentRectangle(
[0, 1],
0,
facecolor='red',
edgecolor='darkred',
)
leg['Unresolvable'] = SegmentRectangle(
[0, 1],
0,
facecolor='magenta',
edgecolor='purple',
)
leg['Overlapping'] = SegmentRectangle(
[0, 1],
0,
facecolor='yellow',
edgecolor='orange',
)
leg['Pending'] = SegmentRectangle(
[0, 1],
0,
facecolor='lightskyblue',
edgecolor='blue',
)
leg['Acknowledged'] = SegmentRectangle(
[0, 1],
0,
facecolor='sandybrown',
edgecolor='brown',
)
logger.debug("Checking archive for each channel...")
# find files
latency = {}
gaps = {}
overlap = {}
pending = {}
plots = {}
for c in channels:
# create data storate
latency[c] = {}
gaps[c] = {}
overlap[c] = {}
pending[c] = {}
# create figure
plot = Plot(figsize=[12, 5])
lax = plot.add_subplot(grid[0, 0], xscale="auto-gps")
sax = plot.add_subplot(grid[1, 0], sharex=lax, projection='segments')
colors = ['lightblue', 'dodgerblue', 'black']
for y, ft in enumerate(filetypes):
# find files
cache = io.find_omicron_files(c, start, end, archive, ext=ft)
cpend = sieve_cache(io.find_pending_files(c, proddir, ext=ft),
segment=Segment(start, end))
# get available segments
avail = segments.cache_segments(cache)
found = avail & segs
pending[c][ft] = segments.cache_segments(cpend) & segs
# remove gaps at the end that represent latency
try:
latency[c][ft] = abs(segs & type(
segs)([type(segs[0])(found[-1][1], segs[-1][1])])) / 3600.
except IndexError:
latency[c][ft] = 0
processed = segs
else:
processed = segs & type(segs)(
[type(segs[0])(start, found[-1][1])])
gaps[c][ft] = type(found)()
lost = type(found)()
for s in processed - found:
if abs(s) < mingap and s in list(segs):
lost.append(s)
else:
gaps[c][ft].append(s)
# remove acknowledged gaps
ack = gaps[c][ft] & acknowledged
gaps[c][ft] -= acknowledged
# print warnings
if abs(gaps[c][ft]):
warnings.warn("Gaps found in %s files for %s:\n%s" %
(c, ft, gaps[c][ft]))
overlap[c][ft] = segments.cache_overlaps(cache)
if abs(overlap[c][ft]):
warnings.warn("Overlap found in %s files for %s:\n%s" %
(c, ft, overlap[c][ft]))
# append archive
times[c][ft] = numpy.concatenate((times[c][ft][-99999:], [NOW]))
ldata[c][ft] = numpy.concatenate(
(ldata[c][ft][-99999:], [latency[c][ft]]))
# plot
line = lax.plot(
times[c][ft],
ldata[c][ft],
label=ft,
color=colors[y],
)[0]
lax.plot(times[c][ft],
ldata[c][ft],
marker='.',
linestyle=' ',
color=line.get_color())
sax.plot_segmentlist(segs,
y=y,
label=ft,
alpha=.5,
facecolor=leg['Analysable'].get_facecolor(),
edgecolor=leg['Analysable'].get_edgecolor())
sax.plot_segmentlist(pending[c][ft],
y=y,
facecolor=leg['Pending'].get_facecolor(),
edgecolor=leg['Pending'].get_edgecolor())
sax.plot_segmentlist(avail,
y=y,
label=ft,
alpha=.2,
height=.1,
facecolor=leg['Available'].get_facecolor(),
edgecolor=leg['Available'].get_edgecolor())
sax.plot_segmentlist(found,
y=y,
label=ft,
alpha=.5,
facecolor=leg['Available'].get_facecolor(),
edgecolor=leg['Available'].get_edgecolor())
sax.plot_segmentlist(lost,
y=y,
facecolor=leg['Unresolvable'].get_facecolor(),
edgecolor=leg['Unresolvable'].get_edgecolor())
sax.plot_segmentlist(gaps[c][ft],
y=y,
facecolor=leg['Missing'].get_facecolor(),
edgecolor=leg['Missing'].get_edgecolor())
sax.plot_segmentlist(overlap[c][ft],
y=y,
facecolor=leg['Overlapping'].get_facecolor(),
edgecolor=leg['Overlapping'].get_edgecolor())
sax.plot_segmentlist(ack,
y=y,
facecolor=leg['Acknowledged'].get_facecolor(),
edgecolor=leg['Acknowledged'].get_edgecolor())
# finalise plot
lax.axhline(args.warning / 3600.,
color=(1.0, 0.7, 0.0),
linestyle='--',
linewidth=2,
label='Warning',
zorder=-1)
lax.axhline(args.error / 3600.,
color='red',
linestyle='--',
linewidth=2,
label='Critical',
zorder=-1)
lax.set_title('Omicron status: {}'.format(c))
lax.set_ylim(0, args.error / 1800.)
lax.set_ylabel('Latency [hours]')
lax.legend(loc='upper left',
bbox_to_anchor=(1.01, 1),
borderaxespad=0,
handlelength=2,
fontsize=12.4)
lax.set_xlabel(' ')
for ax in plot.axes:
ax.set_xlim(args.gps_start_time, args.gps_end_time)
ax.set_epoch(ax.get_xlim()[1])
sax.xaxis.labelpad = 5
sax.set_ylim(-.5, len(filetypes) - .5)
sax.legend(leg.values(),
leg.keys(),
handlelength=1,
fontsize=12.4,
loc='lower left',
bbox_to_anchor=(1.01, 0),
borderaxespad=0)
plots[c] = png = outdir / "nagios-latency-{}.png".format(
c.replace(':', '-'), )
plot.save(png)
plot.close()
logger.debug(" %s" % c)
# update latency and write archive
h5file = h5py.File(latencyfile, 'w')
for c in channels:
g = h5file.create_group(c)
for name, d in zip(['time', 'latency'], [times[c], ldata[c]]):
g2 = g.create_group(name)
for ft in filetypes:
g2.create_dataset(ft, data=d[ft], compression='gzip')
h5file.close()
logger.debug("Stored latency data as HDF in %s" % latencyfile)
# write nagios output for files
status = []
for segset, tag in zip([gaps, overlap], ['gaps', 'overlap']):
chans = [(c, segset[c]) for c in segset
if abs(reduce(operator.or_, segset[c].values()))]
jsonfp = outdir / "nagios-{}-{}.json".format(tag, group)
status.append((tag, jsonfp))
if chans:
gapstr = '\n'.join('%s: %s' % c for c in chans)
code = 1
message = ("%s found in Omicron files for group %r\n%s" %
(tag.title(), group, gapstr))
else:
code = 0
message = ("No %s found in Omicron files for group %r" %
(tag, group))
print_nagios_json(code, message, jsonfp, tag=tag, **{tag: dict(chans)})
# write group JSON
jsonfp = outdir / "nagios-latency-{}.json".format(group)
status.append(('latency', jsonfp))
code = 0
message = 'No channels have high latency for group %r' % group
ldict = dict((c, max(latency[c].values())) for c in latency)
for x, dt in zip([2, 1], [args.error, args.warning]):
dh = dt / 3600.
chans = [c for c in ldict if ldict[c] >= dh]
if chans:
code = x
message = (
"%d channels found with high latency (above %s seconds)" %
(len(chans), dt))
break
print_nagios_json(code, message, jsonfp, tag='latency', latency=ldict)
# auto-detect 'standard' JSON files
for tag, name in zip(
['condor', 'omicron-online'],
['condor', 'processing'],
):
f = outdir / "nagios-{}-{}.json".format(tag, group)
if f.is_file():
status.insert(0, (name, f))
# write HTML summary
if args.html:
page = markup.page()
page.init(
title="%s Omicron Online status" % group,
css=[
('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/'
'bootstrap.min.css'),
('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/'
'jquery.fancybox.min.css'),
],
script=[
'//code.jquery.com/jquery-1.11.2.min.js',
('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/'
'bootstrap.min.js'),
('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/'
'jquery.fancybox.min.js'),
],
)
page.div(class_='container')
# write header
page.div(class_='page-header')
page.h1('Omicron Online status: %s' % group)
page.div.close() # page-header
# write summary
page.div(id_='json')
page.h2("Processing status")
for tag, f in status:
jf = f.name
page.a("%s status" % tag.title(),
href=jf,
role='button',
target="_blank",
id_="nagios-%s" % tag,
class_='btn btn-default json-status')
page.p(style="padding-top: 5px;")
page.small(
"Hover over button for explanation, click to open JSON file", )
page.p.close()
page.div.close() # id=json
# show plots
page.div(id_='plots')
page.h2("Channel details")
page.div(class_='row')
for channel in sorted(channels):
png = plots[channel].name
page.div(class_="col-sm-6 col-md-4")
page.div(class_="panel panel-default")
page.div(class_='panel-heading')
page.h3(channel, class_='panel-title', style="font-size: 14px;")
page.div.close() # panel-heading
page.div(class_='panel-body')
page.a(href=png,
target="_blank",
class_="fancybox",
rel="channel-status-img")
page.img(src=png, class_='img-responsive')
page.a.close()
page.div.close() # panel-body
page.div.close() # panel
page.div.close() # col
page.div.close() # row
page.div.close() # id=plots
# dump parameters
page.div(id_="parameters")
page.h2("Parameters")
for key, val in cp.items(group):
page.p()
page.strong("%s:" % key)
page.add(val)
page.p.close()
page.div.close() # id=parameters
# finish and close
page.div.close() # container
page.script("""
function setStatus(data, id) {
var txt = data.status_intervals[0].txt_status.split("\\n")[0];
$("#"+id).attr("title", txt);
var stat = data.status_intervals[0].num_status;
if (stat == 0) {
$("#"+id).addClass("btn-success"); }
else if (stat == 1) {
$("#"+id).addClass("btn-warning"); }
else if (stat == 2){
$("#"+id).addClass("btn-danger"); }
}
$(document).ready(function() {
$(".json-status").each(function() {
var jsonf = $(this).attr("href");
var id = $(this).attr("id");
$.getJSON(jsonf, function(data) { setStatus(data, id); });
});
$(".fancybox").fancybox({nextEffect: 'none', prevEffect: 'none'});
});""",
type="text/javascript")
with (outdir / "index.html").open("w") as f:
f.write(str(page))
logger.debug("HTML summary written to %s" % f.name)
def _generate_cluster(input_):
"""Generate cluster data for use below
"""
if USETEX:
gwplot.configure_mpl_tex()
currentchan = input_[1][0]
currentts = input_[1][5]
current = input_[0]
p7 = (.135, .15, .95, .9)
plot7 = None
plot7_list = None
if current < len(nonzerodata):
cluster = []
for i, otheritem in enumerate(list(auxdata.items())):
chan_, ts_ = otheritem
if chan_ != currentchan:
pcorr = numpy.corrcoef(currentts.value, ts_.value)[0, 1]
if abs(pcorr) >= cluster_threshold:
stub = re_delim.sub('_', chan_).replace('_', '-', 1)
cluster.append([i, ts_, pcorr, chan_, stub])
if cluster:
# write cluster table to file
cluster = sorted(cluster, key=lambda x: abs(x[2]), reverse=True)
clustertab = Table(data=list(zip(*cluster))[2:4],
names=('Pearson Coefficient', 'Channel'))
plot7_list = '%s_CLUSTER_LIST-%s.csv' % (re_delim.sub(
'_', str(currentchan)).replace('_', '-', 1), gpsstub)
clustertab.write(plot7_list, format='csv', overwrite=True)
ncluster = min(len(cluster), max_correlated_channels)
colors2 = [cmap(i) for i in numpy.linspace(0, 1, ncluster + 1)]
# plot
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p7)
ax = fig.gca(xscale='auto-gps')
ax.plot(times,
scale(currentts.value) * numpy.sign(input_[1][1]),
label=texify(currentchan),
linewidth=line_size_aux,
color=colors[0])
for i in range(0, ncluster):
this = cluster[i]
ax.plot(
times,
scale(this[1].value) * numpy.sign(input_[1][1]) *
numpy.sign(this[2]),
color=colors2[i + 1],
linewidth=line_size_aux,
label=('{0}, r = {1:.2}'.format(texify(cluster[i][3]),
cluster[i][2])),
)
ax.margins(x=0)
ax.set_ylabel('Scaled amplitude [arbitrary units]')
ax.set_title('Highly Correlated Channels')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot7 = gwplot.save_figure(
fig,
'%s_CLUSTER-%s.png' % (re_delim.sub(
'_', str(currentchan)).replace('_', '-', 1), gpsstub),
bbox_inches='tight')
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info(
"Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(currentchan)))
sys.stdout.flush()
return (plot7, plot7_list)
def main(args=None):
"""Run the lasso command-line interface
"""
# declare global variables
# this is needed for multiprocessing utilities
global auxdata, cluster_threshold, cmap, colors, counter, gpsstub
global line_size_aux, line_size_primary, max_correlated_channels
global nonzerocoef, nonzerodata, p1, primary, primary_mean, primary_std
global primaryts, range_is_primary, re_delim, start, target, times
global threshold, trend_type, xlim
parser = create_parser()
args = parser.parse_args(args=args)
# get run params
start = int(args.gpsstart)
end = int(args.gpsend)
pad = args.filter_padding
# set pertinent global variables
cluster_threshold = args.cluster_coefficient
line_size_aux = args.line_size_aux
line_size_primary = args.line_size_primary
threshold = args.threshold
trend_type = args.trend_type
# let's go
LOGGER.info('{} Lasso correlations {}-{}'.format(args.ifo, start, end))
# get primary channel frametype
primary = args.primary_channel.format(ifo=args.ifo)
range_is_primary = 'EFFECTIVE_RANGE_MPC' in args.primary_channel
if args.primary_cache is not None:
LOGGER.info("Using custom primary cache file")
elif args.primary_frametype is None:
try:
args.primary_frametype = DEFAULT_FRAMETYPE[
args.primary_channel.split(':')[1]].format(ifo=args.ifo)
except KeyError as exc:
raise type(exc)("Could not determine primary channel's frametype, "
"please specify with --primary-frametype")
# create output directory
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
os.chdir(args.output_dir)
# multiprocessing for plots
nprocplot = (args.nproc_plot or args.nproc) if USETEX else 1
# bandpass primary
if args.band_pass:
try:
flower, fupper = args.band_pass
except TypeError:
flower, fupper = None
LOGGER.info("-- Loading primary channel data")
bandts = get_data(primary,
start - pad,
end + pad,
verbose='Reading primary:'.rjust(30),
frametype=args.primary_frametype,
source=args.primary_cache,
nproc=args.nproc)
if flower < 0 or fupper >= float((bandts.sample_rate / 2.).value):
raise ValueError(
"bandpass frequency is out of range for this "
"channel, band (Hz): {0}, sample rate: {1}".format(
args.band_pass, bandts.sample_rate))
# get darm BLRMS
LOGGER.debug("-- Filtering data")
if trend_type == 'minute':
stride = 60
else:
stride = 1
if flower:
darmbl = (bandts.highpass(
flower / 2., fstop=flower / 4., filtfilt=False,
ftype='butter').notch(60, filtfilt=False).bandpass(
flower,
fupper,
fstop=[flower / 2., fupper * 1.5],
filtfilt=False,
ftype='butter').crop(start, end))
darmblrms = darmbl.rms(stride)
darmblrms.name = '%s %s-%s Hz BLRMS' % (primary, flower, fupper)
else:
darmbl = bandts.notch(60).crop(start, end)
darmblrms = darmbl.rms(stride)
darmblrms.name = '%s RMS' % primary
primaryts = darmblrms
bandts_asd = bandts.asd(4, 2, method='median')
darmbl_asd = darmbl.asd(4, 2, method='median')
spectrum_plots = gwplot.make_spectrum_plots(start, end, flower, fupper,
args.primary_channel,
bandts_asd, darmbl_asd)
spectrum_plot_zoomed_out = spectrum_plots[0]
spectrum_plot_zoomed_in = spectrum_plots[1]
else:
# load primary channel data
LOGGER.info("-- Loading primary channel data")
primaryts = get_data(primary,
start,
end,
frametype=args.primary_frametype,
source=args.primary_cache,
verbose='Reading:'.rjust(30),
nproc=args.nproc).crop(start, end)
if args.remove_outliers:
LOGGER.debug("-- Removing outliers above %f sigma" %
args.remove_outliers)
gwlasso.remove_outliers(primaryts, args.remove_outliers)
elif args.remove_outliers_pf:
LOGGER.debug("-- Removing outliers in the bottom {} percent "
"of data".format(args.remove_outliers_pf))
gwlasso.remove_outliers(primaryts,
args.remove_outliers_pf,
method='pf')
start = int(primaryts.span()[0])
end = int(primaryts.span()[1])
primary_mean = numpy.mean(primaryts.value)
primary_std = numpy.std(primaryts.value)
# get aux data
LOGGER.info("-- Loading auxiliary channel data")
if args.channel_file is None:
host, port = io_nds2.host_resolution_order(args.ifo)[0]
channels = ChannelList.query_nds2('*.mean',
host=host,
port=port,
type='m-trend')
else:
with open(args.channel_file, 'r') as f:
channels = [name.rstrip('\n') for name in f]
nchan = len(channels)
LOGGER.debug("Identified %d channels" % nchan)
if trend_type == 'minute':
frametype = '%s_M' % args.ifo # for minute trends
else:
frametype = '%s_T' % args.ifo # for second trends
# read aux channels
auxdata = get_data(channels,
start,
end,
verbose='Reading:'.rjust(30),
frametype=frametype,
nproc=args.nproc,
pad=0).crop(start, end)
# -- removes flat data to be re-introdused later
LOGGER.info('-- Pre-processing auxiliary channel data')
auxdata = gwlasso.remove_flat(auxdata)
flatable = Table(data=(list(set(channels) - set(auxdata.keys())), ),
names=('Channels', ))
LOGGER.debug('Removed {0} channels with flat data'.format(len(flatable)))
LOGGER.debug('{0} channels remaining'.format(len(auxdata)))
# -- remove bad data
LOGGER.info("Removing any channels with bad data...")
nbefore = len(auxdata)
auxdata = gwlasso.remove_bad(auxdata)
nafter = len(auxdata)
LOGGER.debug('Removed {0} channels with bad data'.format(nbefore - nafter))
LOGGER.debug('{0} channels remaining'.format(nafter))
data = numpy.array([scale(ts.value) for ts in auxdata.values()]).T
# -- perform lasso regression -------------------
# create model
LOGGER.info('-- Fitting data to target')
target = scale(primaryts.value)
model = gwlasso.fit(data, target, alpha=args.alpha)
LOGGER.info('Alpha: {}'.format(model.alpha))
# restructure results for convenience
allresults = Table(data=(list(auxdata.keys()), model.coef_,
numpy.abs(model.coef_)),
names=('Channel', 'Lasso coefficient', 'rank'))
allresults.sort('rank')
allresults.reverse()
useful = allresults['rank'] > 0
allresults.remove_column('rank')
results = allresults[useful] # non-zero coefficient
zeroed = allresults[numpy.invert(useful)] # zero coefficient
# extract data for useful channels
nonzerodata = {name: auxdata[name] for name in results['Channel']}
nonzerocoef = {name: coeff for name, coeff in results.as_array()}
# print results
LOGGER.info('Found {} channels with |Lasso coefficient| >= {}:\n\n'.format(
len(results), threshold))
print(results)
print('\n\n')
# convert to pandas
set_option('max_colwidth', -1)
df = results.to_pandas()
df.index += 1
# write results to files
gpsstub = '%d-%d' % (start, end - start)
resultsfile = '%s-LASSO_RESULTS-%s.csv' % (args.ifo, gpsstub)
results.write(resultsfile, format='csv', overwrite=True)
zerofile = '%s-ZERO_COEFFICIENT_CHANNELS-%s.csv' % (args.ifo, gpsstub)
zeroed.write(zerofile, format='csv', overwrite=True)
flatfile = '%s-FLAT_CHANNELS-%s.csv' % (args.ifo, gpsstub)
flatable.write(flatfile, format='csv', overwrite=True)
# -- generate lasso plots
modelFit = model.predict(data)
re_delim = re.compile(r'[:_-]')
p1 = (.1, .15, .9, .9) # global plot defaults for plot1, lasso model
times = primaryts.times.value
xlim = primaryts.span
cmap = get_cmap('tab20')
colors = [cmap(i) for i in numpy.linspace(0, 1, len(nonzerodata) + 1)]
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax.plot(times,
_descaler(modelFit),
label='Lasso model',
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
ax.set_title('Lasso Model of Range')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Lasso Model of Primary Channel')
ax.legend(loc='best')
plot1 = gwplot.save_figure(plot,
'%s-LASSO_MODEL-%s.png' % (args.ifo, gpsstub),
bbox_inches='tight')
# summed contributions
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
summed = 0
for i, name in enumerate(results['Channel']):
summed += scale(nonzerodata[name].value) * nonzerocoef[name]
if i:
label = 'Channels 1-{0}'.format(i + 1)
else:
label = 'Channel 1'
ax.plot(times,
_descaler(summed),
label=label,
color=colors[i],
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Summations of Channel Contributions to Model')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot2 = gwplot.save_figure(plot,
'%s-LASSO_CHANNEL_SUMMATION-%s.png' %
(args.ifo, gpsstub),
bbox_inches='tight')
# individual contributions
plot = Plot(figsize=(12, 4))
plot.subplots_adjust(*p1)
ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
for i, name in enumerate(results['Channel']):
this = _descaler(scale(nonzerodata[name].value) * nonzerocoef[name])
if i:
label = 'Channels 1-{0}'.format(i + 1)
else:
label = 'Channel 1'
ax.plot(times,
this,
label=texify(name),
color=colors[i],
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.set_title('Individual Channel Contributions to Model')
ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5))
plot3 = gwplot.save_figure(plot,
'%s-LASSO_CHANNEL_CONTRIBUTIONS-%s.png' %
(args.ifo, gpsstub),
bbox_inches='tight')
# -- process aux channels, making plots
LOGGER.info("-- Processing channels")
counter = multiprocessing.Value('i', 0)
# process channels
pool = multiprocessing.Pool(nprocplot)
results = pool.map(_process_channel, enumerate(list(nonzerodata.items())))
results = sorted(results, key=lambda x: abs(x[1]), reverse=True)
# generate clustered time series plots
counter = multiprocessing.Value('i', 0)
max_correlated_channels = 20
if args.no_cluster is False:
LOGGER.info("-- Generating clusters")
pool = multiprocessing.Pool(nprocplot)
clusters = pool.map(_generate_cluster, enumerate(results))
channelsfile = '%s-CHANNELS-%s.csv' % (args.ifo, gpsstub)
numpy.savetxt(channelsfile, channels, delimiter=',', fmt='%s')
# write html
trange = '%d-%d' % (start, end)
title = '%s Lasso Correlation: %s' % (args.ifo, trange)
if args.band_pass:
links = [trange
] + [(s, '#%s' % s.lower())
for s in ['Parameters', 'Spectra', 'Model', 'Results']]
else:
links = [trange] + [(s, '#%s' % s.lower())
for s in ['Parameters', 'Model', 'Results']]
(brand, class_) = htmlio.get_brand(args.ifo, 'Lasso', start)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(title='%s Lasso | %s' %
(args.ifo, trange),
navbar=navbar)
page.h1(title, class_='pb-2 mt-3 mb-2 border-bottom')
# -- summary table
content = [
('Primary channel', markup.oneliner.code(primary)),
('Primary frametype', markup.oneliner.code(args.primary_frametype)
or '-'),
('Primary cache file', markup.oneliner.code(args.primary_cache)
or '-'), ('Outlier threshold', '%s sigma' % args.remove_outliers),
('Lasso coefficient threshold', str(threshold)),
('Cluster coefficient threshold', str(args.cluster_coefficient)),
('Non-zero coefficients', str(numpy.count_nonzero(model.coef_))),
('α (model)', '%.4f' % model.alpha)
]
if args.band_pass:
content.insert(
2, ('Primary bandpass', '{0}-{1} Hz'.format(flower, fupper)))
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(content, start=start, end=end))
page.div.close() # col-md-9 col-sm-12
# -- download button
files = [('%s analyzed channels (CSV)' % nchan, channelsfile),
('%s flat channels (CSV)' % len(flatable), flatfile),
('%s zeroed channels (CSV)' % len(zeroed), zerofile)]
page.div(class_='col-md-3 col-sm-12')
page.add(
htmlio.download_btn(files,
label='Channel information',
btnclass='btn btn-%s dropdown-toggle' %
args.ifo.lower()))
page.div.close() # col-md-3 col-sm-12
page.div.close() # rowa
# -- command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
if args.band_pass:
page.h2('Primary channel spectra', class_='mt-4', id_='spectra')
page.div(class_='card border-light card-body shadow-sm')
page.div(class_='row')
page.div(class_='col-md-6')
spectra_img1 = htmlio.FancyPlot(spectrum_plot_zoomed_out)
page.add(htmlio.fancybox_img(spectra_img1))
page.div.close() # col-md-6
page.div(class_='col-md-6')
spectra_img2 = htmlio.FancyPlot(spectrum_plot_zoomed_in)
page.add(htmlio.fancybox_img(spectra_img2))
page.div.close() # col-md-6
page.div.close() # row
page.div.close() # card border-light card-body shadow-sm
# -- model information
page.h2('Model information', class_='mt-4', id_='model')
page.div(class_='card card-%s card-body shadow-sm' % args.ifo.lower())
page.div(class_='row')
page.div(class_='col-md-8 offset-md-2', id_='results-table')
page.p('Below are the top {} mean minute-trend channels, ranked by '
'Lasso correlation with the primary.'.format(df.shape[0]))
page.add(
df.to_html(classes=('table', 'table-sm', 'table-hover'),
formatters={
'Lasso coefficient': lambda x: "%.4f" % x,
'Channel':
lambda x: str(htmlio.cis_link(x.split('.')[0])),
'__index__': lambda x: str(x)
},
escape=False,
border=0).replace(' style="text-align: right;"', ''))
page.div.close() # col-md-10 offset-md-1
page.div.close() # row
page.div(class_='row', id_='primary-lasso')
page.div(class_='col-md-8 offset-md-2')
img1 = htmlio.FancyPlot(plot1)
page.add(htmlio.fancybox_img(img1)) # primary lasso plot
page.div.close() # col-md-8 offset-md-2
page.div.close() # primary-lasso
page.div(class_='row', id_='channel-summation')
img2 = htmlio.FancyPlot(plot2)
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img2))
page.div.close() # col-md-8 offset-md-2
page.div.close() # channel-summation
page.div(class_='row', id_='channels-and-primary')
img3 = htmlio.FancyPlot(plot3)
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img3))
page.div.close() # col-md-8 offset-md-2
page.div.close() # channels-and-primary
page.div.close() # card card-<ifo> card-body shadow-sm
# -- results
page.h2('Top channels', class_='mt-4', id_='results')
page.div(id_='results')
# for each aux channel, create information container and put plots in it
for i, (ch, lassocoef, plot4, plot5, plot6, ts) in enumerate(results):
# set container color/context based on lasso coefficient
if lassocoef == 0:
break
elif abs(lassocoef) < threshold:
h = '%s [lasso coefficient = %.4f] (Below threshold)' % (ch,
lassocoef)
else:
h = '%s [lasso coefficient = %.4f]' % (ch, lassocoef)
if ((lassocoef is None) or (lassocoef == 0)
or (abs(lassocoef) < threshold)):
card = 'card border-light mb-1 shadow-sm'
card_header = 'card-header bg-light'
elif abs(lassocoef) >= .5:
card = 'card border-danger mb-1 shadow-sm'
card_header = 'card-header text-white bg-danger'
elif abs(lassocoef) >= .2:
card = 'card border-warning mb-1 shadow-sm'
card_header = 'card-header text-white bg-warning'
else:
card = 'card border-info mb-1 shadow-sm'
card_header = 'card-header text-white bg-info'
page.div(class_=card)
# heading
page.div(class_=card_header)
page.a(h,
class_='collapsed card-link cis-link',
href='#channel%d' % i,
**{'data-toggle': 'collapse'})
page.div.close() # card-header
# body
page.div(id_='channel%d' % i,
class_='collapse',
**{'data-parent': '#results'})
page.div(class_='card-body')
if lassocoef is None:
page.p('The amplitude data for this channel is flat (does not '
'change) within the chosen time period.')
elif abs(lassocoef) < threshold:
page.p('Lasso coefficient below the threshold of %g.' %
(threshold))
else:
for image in [plot4, plot5, plot6]:
img = htmlio.FancyPlot(image)
page.div(class_='row')
page.div(class_='col-md-8 offset-md-2')
page.add(htmlio.fancybox_img(img))
page.div.close() # col-md-8 offset-md-2
page.div.close() # row
page.add('<hr class="row-divider">')
if args.no_cluster is False:
if clusters[i][0] is None:
page.p("<font size='3'><br />No channels were highly "
"correlated with this channel.</font>")
else:
page.div(class_='row', id_='clusters')
page.div(class_='col-md-12')
cimg = htmlio.FancyPlot(clusters[i][0])
page.add(htmlio.fancybox_img(cimg))
page.div.close() # col-md-12
page.div.close() # clusters
if clusters[i][1] is not None:
corr_link = markup.oneliner.a(
'Export {} channels (CSV)'.format(
max_correlated_channels),
href=clusters[i][1],
download=clusters[i][1],
)
page.button(
corr_link,
class_='btn btn-%s' % args.ifo.lower(),
)
page.div.close() # card-body
page.div.close() # collapse
page.div.close() # card
page.div.close() # results
htmlio.close_page(page, 'index.html') # save and close
LOGGER.info("-- Process Completed")
def _process_channel(input_):
"""Handle individual channels for multiprocessing
"""
if USETEX:
gwplot.configure_mpl_tex()
p4 = (.1, .1, .9, .95)
chan = input_[1][0]
ts = input_[1][1]
lassocoef = nonzerocoef[chan]
zeroed = lassocoef == 0
if zeroed:
plot4 = None
plot5 = None
plot6 = None
pcorr = None
else:
plot4 = None
plot5 = None
plot6 = None
if trend_type == 'minute':
pcorr = numpy.corrcoef(ts.value, primaryts.value)[0, 1]
else:
pcorr = 0.0
if abs(lassocoef) < threshold:
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info(
"Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(chan)))
sys.stdout.flush()
return (chan, lassocoef, plot4, plot5, plot6, ts)
# create time series subplots
fig = Plot(figsize=(12, 8))
fig.subplots_adjust(*p4)
ax1 = fig.add_subplot(2, 1, 1, xscale='auto-gps', epoch=start)
ax1.plot(primaryts,
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax1.set_xlabel(None)
ax2 = fig.add_subplot(2, 1, 2, sharex=ax1, xlim=xlim)
ax2.plot(ts, label=texify(chan), linewidth=line_size_aux)
if range_is_primary:
ax1.set_ylabel('Sensitive range [Mpc]')
else:
ax1.set_ylabel('Primary channel units')
ax2.set_ylabel('Channel units')
for ax in fig.axes:
ax.legend(loc='best')
channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1)
plot4 = gwplot.save_figure(fig,
'%s_TRENDS-%s.png' % (channelstub, gpsstub),
bbox_inches='tight')
# create scaled, sign-corrected, and overlayed timeseries
tsscaled = scale(ts.value)
if lassocoef < 0:
tsscaled = numpy.negative(tsscaled)
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p1)
ax = fig.gca(xscale='auto-gps', epoch=start, xlim=xlim)
ax.plot(times,
_descaler(target),
label=texify(primary),
color='black',
linewidth=line_size_primary)
ax.plot(times,
_descaler(tsscaled),
label=texify(chan),
linewidth=line_size_aux)
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary Channel Units')
ax.legend(loc='best')
plot5 = gwplot.save_figure(fig,
'%s_COMPARISON-%s.png' %
(channelstub, gpsstub),
bbox_inches='tight')
# scatter plot
tsCopy = ts.value.reshape(-1, 1)
primarytsCopy = primaryts.value.reshape(-1, 1)
primaryReg = linear_model.LinearRegression()
primaryReg.fit(tsCopy, primarytsCopy)
primaryFit = primaryReg.predict(tsCopy)
fig = Plot(figsize=(12, 4))
fig.subplots_adjust(*p1)
ax = fig.gca()
ax.set_xlabel(texify(chan) + ' [Channel units]')
if range_is_primary:
ax.set_ylabel('Sensitive range [Mpc]')
else:
ax.set_ylabel('Primary channel units')
y_min = min(primaryts.value)
y_max = max(primaryts.value)
y_range = y_max - y_min
ax.set_ylim(y_min - (y_range * 0.1), y_max + (y_range * 0.1))
ax.text(.9,
.1,
'r = ' + str('{0:.2}'.format(pcorr)),
verticalalignment='bottom',
horizontalalignment='right',
transform=ax.transAxes,
color='black',
size=20,
bbox=dict(boxstyle='square',
facecolor='white',
alpha=.75,
edgecolor='black'))
ax.scatter(ts.value, primaryts.value, color='red')
ax.plot(ts.value, primaryFit, color='black')
ax.autoscale_view(tight=False, scalex=True, scaley=True)
plot6 = gwplot.save_figure(fig,
'%s_SCATTER-%s.png' %
(channelstub, gpsstub),
bbox_inches='tight')
# increment counter and print status
with counter.get_lock():
counter.value += 1
pc = 100 * counter.value / len(nonzerodata)
LOGGER.info("Completed [%d/%d] %3d%% %-50s" %
(counter.value, len(nonzerodata), pc, '(%s)' % str(chan)))
sys.stdout.flush()
return (chan, lassocoef, plot4, plot5, plot6, ts)
def main(args=None):
"""Run the primary scattering command-line tool
"""
parser = create_parser()
args = parser.parse_args(args=args)
# set up logger
logger = cli.logger(
name=PROG.split('python -m ').pop(),
level='DEBUG' if args.verbose else 'INFO',
)
# useful variables
fthresh = (
int(args.frequency_threshold) if args.frequency_threshold.is_integer()
else args.frequency_threshold)
multiplier = args.multiplier_for_threshold
tstr = str(fthresh).replace('.', '_')
gpsstr = '%s-%s' % (int(args.gpsstart), int(args.gpsend - args.gpsstart))
args.optic = args.optic or list(OPTIC_MOTION_CHANNELS.keys())
# go to working directory
indir = os.getcwd()
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
os.chdir(args.output_dir)
# set up output files
summfile = '{}-SCATTERING_SUMMARY-{}.csv'.format(
args.ifo, gpsstr)
segfile = '{}-SCATTERING_SEGMENTS_{}_HZ-{}.h5'.format(
args.ifo, tstr, gpsstr)
# log start of process
logger.info('{} Scattering {}-{}'.format(
args.ifo, int(args.gpsstart), int(args.gpsend)))
# -- get state segments -----------
span = Segment(args.gpsstart, args.gpsend)
# get segments
if args.state_flag is not None:
state = DataQualityFlag.query(
args.state_flag, int(args.gpsstart), int(args.gpsend),
url=DEFAULT_SEGMENT_SERVER,
).coalesce()
statea = []
padding = args.segment_start_pad + args.segment_end_pad
for i, seg in enumerate(state.active):
if abs(seg) > padding:
statea.append(Segment(
seg[0] + args.segment_start_pad,
seg[1] - args.segment_end_pad,
))
else:
logger.debug(
"Segment length {} shorter than padding length {}, "
"skipping segment {}-{}".format(abs(seg), padding, *seg),
)
statea = SegmentList(statea)
logger.debug("Downloaded %d segments for %s"
% (len(statea), args.state_flag))
else:
statea = SegmentList([span])
livetime = float(abs(statea))
logger.debug("Processing %.2f s of livetime" % livetime)
# -- load h(t) --------------------
args.main_channel = args.main_channel.format(IFO=args.ifo)
logger.debug("Loading Omicron triggers for %s" % args.main_channel)
if args.gpsstart >= 1230336018: # Jan 1 2019
ext = "h5"
names = ["time", "frequency", "snr"]
read_kw = {
"columns": names,
"selection": [
"{0} < frequency < {1}".format(
args.fmin, multiplier * fthresh),
("time", in_segmentlist, statea),
],
"format": "hdf5",
"path": "triggers",
}
else:
ext = "xml.gz"
names = ['peak', 'peak_frequency', 'snr']
read_kw = {
"columns": names,
"selection": [
"{0} < peak_frequency < {1}".format(
args.fmin, multiplier * fthresh),
('peak', in_segmentlist, statea),
],
"format": 'ligolw',
"tablename": "sngl_burst",
}
fullcache = []
for seg in statea:
cache = gwtrigfind.find_trigger_files(
args.main_channel, 'omicron', seg[0], seg[1], ext=ext,
)
if len(cache) == 0:
warnings.warn(
"No Omicron triggers found for %s in segment [%d .. %d)"
% (args.main_channel, seg[0], seg[1]),
)
continue
fullcache.extend(cache)
# read triggers
if fullcache:
trigs = EventTable.read(fullcache, nproc=args.nproc, **read_kw)
else: # no files (no livetime?)
trigs = EventTable(names=names)
highsnrtrigs = trigs[trigs['snr'] >= 8]
logger.debug("%d read" % len(trigs))
# -- prepare HTML -----------------
links = [
'%d-%d' % (int(args.gpsstart), int(args.gpsend)),
('Parameters', '#parameters'),
('Segments', (
('State flag', '#state-flag'),
('Optical sensors', '#osems'),
('Transmons', '#transmons'),
)),
]
if args.omega_scans:
links.append(('Scans', '#omega-scans'))
(brand, class_) = htmlio.get_brand(args.ifo, 'Scattering', args.gpsstart)
navbar = htmlio.navbar(links, class_=class_, brand=brand)
page = htmlio.new_bootstrap_page(
title='%s Scattering | %d-%d' % (
args.ifo, int(args.gpsstart), int(args.gpsend)),
navbar=navbar)
page.div(class_='pb-2 mt-3 mb-2 border-bottom')
page.h1('%s Scattering: %d-%d'
% (args.ifo, int(args.gpsstart), int(args.gpsend)))
page.div.close() # pb-2 mt-3 mb-2 border-bottom
page.h2('Parameters', class_='mt-4 mb-4', id_='parameters')
page.div(class_='row')
page.div(class_='col-md-9 col-sm-12')
page.add(htmlio.parameter_table(
start=int(args.gpsstart), end=int(args.gpsend), flag=args.state_flag))
page.div.close() # col-md-9 col-sm-12
# link to summary files
page.div(class_='col-md-3 col-sm-12')
page.add(htmlio.download_btn(
[('Segments (HDF)', segfile),
('Triggers (CSV)', summfile)],
btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(),
))
page.div.close() # col-md-3 col-sm-12
page.div.close() # row
# command-line
page.h5('Command-line:')
page.add(htmlio.get_command_line(about=False, prog=PROG))
# section header
page.h2('Segments', class_='mt-4', id_='segments')
if statea: # contextual information
paper = markup.oneliner.a(
'Accadia et al. (2010)', target='_blank', class_='alert-link',
href='http://iopscience.iop.org/article/10.1088/0264-9381/27'
'/19/194011')
msg = (
"Segments marked \"optical sensors\" below show evidence of beam "
"scattering between {0} and {1} Hz based on the velocity of optic "
"motion, with fringe frequencies projected using equation (3) of "
"{2}. Segments marked \"transmons\" are based on whitened, "
"band-limited RMS trends of transmon sensors. In both cases, "
"yellow panels denote weak evidence for scattering, while red "
"panels denote strong evidence."
).format(args.fmin, multiplier * fthresh, str(paper))
page.add(htmlio.alert(msg, context=args.ifo.lower()))
else: # null segments
page.add(htmlio.alert('No active analysis segments were found',
context='warning', dismiss=False))
# record state segments
if args.state_flag is not None:
page.h3('State flag', class_='mt-3', id_='state-flag')
page.div(id_='accordion1')
page.add(htmlio.write_flag_html(
state, span, 'state', parent='accordion1', context='success',
plotdir='', facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen',
known={'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4}))
page.div.close()
# -- find scattering evidence -----
# read data for OSEMs and transmons
osems = ['%s:%s' % (args.ifo, c) for optic in args.optic for
c in OPTIC_MOTION_CHANNELS[optic]]
transmons = ['%s:%s' % (args.ifo, c) for c in TRANSMON_CHANNELS]
allchannels = osems + transmons
logger.info("Reading all timeseries data")
alldata = []
n = len(statea)
for i, seg in enumerate(statea):
msg = "{0}/{1} {2}:".rjust(30).format(
str(i + 1).rjust(len(str(n))),
n,
str(seg),
) if args.verbose else False
alldata.append(
get_data(allchannels, seg[0], seg[1],
frametype=args.frametype.format(IFO=args.ifo),
verbose=msg, nproc=args.nproc).resample(128))
try: # ensure that only available channels are analyzed
osems = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(osems))
transmons = list(
set(alldata[0].keys()) & set(alldata[-1].keys()) & set(transmons))
except IndexError:
osems = []
transmons = []
# initialize scattering segments
scatter_segments = DataQualityDict()
actives = SegmentList()
# scattering based on OSEM velocity
if statea:
page.h3('Optical sensors (OSEMs)', class_='mt-3', id_='osems')
page.div(id_='osems-group')
logger.info('Searching for scatter based on OSEM velocity')
for i, channel in enumerate(sorted(osems)):
logger.info("-- Processing %s --" % channel)
chanstr = re.sub('[:-]', '_', channel).replace('_', '-', 1)
optic = channel.split('-')[1].split('_')[0]
flag = '%s:DCH-%s_SCATTERING_GE_%s_HZ:1' % (args.ifo, optic, tstr)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering above {0} Hz from {1} in "
"{2}".format(fthresh, optic, channel),
)
# set up plot(s)
plot = Plot(figsize=[12, 12])
axes = {}
axes['position'] = plot.add_subplot(
411, xscale='auto-gps', xlabel='')
axes['fringef'] = plot.add_subplot(
412, sharex=axes['position'], xlabel='')
axes['triggers'] = plot.add_subplot(
413, sharex=axes['position'], xlabel='')
axes['segments'] = plot.add_subplot(
414, projection='segments', sharex=axes['position'])
plot.subplots_adjust(bottom=.07, top=.95)
fringecolors = [None] * len(FREQUENCY_MULTIPLIERS)
histdata = dict((x, numpy.ndarray((0,))) for
x in FREQUENCY_MULTIPLIERS)
linecolor = None
# loop over state segments and find scattering fringes
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
ts = alldata[j][channel]
# get raw data and plot
line = axes['position'].plot(ts, color=linecolor)[0]
linecolor = line.get_color()
# get fringe frequency and plot
fringef = get_fringe_frequency(ts, multiplier=1)
for k, m in list(enumerate(FREQUENCY_MULTIPLIERS))[::-1]:
fm = fringef * m
line = axes['fringef'].plot(
fm, color=fringecolors[k],
label=(j == 0 and r'$f\times%d$' % m or None))[0]
fringecolors[k] = line.get_color()
histdata[m] = numpy.resize(
histdata[m], (histdata[m].size + fm.size,))
histdata[m][-fm.size:] = fm.value
# get segments and plot
scatter = get_segments(
fringef * multiplier,
fthresh,
name=flag,
pad=args.segment_padding
)
axes['segments'].plot(
scatter, facecolor='red', edgecolor='darkred',
known={'alpha': 0.6, 'facecolor': 'lightgray',
'edgecolor': 'gray', 'height': 0.4},
height=0.8, y=0, label=' ',
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# finalize plot
logger.debug("Plotting")
name = texify(channel)
axes['position'].set_title("Scattering evidence in %s" % name)
axes['position'].set_xlabel('')
axes['position'].set_ylabel(r'Position [$\mu$m]')
axes['position'].text(
0.01, 0.95, 'Optic position',
transform=axes['position'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['fringef'].plot(
span, [fthresh, fthresh], 'k--')
axes['fringef'].set_xlabel('')
axes['fringef'].set_ylabel(r'Frequency [Hz]')
axes['fringef'].yaxis.tick_right()
axes['fringef'].yaxis.set_label_position("right")
axes['fringef'].set_ylim(0, multiplier * fthresh)
axes['fringef'].text(
0.01, 0.95, 'Calculated fringe frequency',
transform=axes['fringef'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
handles, labels = axes['fringef'].get_legend_handles_labels()
axes['fringef'].legend(handles[::-1], labels[::-1], loc='upper right',
borderaxespad=0, bbox_to_anchor=(-0.01, 1.),
handlelength=1)
axes['triggers'].scatter(
trigs[names[0]],
trigs[names[1]],
c=trigs[names[2]],
edgecolor='none',
)
name = texify(args.main_channel)
axes['triggers'].text(
0.01, 0.95,
'%s event triggers (Omicron)' % name,
transform=axes['triggers'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
axes['triggers'].set_ylabel('Frequency [Hz]')
axes['triggers'].set_ylim(args.fmin, multiplier * fthresh)
axes['triggers'].colorbar(cmap='YlGnBu', clim=(3, 100), norm='log',
label='Signal-to-noise ratio')
axes['segments'].set_ylim(-.55, .55)
axes['segments'].text(
0.01, 0.95,
r'Time segments with $f\times%d > %.2f$ Hz' % (
multiplier, fthresh),
transform=axes['segments'].transAxes, va='top', ha='left',
bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5})
for ax in axes.values():
ax.set_epoch(int(args.gpsstart))
ax.set_xlim(*span)
png = '%s_SCATTERING_%s_HZ-%s.png' % (chanstr, tstr, gpsstr)
try:
plot.save(png)
except OverflowError as e:
warnings.warn(str(e))
plot.axes[1].set_ylim(0, multiplier * fthresh)
plot.refresh()
plot.save(png)
plot.close()
logger.debug("%s written." % png)
# make histogram
histogram = Plot(figsize=[12, 6])
ax = histogram.gca()
hrange = (0, multiplier * fthresh)
for m, color in list(zip(histdata, fringecolors))[::-1]:
if histdata[m].size:
ax.hist(
histdata[m], facecolor=color, alpha=.6, range=hrange,
bins=50, histtype='stepfilled', label=r'$f\times%d$' % m,
cumulative=-1, weights=ts.dx.value, bottom=1e-100,
log=True)
else:
ax.plot(histdata[m], color=color, label=r'$f\times%d$' % m)
ax.set_yscale('log')
ax.set_ylim(.01, float(livetime))
ax.set_ylabel('Time with fringe above frequency [s]')
ax.set_xlim(*hrange)
ax.set_xlabel('Frequency [Hz]')
ax.set_title(axes['position'].get_title())
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles[::-1], labels[::-1], loc='upper right')
hpng = '%s_SCATTERING_HISTOGRAM-%s.png' % (chanstr, gpsstr)
histogram.save(hpng)
histogram.close()
logger.debug("%s written." % hpng)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif ((deadtime != 0 and effdt < 2) or
(histdata[multiplier].size and
histdata[multiplier].max() >=
fthresh/2.)):
context = 'warning'
else:
continue
page.div(class_='card border-%s mb-1 shadow-sm' % context)
page.div(class_='card-header text-white bg-%s' % context)
page.a(channel, class_='collapsed card-link cis-link',
href='#osem%s' % i, **{'data-toggle': 'collapse'})
page.div.close() # card-header
page.div(id_='osem%s' % i, class_='collapse',
**{'data-parent': '#osems-group'})
page.div(class_='card-body')
page.div(class_='row')
img = htmlio.FancyPlot(
png, caption=SCATTER_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(img))
page.div.close() # col-md-10 offset-md-1
himg = htmlio.FancyPlot(
hpng, caption=HIST_CAPTION.format(CHANNEL=channel))
page.div(class_='col-md-10 offset-md-1')
page.add(htmlio.fancybox_img(himg))
page.div.close() # col-md-10 offset-md-1
page.div.close() # row
segs = StringIO()
if deadtime:
page.p("%d segments were found predicting a scattering fringe "
"above %.2f Hz." % (
len(scatter_segments[channel].active),
fthresh))
page.table(class_='table table-sm table-hover')
page.tbody()
page.tr()
page.th('Deadtime')
page.td('%.2f/%d seconds' % (deadtime, livetime))
page.td('%.2f%%' % deadtimepc)
page.tr.close()
page.tr()
page.th('Efficiency<br><small>(SNR≥8 and '
'%.2f Hz</sub><f<sub>peak</sub><%.2f Hz)</small>'
% (args.fmin, multiplier * fthresh))
page.td('%d/%d events' % (efficiency, len(highsnrtrigs)))
page.td('%.2f%%' % efficiencypc)
page.tr.close()
page.tr()
page.th('Efficiency/Deadtime')
page.td()
page.td('%.2f' % effdt)
page.tr.close()
page.tbody.close()
page.table.close()
scatter_segments[channel].active.write(segs, format='segwizard',
coltype=float)
page.pre(segs.getvalue())
else:
page.p("No segments were found with scattering above %.2f Hz."
% fthresh)
page.div.close() # card-body
page.div.close() # collapse
page.div.close() # card
if statea: # close accordion
page.div.close() # osems-group
# scattering based on transmon BLRMS
if statea:
page.h3('Transmons', class_='mt-3', id_='transmons')
page.div(id_='transmons-group')
logger.info('Searching for scatter based on band-limited RMS of transmons')
for i, channel in enumerate(sorted(transmons)):
logger.info("-- Processing %s --" % channel)
optic = channel.split('-')[1][:6]
flag = '%s:DCH-%s_SCATTERING_BLRMS:1' % (args.ifo, optic)
scatter_segments[channel] = DataQualityFlag(
flag,
isgood=False,
description="Evidence for scattering from whitened, band-limited "
"RMS trends of {0}".format(channel),
)
# loop over state segments and compute BLRMS
for j, seg in enumerate(statea):
logger.debug("Processing segment [%d .. %d)" % seg)
wblrms = get_blrms(
alldata[j][channel],
flow=args.bandpass_flow,
fhigh=args.bandpass_fhigh,
)
scatter = get_segments(
wblrms,
numpy.mean(wblrms) + args.sigma * numpy.std(wblrms),
name=flag,
)
scatter_segments[channel] += scatter
logger.debug(
" Found %d scattering segments" % (len(scatter.active)))
logger.debug("Completed channel %s, found %d segments in total"
% (channel, len(scatter_segments[channel].active)))
# calculate efficiency and deadtime of veto
deadtime = abs(scatter_segments[channel].active)
try:
deadtimepc = deadtime / livetime * 100
except ZeroDivisionError:
deadtimepc = 0.
logger.info("Deadtime: %.2f%% (%.2f/%ds)"
% (deadtimepc, deadtime, livetime))
highsnrtrigs = trigs[trigs['snr'] <= 200]
efficiency = in_segmentlist(highsnrtrigs[names[0]],
scatter_segments[channel].active).sum()
try:
efficiencypc = efficiency / len(highsnrtrigs) * 100
except ZeroDivisionError:
efficiencypc = 0.
logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)"
% (efficiencypc, efficiency, len(highsnrtrigs)))
if deadtimepc == 0.:
effdt = 0
else:
effdt = efficiencypc/deadtimepc
logger.info("Efficiency/Deadtime: %.2f" % effdt)
if abs(scatter_segments[channel].active):
actives.extend(scatter_segments[channel].active)
# write HTML
if deadtime != 0 and effdt > 2:
context = 'danger'
elif deadtime != 0 and effdt < 2:
context = 'warning'
else:
continue
page.add(htmlio.write_flag_html(
scatter_segments[channel], span, i, parent='transmons-group',
title=channel, context=context, plotdir=''))
if statea: # close accordion
page.div.close() # transmons-group
actives = actives.coalesce() # merge contiguous segments
if statea and not actives:
page.add(htmlio.alert(
'No evidence of scattering found in the channels analyzed',
context=args.ifo.lower(), dismiss=False))
# identify triggers during active segments
logger.debug('Writing a summary CSV record')
ind = [i for i, trigtime in enumerate(highsnrtrigs[names[0]])
if trigtime in actives]
gps = highsnrtrigs[names[0]][ind]
freq = highsnrtrigs[names[1]][ind]
snr = highsnrtrigs[names[2]][ind]
segs = [y for x in gps for y in actives if x in y]
table = EventTable(
[gps, freq, snr, [seg[0] for seg in segs], [seg[1] for seg in segs]],
names=('trigger_time', 'trigger_frequency', 'trigger_snr',
'segment_start', 'segment_end'))
logger.info('The following {} triggers fell within active scattering '
'segments:\n\n'.format(len(table)))
print(table)
print('\n\n')
table.write(summfile, overwrite=True)
# -- launch omega scans -----------
nscans = min(args.omega_scans, len(table))
if nscans > 0:
# launch scans
scandir = 'scans'
ind = random.sample(range(0, len(table)), nscans)
omegatimes = [str(t) for t in table['trigger_time'][ind]]
logger.debug('Collected {} event times to omega scan: {}'.format(
nscans, ', '.join(omegatimes)))
logger.info('Creating workflow for omega scans')
flags = batch.get_command_line_flags(
ifo=args.ifo, ignore_state_flags=True)
condorcmds = batch.get_condor_arguments(timeout=4, gps=args.gpsstart)
batch.generate_dag(omegatimes, flags=flags, submit=True,
outdir=scandir, condor_commands=condorcmds)
logger.info('Launched {} omega scans to condor'.format(nscans))
# render HTML
page.h2('Omega scans', class_='mt-4', id_='omega-scans')
msg = (
'The following event times correspond to significant Omicron '
'triggers that occur during the scattering segments found above. '
'To compare these against fringe frequency projections, please '
'use the "simple scattering" module:',
markup.oneliner.pre(
'$ python -m gwdetchar.scattering.simple --help',
),
)
page.add(htmlio.alert(msg, context=args.ifo.lower()))
page.add(htmlio.scaffold_omega_scans(
omegatimes, args.main_channel, scandir=scandir))
elif args.omega_scans:
logger.info('No events found during active scattering segments')
# -- finalize ---------------------
# write segments
scatter_segments.write(segfile, path="segments", overwrite=True)
logger.debug("%s written" % segfile)
# write HTML
htmlio.close_page(page, 'index.html')
logger.info("-- index.html written, all done --")
# return to original directory
os.chdir(indir)
