def tic_gauge(self, fig, gridspec):
"""TIC gauge, showing the TIC versus iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "TIC"
if check_data(self.tracking_data, ["tic_diag"]):
plot_args = {
"x": "iteration",
"y": "tic_diag",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
if check_data(self.tracking_data, ["tic_trace"]):
if "ax" in locals():
ax2 = ax.twinx()
else:
ax2 = fig.add_subplot(gridspec)
plot_args = {
"x": "iteration",
"y": "tic_trace",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap_backup,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2_backup,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
create_basic_plot(**plot_args, ax=ax2)
def gradient_tests_gauge(self, fig, gridspec):
"""Gauge, showing the the status of several gradient tests.
All three gradient tests (the norm test, the inner product test, and the
orthogonality test) indicate how strongly individual gradients in a mini-batch
scatter around the mean gradient. This information can be used to adapt the
batch size whenever the information becomes to noisy, as indicated by large
values.
The central plot visualizes all three tests in different colors. Each area shows
how far the individual gradients scatter. The smaller plots show their evolution
over time.
**Preview**
.. image:: ../../_static/instrument_previews/GradientTests.png
:alt: Preview GradientTests Gauge
**Requires**
The gradient test instrument requires data from all three gradient test quantities,
namely the :class:`~cockpit.quantities.InnerTest`,
:class:`~cockpit.quantities.NormTest`, and :class:`~cockpit.quantities.OrthoTest`
quantity classes.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot
title = "Gradient Tests"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "InnerTest", "NormTest", "OrthoTest"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
ax.set_title(title, fontweight="bold", fontsize="large")
ax.set_axis_off()
# Gridspecs (inside gridspec)
gs = gridspec.subgridspec(3, 3, wspace=0.05, hspace=0.1)
ax_all = fig.add_subplot(gs[1:, 1:])
ax_norm = fig.add_subplot(gs[1, 0])
ax_inner = fig.add_subplot(gs[2, 0])
ax_ortho = fig.add_subplot(gs[0, 2])
_format(self, ax_all, ax_norm, ax_inner, ax_ortho)
_plot(self, ax_all, ax_norm, ax_inner, ax_ortho)
def mean_gsnr_gauge(self, fig, gridspec):
"""Mean GSNR gauge, showing the mean GSNR versus iteration.
The mean GSNR describes the average gradient signal-to-noise-ratio. `Recent
work <https://arxiv.org/abs/2001.07384>`_ used this quantity to study the
generalization performances of neural networks, noting "that larger GSNR during
training process leads to better generalization performance. The instrument
shows the mean GSNR versus iteration, overlayed with an exponentially weighted
average.
**Preview**
.. image:: ../../_static/instrument_previews/MeanGSNR.png
:alt: Preview MeanGSNR Gauge
**Requires**
This instrument requires data from the :class:`~cockpit.quantities.MeanGSNR`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Mean GSNR"
# Check if the required data is available, else skip this instrument
requires = ["MeanGSNR"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "MeanGSNR",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def histogram_1d_gauge(self, fig, gridspec, y_scale="log"):
"""One-dimensional histogram of the individual gradient elements.
This instrument provides a histogram of the gradient element values across all
individual gradients in a mini-batch. The histogram shows the distribution for
the last tracked iteration only.
**Preview**
.. image:: ../../_static/instrument_previews/Hist1d.png
:alt: Preview Hist1d Gauge
**Requires**
This two dimensional histogram instrument requires data from the
:class:`~cockpit.quantities.GradHist1d` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
y_scale (str, optional): Scale of the y-axis. Defaults to "log".
"""
# Plot
title = "Gradient Element Histogram"
# Check if the required data is available, else skip this instrument
requires = ["GradHist1d"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
plot_args = {
"title": title,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"xlabel": "Gradient Element Value",
"ylabel": "Frequency",
"y_scale": y_scale,
}
vals, mid_points, width = _get_histogram_data(self.tracking_data)
ax.bar(mid_points, vals, width=width, color=self.primary_color)
_beautify_plot(ax=ax, **plot_args)
ax.set_title(title, fontweight="bold", fontsize="large")
def hyperparameter_gauge(self, fig, gridspec):
"""Hyperparameter gauge, currently showing the learning rate over time.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Hyperparameters"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "learning_rate"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
clean_learning_rate = self.tracking_data[["iteration",
"learning_rate"]].dropna()
# Plot Settings
plot_args = {
"x": "iteration",
"y": "learning_rate",
"data": clean_learning_rate,
}
ylabel = plot_args["y"].replace("_", " ").title()
sns.lineplot(**plot_args,
ax=ax,
label=ylabel,
linewidth=2,
color=self.secondary_color)
_beautify_plot(
ax=ax,
xlabel=plot_args["x"],
ylabel=ylabel,
x_scale="symlog" if self.show_log_iter else "linear",
title=title,
xlim="tight",
fontweight="bold",
facecolor=self.bg_color_instruments2,
)
ax.legend()
_add_last_value_to_legend(ax)
def cabs_gauge(self, fig, gridspec):
"""CABS gauge, showing the CABS rule versus iteration.
If the stopping criterion becomes positive, this suggests stopping the training
according to
- Balles, L., Romero, J., & Hennig, P.,
Coupling adaptive batch sizes with learning rates (2017).
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "CABS"
# Check if the required data is available, else skip this instrument
requires = ["cabs"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "cabs",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def grad_norm_gauge(self, fig, gridspec):
"""Showing the gradient norm versus iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Gradient Norm"
# Check if the required data is available, else skip this instrument
requires = ["grad_norm"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["grad_norm_all"] = self.tracking_data.grad_norm.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "grad_norm_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def histogram_1d_gauge(self, fig, gridspec, y_scale="log"):
"""One-dimensional histogram of the individual gradient elements.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
y_scale (str, optional): Scale of the y-axis. Defaults to "log".
"""
# Plot
title = "Gradient Element Histogram"
# Check if the required data is available, else skip this instrument
requires = ["edges", "hist_1d"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
plot_args = {
"title": title,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"xlabel": "gradient element value",
"ylabel": "frequency",
"y_scale": y_scale,
}
vals, mid_points, width = _get_histogram_data(self.tracking_data)
ax.bar(mid_points, vals, width=width, color=self.primary_color)
_beautify_plot(ax=ax, **plot_args)
ax.set_title(title, fontweight="bold", fontsize="large")
def max_ev_gauge(self, fig, gridspec):
"""Showing the largest eigenvalue of the Hessian versus iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed.
"""
# Plot Trace vs iteration
title = "Max Eigenvalue"
# Check if the required data is available, else skip this instrument
requires = ["max_ev"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "max_ev",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "log",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
# part that should be plotted
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def gradient_tests_gauge(self, fig, gridspec):
"""Gauge, showing the the status of several gradient tests.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot
title = "Gradient Tests"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "inner_product_test", "norm_test", "orthogonality_test"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
ax.set_title(title, fontweight="bold", fontsize="large")
ax.set_axis_off()
# Gridspecs (inside gridspec)
gs = gridspec.subgridspec(3, 3, wspace=0.05, hspace=0.1)
ax_all = fig.add_subplot(gs[1:, 1:])
ax_norm = fig.add_subplot(gs[1, 0])
ax_inner = fig.add_subplot(gs[2, 0])
ax_ortho = fig.add_subplot(gs[0, 2])
_format(self, ax_all, ax_norm, ax_inner, ax_ortho)
_plot(self, ax_all, ax_norm, ax_inner, ax_ortho)
def hyperparameter_gauge(self, fig, gridspec):
"""Hyperparameter gauge, currently showing the learning rate over time.
This instrument visualizes the hyperparameters values over the course of the
training. Currently, it shows the learning rate, the most likely parameter to
be adapted during training. The current learning rate is additionally shown
in the figure's legend.
**Preview**
.. image:: ../../_static/instrument_previews/Hyperparameters.png
:alt: Preview Hyperparameter Gauge
**Requires**
This instrument requires the learning rate data passed via the
:func:`cockpit.Cockpit.log()` method.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Hyperparameters"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "learning_rate"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
clean_learning_rate = self.tracking_data[["iteration",
"learning_rate"]].dropna()
# Plot Settings
plot_args = {
"x": "iteration",
"y": "learning_rate",
"data": clean_learning_rate,
}
ylabel = plot_args["y"].replace("_", " ").title()
sns.lineplot(**plot_args,
ax=ax,
label=ylabel,
linewidth=2,
color=self.secondary_color)
_beautify_plot(
ax=ax,
xlabel=plot_args["x"],
ylabel=ylabel,
x_scale="symlog" if self.show_log_iter else "linear",
title=title,
xlim="tight",
fontweight="bold",
facecolor=self.bg_color_instruments2,
)
ax.legend()
_add_last_value_to_legend(ax)
def trace_gauge(self, fig, gridspec):
"""Trace gauge, showing the trace of the Hessian versus iteration.
The trace of the hessian is the sum of its eigenvalues and thus can indicate
the overall or average curvature of the loss landscape at the current point.
Increasing values for the trace indicate a steeper curvature, for example, a
narrower valley. This instrument shows the trace versus iteration, overlayed
with an exponentially weighted average.
**Preview**
.. image:: ../../_static/instrument_previews/HessTrace.png
:alt: Preview HessTrace Gauge
**Requires**
The trace instrument requires data from the :class:`~cockpit.quantities.HessTrace`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Trace"
# Check if the required data is available, else skip this instrument
requires = ["HessTrace"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["HessTrace_all"] = self.tracking_data.HessTrace.map(
lambda x: sum(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "HessTrace_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def grad_norm_gauge(self, fig, gridspec):
"""Showing the gradient norm versus iteration.
If the training gets stuck, due to a small
:class:`~cockpit.quantities.UpdateSize` it can be the result of both a badly
chosen learning rate, or from a flat plateau in the loss landscape.
This instrument shows the gradient norm at each iteration, overlayed with an
exponentially weighted average, and can thus distinguish these two cases.
**Preview**
.. image:: ../../_static/instrument_previews/GradientNorm.png
:alt: Preview GradientNorm Gauge
**Requires**
The gradient norm instrument requires data from the
:class:`~cockpit.quantities.GradNorm` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Gradient Norm"
# Check if the required data is available, else skip this instrument
requires = ["GradNorm"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["GradNorm_all"] = self.tracking_data.GradNorm.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "GradNorm_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def histogram_2d_gauge(self,
fig,
gridspec,
transformation=None,
marginals=True,
idx=None):
"""Two-dimensional histogram of the individual gradient and parameter elements.
This instrument provides a combined histogram of parameter-gradient pairs of
the network. The values are collected across an entire mini-batch and thus
captures indvidual gradients as well. The marignal distributions across the
parameters and gradient values are shown at the top and right respectively.
The histogram shows the distribution of gradient and parameter elements for
the last tracked iteration only.
**Preview**
.. image:: ../../_static/instrument_previews/Hist2d.png
:alt: Preview Hist2d Gauge
**Requires**
This two dimensional histogram instrument requires data from the
:class:`~cockpit.quantities.GradHist2d` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
transformation (callable): Some map applied to the bin values as a
transformation for the plot. Defaults to `None` which means no
transformation.
marginals (bool): Whether to plot the marginal histograms as well.
idx (int): Index of parameter whose histogram data should be used.
If ``None`` (default), uses data of all parameters.
"""
# Plot
title_suffix = "(all)" if idx is None else f"(parameter {idx})"
title = f"Gradient/Parameter Element Histogram {title_suffix}"
# Check if the required data is available, else skip this instrument
requires = ["GradHist2d"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
ax.set_axis_off()
ax.set_title(title, fontweight="bold", fontsize="large")
# Gridspecs (inside gridspec)
gs = gridspec.subgridspec(3, 3, wspace=0, hspace=0)
# plot the joint
if marginals:
ax_joint = fig.add_subplot(gs[1:, :2])
else:
ax_joint = fig.add_subplot(gs[:, :])
joint_plot_args = {
"facecolor": self.bg_color_instruments,
"xlabel": "Parameter Element Value",
"ylabel": "Gradient Element\nValue",
}
df = _get_2d_histogram_data(self.tracking_data,
transformation=transformation,
idx=idx)
cmap = self.alpha_cmap
sns.heatmap(data=df, cbar=False, cmap=cmap, ax=ax_joint)
_beautify_plot(ax=ax_joint, **joint_plot_args)
ax_joint.set_xticklabels(_ticks_formatter(ax_joint.get_xticklabels()))
ax_joint.set_yticklabels(_ticks_formatter(ax_joint.get_yticklabels()))
# "Zero lines
# TODO This assumes that the bins are symmetrical!
ax_joint.axvline(df.shape[1] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
ax_joint.axhline(df.shape[0] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
# plot the marginals
if marginals:
ax_xmargin = fig.add_subplot(gs[1:, 2])
ax_xmargin.set_xscale("log")
ax_xmargin.get_yaxis().set_visible(False)
vals, mid_points, bin_size = _get_xmargin_histogram_data(
self.tracking_data, idx=idx)
ax_xmargin.set_ylim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_xmargin.barh(mid_points,
vals,
height=bin_size,
color=self.primary_color,
linewidth=0.1)
ax_xmargin.xaxis.set_minor_locator(ticker.MaxNLocator(3))
ax_ymargin = fig.add_subplot(gs[0, :2])
ax_ymargin.set_yscale("log")
ax_ymargin.get_xaxis().set_visible(False)
vals, mid_points, bin_size = _get_ymargin_histogram_data(
self.tracking_data, idx=idx)
ax_ymargin.set_xlim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_ymargin.bar(
mid_points,
vals,
width=bin_size,
color=self.primary_color,
linewidth=0.2,
)
ax_ymargin.yaxis.set_minor_locator(ticker.MaxNLocator(3))
ax_ymargin.yaxis.set_minor_formatter(ticker.FormatStrFormatter("%.3g"))
def histogram_2d_gauge(self,
fig,
gridspec,
transformation=None,
marginals=True,
idx=None):
"""Two-dimensional histogram of the individual gradient and parameter elements.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
transformation (method): Some map applied to the bin values as a
transformation for the plot. Defaults to `None` which means no
transformation.
marginal (bool): Whether to plot the marginal histograms as well.
idx (int): Index of parameter whose histogram data should be used.
If ``None`` (default), uses data of all parameters.
"""
# Plot
title_suffix = "(all)" if idx is None else f"(parameter {idx})"
title = f"Gradient/Parameter Element Histogram {title_suffix}"
# Check if the required data is available, else skip this instrument
key_prefix = "" if idx is None else f"param_{idx}_"
x_key = key_prefix + "x_edges"
y_key = key_prefix + "y_edges"
hist_key = key_prefix + "hist_2d"
requires = [x_key, y_key, hist_key]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
ax.set_axis_off()
ax.set_title(title, fontweight="bold", fontsize="large")
# Gridspecs (inside gridspec)
gs = gridspec.subgridspec(3, 3, wspace=0, hspace=0)
# plot the joint
if marginals:
ax_joint = fig.add_subplot(gs[1:, :2])
else:
ax_joint = fig.add_subplot(gs[:, :])
joint_plot_args = {
"facecolor": self.bg_color_instruments,
"xlabel": "parameter element value",
"ylabel": "gradient element value",
}
df = _get_2d_histogram_data(self.tracking_data,
transformation=transformation,
idx=idx)
cmap = self.alpha_cmap
sns.heatmap(data=df, cbar=False, cmap=cmap, ax=ax_joint)
_beautify_plot(ax=ax_joint, **joint_plot_args)
# "Zero lines
# TODO This assumes that the bins are symmetrical!
ax_joint.axvline(df.shape[1] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
ax_joint.axhline(df.shape[0] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
# plot the marginals
if marginals:
ax_xmargin = fig.add_subplot(gs[1:, 2])
ax_xmargin.set_xscale("log")
ax_xmargin.get_yaxis().set_visible(False)
vals, mid_points, bin_size = _get_xmargin_histogram_data(
self.tracking_data, idx=idx)
ax_xmargin.set_ylim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_xmargin.barh(mid_points,
vals,
height=bin_size,
color=self.primary_color,
linewidth=0.1)
ax_ymargin = fig.add_subplot(gs[0, :2])
ax_ymargin.set_yscale("log")
ax_ymargin.get_xaxis().set_visible(False)
vals, mid_points, bin_size = _get_ymargin_histogram_data(
self.tracking_data, idx=idx)
ax_ymargin.set_xlim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_ymargin.bar(
mid_points,
vals,
width=bin_size,
color=self.primary_color,
linewidth=0.2,
)
def alpha_gauge(self, fig, gridspec):
"""Showing a distribution of the alpha values since the last plot.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed.
"""
# Plot Alpha Distribution
title = "Alpha Distribution"
# Check if the required data is available, else skip this instrument
requires = ["alpha"]
plot_possible = check_data(self.tracking_data, requires, min_elements=2)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"xlabel": "Local Step Length",
"ylabel": "Stand. Loss",
"title": title,
"xlim": [-1.5, 1.5],
"ylim": [0, 1.75],
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"zero_lines": True,
"center": [True, False],
}
color_all = "gray"
color_last = self.primary_color
color_parabola = self.secondary_color
ax = fig.add_subplot(gridspec)
# Plot unit parabola
x = np.linspace(plot_args["xlim"][0], plot_args["xlim"][1], 100)
y = x ** 2
ax.plot(x, y, linewidth=2, color=color_parabola)
_beautify_plot(**plot_args, ax=ax)
# Alpha Histogram
ax2 = ax.twinx()
# All alphas
sns.distplot(
self.tracking_data["alpha"].dropna(),
ax=ax2,
# norm_hist=True,
fit=stats.norm,
kde=False,
color=color_all,
fit_kws={"color": color_all},
hist_kws={"linewidth": 0, "alpha": 0.5},
label="all",
)
(mu_all, _) = stats.norm.fit(self.tracking_data["alpha"].dropna())
# Last 10% alphas
len_last_elements = int(len(self.tracking_data["alpha"]) / 10)
try:
sns.distplot(
self.tracking_data["alpha"][-len_last_elements:].dropna(),
ax=ax2,
# norm_hist=True,
fit=stats.norm,
kde=False,
color=color_last,
fit_kws={"color": color_last},
hist_kws={"linewidth": 0, "alpha": 0.85},
label="last 10 %",
)
(mu_last, _) = stats.norm.fit(
self.tracking_data["alpha"][-len_last_elements:].dropna()
)
except ValueError:
mu_last = None
# Manually beautify the plot:
# Adding Zone Lines
ax.axvline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(-1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(1, ls="-", color="#ababba", linewidth=0.5, zorder=0)
# Labels
ax.set_xlabel(r"Local step length $\alpha$")
ax2.set_ylabel(r"$\alpha$ density")
# Add indicator for outliers
if max(self.tracking_data["alpha"][-len_last_elements:]) > plot_args["xlim"][1]:
ax.annotate(
"",
xy=(1.8, 0.3),
xytext=(1.7, 0.3),
size=20,
arrowprops=dict(color=color_last),
)
elif max(self.tracking_data["alpha"]) > plot_args["xlim"][1]:
ax.annotate(
"",
xy=(1.8, 0.3),
xytext=(1.7, 0.3),
size=20,
arrowprops=dict(color=color_all),
)
if min(self.tracking_data["alpha"][-len_last_elements:]) < plot_args["xlim"][0]:
ax.annotate(
"",
xy=(-1.8, 0.3),
xytext=(-1.7, 0.3),
size=20,
arrowprops=dict(color=color_last),
)
elif min(self.tracking_data["alpha"]) < plot_args["xlim"][0]:
ax.annotate(
"",
xy=(-1.8, 0.3),
xytext=(-1.7, 0.3),
size=20,
arrowprops=dict(color=color_all),
)
# Legend
# Get the fitted parameters used by sns
lines2, labels2 = ax2.get_legend_handles_labels()
legend = []
if mu_all is not None:
legend.append("{0} ($\mu=${1:.2f})".format(labels2[0], mu_all)) # noqa: W605
if mu_last is not None:
legend.append("{0} ($\mu=${1:.2f})".format(labels2[1], mu_last)) # noqa: W605
ax2.legend(legend)
def alpha_gauge(self, fig, gridspec):
r"""Showing a distribution of the alpha values.
This alpha instruments provides a summary of the alpha values of all tracked
iterations (shown in gray) as well as the last 10% of them (shown in blue).
The alpha value uses loss and gradient information to build a noise-informed
univariate quadratic approximation of the loss function to assess to which
point on this parabola the optimizer moves. The parabola (shown in orange) is
always normalized such that the starting point of each iteration is at
:math:`x=-1`. If the optimzer takes a single step to the local minimum (e.g.
the valley floor) it would be indicated by an :math:`\alpha` value of :math:`0`.
Analogously, taking a step to the exact opposite side of the valley, will be
descrbied by :math:`\alpha=1`.
The instruments shows a histogram and a distribution fit of all alpha values,
as well as the last 10% of tracked iterations. The mean values of both distributions
are also shown in the figure's legend.
For a more detailed explanation of the alpha value and why - perhaps against
intuition - values larger than zero might be desirable, have a look at the
Cockpit paper:
- `Schneider, F., Dangel, F., & Hennig, P.,
Cockpit: A Practical Debugging Tool for Training Deep Neural Networks (2021).
<https://arxiv.org/abs/2102.06604>`_
.. image:: ../../_static/instrument_previews/Alpha.png
:alt: Preview Alpha Gauge
**Requires**
This instrument requires data from the :class:`~cockpit.quantities.Alpha`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument
should be placed.
"""
# Plot Alpha Distribution
title = "Alpha Distribution"
# Check if the required data is available, else skip this instrument
requires = ["Alpha"]
plot_possible = check_data(self.tracking_data, requires, min_elements=2)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
plot_args = {
"xlabel": "Local Step Length",
"ylabel": "Stand. Loss",
"title": title,
"xlim": [-1.5, 1.5],
"ylim": [0, 1.75],
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"zero_lines": True,
"center": [True, False],
}
color_all = "gray"
color_last = self.primary_color
color_parabola = self.secondary_color
ax = fig.add_subplot(gridspec)
# Plot unit parabola
x = np.linspace(plot_args["xlim"][0], plot_args["xlim"][1], 100)
y = x**2
ax.plot(x, y, linewidth=2, color=color_parabola)
_beautify_plot(**plot_args, ax=ax)
# Alpha Histogram
ax2 = ax.twinx()
# All alphas
sns.histplot(
self.tracking_data["Alpha"].dropna(),
ax=ax2,
kde=True,
color=color_all,
kde_kws={"cut": 10},
alpha=0.5,
stat="probability",
label="all",
)
(mu_all, _) = stats.norm.fit(self.tracking_data["Alpha"].dropna())
# Last 10% alphas
len_last_elements = int(len(self.tracking_data["Alpha"]) / 10)
sns.histplot(
self.tracking_data["Alpha"].dropna().tail(len_last_elements),
ax=ax2,
kde=True,
color=color_last,
kde_kws={"cut": 10},
alpha=0.5,
stat="probability",
label="last 10 %",
)
if len_last_elements == 0:
mu_last = math.nan
else:
(mu_last, _) = stats.norm.fit(
self.tracking_data["Alpha"].dropna().tail(len_last_elements))
# Manually beautify the plot:
# Adding Zone Lines
ax.axvline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(-1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(1, ls="-", color="#ababba", linewidth=0.5, zorder=0)
# Labels
ax.set_xlabel(r"Local step length $\alpha$")
ax2.set_ylabel(r"$\alpha$ density")
_add_indicators(self, ax, mu_last, plot_args, color_all, color_last,
len_last_elements)
# Legend
# Get the fitted parameters used by sns
lines2, labels2 = ax2.get_legend_handles_labels()
for idx, lab in enumerate(labels2):
if "all" in lab and not math.isnan(mu_all):
labels2[idx] = lab + " ($\mu=${0:.2f})".format(
mu_all) # noqa: W605
if "last 10 %" in lab and not math.isnan(mu_last):
labels2[idx] = lab + " ($\mu=${0:.2f})".format(
mu_last) # noqa: W605
ax2.legend(lines2, labels2)
def tic_gauge(self, fig, gridspec):
"""TIC gauge, showing the TIC versus iteration.
The TIC (either approximated via traces or using a diagonal approximation)
describes the relation between the curvature and the gradient noise. `Recent
work <https://arxiv.org/abs/1906.07774>`_ suggested that *at a local minimum*,
this quantitiy can estimate the generalization gap. This instrument shows the
TIC versus iteration, overlayed with an exponentially weighted average.
**Preview**
.. image:: ../../_static/instrument_previews/TIC.png
:alt: Preview TIC Gauge
**Requires**
The trace instrument requires data from the :class:`~cockpit.quantities.TICDiag`
or :class:`~cockpit.quantities.TICTrace` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "TIC"
if check_data(self.tracking_data, ["TICDiag"]):
plot_args = {
"x": "iteration",
"y": "TICDiag",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
if check_data(self.tracking_data, ["TICTrace"]):
if "ax" in locals():
ax2 = ax.twinx()
else:
ax2 = fig.add_subplot(gridspec)
plot_args = {
"x": "iteration",
"y": "TICTrace",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap_backup,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2_backup,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
create_basic_plot(**plot_args, ax=ax2)
def max_ev_gauge(self, fig, gridspec):
"""Showing the largest eigenvalue of the Hessian versus iteration.
The largest eigenvalue of the Hessian indicates the loss surface's sharpest
valley. Together with the :func:`~cockpit.instruments.trace_gauge()`, which
provides a notion of "average curvature", it can help understand the "average
condition number" of the loss landscape at the current point. The instrument
shows the largest eigenvalue of the Hessian versus iteration, overlayed with
an exponentially weighted average.
**Preview**
.. image:: ../../_static/instrument_previews/HessMaxEV.png
:alt: Preview HessMaxEV Gauge
**Requires**
The trace instrument requires data from the :class:`~cockpit.quantities.HessMaxEv`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed.
"""
# Plot Trace vs iteration
title = "Max Eigenvalue"
# Check if the required data is available, else skip this instrument
requires = ["HessMaxEV"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "HessMaxEV",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "log",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
# part that should be plotted
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
ax.yaxis.set_minor_formatter(ticker.FormatStrFormatter("%.2g"))
def distance_gauge(self, fig, gridspec):
"""Distance gauge showing two different quantities related to distance.
This instruments shows two quantities at once. Firstly, the :math:`L_2`-distance
of the current parameters to their initialization. This describes the total distance
that the optimization trajectory "has traveled so far" and can be seen via the
blue-to-green dots (and the left y-axis).
Secondly, the update sizes of individual steps are shown via the yellow-to-blue
dots (and the right y-axis). It measure the distance that a single parameter
update covers.
Both quantities are overlayed with an exponentially weighted average.
.. image:: ../../_static/instrument_previews/Distances.png
:alt: Preview Distances Gauge
**Requires**
The distance instrument requires data from both, the
:class:`~cockpit.quantities.UpdateSize` and the
:class:`~cockpit.quantities.Distance` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Distance"
# Check if the required data is available, else skip this instrument
requires = ["Distance", "UpdateSize"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["Distance_all"] = self.tracking_data.Distance.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
self.tracking_data["UpdateSize_all"] = self.tracking_data.UpdateSize.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "Distance_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
ax2 = ax.twinx()
plot_args = {
"x": "iteration",
"y": "UpdateSize_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap.reversed(),
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2.reversed(),
"xlim": "tight",
"ylim": None,
"marker": ",",
}
create_basic_plot(**plot_args, ax=ax2)
def cabs_gauge(self, fig, gridspec):
"""CABS gauge, showing the CABS rule versus iteration.
The batch size trades-off more accurate gradient approximations with longer
computation. The `CABS criterion <https://arxiv.org/abs/1612.05086>`_ describes
the optimal batch size under certain assumptions.
The instruments shows the suggested batch size (and an exponential weighted
average) over the course of training, according to
- `Balles, L., Romero, J., & Hennig, P.,
Coupling adaptive batch sizes with learning rates (2017).
<https://arxiv.org/abs/1612.05086>`_
**Preview**
.. image:: ../../_static/instrument_previews/CABS.png
:alt: Preview CABS Gauge
**Requires**
This instrument requires data from the :class:`~cockpit.quantities.CABS`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "CABS"
# Check if the required data is available, else skip this instrument
requires = ["CABS"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "CABS",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def performance_gauge(self, fig, gridspec):
"""Plotting train/valid accuracy vs. epoch and mini-batch loss vs. iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Performance Plot"
# Check if the required data is available, else skip this instrument
requires = [
"iteration", "train_accuracy", "valid_accuracy", "mini_batch_loss"
]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
# Mini-batch train loss
plot_args = {
"x": "iteration",
"y": "mini_batch_loss",
"data": self.tracking_data,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments2,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
clean_accuracies = self.tracking_data[[
"iteration", "train_accuracy", "valid_accuracy"
]].dropna()
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "train_accuracy",
"data": clean_accuracies,
}
ax2 = ax.twinx()
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.primary_color,
)
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "valid_accuracy",
"data": clean_accuracies,
}
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.secondary_color,
)
# Customization
ax2.set_ylim([0, 1])
ax2.set_ylabel("Accuracy")
_add_last_value_to_legend(ax2, percentage=True)
def early_stopping_gauge(self, fig, gridspec):
"""Early Stopping gauge, showing the LHS of the stopping criterion versus iteration.
Early stopping the training has been widely used to prevent poor generalization
due to over-fitting. `Mahsereci et al. (2017) <https://arxiv.org/abs/1703.09580>`_
proposed an evidence-based stopping criterion based on mini-batch statistics.
This instruments visualizes this criterion versus iteration, overlayed
with an exponentially weighted average. If the stopping criterion becomes
positive, this suggests stopping the training according to
- `Mahsereci, M., Balles, L., Lassner, C., & Hennig, P.,
Early stopping without a validation set (2017).
<https://arxiv.org/abs/1703.09580>`_
**Preview**
.. image:: ../../_static/instrument_previews/EarlyStopping.png
:alt: Preview EarlyStopping Gauge
**Requires**
This instrument requires data from the :class:`~cockpit.quantities.EarlyStopping`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Early stopping"
# Check if the required data is available, else skip this instrument
requires = ["EarlyStopping"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "EarlyStopping",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def performance_gauge(self, fig, gridspec):
"""Plotting train/valid accuracy vs. epoch and mini-batch loss vs. iteration.
This instruments visualizes the currently most popular diagnostic metrics. It
shows the mini-batch loss in each iteration (overlayed with an exponentially
weighted average) as well as accuracies for both the training as well as the
validation set. The current accuracy numbers are also shown in the legend.
**Preview**
.. image:: ../../_static/instrument_previews/Performance.png
:alt: Preview Performance Gauge
**Requires**
This instrument visualizes quantities passed via the
:func:`cockpit.Cockpit.log()` method.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Performance Plot"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "Loss"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the loss data for the " + title + " instrument",
stacklevel=1,
)
return
# Mini-batch train loss
plot_args = {
"x": "iteration",
"y": "Loss",
"data": self.tracking_data,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments2,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
requires = ["iteration", "train_accuracy", "valid_accuracy"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the accuracy data for the " + title +
" instrument",
stacklevel=1,
)
return
else:
clean_accuracies = self.tracking_data[[
"iteration", "train_accuracy", "valid_accuracy"
]].dropna()
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "train_accuracy",
"data": clean_accuracies,
}
ax2 = ax.twinx()
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.primary_color,
)
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "valid_accuracy",
"data": clean_accuracies,
}
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.secondary_color,
)
# Customization
ax2.set_ylim([0, 1])
ax2.set_ylabel("Accuracy")
_add_last_value_to_legend(ax2, percentage=True)
