def trace_gauge(self, fig, gridspec):
"""Trace gauge, showing the trace of the Hessian versus iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Trace"
# Check if the required data is available, else skip this instrument
requires = ["trace"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["trace_all"] = self.tracking_data.trace.map(
lambda x: sum(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "trace_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def max_ev_gauge(self, fig, gridspec):
"""Showing the largest eigenvalue of the Hessian versus iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed.
"""
# Plot Trace vs iteration
title = "Max Eigenvalue"
# Check if the required data is available, else skip this instrument
requires = ["max_ev"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "max_ev",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "log",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
# part that should be plotted
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def grad_norm_gauge(self, fig, gridspec):
"""Showing the gradient norm versus iteration.
If the training gets stuck, due to a small
:class:`~cockpit.quantities.UpdateSize` it can be the result of both a badly
chosen learning rate, or from a flat plateau in the loss landscape.
This instrument shows the gradient norm at each iteration, overlayed with an
exponentially weighted average, and can thus distinguish these two cases.
**Preview**
.. image:: ../../_static/instrument_previews/GradientNorm.png
:alt: Preview GradientNorm Gauge
**Requires**
The gradient norm instrument requires data from the
:class:`~cockpit.quantities.GradNorm` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Gradient Norm"
# Check if the required data is available, else skip this instrument
requires = ["GradNorm"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["GradNorm_all"] = self.tracking_data.GradNorm.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "GradNorm_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def distance_gauge(self, fig, gridspec):
"""Showing the parameter L2-distance to the initialization versus iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Distance"
# Check if the required data is available, else skip this instrument
requires = ["d2init", "update_size"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["d2init_all"] = self.tracking_data.d2init.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x
)
self.tracking_data["update_size_all"] = self.tracking_data.update_size.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x
)
plot_args = {
"x": "iteration",
"y": "d2init_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
ax2 = ax.twinx()
plot_args = {
"x": "iteration",
"y": "update_size_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap.reversed(),
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2.reversed(),
"xlim": "tight",
"ylim": None,
"marker": ",",
}
create_basic_plot(**plot_args, ax=ax2)
def early_stopping_gauge(self, fig, gridspec):
"""Early Stopping gauge, showing the LHS of the stopping criterion versus iteration.
Early stopping the training has been widely used to prevent poor generalization
due to over-fitting. `Mahsereci et al. (2017) <https://arxiv.org/abs/1703.09580>`_
proposed an evidence-based stopping criterion based on mini-batch statistics.
This instruments visualizes this criterion versus iteration, overlayed
with an exponentially weighted average. If the stopping criterion becomes
positive, this suggests stopping the training according to
- `Mahsereci, M., Balles, L., Lassner, C., & Hennig, P.,
Early stopping without a validation set (2017).
<https://arxiv.org/abs/1703.09580>`_
**Preview**
.. image:: ../../_static/instrument_previews/EarlyStopping.png
:alt: Preview EarlyStopping Gauge
**Requires**
This instrument requires data from the :class:`~cockpit.quantities.EarlyStopping`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Early stopping"
# Check if the required data is available, else skip this instrument
requires = ["EarlyStopping"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "EarlyStopping",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def distance_gauge(self, fig, gridspec):
"""Distance gauge showing two different quantities related to distance.
This instruments shows two quantities at once. Firstly, the :math:`L_2`-distance
of the current parameters to their initialization. This describes the total distance
that the optimization trajectory "has traveled so far" and can be seen via the
blue-to-green dots (and the left y-axis).
Secondly, the update sizes of individual steps are shown via the yellow-to-blue
dots (and the right y-axis). It measure the distance that a single parameter
update covers.
Both quantities are overlayed with an exponentially weighted average.
.. image:: ../../_static/instrument_previews/Distances.png
:alt: Preview Distances Gauge
**Requires**
The distance instrument requires data from both, the
:class:`~cockpit.quantities.UpdateSize` and the
:class:`~cockpit.quantities.Distance` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Distance"
# Check if the required data is available, else skip this instrument
requires = ["Distance", "UpdateSize"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["Distance_all"] = self.tracking_data.Distance.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
self.tracking_data["UpdateSize_all"] = self.tracking_data.UpdateSize.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "Distance_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
ax2 = ax.twinx()
plot_args = {
"x": "iteration",
"y": "UpdateSize_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap.reversed(),
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2.reversed(),
"xlim": "tight",
"ylim": None,
"marker": ",",
}
create_basic_plot(**plot_args, ax=ax2)
def performance_gauge(self, fig, gridspec):
"""Plotting train/valid accuracy vs. epoch and mini-batch loss vs. iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Performance Plot"
# Check if the required data is available, else skip this instrument
requires = [
"iteration", "train_accuracy", "valid_accuracy", "mini_batch_loss"
]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
# Mini-batch train loss
plot_args = {
"x": "iteration",
"y": "mini_batch_loss",
"data": self.tracking_data,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments2,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
clean_accuracies = self.tracking_data[[
"iteration", "train_accuracy", "valid_accuracy"
]].dropna()
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "train_accuracy",
"data": clean_accuracies,
}
ax2 = ax.twinx()
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.primary_color,
)
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "valid_accuracy",
"data": clean_accuracies,
}
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.secondary_color,
)
# Customization
ax2.set_ylim([0, 1])
ax2.set_ylabel("Accuracy")
_add_last_value_to_legend(ax2, percentage=True)
def cabs_gauge(self, fig, gridspec):
"""CABS gauge, showing the CABS rule versus iteration.
The batch size trades-off more accurate gradient approximations with longer
computation. The `CABS criterion <https://arxiv.org/abs/1612.05086>`_ describes
the optimal batch size under certain assumptions.
The instruments shows the suggested batch size (and an exponential weighted
average) over the course of training, according to
- `Balles, L., Romero, J., & Hennig, P.,
Coupling adaptive batch sizes with learning rates (2017).
<https://arxiv.org/abs/1612.05086>`_
**Preview**
.. image:: ../../_static/instrument_previews/CABS.png
:alt: Preview CABS Gauge
**Requires**
This instrument requires data from the :class:`~cockpit.quantities.CABS`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "CABS"
# Check if the required data is available, else skip this instrument
requires = ["CABS"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "CABS",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def max_ev_gauge(self, fig, gridspec):
"""Showing the largest eigenvalue of the Hessian versus iteration.
The largest eigenvalue of the Hessian indicates the loss surface's sharpest
valley. Together with the :func:`~cockpit.instruments.trace_gauge()`, which
provides a notion of "average curvature", it can help understand the "average
condition number" of the loss landscape at the current point. The instrument
shows the largest eigenvalue of the Hessian versus iteration, overlayed with
an exponentially weighted average.
**Preview**
.. image:: ../../_static/instrument_previews/HessMaxEV.png
:alt: Preview HessMaxEV Gauge
**Requires**
The trace instrument requires data from the :class:`~cockpit.quantities.HessMaxEv`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed.
"""
# Plot Trace vs iteration
title = "Max Eigenvalue"
# Check if the required data is available, else skip this instrument
requires = ["HessMaxEV"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
plot_args = {
"x": "iteration",
"y": "HessMaxEV",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "log",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
# part that should be plotted
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
ax.yaxis.set_minor_formatter(ticker.FormatStrFormatter("%.2g"))
def tic_gauge(self, fig, gridspec):
"""TIC gauge, showing the TIC versus iteration.
The TIC (either approximated via traces or using a diagonal approximation)
describes the relation between the curvature and the gradient noise. `Recent
work <https://arxiv.org/abs/1906.07774>`_ suggested that *at a local minimum*,
this quantitiy can estimate the generalization gap. This instrument shows the
TIC versus iteration, overlayed with an exponentially weighted average.
**Preview**
.. image:: ../../_static/instrument_previews/TIC.png
:alt: Preview TIC Gauge
**Requires**
The trace instrument requires data from the :class:`~cockpit.quantities.TICDiag`
or :class:`~cockpit.quantities.TICTrace` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "TIC"
if check_data(self.tracking_data, ["TICDiag"]):
plot_args = {
"x": "iteration",
"y": "TICDiag",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
if check_data(self.tracking_data, ["TICTrace"]):
if "ax" in locals():
ax2 = ax.twinx()
else:
ax2 = fig.add_subplot(gridspec)
plot_args = {
"x": "iteration",
"y": "TICTrace",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap_backup,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2_backup,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
create_basic_plot(**plot_args, ax=ax2)
def trace_gauge(self, fig, gridspec):
"""Trace gauge, showing the trace of the Hessian versus iteration.
The trace of the hessian is the sum of its eigenvalues and thus can indicate
the overall or average curvature of the loss landscape at the current point.
Increasing values for the trace indicate a steeper curvature, for example, a
narrower valley. This instrument shows the trace versus iteration, overlayed
with an exponentially weighted average.
**Preview**
.. image:: ../../_static/instrument_previews/HessTrace.png
:alt: Preview HessTrace Gauge
**Requires**
The trace instrument requires data from the :class:`~cockpit.quantities.HessTrace`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Trace"
# Check if the required data is available, else skip this instrument
requires = ["HessTrace"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["HessTrace_all"] = self.tracking_data.HessTrace.map(
lambda x: sum(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "HessTrace_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def performance_gauge(self, fig, gridspec):
"""Plotting train/valid accuracy vs. epoch and mini-batch loss vs. iteration.
This instruments visualizes the currently most popular diagnostic metrics. It
shows the mini-batch loss in each iteration (overlayed with an exponentially
weighted average) as well as accuracies for both the training as well as the
validation set. The current accuracy numbers are also shown in the legend.
**Preview**
.. image:: ../../_static/instrument_previews/Performance.png
:alt: Preview Performance Gauge
**Requires**
This instrument visualizes quantities passed via the
:func:`cockpit.Cockpit.log()` method.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Performance Plot"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "Loss"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the loss data for the " + title + " instrument",
stacklevel=1,
)
return
# Mini-batch train loss
plot_args = {
"x": "iteration",
"y": "Loss",
"data": self.tracking_data,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments2,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
requires = ["iteration", "train_accuracy", "valid_accuracy"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the accuracy data for the " + title +
" instrument",
stacklevel=1,
)
return
else:
clean_accuracies = self.tracking_data[[
"iteration", "train_accuracy", "valid_accuracy"
]].dropna()
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "train_accuracy",
"data": clean_accuracies,
}
ax2 = ax.twinx()
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.primary_color,
)
# Train Accuracy
plot_args = {
"x": "iteration",
"y": "valid_accuracy",
"data": clean_accuracies,
}
sns.lineplot(
**plot_args,
ax=ax2,
label=plot_args["y"].title().replace("_", " "),
linewidth=2,
color=self.secondary_color,
)
# Customization
ax2.set_ylim([0, 1])
ax2.set_ylabel("Accuracy")
_add_last_value_to_legend(ax2, percentage=True)