def histogram_1d_gauge(self, fig, gridspec, y_scale="log"):
"""One-dimensional histogram of the individual gradient elements.
This instrument provides a histogram of the gradient element values across all
individual gradients in a mini-batch. The histogram shows the distribution for
the last tracked iteration only.
**Preview**
.. image:: ../../_static/instrument_previews/Hist1d.png
:alt: Preview Hist1d Gauge
**Requires**
This two dimensional histogram instrument requires data from the
:class:`~cockpit.quantities.GradHist1d` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
y_scale (str, optional): Scale of the y-axis. Defaults to "log".
"""
# Plot
title = "Gradient Element Histogram"
# Check if the required data is available, else skip this instrument
requires = ["GradHist1d"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
plot_args = {
"title": title,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"xlabel": "Gradient Element Value",
"ylabel": "Frequency",
"y_scale": y_scale,
}
vals, mid_points, width = _get_histogram_data(self.tracking_data)
ax.bar(mid_points, vals, width=width, color=self.primary_color)
_beautify_plot(ax=ax, **plot_args)
ax.set_title(title, fontweight="bold", fontsize="large")
def hyperparameter_gauge(self, fig, gridspec):
"""Hyperparameter gauge, currently showing the learning rate over time.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Hyperparameters"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "learning_rate"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
clean_learning_rate = self.tracking_data[["iteration",
"learning_rate"]].dropna()
# Plot Settings
plot_args = {
"x": "iteration",
"y": "learning_rate",
"data": clean_learning_rate,
}
ylabel = plot_args["y"].replace("_", " ").title()
sns.lineplot(**plot_args,
ax=ax,
label=ylabel,
linewidth=2,
color=self.secondary_color)
_beautify_plot(
ax=ax,
xlabel=plot_args["x"],
ylabel=ylabel,
x_scale="symlog" if self.show_log_iter else "linear",
title=title,
xlim="tight",
fontweight="bold",
facecolor=self.bg_color_instruments2,
)
ax.legend()
_add_last_value_to_legend(ax)
def histogram_1d_gauge(self, fig, gridspec, y_scale="log"):
"""One-dimensional histogram of the individual gradient elements.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
y_scale (str, optional): Scale of the y-axis. Defaults to "log".
"""
# Plot
title = "Gradient Element Histogram"
# Check if the required data is available, else skip this instrument
requires = ["edges", "hist_1d"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
plot_args = {
"title": title,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"xlabel": "gradient element value",
"ylabel": "frequency",
"y_scale": y_scale,
}
vals, mid_points, width = _get_histogram_data(self.tracking_data)
ax.bar(mid_points, vals, width=width, color=self.primary_color)
_beautify_plot(ax=ax, **plot_args)
ax.set_title(title, fontweight="bold", fontsize="large")
def hyperparameter_gauge(self, fig, gridspec):
"""Hyperparameter gauge, currently showing the learning rate over time.
This instrument visualizes the hyperparameters values over the course of the
training. Currently, it shows the learning rate, the most likely parameter to
be adapted during training. The current learning rate is additionally shown
in the figure's legend.
**Preview**
.. image:: ../../_static/instrument_previews/Hyperparameters.png
:alt: Preview Hyperparameter Gauge
**Requires**
This instrument requires the learning rate data passed via the
:func:`cockpit.Cockpit.log()` method.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Hyperparameters"
# Check if the required data is available, else skip this instrument
requires = ["iteration", "learning_rate"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
clean_learning_rate = self.tracking_data[["iteration",
"learning_rate"]].dropna()
# Plot Settings
plot_args = {
"x": "iteration",
"y": "learning_rate",
"data": clean_learning_rate,
}
ylabel = plot_args["y"].replace("_", " ").title()
sns.lineplot(**plot_args,
ax=ax,
label=ylabel,
linewidth=2,
color=self.secondary_color)
_beautify_plot(
ax=ax,
xlabel=plot_args["x"],
ylabel=ylabel,
x_scale="symlog" if self.show_log_iter else "linear",
title=title,
xlim="tight",
fontweight="bold",
facecolor=self.bg_color_instruments2,
)
ax.legend()
_add_last_value_to_legend(ax)
def histogram_2d_gauge(self,
fig,
gridspec,
transformation=None,
marginals=True,
idx=None):
"""Two-dimensional histogram of the individual gradient and parameter elements.
This instrument provides a combined histogram of parameter-gradient pairs of
the network. The values are collected across an entire mini-batch and thus
captures indvidual gradients as well. The marignal distributions across the
parameters and gradient values are shown at the top and right respectively.
The histogram shows the distribution of gradient and parameter elements for
the last tracked iteration only.
**Preview**
.. image:: ../../_static/instrument_previews/Hist2d.png
:alt: Preview Hist2d Gauge
**Requires**
This two dimensional histogram instrument requires data from the
:class:`~cockpit.quantities.GradHist2d` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
transformation (callable): Some map applied to the bin values as a
transformation for the plot. Defaults to `None` which means no
transformation.
marginals (bool): Whether to plot the marginal histograms as well.
idx (int): Index of parameter whose histogram data should be used.
If ``None`` (default), uses data of all parameters.
"""
# Plot
title_suffix = "(all)" if idx is None else f"(parameter {idx})"
title = f"Gradient/Parameter Element Histogram {title_suffix}"
# Check if the required data is available, else skip this instrument
requires = ["GradHist2d"]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
ax.set_axis_off()
ax.set_title(title, fontweight="bold", fontsize="large")
# Gridspecs (inside gridspec)
gs = gridspec.subgridspec(3, 3, wspace=0, hspace=0)
# plot the joint
if marginals:
ax_joint = fig.add_subplot(gs[1:, :2])
else:
ax_joint = fig.add_subplot(gs[:, :])
joint_plot_args = {
"facecolor": self.bg_color_instruments,
"xlabel": "Parameter Element Value",
"ylabel": "Gradient Element\nValue",
}
df = _get_2d_histogram_data(self.tracking_data,
transformation=transformation,
idx=idx)
cmap = self.alpha_cmap
sns.heatmap(data=df, cbar=False, cmap=cmap, ax=ax_joint)
_beautify_plot(ax=ax_joint, **joint_plot_args)
ax_joint.set_xticklabels(_ticks_formatter(ax_joint.get_xticklabels()))
ax_joint.set_yticklabels(_ticks_formatter(ax_joint.get_yticklabels()))
# "Zero lines
# TODO This assumes that the bins are symmetrical!
ax_joint.axvline(df.shape[1] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
ax_joint.axhline(df.shape[0] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
# plot the marginals
if marginals:
ax_xmargin = fig.add_subplot(gs[1:, 2])
ax_xmargin.set_xscale("log")
ax_xmargin.get_yaxis().set_visible(False)
vals, mid_points, bin_size = _get_xmargin_histogram_data(
self.tracking_data, idx=idx)
ax_xmargin.set_ylim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_xmargin.barh(mid_points,
vals,
height=bin_size,
color=self.primary_color,
linewidth=0.1)
ax_xmargin.xaxis.set_minor_locator(ticker.MaxNLocator(3))
ax_ymargin = fig.add_subplot(gs[0, :2])
ax_ymargin.set_yscale("log")
ax_ymargin.get_xaxis().set_visible(False)
vals, mid_points, bin_size = _get_ymargin_histogram_data(
self.tracking_data, idx=idx)
ax_ymargin.set_xlim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_ymargin.bar(
mid_points,
vals,
width=bin_size,
color=self.primary_color,
linewidth=0.2,
)
ax_ymargin.yaxis.set_minor_locator(ticker.MaxNLocator(3))
ax_ymargin.yaxis.set_minor_formatter(ticker.FormatStrFormatter("%.3g"))
def histogram_2d_gauge(self,
fig,
gridspec,
transformation=None,
marginals=True,
idx=None):
"""Two-dimensional histogram of the individual gradient and parameter elements.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
transformation (method): Some map applied to the bin values as a
transformation for the plot. Defaults to `None` which means no
transformation.
marginal (bool): Whether to plot the marginal histograms as well.
idx (int): Index of parameter whose histogram data should be used.
If ``None`` (default), uses data of all parameters.
"""
# Plot
title_suffix = "(all)" if idx is None else f"(parameter {idx})"
title = f"Gradient/Parameter Element Histogram {title_suffix}"
# Check if the required data is available, else skip this instrument
key_prefix = "" if idx is None else f"param_{idx}_"
x_key = key_prefix + "x_edges"
y_key = key_prefix + "y_edges"
hist_key = key_prefix + "hist_2d"
requires = [x_key, y_key, hist_key]
plot_possible = check_data(self.tracking_data, requires, min_elements=1)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
ax = fig.add_subplot(gridspec)
ax.set_axis_off()
ax.set_title(title, fontweight="bold", fontsize="large")
# Gridspecs (inside gridspec)
gs = gridspec.subgridspec(3, 3, wspace=0, hspace=0)
# plot the joint
if marginals:
ax_joint = fig.add_subplot(gs[1:, :2])
else:
ax_joint = fig.add_subplot(gs[:, :])
joint_plot_args = {
"facecolor": self.bg_color_instruments,
"xlabel": "parameter element value",
"ylabel": "gradient element value",
}
df = _get_2d_histogram_data(self.tracking_data,
transformation=transformation,
idx=idx)
cmap = self.alpha_cmap
sns.heatmap(data=df, cbar=False, cmap=cmap, ax=ax_joint)
_beautify_plot(ax=ax_joint, **joint_plot_args)
# "Zero lines
# TODO This assumes that the bins are symmetrical!
ax_joint.axvline(df.shape[1] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
ax_joint.axhline(df.shape[0] / 2,
ls="-",
color="#ababba",
linewidth=1.5,
zorder=0)
# plot the marginals
if marginals:
ax_xmargin = fig.add_subplot(gs[1:, 2])
ax_xmargin.set_xscale("log")
ax_xmargin.get_yaxis().set_visible(False)
vals, mid_points, bin_size = _get_xmargin_histogram_data(
self.tracking_data, idx=idx)
ax_xmargin.set_ylim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_xmargin.barh(mid_points,
vals,
height=bin_size,
color=self.primary_color,
linewidth=0.1)
ax_ymargin = fig.add_subplot(gs[0, :2])
ax_ymargin.set_yscale("log")
ax_ymargin.get_xaxis().set_visible(False)
vals, mid_points, bin_size = _get_ymargin_histogram_data(
self.tracking_data, idx=idx)
ax_ymargin.set_xlim(
[mid_points[0] - bin_size / 2, mid_points[-1] + bin_size / 2])
ax_ymargin.bar(
mid_points,
vals,
width=bin_size,
color=self.primary_color,
linewidth=0.2,
)
def alpha_gauge(self, fig, gridspec):
"""Showing a distribution of the alpha values since the last plot.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed.
"""
# Plot Alpha Distribution
title = "Alpha Distribution"
# Check if the required data is available, else skip this instrument
requires = ["alpha"]
plot_possible = check_data(self.tracking_data, requires, min_elements=2)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
plot_args = {
"xlabel": "Local Step Length",
"ylabel": "Stand. Loss",
"title": title,
"xlim": [-1.5, 1.5],
"ylim": [0, 1.75],
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"zero_lines": True,
"center": [True, False],
}
color_all = "gray"
color_last = self.primary_color
color_parabola = self.secondary_color
ax = fig.add_subplot(gridspec)
# Plot unit parabola
x = np.linspace(plot_args["xlim"][0], plot_args["xlim"][1], 100)
y = x ** 2
ax.plot(x, y, linewidth=2, color=color_parabola)
_beautify_plot(**plot_args, ax=ax)
# Alpha Histogram
ax2 = ax.twinx()
# All alphas
sns.distplot(
self.tracking_data["alpha"].dropna(),
ax=ax2,
# norm_hist=True,
fit=stats.norm,
kde=False,
color=color_all,
fit_kws={"color": color_all},
hist_kws={"linewidth": 0, "alpha": 0.5},
label="all",
)
(mu_all, _) = stats.norm.fit(self.tracking_data["alpha"].dropna())
# Last 10% alphas
len_last_elements = int(len(self.tracking_data["alpha"]) / 10)
try:
sns.distplot(
self.tracking_data["alpha"][-len_last_elements:].dropna(),
ax=ax2,
# norm_hist=True,
fit=stats.norm,
kde=False,
color=color_last,
fit_kws={"color": color_last},
hist_kws={"linewidth": 0, "alpha": 0.85},
label="last 10 %",
)
(mu_last, _) = stats.norm.fit(
self.tracking_data["alpha"][-len_last_elements:].dropna()
)
except ValueError:
mu_last = None
# Manually beautify the plot:
# Adding Zone Lines
ax.axvline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(-1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(1, ls="-", color="#ababba", linewidth=0.5, zorder=0)
# Labels
ax.set_xlabel(r"Local step length $\alpha$")
ax2.set_ylabel(r"$\alpha$ density")
# Add indicator for outliers
if max(self.tracking_data["alpha"][-len_last_elements:]) > plot_args["xlim"][1]:
ax.annotate(
"",
xy=(1.8, 0.3),
xytext=(1.7, 0.3),
size=20,
arrowprops=dict(color=color_last),
)
elif max(self.tracking_data["alpha"]) > plot_args["xlim"][1]:
ax.annotate(
"",
xy=(1.8, 0.3),
xytext=(1.7, 0.3),
size=20,
arrowprops=dict(color=color_all),
)
if min(self.tracking_data["alpha"][-len_last_elements:]) < plot_args["xlim"][0]:
ax.annotate(
"",
xy=(-1.8, 0.3),
xytext=(-1.7, 0.3),
size=20,
arrowprops=dict(color=color_last),
)
elif min(self.tracking_data["alpha"]) < plot_args["xlim"][0]:
ax.annotate(
"",
xy=(-1.8, 0.3),
xytext=(-1.7, 0.3),
size=20,
arrowprops=dict(color=color_all),
)
# Legend
# Get the fitted parameters used by sns
lines2, labels2 = ax2.get_legend_handles_labels()
legend = []
if mu_all is not None:
legend.append("{0} ($\mu=${1:.2f})".format(labels2[0], mu_all)) # noqa: W605
if mu_last is not None:
legend.append("{0} ($\mu=${1:.2f})".format(labels2[1], mu_last)) # noqa: W605
ax2.legend(legend)
def alpha_gauge(self, fig, gridspec):
r"""Showing a distribution of the alpha values.
This alpha instruments provides a summary of the alpha values of all tracked
iterations (shown in gray) as well as the last 10% of them (shown in blue).
The alpha value uses loss and gradient information to build a noise-informed
univariate quadratic approximation of the loss function to assess to which
point on this parabola the optimizer moves. The parabola (shown in orange) is
always normalized such that the starting point of each iteration is at
:math:`x=-1`. If the optimzer takes a single step to the local minimum (e.g.
the valley floor) it would be indicated by an :math:`\alpha` value of :math:`0`.
Analogously, taking a step to the exact opposite side of the valley, will be
descrbied by :math:`\alpha=1`.
The instruments shows a histogram and a distribution fit of all alpha values,
as well as the last 10% of tracked iterations. The mean values of both distributions
are also shown in the figure's legend.
For a more detailed explanation of the alpha value and why - perhaps against
intuition - values larger than zero might be desirable, have a look at the
Cockpit paper:
- `Schneider, F., Dangel, F., & Hennig, P.,
Cockpit: A Practical Debugging Tool for Training Deep Neural Networks (2021).
<https://arxiv.org/abs/2102.06604>`_
.. image:: ../../_static/instrument_previews/Alpha.png
:alt: Preview Alpha Gauge
**Requires**
This instrument requires data from the :class:`~cockpit.quantities.Alpha`
quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument
should be placed.
"""
# Plot Alpha Distribution
title = "Alpha Distribution"
# Check if the required data is available, else skip this instrument
requires = ["Alpha"]
plot_possible = check_data(self.tracking_data, requires, min_elements=2)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
plot_args = {
"xlabel": "Local Step Length",
"ylabel": "Stand. Loss",
"title": title,
"xlim": [-1.5, 1.5],
"ylim": [0, 1.75],
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
"zero_lines": True,
"center": [True, False],
}
color_all = "gray"
color_last = self.primary_color
color_parabola = self.secondary_color
ax = fig.add_subplot(gridspec)
# Plot unit parabola
x = np.linspace(plot_args["xlim"][0], plot_args["xlim"][1], 100)
y = x**2
ax.plot(x, y, linewidth=2, color=color_parabola)
_beautify_plot(**plot_args, ax=ax)
# Alpha Histogram
ax2 = ax.twinx()
# All alphas
sns.histplot(
self.tracking_data["Alpha"].dropna(),
ax=ax2,
kde=True,
color=color_all,
kde_kws={"cut": 10},
alpha=0.5,
stat="probability",
label="all",
)
(mu_all, _) = stats.norm.fit(self.tracking_data["Alpha"].dropna())
# Last 10% alphas
len_last_elements = int(len(self.tracking_data["Alpha"]) / 10)
sns.histplot(
self.tracking_data["Alpha"].dropna().tail(len_last_elements),
ax=ax2,
kde=True,
color=color_last,
kde_kws={"cut": 10},
alpha=0.5,
stat="probability",
label="last 10 %",
)
if len_last_elements == 0:
mu_last = math.nan
else:
(mu_last, _) = stats.norm.fit(
self.tracking_data["Alpha"].dropna().tail(len_last_elements))
# Manually beautify the plot:
# Adding Zone Lines
ax.axvline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(-1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axvline(1, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(0, ls="-", color="#ababba", linewidth=1.5, zorder=0)
ax.axhline(1, ls="-", color="#ababba", linewidth=0.5, zorder=0)
# Labels
ax.set_xlabel(r"Local step length $\alpha$")
ax2.set_ylabel(r"$\alpha$ density")
_add_indicators(self, ax, mu_last, plot_args, color_all, color_last,
len_last_elements)
# Legend
# Get the fitted parameters used by sns
lines2, labels2 = ax2.get_legend_handles_labels()
for idx, lab in enumerate(labels2):
if "all" in lab and not math.isnan(mu_all):
labels2[idx] = lab + " ($\mu=${0:.2f})".format(
mu_all) # noqa: W605
if "last 10 %" in lab and not math.isnan(mu_last):
labels2[idx] = lab + " ($\mu=${0:.2f})".format(
mu_last) # noqa: W605
ax2.legend(lines2, labels2)