def _compute_step_length(self):
"""Return distance between start and end point."""
start_params, end_params = self._get_info("params")
dists = [(end_params[key] - start_params[key]).norm(2).item()
for key in start_params.keys()]
return _root_sum_of_squares(dists)
def _projected_gradient(u, v):
"""Computes magnitude of a projection of one vector onto another.
Args:
u ([torch.Tensor]): First parameter vector, the vector to be projected.
v ([torch.Tensor]): Second parameter vector, the direction.
Returns:
float: Scalar magnitude of the projection.
"""
dot_product = sum(_layerwise_dot_product(u, v))
norm = _root_sum_of_squares([v_el.norm(2).item() for v_el in v])
return dot_product / norm
def grad_norm_gauge(self, fig, gridspec):
"""Showing the gradient norm versus iteration.
Args:
self (cockpit.plotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Gradient Norm"
# Check if the required data is available, else skip this instrument
requires = ["grad_norm"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
warnings.warn(
"Couldn't get the required data for the " + title + " instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["grad_norm_all"] = self.tracking_data.grad_norm.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "grad_norm_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def grad_norm_gauge(self, fig, gridspec):
"""Showing the gradient norm versus iteration.
If the training gets stuck, due to a small
:class:`~cockpit.quantities.UpdateSize` it can be the result of both a badly
chosen learning rate, or from a flat plateau in the loss landscape.
This instrument shows the gradient norm at each iteration, overlayed with an
exponentially weighted average, and can thus distinguish these two cases.
**Preview**
.. image:: ../../_static/instrument_previews/GradientNorm.png
:alt: Preview GradientNorm Gauge
**Requires**
The gradient norm instrument requires data from the
:class:`~cockpit.quantities.GradNorm` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Gradient Norm"
# Check if the required data is available, else skip this instrument
requires = ["GradNorm"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["GradNorm_all"] = self.tracking_data.GradNorm.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "GradNorm_all",
"data": self.tracking_data,
"x_scale": "symlog" if self.show_log_iter else "linear",
"y_scale": "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
def distance_gauge(self, fig, gridspec):
"""Distance gauge showing two different quantities related to distance.
This instruments shows two quantities at once. Firstly, the :math:`L_2`-distance
of the current parameters to their initialization. This describes the total distance
that the optimization trajectory "has traveled so far" and can be seen via the
blue-to-green dots (and the left y-axis).
Secondly, the update sizes of individual steps are shown via the yellow-to-blue
dots (and the right y-axis). It measure the distance that a single parameter
update covers.
Both quantities are overlayed with an exponentially weighted average.
.. image:: ../../_static/instrument_previews/Distances.png
:alt: Preview Distances Gauge
**Requires**
The distance instrument requires data from both, the
:class:`~cockpit.quantities.UpdateSize` and the
:class:`~cockpit.quantities.Distance` quantity class.
Args:
self (CockpitPlotter): The cockpit plotter requesting this instrument.
fig (matplotlib.figure.Figure): Figure of the Cockpit.
gridspec (matplotlib.gridspec.GridSpec): GridSpec where the instrument should be
placed
"""
# Plot Trace vs iteration
title = "Distance"
# Check if the required data is available, else skip this instrument
requires = ["Distance", "UpdateSize"]
plot_possible = check_data(self.tracking_data, requires)
if not plot_possible:
if self.debug:
warnings.warn(
"Couldn't get the required data for the " + title +
" instrument",
stacklevel=1,
)
return
# Compute
self.tracking_data["Distance_all"] = self.tracking_data.Distance.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
self.tracking_data["UpdateSize_all"] = self.tracking_data.UpdateSize.map(
lambda x: _root_sum_of_squares(x) if type(x) == list else x)
plot_args = {
"x": "iteration",
"y": "Distance_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap,
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2,
"title": title,
"xlim": "tight",
"ylim": None,
"fontweight": "bold",
"facecolor": self.bg_color_instruments,
}
ax = fig.add_subplot(gridspec)
create_basic_plot(**plot_args, ax=ax)
ax2 = ax.twinx()
plot_args = {
"x": "iteration",
"y": "UpdateSize_all",
"data": self.tracking_data,
"y_scale": "linear",
"x_scale": "symlog" if self.show_log_iter else "linear",
"cmap": self.cmap.reversed(),
"EMA": "y",
"EMA_alpha": self.EMA_alpha,
"EMA_cmap": self.cmap2.reversed(),
"xlim": "tight",
"ylim": None,
"marker": ",",
}
create_basic_plot(**plot_args, ax=ax2)