def plot_lr(self,
show_moms=False,
skip_start: int = 0,
skip_end: int = 0,
return_fig: bool = None) -> Optional[plt.Figure]:
"Plot learning rate, `show_moms` to include momentum."
iterations = range_of(self.lrs)
lrs = self.lrs[skip_start:-skip_end] if skip_end > 0 else self.lrs[
skip_start:]
iterations = iterations[
skip_start:-skip_end] if skip_end > 0 else iterations[skip_start:]
if show_moms:
moms = self.moms[
skip_start:-skip_end] if skip_end > 0 else self.moms[
skip_start:]
fig, axs = plt.subplots(1, 2, figsize=(12, 4))
axs[0].plot(iterations, lrs)
axs[0].set_xlabel('Iterations')
axs[0].set_ylabel('Learning Rate')
axs[1].plot(iterations, moms)
axs[1].set_xlabel('Iterations')
axs[1].set_ylabel('Momentum')
else:
fig, ax = plt.subplots()
ax.plot(iterations, lrs)
ax.set_xlabel('Iterations')
ax.set_ylabel('Learning Rate')
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot(self, skip_start:int=10, skip_end:int=5, suggestion:bool=False, return_fig:bool=None,
**kwargs)->Optional[plt.Figure]:
"Plot learning rate and losses, trimmed between `skip_start` and `skip_end`. Optionally plot and return min gradient"
lrs = self._split_list(self.lrs, skip_start, skip_end)
losses = self._split_list(self.losses, skip_start, skip_end)
losses = [x.item() for x in losses]
if 'k' in kwargs: losses = self.smoothen_by_spline(lrs, losses, **kwargs)
fig, ax = plt.subplots(1,1)
ax.plot(lrs, losses)
ax.set_ylabel("Loss")
ax.set_xlabel("Learning Rate")
ax.set_xscale('log')
ax.xaxis.set_major_formatter(plt.FormatStrFormatter('%.0e'))
if suggestion:
try: mg = (np.gradient(np.array(losses))).argmin()
except:
print("Failed to compute the gradients, there might not be enough points.")
return
print(f"Min numerical gradient: {lrs[mg]:.2E}")
ax.plot(lrs[mg],losses[mg],markersize=10,marker='o',color='red')
self.min_grad_lr = lrs[mg]
ml = np.argmin(losses)
print(f"Min loss divided by 10: {lrs[ml]/10:.2E}")
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_losses(self,
skip_start: int = 0,
skip_end: int = 0,
return_fig: bool = None) -> Optional[plt.Figure]:
"Plot training and validation losses."
fig, ax = plt.subplots(1, 1)
iterations = range_of(self.losses)
losses = self.losses[
skip_start:-skip_end] if skip_end > 0 else self.losses[skip_start:]
iterations = iterations[
skip_start:-skip_end] if skip_end > 0 else iterations[skip_start:]
ax.plot(iterations, losses, label='Train')
val_iter = np.cumsum(self.nb_batches)
start_val = (val_iter - skip_start >= 0).nonzero()[0].min()
end_val = (val_iter[-1] - val_iter - skip_end >=
0).nonzero()[0].max() + 1
val_iter = val_iter[start_val:end_val] if skip_end > 0 else val_iter[
start_val:]
val_losses = self.val_losses[
start_val:end_val] if skip_end > 0 else self.val_losses[start_val:]
ax.plot(val_iter, val_losses, label='Validation')
ax.set_ylabel('Loss')
ax.set_xlabel('Batches processed')
ax.legend()
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_rewards(self,
per_episode=False,
return_fig: bool = None,
group_name=None,
cumulative=False,
no_show=False,
smooth_const: Union[None, float] = None,
**kwargs):
values = self.get_values(self.ds.x, 'reward', per_episode)
processed_values = cumulate_squash(values,
squash_episodes=per_episode,
cumulative=cumulative,
**kwargs)
if group_name:
self.groups.append(
GroupField(processed_values, self.learn.model.name, group_name,
'reward', per_episode))
if no_show: return
if smooth_const:
processed_values = smooth(processed_values, smooth_const)
fig = self.line_figure(processed_values,
per_episode=per_episode,
cumulative=cumulative,
**kwargs)
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_metrics(self, skip_start:int=0, skip_end:int=0, return_fig:bool=None)->Optional[plt.Figure]:
"Plot metrics collected during training."
assert len(self.metrics) != 0, "There are no metrics to plot."
fig, axes = plt.subplots(len(self.metrics[0]),1,figsize=(6, 4*len(self.metrics[0])))
val_iter = self._split_list_val(np.cumsum(self.nb_batches), skip_start, skip_end)
axes = axes.flatten() if len(self.metrics[0]) != 1 else [axes]
for i, ax in enumerate(axes):
values = [met[i] for met in self.metrics]
values = self._split_list_val(values, skip_start, skip_end)
ax.plot(val_iter, values)
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_reward_bounds(self,
title=None,
return_fig: bool = None,
per_episode=False,
smooth_groups: Union[None, float] = None,
figsize=(5, 5),
show_average=False,
hide_edges=False):
groups = self.filter_by(per_episode, 'reward')
if smooth_groups is not None: [g.smooth(smooth_groups) for g in groups]
unique_values = list(set([g.unique_tuple for g in groups]))
colors = list(
islice(cycle(plt.rcParams['axes.prop_cycle'].by_key()['color']),
len(unique_values)))
fig, ax = plt.subplots(1, 1, figsize=figsize) # type: Figure, Axes
for grouped, c in zip(self.group_by(groups, unique_values), colors):
min_len = min([len(v.values) for v in grouped])
min_b = np.min([v.values[:min_len] for v in grouped], axis=0)
max_b = np.max([v.values[:min_len] for v in grouped], axis=0)
if show_average:
average = np.average([v.values[:min_len] for v in grouped],
axis=0)
ax.plot(average, c=c, linestyle=':')
# TODO fit function sometimes does +1 more episodes... WHY?
overflow = [
v.values for v in grouped if len(v.values) - min_len > 2
]
if not hide_edges:
ax.plot(min_b, c=c)
ax.plot(max_b, c=c)
for v in overflow:
ax.plot(v, c=c)
ax.fill_between(list(range(min_len)),
min_b,
max_b,
where=max_b > min_b,
color=c,
alpha=0.3,
label=f'{grouped[0].meta} {grouped[0].model}')
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
ax.set_title(
ifnone(title,
f'{"Per Episode" if per_episode else "Per Step"} Rewards'))
ax.set_ylabel('Rewards')
ax.set_xlabel(f'{"Episodes" if per_episode else "Steps"}')
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_losses(self, skip_start:int=0, skip_end:int=0, return_fig:bool=None)->Optional[plt.Figure]:
"Plot training and validation losses."
fig, ax = plt.subplots(1,1)
losses = self._split_list(self.losses, skip_start, skip_end)
iterations = self._split_list(range_of(self.losses), skip_start, skip_end)
ax.plot(iterations, losses, label='Train')
val_iter = self._split_list_val(np.cumsum(self.nb_batches), skip_start, skip_end)
val_losses = self._split_list_val(self.val_losses, skip_start, skip_end)
ax.plot(val_iter, val_losses, label='Validation')
ax.set_ylabel('Loss')
ax.set_xlabel('Batches processed')
ax.legend()
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_heat_map(self, action=-1, figsize=(7, 7), return_fig=None):
exploring = self.learn.exploration_method.explore
self.learn.exploration_method.explore = False
heat_map, chosen_actions = self.heat_map(action)
fig, ax = plt.subplots(1, 1, figsize=figsize) # type: Figure, Axes
im = ax.imshow(heat_map)
fig.colorbar(im)
title = f'Heat mapped values {"maximum" if action == -1 else "for action " + str(action)}'
title += '\nText: Chosen action for a given state'
ax.set_title(title)
self.add_text_to_image(ax, chosen_actions)
self.learn.exploration_method.explore = exploring
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_metrics(self, skip_start:int=0, skip_end:int=0, return_fig:bool=None)->Optional[plt.Figure]:
"Plot metrics collected during training."
assert len(self.metrics) != 0, "There are no metrics to plot."
fig, axes = plt.subplots(len(self.metrics[0]),1,figsize=(6, 4*len(self.metrics[0])))
val_iter = np.cumsum(self.nb_batches)
start_val = (val_iter - skip_start >= 0).nonzero()[0].min()
end_val = (val_iter[-1] - val_iter - skip_end >= 0).nonzero()[0].max()+1
val_iter = val_iter[start_val:end_val] if skip_end > 0 else val_iter[start_val:]
axes = axes.flatten() if len(self.metrics[0]) != 1 else [axes]
for i, ax in enumerate(axes):
values = [met[i] for met in self.metrics]
values = values[start_val:end_val] if skip_end > 0 else values[start_val:]
ax.plot(val_iter, values)
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
def plot_q(self, figsize=(8, 8), return_fig=None):
r"""
Heat maps the density of actual vs estimated q v. Good reference for this is at [1].
References:
[1] "Simple Example Of 2D Density Plots In Python." Medium. N. p., 2019. Web. 31 Aug. 2019.
https://towardsdatascience.com/simple-example-of-2d-density-plots-in-python-83b83b934f67
Returns:
"""
q_action, q_predicted = self.q(self.items)
# Define the borders
deltaX = (np.max(q_action) - np.min(q_action)) / 10
deltaY = (np.max(q_predicted) - np.min(q_predicted)) / 10
xmin = np.min(q_action) - deltaX
xmax = np.max(q_action) + deltaX
ymin = np.min(q_predicted) - deltaY
ymax = np.max(q_predicted) + deltaY
# Create meshgrid
xx, yy = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]
positions = np.vstack([xx.ravel(), yy.ravel()])
values = np.vstack([q_action, q_predicted])
kernel = st.gaussian_kde(values)
f = np.reshape(kernel(positions).T, xx.shape)
fig = plt.figure(figsize=figsize)
ax = fig.gca()
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
cfset = ax.contourf(xx, yy, f, cmap='coolwarm')
ax.imshow(np.rot90(f),
cmap='coolwarm',
extent=[xmin, xmax, ymin, ymax])
cset = ax.contour(xx, yy, f, colors='k')
ax.clabel(cset, inline=1, fontsize=10)
ax.set_xlabel('Actual Returns')
ax.set_ylabel('Estimated Q')
ax.set_title('2D Gaussian Kernel Q Density Estimation')
if ifnone(return_fig, defaults.return_fig): return fig
if not IN_NOTEBOOK: plot_sixel(fig)
