def _test_outputs_gradients(model):
x, y, _ = make_data(K.int_shape(model.input), model.layers[2].units)
name = model.layers[1].name
grads_all = get_gradients(model, name, x, y, mode='outputs')
grads_last = get_gradients(model, 2, x, y, mode='outputs')
kwargs1 = dict(n_rows=None,
show_xy_ticks=[0, 0],
show_borders=True,
max_timesteps=50,
title_mode='grads')
kwargs2 = dict(n_rows=2,
show_xy_ticks=[1, 1],
show_borders=False,
max_timesteps=None)
features_1D(grads_all[0], **kwargs1)
features_1D(grads_all[:1], **kwargs1)
features_1D(grads_all, **kwargs2)
features_2D(grads_all[0], norm=(-.01, .01), show_colorbar=True, **kwargs1)
features_2D(grads_all, norm=None, reflect_half=True, **kwargs2)
features_0D(grads_last, marker='o', color=None, title_mode='grads')
features_0D(grads_last, marker='x', color='blue', ylims=(-.1, .1))
features_hist(grads_all, bins=100, xlims=(-.01, .01), title="Outs hists")
features_hist(grads_all, bins=100, n_rows=4)
print('\n') # improve separation
def _plot(grads_sum, plot_kw):
defaults = {'share_xy': False, 'center_zero': True}
for k, v in defaults.items():
if k not in plot_kw:
plot_kw[k] = v
data = list(grads_sum.values())
features_hist(data, annotations=list(grads_sum), **plot_kw)
def test_misc(): # test miscellaneous functionalities
units = 6
batch_shape = (8, 100, 2 * units)
reset_seeds(reset_graph_with_backend=K)
model = make_model(GRU,
batch_shape,
activation='relu',
recurrent_dropout=0.3,
IMPORTS=IMPORTS)
x, y, sw = make_data(batch_shape, units)
model.train_on_batch(x, y, sw)
weights_norm(model, 'gru', omit_names='bias', verbose=1)
weights_norm(model, ['gru', 1, (1, 1)], norm_fn=np.abs)
stats = weights_norm(model, 'gru')
weights_norm(model, 'gru', _dict=stats)
grads = get_gradients(model, 1, x, y)
get_gradients(model, 1, x, y, as_dict=True)
get_gradients(model, ['gru', 1], x, y)
get_outputs(model, ['gru', 1], x)
features_1D(grads,
subplot_samples=True,
tight=True,
borderwidth=2,
share_xy=False)
with tempdir() as dirpath:
features_0D(grads[0], savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_1D(grads[0],
subplot_samples=True,
annotations=[1, 'pi'],
savepath=os.path.join(dirpath, 'img.png'))
features_2D(grads.T, n_rows=1.5, tight=True, borderwidth=2)
with tempdir() as dirpath:
features_2D(grads.T[:, :, 0],
norm='auto',
savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_hist(grads,
show_borders=False,
borderwidth=1,
annotations=[0],
show_xy_ticks=[0, 0],
share_xy=(1, 1),
title="grads",
savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_hist_v2(list(grads[:, :4, :3]),
colnames=list('abcd'),
show_borders=False,
xlims=(-.01, .01),
ylim=100,
borderwidth=1,
show_xy_ticks=[0, 0],
side_annot='row',
share_xy=True,
title="Grads",
savepath=os.path.join(dirpath, 'img.png'))
features_hist(grads, center_zero=True, xlims=(-1, 1), share_xy=0)
features_hist_v2(list(grads[:, :4, :3]),
center_zero=True,
xlims=(-1, 1),
share_xy=(False, False))
with tempdir() as dirpath:
rnn_histogram(model,
1,
show_xy_ticks=[0, 0],
equate_axes=2,
savepath=os.path.join(dirpath, 'img.png'))
rnn_histogram(model,
1,
equate_axes=False,
configs={
'tight': dict(left=0, right=1),
'plot': dict(color='red'),
'title': dict(fontsize=14),
})
rnn_heatmap(model, 1, cmap=None, normalize=True, show_borders=False)
rnn_heatmap(model, 1, cmap=None, norm='auto', absolute_value=True)
rnn_heatmap(model, 1, norm=None)
with tempdir() as dirpath:
rnn_heatmap(model,
1,
norm=(-.004, .004),
savepath=os.path.join(dirpath, 'img.png'))
hist_clipped(grads, peaks_to_clip=2)
_, ax = plt.subplots(1, 1)
hist_clipped(grads, peaks_to_clip=2, ax=ax, annot_kw=dict(fontsize=15))
get_full_name(model, 'gru')
get_full_name(model, 1)
pass_on_error(get_full_name, model, 'croc')
get_weights(model, 'gru', as_dict=False)
get_weights(model, 'gru', as_dict=True)
get_weights(model, 'gru/bias')
get_weights(model, ['gru', 1, (1, 1)])
pass_on_error(get_weights, model, 'gru/goo')
get_weights(model, '*')
get_gradients(model, '*', x, y)
get_outputs(model, '*', x)
from see_rnn.utils import _filter_duplicates_by_keys
keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4])
assert keys == ['a', 'b', 'c']
assert data == [1, 2, 4]
keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4],
[5, 6, 7, 8])
assert keys == ['a', 'b', 'c']
assert data[0] == [1, 2, 4] and data[1] == [5, 6, 8]
from see_rnn.inspect_gen import get_layer, detect_nans
get_layer(model, 'gru')
get_rnn_weights(model, 1, concat_gates=False, as_tensors=True)
rnn_heatmap(model, 1, input_data=x, labels=y, mode='weights')
_test_prefetched_data(model)
# test NaN/Inf detection
nan_txt = detect_nans(np.array([1] * 9999 + [np.nan])).replace('\n', ' ')
print(nan_txt) # case: print as quantity
data = np.array([np.nan, np.inf, -np.inf, 0])
print(detect_nans(data, include_inf=True))
print(detect_nans(data, include_inf=False))
data = np.array([np.inf, 0])
print(detect_nans(data, include_inf=True))
detect_nans(np.array([0]))
K.set_value(model.optimizer.lr, 1e12)
train_model(model, iterations=10)
rnn_histogram(model, 1)
rnn_heatmap(model, 1)
del model
reset_seeds(reset_graph_with_backend=K)
# test SimpleRNN & other
_model = make_model(SimpleRNN,
batch_shape,
units=128,
use_bias=False,
IMPORTS=IMPORTS)
train_model(_model, iterations=1) # TF2-Keras-Graph bug workaround
rnn_histogram(_model, 1) # test _pretty_hist
K.set_value(_model.optimizer.lr, 1e50) # force NaNs
train_model(_model, iterations=20)
rnn_heatmap(_model, 1)
data = get_rnn_weights(_model, 1)
rnn_heatmap(_model, 1, input_data=x, labels=y, data=data)
os.environ["TF_KERAS"] = '0'
get_rnn_weights(_model, 1, concat_gates=False)
del _model
assert True
cprint("\n<< MISC TESTS PASSED >>\n", 'green')
def layer_hists(model, _id='*', mode='weights', input_data=None, labels=None,
omit_names='bias', share_xy=(0, 0), configs=None, **kw):
"""Histogram grid of layer weights, outputs, or gradients.
Arguments:
model: models.Model / models.Sequential (keras / tf.keras)
The model.
_id: str / int / list[str/int].
- int -> idx; str -> name
- idx: int. Index of layer to fetch, via model.layers[idx].
- name: str. Name of layer (full or substring) to be fetched.
Returns earliest match if multiple found.
- list[str/int] -> treat each str element as name, int as idx.
Ex: `['gru', 2]` gets (e.g.) weights of first layer with name
substring 'gru', then of layer w/ idx 2.
- `'*'` (wildcard) -> get (e.g.) outputs of all layers (except input)
with 'output' attribute.
mode: str in ('weights', 'outputs', 'gradients:weights',\
'gradients:outputs')
Whether to fetch layer weights, outputs, or gradients (w.r.t.
outputs or weights).
input_data: np.ndarray / list[np.ndarray] / None
Data to feed to `model` to fetch outputs / gradients. List of arrays
for multi-input networks. Ignored for `mode='weights'`.
labels: np.ndarray / list[np.ndarray]
Labels to feed to `model` to fetch gradients. List of arrays
for multi-output networks.
Ignored for `mode in ('weights', 'outputs')`.
omit_names: str / list[str] / tuple[str]
Names of weights to omit for `_id` specifying layer names. E.g.
for `Dense`, `omit_names='bias'` will fetch only kernel weights.
Ignored for `mode != 'weights'`.
share_xy: tuple[bool, bool] / tuple[str, str]
Whether to share x or y limits in histogram grid, respectively.
kwarg to `plt.subplots()`; can be `'col'` or `'row'` for sharing
along rows or columns, respectively.
configs: dict / None
kwargs to customize various plot schemes:
- `'plot'`: passed partly to `ax.hist()` in `see_rnn.hist_clipped()`;
include `peaks_to_clip` to adjust ylims with a
number of peaks disregarded. See `help(see_rnn.hist_clipped)`.
ax = subplots axis
- `'subplot'`: passed to `plt.subplots()`
- `'title'`: passed to `fig.suptitle()`; fig = subplots figure
- `'tight'`: passed to `fig.subplots_adjust()`
- `'annot'`: passed to `ax.annotate()`
- `'save'`: passed to `fig.savefig()` if `savepath` is not None.
kw: dict / kwargs
kwargs passed to `see_rnn.features_hist`.
"""
def _process_configs(configs, mode):
defaults = {}
if 'gradients' in mode or mode == 'outputs':
defaults.update({'plot': dict(peaks_to_clip=2, annot_kw=None),
'subplot': dict(sharex=False, sharey=False)})
if not configs:
return defaults
for name, _dict in defaults.items():
if name not in configs:
configs[name] = _dict
else:
for k, v in _dict.items():
if k not in configs[name]:
configs[name][k] = v
return configs
def _prevalidate(mode, input_data, labels):
supported = ('weights', 'outputs', 'gradients',
'gradients:outputs', 'gradients:weights')
if mode not in supported:
raise ValueError(("unsupported `mode`: '{}'; supported are: {}"
).format(mode, ', '.join(supported)))
if 'gradients' in mode and (input_data is None or labels is None):
raise ValueError("`input_data` or `labels` cannot be None for "
"'gradients'-based `mode`")
if mode == 'outputs' and input_data is None:
raise ValueError("`input_data` cannot be None for `mode='outputs'`")
def _get_data(model, _id, mode, input_data, labels, omit_names, kw):
if mode == 'weights':
data = get_weights(model, _id, omit_names, as_dict=True)
elif 'gradients' in mode:
if mode in ('gradients', 'gradients:outputs'):
data = get_gradients(model, _id, input_data, labels,
mode='outputs', as_dict=True)
else:
data = get_gradients(model, _id, input_data, labels,
mode='weights', as_dict=True)
elif mode == 'outputs':
data = get_outputs(model, _id, input_data, as_dict=True)
data_flat = [x.ravel() for x in data.values()]
return data_flat, list(data)
configs = _process_configs(configs, mode)
_prevalidate(mode, input_data, labels)
data_flat, data_names = _get_data(model, _id, mode, input_data, labels,
omit_names, kw)
features_hist(data_flat, annotations=data_names, configs=configs,
share_xy=share_xy, **kw)