def infer_train_hist(model, input_data, layer=None, keep_borders=True,
bins=100, xlims=None, fontsize=14, vline=None,
w=1, h=1):
"""Histograms of flattened layer output values in inference and train mode.
Useful for comparing effect of layers that behave differently in train vs
infrence modes (Dropout, BatchNormalization, etc) on model prediction (or
intermediate activations).
Arguments:
model: models.Model / models.Sequential (keras / tf.keras)
The model.
input_data: np.ndarray / list[np.ndarray]
Data to feed to `model` to fetch outputs. List of arrays for
multi-input networks.
layer: layers.Layer / None
Layer whose outputs to fetch; defaults to last layer (output).
keep_borders: bool
Whether to keep the plots' bounding box.
bins: int
Number of histogram bins; kwarg to `plt.hist()`
xlims: tuple[float, float] / None
Histogram x limits. Defaults to min/max of flattened data per plot.
fontsize: int
Title font size.
vline: float / None
x-coordinate of vertical line to draw (e.g. predict threshold).
w, h: float
Scale figure width & height, respectively.
"""
layer = layer or model.layers[-1]
outs = [get_outputs(model, '', input_data, layer, learning_phase=0),
get_outputs(model, '', input_data, layer, learning_phase=1)]
fig, axes = plt.subplots(2, 1, sharex=True, sharey=True,
figsize=(13 * w, 6 * h))
for i, (ax, out) in enumerate(zip(axes.flat, outs)):
out = np.asarray(out).ravel()
ax.hist(out, bins=bins)
mode = "ON" if i == 0 else "OFF"
ax.set_title("Train mode " + mode, weight='bold', fontsize=fontsize)
if not keep_borders:
ax.box(on=None)
if vline is not None:
ax.axvline(vline, color='r', linewidth=2)
_xlims = xlims or (out.min(), out.max())
ax.set_xlim(*_xlims)
scalefig(fig)
plt.show()
def test_multi_io():
def _make_multi_io_model():
ipt1 = Input((40, 8))
ipt2 = Input((40, 16))
ipts = concatenate([ipt1, ipt2])
out1 = GRU(6, return_sequences=True)(ipts)
out2 = GRU(12, return_sequences=True)(ipts)
model = Model([ipt1, ipt2], [out1, out2])
model.compile('adam', 'mse')
return model
def _make_multi_io_data():
x1 = np.random.randn(8, 40, 8)
x2 = np.random.randn(8, 40, 16)
y1 = np.random.randn(8, 40, 6)
y2 = np.random.randn(8, 40, 12)
return x1, x2, y1, y2
model = _make_multi_io_model()
x1, x2, y1, y2 = _make_multi_io_data()
grads = get_gradients(model, '*', [x1, x2], [y1, y2])
outs = get_outputs(model, '*', [x1, x2])
assert outs[0].shape == grads[0].shape == (8, 40, 24)
assert outs[1].shape == grads[1].shape == (8, 40, 6)
assert outs[2].shape == grads[2].shape == (8, 40, 12)
cprint("\n<< MULTI_IO TESTS PASSED >>\n", 'green')
def test_errors(): # test Exception cases
units = 6
batch_shape = (8, 100, 2 * units)
reset_seeds(reset_graph_with_backend=K)
model = make_model(GRU,
batch_shape,
activation='relu',
recurrent_dropout=0.3,
IMPORTS=IMPORTS)
x, y, sw = make_data(batch_shape, units)
model.train_on_batch(x, y, sw)
grads = get_gradients(model, 1, x, y)
grads_4D = np.expand_dims(grads, -1)
from see_rnn.inspect_gen import get_layer, _make_grads_fn
pass_on_error(features_0D, grads)
pass_on_error(features_0D, grads_4D)
pass_on_error(features_1D, grads_4D)
pass_on_error(features_2D, grads_4D)
pass_on_error(features_2D, grads)
pass_on_error(get_gradients, model, 1, x, y, mode='cactus')
pass_on_error(get_gradients, model, 1, x, y, layer=model.layers[1])
pass_on_error(_make_grads_fn, model, model.layers[1], mode='banana')
pass_on_error(features_hist, grads[:, :4, :3], po='tato')
pass_on_error(features_hist_v2, grads[:, :4, :3], po='tato')
pass_on_error(get_layer, model)
pass_on_error(get_layer, model, 'capsule')
pass_on_error(rnn_heatmap, model, 1, input_data=x, labels=y, mode='coffee')
pass_on_error(rnn_heatmap, model, 1, co='vid')
pass_on_error(rnn_heatmap, model, 1, norm=(0, 1, 2))
pass_on_error(rnn_heatmap, model, 1, mode='grads')
pass_on_error(rnn_histogram, model, 1, norm=None)
pass_on_error(rnn_heatmap, model, layer_index=9001)
pass_on_error(features_0D, grads, cake='lie')
pass_on_error(features_1D, grads, pup='not just any')
pass_on_error(features_2D, grads, true=False)
outs = list(get_outputs(model, 1, x, as_dict=True).values())
pass_on_error(rnn_histogram, model, 1, data=outs)
pass_on_error(rnn_histogram, model, 1, data=[1])
pass_on_error(rnn_histogram, model, 1, data=[[1]])
pass_on_error(features_hist, grads, co='vid')
pass_on_error(features_0D, grads, configs={'x': {}})
pass_on_error(features_1D, grads, configs={'x': {}})
pass_on_error(features_2D, grads, configs={'x': {}})
pass_on_error(features_hist, grads, configs={'x': {}})
pass_on_error(features_hist_v2, grads, configs={'x': {}})
cprint("\n<< EXCEPTION TESTS PASSED >>\n", 'green')
assert True
def _get_data(model, _id, mode, input_data, labels, omit_names, kw):
if mode == 'weights':
data = get_weights(model, _id, omit_names, as_dict=True)
elif 'gradients' in mode:
if mode in ('gradients', 'gradients:outputs'):
data = get_gradients(model, _id, input_data, labels,
mode='outputs', as_dict=True)
else:
data = get_gradients(model, _id, input_data, labels,
mode='weights', as_dict=True)
elif mode == 'outputs':
data = get_outputs(model, _id, input_data, as_dict=True)
data_flat = [x.ravel() for x in data.values()]
return data_flat, list(data)
def _test_outputs(model):
x, *_ = make_data(K.int_shape(model.input), model.layers[2].units)
outs = get_outputs(model, 1, x)
features_1D(outs[:1], show_y_zero=True)
features_1D(outs[0])
features_2D(outs)
def test_misc(): # test miscellaneous functionalities
units = 6
batch_shape = (8, 100, 2 * units)
reset_seeds(reset_graph_with_backend=K)
model = make_model(GRU,
batch_shape,
activation='relu',
recurrent_dropout=0.3,
IMPORTS=IMPORTS)
x, y, sw = make_data(batch_shape, units)
model.train_on_batch(x, y, sw)
weights_norm(model, 'gru', omit_names='bias', verbose=1)
weights_norm(model, ['gru', 1, (1, 1)], norm_fn=np.abs)
stats = weights_norm(model, 'gru')
weights_norm(model, 'gru', _dict=stats)
grads = get_gradients(model, 1, x, y)
get_gradients(model, 1, x, y, as_dict=True)
get_gradients(model, ['gru', 1], x, y)
get_outputs(model, ['gru', 1], x)
features_1D(grads,
subplot_samples=True,
tight=True,
borderwidth=2,
share_xy=False)
with tempdir() as dirpath:
features_0D(grads[0], savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_1D(grads[0],
subplot_samples=True,
annotations=[1, 'pi'],
savepath=os.path.join(dirpath, 'img.png'))
features_2D(grads.T, n_rows=1.5, tight=True, borderwidth=2)
with tempdir() as dirpath:
features_2D(grads.T[:, :, 0],
norm='auto',
savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_hist(grads,
show_borders=False,
borderwidth=1,
annotations=[0],
show_xy_ticks=[0, 0],
share_xy=(1, 1),
title="grads",
savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_hist_v2(list(grads[:, :4, :3]),
colnames=list('abcd'),
show_borders=False,
xlims=(-.01, .01),
ylim=100,
borderwidth=1,
show_xy_ticks=[0, 0],
side_annot='row',
share_xy=True,
title="Grads",
savepath=os.path.join(dirpath, 'img.png'))
features_hist(grads, center_zero=True, xlims=(-1, 1), share_xy=0)
features_hist_v2(list(grads[:, :4, :3]),
center_zero=True,
xlims=(-1, 1),
share_xy=(False, False))
with tempdir() as dirpath:
rnn_histogram(model,
1,
show_xy_ticks=[0, 0],
equate_axes=2,
savepath=os.path.join(dirpath, 'img.png'))
rnn_histogram(model,
1,
equate_axes=False,
configs={
'tight': dict(left=0, right=1),
'plot': dict(color='red'),
'title': dict(fontsize=14),
})
rnn_heatmap(model, 1, cmap=None, normalize=True, show_borders=False)
rnn_heatmap(model, 1, cmap=None, norm='auto', absolute_value=True)
rnn_heatmap(model, 1, norm=None)
with tempdir() as dirpath:
rnn_heatmap(model,
1,
norm=(-.004, .004),
savepath=os.path.join(dirpath, 'img.png'))
hist_clipped(grads, peaks_to_clip=2)
_, ax = plt.subplots(1, 1)
hist_clipped(grads, peaks_to_clip=2, ax=ax, annot_kw=dict(fontsize=15))
get_full_name(model, 'gru')
get_full_name(model, 1)
pass_on_error(get_full_name, model, 'croc')
get_weights(model, 'gru', as_dict=False)
get_weights(model, 'gru', as_dict=True)
get_weights(model, 'gru/bias')
get_weights(model, ['gru', 1, (1, 1)])
pass_on_error(get_weights, model, 'gru/goo')
get_weights(model, '*')
get_gradients(model, '*', x, y)
get_outputs(model, '*', x)
from see_rnn.utils import _filter_duplicates_by_keys
keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4])
assert keys == ['a', 'b', 'c']
assert data == [1, 2, 4]
keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4],
[5, 6, 7, 8])
assert keys == ['a', 'b', 'c']
assert data[0] == [1, 2, 4] and data[1] == [5, 6, 8]
from see_rnn.inspect_gen import get_layer, detect_nans
get_layer(model, 'gru')
get_rnn_weights(model, 1, concat_gates=False, as_tensors=True)
rnn_heatmap(model, 1, input_data=x, labels=y, mode='weights')
_test_prefetched_data(model)
# test NaN/Inf detection
nan_txt = detect_nans(np.array([1] * 9999 + [np.nan])).replace('\n', ' ')
print(nan_txt) # case: print as quantity
data = np.array([np.nan, np.inf, -np.inf, 0])
print(detect_nans(data, include_inf=True))
print(detect_nans(data, include_inf=False))
data = np.array([np.inf, 0])
print(detect_nans(data, include_inf=True))
detect_nans(np.array([0]))
K.set_value(model.optimizer.lr, 1e12)
train_model(model, iterations=10)
rnn_histogram(model, 1)
rnn_heatmap(model, 1)
del model
reset_seeds(reset_graph_with_backend=K)
# test SimpleRNN & other
_model = make_model(SimpleRNN,
batch_shape,
units=128,
use_bias=False,
IMPORTS=IMPORTS)
train_model(_model, iterations=1) # TF2-Keras-Graph bug workaround
rnn_histogram(_model, 1) # test _pretty_hist
K.set_value(_model.optimizer.lr, 1e50) # force NaNs
train_model(_model, iterations=20)
rnn_heatmap(_model, 1)
data = get_rnn_weights(_model, 1)
rnn_heatmap(_model, 1, input_data=x, labels=y, data=data)
os.environ["TF_KERAS"] = '0'
get_rnn_weights(_model, 1, concat_gates=False)
del _model
assert True
cprint("\n<< MISC TESTS PASSED >>\n", 'green')
def viz_outs(model, idx=1):
x, y, _ = make_data(K.int_shape(model.input), model.layers[2].units)
outs = get_outputs(model, idx, x)
features_1D(outs[:1], n_rows=8, show_borders=False)
features_2D(outs, n_rows=8, norm=(-1, 1))
