def viz_weights_grads(model, layer_idx=1):
x, y = make_data(K.int_shape(model.input), model.layers[2].units)
kws = dict(layer_idx=layer_idx, input_data=x, labels=y)
rnn_histogram(model, mode='grads', bins=400, **kws)
print('\n')
rnn_heatmap(model, mode='grads', cmap=None, absolute_value=True, **kws)
def _test_weights_gradients(model):
x, y = make_data(K.int_shape(model.input), model.layers[2].units)
name = model.layers[1].name
kws = dict(input_data=x, labels=y, mode='grads')
rnn_histogram(model, layer_name=name, bins=100, **kws)
rnn_heatmap(model, layer_name=name, **kws)
def _test_weights_gradients(model):
x, y, _ = make_data(K.int_shape(model.input), model.layers[2].units)
name = model.layers[1].name
with tempdir() as dirpath:
kws = dict(input_data=x, labels=y, mode='grads')
if hasattr(model.layers[1], 'backward_layer'):
kws['savepath'] = dirpath
rnn_histogram(model, name, bins=100, **kws)
rnn_heatmap(model, name, **kws)
def test_misc(): # test miscellaneous functionalities
units = 6
batch_shape = (8, 100, 2*units)
reset_seeds(reset_graph_with_backend=K)
model = make_model(GRU, batch_shape, activation='relu', recurrent_dropout=0.3)
x, y = make_data(batch_shape, units)
model.train_on_batch(x, y)
grads = get_layer_gradients(model, x, y, layer_idx=1)
show_features_1D(grads, subplot_samples=True, tight=True, borderwidth=2)
show_features_1D(grads[0], subplot_samples=True)
show_features_2D(grads.T, n_rows=1.5, tight=True, borderwidth=2)
show_features_2D(grads.T[:, :, 0])
rnn_histogram(model, layer_idx=1, show_xy_ticks=[0, 0], equate_axes=2)
rnn_heatmap(model, layer_idx=1, cmap=None, normalize=True, show_borders=False)
rnn_heatmap(model, layer_idx=1, cmap=None, absolute_value=True)
rnn_heatmap(model, layer_idx=1, norm=None)
rnn_heatmap(model, layer_idx=1, norm=(-.004, .004))
from see_rnn.inspect_gen import get_layer, _detect_nans
get_layer(model, layer_name='gru')
get_rnn_weights(model, layer_idx=1, concat_gates=False, as_tensors=True)
rnn_heatmap(model, layer_idx=1, input_data=x, labels=y, mode='weights')
_test_prefetched_data(model)
# test NaN detection
nan_txt = _detect_nans(np.array([1]*9999 + [np.nan])).replace('\n', ' ')
print(nan_txt) # case: print as quantity
K.set_value(model.optimizer.lr, 1e12)
train_model(model, iterations=10)
rnn_histogram(model, layer_idx=1)
rnn_heatmap(model, layer_idx=1)
del model
reset_seeds(reset_graph_with_backend=K)
# test SimpleRNN & other
_model = make_model(SimpleRNN, batch_shape, units=128, use_bias=False)
train_model(_model, iterations=1) # TF2-Keras-Graph bug workaround
rnn_histogram(_model, layer_idx=1) # test _pretty_hist
K.set_value(_model.optimizer.lr, 1e50) # SimpleRNNs seem ridiculously robust
train_model(_model, iterations=20)
rnn_heatmap(_model, layer_idx=1)
data = get_rnn_weights(_model, layer_idx=1)
rnn_heatmap(_model, layer_idx=1, input_data=x, labels=y, data=data)
os.environ["TF_KERAS"] = '0'
get_rnn_weights(_model, layer_idx=1, concat_gates=False)
del _model
assert True
cprint("\n<< MISC TESTS PASSED >>\n", 'green')
def test_misc(): # test miscellaneous functionalities
units = 6
batch_shape = (8, 100, 2 * units)
reset_seeds(reset_graph_with_backend=K)
model = make_model(GRU,
batch_shape,
activation='relu',
recurrent_dropout=0.3,
IMPORTS=IMPORTS)
x, y, sw = make_data(batch_shape, units)
model.train_on_batch(x, y, sw)
weights_norm(model, 'gru', omit_names='bias', verbose=1)
weights_norm(model, ['gru', 1, (1, 1)], norm_fn=np.abs)
stats = weights_norm(model, 'gru')
weights_norm(model, 'gru', _dict=stats)
grads = get_gradients(model, 1, x, y)
get_gradients(model, 1, x, y, as_dict=True)
get_gradients(model, ['gru', 1], x, y)
get_outputs(model, ['gru', 1], x)
features_1D(grads,
subplot_samples=True,
tight=True,
borderwidth=2,
share_xy=False)
with tempdir() as dirpath:
features_0D(grads[0], savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_1D(grads[0],
subplot_samples=True,
annotations=[1, 'pi'],
savepath=os.path.join(dirpath, 'img.png'))
features_2D(grads.T, n_rows=1.5, tight=True, borderwidth=2)
with tempdir() as dirpath:
features_2D(grads.T[:, :, 0],
norm='auto',
savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_hist(grads,
show_borders=False,
borderwidth=1,
annotations=[0],
show_xy_ticks=[0, 0],
share_xy=(1, 1),
title="grads",
savepath=os.path.join(dirpath, 'img.png'))
with tempdir() as dirpath:
features_hist_v2(list(grads[:, :4, :3]),
colnames=list('abcd'),
show_borders=False,
xlims=(-.01, .01),
ylim=100,
borderwidth=1,
show_xy_ticks=[0, 0],
side_annot='row',
share_xy=True,
title="Grads",
savepath=os.path.join(dirpath, 'img.png'))
features_hist(grads, center_zero=True, xlims=(-1, 1), share_xy=0)
features_hist_v2(list(grads[:, :4, :3]),
center_zero=True,
xlims=(-1, 1),
share_xy=(False, False))
with tempdir() as dirpath:
rnn_histogram(model,
1,
show_xy_ticks=[0, 0],
equate_axes=2,
savepath=os.path.join(dirpath, 'img.png'))
rnn_histogram(model,
1,
equate_axes=False,
configs={
'tight': dict(left=0, right=1),
'plot': dict(color='red'),
'title': dict(fontsize=14),
})
rnn_heatmap(model, 1, cmap=None, normalize=True, show_borders=False)
rnn_heatmap(model, 1, cmap=None, norm='auto', absolute_value=True)
rnn_heatmap(model, 1, norm=None)
with tempdir() as dirpath:
rnn_heatmap(model,
1,
norm=(-.004, .004),
savepath=os.path.join(dirpath, 'img.png'))
hist_clipped(grads, peaks_to_clip=2)
_, ax = plt.subplots(1, 1)
hist_clipped(grads, peaks_to_clip=2, ax=ax, annot_kw=dict(fontsize=15))
get_full_name(model, 'gru')
get_full_name(model, 1)
pass_on_error(get_full_name, model, 'croc')
get_weights(model, 'gru', as_dict=False)
get_weights(model, 'gru', as_dict=True)
get_weights(model, 'gru/bias')
get_weights(model, ['gru', 1, (1, 1)])
pass_on_error(get_weights, model, 'gru/goo')
get_weights(model, '*')
get_gradients(model, '*', x, y)
get_outputs(model, '*', x)
from see_rnn.utils import _filter_duplicates_by_keys
keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4])
assert keys == ['a', 'b', 'c']
assert data == [1, 2, 4]
keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4],
[5, 6, 7, 8])
assert keys == ['a', 'b', 'c']
assert data[0] == [1, 2, 4] and data[1] == [5, 6, 8]
from see_rnn.inspect_gen import get_layer, detect_nans
get_layer(model, 'gru')
get_rnn_weights(model, 1, concat_gates=False, as_tensors=True)
rnn_heatmap(model, 1, input_data=x, labels=y, mode='weights')
_test_prefetched_data(model)
# test NaN/Inf detection
nan_txt = detect_nans(np.array([1] * 9999 + [np.nan])).replace('\n', ' ')
print(nan_txt) # case: print as quantity
data = np.array([np.nan, np.inf, -np.inf, 0])
print(detect_nans(data, include_inf=True))
print(detect_nans(data, include_inf=False))
data = np.array([np.inf, 0])
print(detect_nans(data, include_inf=True))
detect_nans(np.array([0]))
K.set_value(model.optimizer.lr, 1e12)
train_model(model, iterations=10)
rnn_histogram(model, 1)
rnn_heatmap(model, 1)
del model
reset_seeds(reset_graph_with_backend=K)
# test SimpleRNN & other
_model = make_model(SimpleRNN,
batch_shape,
units=128,
use_bias=False,
IMPORTS=IMPORTS)
train_model(_model, iterations=1) # TF2-Keras-Graph bug workaround
rnn_histogram(_model, 1) # test _pretty_hist
K.set_value(_model.optimizer.lr, 1e50) # force NaNs
train_model(_model, iterations=20)
rnn_heatmap(_model, 1)
data = get_rnn_weights(_model, 1)
rnn_heatmap(_model, 1, input_data=x, labels=y, data=data)
os.environ["TF_KERAS"] = '0'
get_rnn_weights(_model, 1, concat_gates=False)
del _model
assert True
cprint("\n<< MISC TESTS PASSED >>\n", 'green')
def _test_prefetched_data(model):
weights = get_rnn_weights(model, 1)
rnn_histogram(model, 1, data=weights)
rnn_heatmap(model, 1, data=weights)
def _test_weights(model):
name = model.layers[1].name
rnn_histogram(model, name, mode='weights', bins=100)
rnn_heatmap(model, name, mode='weights')
from see_rnn import rnn_histogram, rnn_heatmap
#%%# Case 1 ###################################################################
# We build a model prone to large but not exploding/vanishing gradients
C['model']['optimizer'] = Adam(10)
C['traingen']['epochs'] = 1
tg = init_session(C, make_classifier)
#%%# Train ####################################################################
tg.train()
#%%# Case 2 ###################################################################
# Now a model prone to exploding / vanishing gradients
from utils import TS_CONFIGS as C
from utils import make_timeseries_classifier
C['model']['activation'] = 'relu'
C['model']['optimizer'] = Adam(.3)
C['traingen']['epochs'] = 1
C['traingen']['eval_fn'] = 'predict'
C['traingen']['val_freq'] = {'epoch': 1}
tg = init_session(C, make_timeseries_classifier)
#%%# Train ####################################################################
tg.train()
#%%# Visualize ############################
rnn_histogram(tg.model, 1)
rnn_heatmap(tg.model, 1)
rnn_histogram(tg.model, 2)
rnn_heatmap(tg.model, 2)
def viz_weights(model, layer_idx=1):
rnn_histogram(model, layer_idx=layer_idx, mode='weights', bins=400)
print('\n')
rnn_heatmap(model, layer_idx=layer_idx, mode='weights', norm='auto')
def viz_prefetched_data(model, data, layer_idx=1):
rnn_histogram(model, layer_idx=layer_idx, data=data)
rnn_heatmap(model, layer_idx=layer_idx, data=data)
