def test_envs(): # pseudo-tests for coverage for different env flags reset_seeds(reset_graph_with_backend=K) units = 6 batch_shape = (8, 100, 2 * units) x, y, sw = make_data(batch_shape, units) from importlib import reload from see_rnn import inspect_gen, inspect_rnn, utils, _backend for flag in ['1', '0']: os.environ["TF_KERAS"] = flag TF_KERAS = os.environ["TF_KERAS"] == '1' reload(_backend) reload(utils) reload(inspect_gen) reload(inspect_rnn) from see_rnn.inspect_gen import get_gradients as glg from see_rnn.inspect_rnn import rnn_summary as rs from see_rnn.utils import _validate_rnn_type as _vrt reset_seeds(reset_graph_with_backend=K) if TF_KERAS: from tensorflow.keras.layers import Input, Bidirectional from tensorflow.keras.layers import GRU as _GRU from tensorflow.keras.models import Model import tensorflow.keras.backend as _K else: from keras.layers import Input, Bidirectional from keras.layers import GRU as _GRU from keras.models import Model import keras.backend as _K reset_seeds(reset_graph_with_backend=_K) new_imports = dict(Input=Input, Bidirectional=Bidirectional, Model=Model) model = make_model(_GRU, batch_shape, new_imports=new_imports, IMPORTS=IMPORTS) pass_on_error(model, x, y, 1) # possibly _backend-induced err pass_on_error(glg, model, 1, x, y) rs(model.layers[1]) from see_rnn.inspect_rnn import get_rnn_weights as grw grw(model, 1, concat_gates=False, as_tensors=True) grw(model, 1, concat_gates=False, as_tensors=False) _test_outputs(model) setattr(model.layers[2].cell, 'get_weights', None) get_rnn_weights(model, 2, concat_gates=True, as_tensors=False) _model = _make_nonrnn_model() pass_on_error(_vrt, _model.layers[1]) del model, _model assert True cprint("\n<< ENV TESTS PASSED >>\n", 'green')
def test_misc(): # test miscellaneous functionalities units = 6 batch_shape = (8, 100, 2*units) reset_seeds(reset_graph_with_backend=K) model = make_model(GRU, batch_shape, activation='relu', recurrent_dropout=0.3) x, y = make_data(batch_shape, units) model.train_on_batch(x, y) grads = get_layer_gradients(model, x, y, layer_idx=1) show_features_1D(grads, subplot_samples=True, tight=True, borderwidth=2) show_features_1D(grads[0], subplot_samples=True) show_features_2D(grads.T, n_rows=1.5, tight=True, borderwidth=2) show_features_2D(grads.T[:, :, 0]) rnn_histogram(model, layer_idx=1, show_xy_ticks=[0, 0], equate_axes=2) rnn_heatmap(model, layer_idx=1, cmap=None, normalize=True, show_borders=False) rnn_heatmap(model, layer_idx=1, cmap=None, absolute_value=True) rnn_heatmap(model, layer_idx=1, norm=None) rnn_heatmap(model, layer_idx=1, norm=(-.004, .004)) from see_rnn.inspect_gen import get_layer, _detect_nans get_layer(model, layer_name='gru') get_rnn_weights(model, layer_idx=1, concat_gates=False, as_tensors=True) rnn_heatmap(model, layer_idx=1, input_data=x, labels=y, mode='weights') _test_prefetched_data(model) # test NaN detection nan_txt = _detect_nans(np.array([1]*9999 + [np.nan])).replace('\n', ' ') print(nan_txt) # case: print as quantity K.set_value(model.optimizer.lr, 1e12) train_model(model, iterations=10) rnn_histogram(model, layer_idx=1) rnn_heatmap(model, layer_idx=1) del model reset_seeds(reset_graph_with_backend=K) # test SimpleRNN & other _model = make_model(SimpleRNN, batch_shape, units=128, use_bias=False) train_model(_model, iterations=1) # TF2-Keras-Graph bug workaround rnn_histogram(_model, layer_idx=1) # test _pretty_hist K.set_value(_model.optimizer.lr, 1e50) # SimpleRNNs seem ridiculously robust train_model(_model, iterations=20) rnn_heatmap(_model, layer_idx=1) data = get_rnn_weights(_model, layer_idx=1) rnn_heatmap(_model, layer_idx=1, input_data=x, labels=y, data=data) os.environ["TF_KERAS"] = '0' get_rnn_weights(_model, layer_idx=1, concat_gates=False) del _model assert True cprint("\n<< MISC TESTS PASSED >>\n", 'green')
def test_misc(): # test miscellaneous functionalities units = 6 batch_shape = (8, 100, 2 * units) reset_seeds(reset_graph_with_backend=K) model = make_model(GRU, batch_shape, activation='relu', recurrent_dropout=0.3, IMPORTS=IMPORTS) x, y, sw = make_data(batch_shape, units) model.train_on_batch(x, y, sw) weights_norm(model, 'gru', omit_names='bias', verbose=1) weights_norm(model, ['gru', 1, (1, 1)], norm_fn=np.abs) stats = weights_norm(model, 'gru') weights_norm(model, 'gru', _dict=stats) grads = get_gradients(model, 1, x, y) get_gradients(model, 1, x, y, as_dict=True) get_gradients(model, ['gru', 1], x, y) get_outputs(model, ['gru', 1], x) features_1D(grads, subplot_samples=True, tight=True, borderwidth=2, share_xy=False) with tempdir() as dirpath: features_0D(grads[0], savepath=os.path.join(dirpath, 'img.png')) with tempdir() as dirpath: features_1D(grads[0], subplot_samples=True, annotations=[1, 'pi'], savepath=os.path.join(dirpath, 'img.png')) features_2D(grads.T, n_rows=1.5, tight=True, borderwidth=2) with tempdir() as dirpath: features_2D(grads.T[:, :, 0], norm='auto', savepath=os.path.join(dirpath, 'img.png')) with tempdir() as dirpath: features_hist(grads, show_borders=False, borderwidth=1, annotations=[0], show_xy_ticks=[0, 0], share_xy=(1, 1), title="grads", savepath=os.path.join(dirpath, 'img.png')) with tempdir() as dirpath: features_hist_v2(list(grads[:, :4, :3]), colnames=list('abcd'), show_borders=False, xlims=(-.01, .01), ylim=100, borderwidth=1, show_xy_ticks=[0, 0], side_annot='row', share_xy=True, title="Grads", savepath=os.path.join(dirpath, 'img.png')) features_hist(grads, center_zero=True, xlims=(-1, 1), share_xy=0) features_hist_v2(list(grads[:, :4, :3]), center_zero=True, xlims=(-1, 1), share_xy=(False, False)) with tempdir() as dirpath: rnn_histogram(model, 1, show_xy_ticks=[0, 0], equate_axes=2, savepath=os.path.join(dirpath, 'img.png')) rnn_histogram(model, 1, equate_axes=False, configs={ 'tight': dict(left=0, right=1), 'plot': dict(color='red'), 'title': dict(fontsize=14), }) rnn_heatmap(model, 1, cmap=None, normalize=True, show_borders=False) rnn_heatmap(model, 1, cmap=None, norm='auto', absolute_value=True) rnn_heatmap(model, 1, norm=None) with tempdir() as dirpath: rnn_heatmap(model, 1, norm=(-.004, .004), savepath=os.path.join(dirpath, 'img.png')) hist_clipped(grads, peaks_to_clip=2) _, ax = plt.subplots(1, 1) hist_clipped(grads, peaks_to_clip=2, ax=ax, annot_kw=dict(fontsize=15)) get_full_name(model, 'gru') get_full_name(model, 1) pass_on_error(get_full_name, model, 'croc') get_weights(model, 'gru', as_dict=False) get_weights(model, 'gru', as_dict=True) get_weights(model, 'gru/bias') get_weights(model, ['gru', 1, (1, 1)]) pass_on_error(get_weights, model, 'gru/goo') get_weights(model, '*') get_gradients(model, '*', x, y) get_outputs(model, '*', x) from see_rnn.utils import _filter_duplicates_by_keys keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4]) assert keys == ['a', 'b', 'c'] assert data == [1, 2, 4] keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4], [5, 6, 7, 8]) assert keys == ['a', 'b', 'c'] assert data[0] == [1, 2, 4] and data[1] == [5, 6, 8] from see_rnn.inspect_gen import get_layer, detect_nans get_layer(model, 'gru') get_rnn_weights(model, 1, concat_gates=False, as_tensors=True) rnn_heatmap(model, 1, input_data=x, labels=y, mode='weights') _test_prefetched_data(model) # test NaN/Inf detection nan_txt = detect_nans(np.array([1] * 9999 + [np.nan])).replace('\n', ' ') print(nan_txt) # case: print as quantity data = np.array([np.nan, np.inf, -np.inf, 0]) print(detect_nans(data, include_inf=True)) print(detect_nans(data, include_inf=False)) data = np.array([np.inf, 0]) print(detect_nans(data, include_inf=True)) detect_nans(np.array([0])) K.set_value(model.optimizer.lr, 1e12) train_model(model, iterations=10) rnn_histogram(model, 1) rnn_heatmap(model, 1) del model reset_seeds(reset_graph_with_backend=K) # test SimpleRNN & other _model = make_model(SimpleRNN, batch_shape, units=128, use_bias=False, IMPORTS=IMPORTS) train_model(_model, iterations=1) # TF2-Keras-Graph bug workaround rnn_histogram(_model, 1) # test _pretty_hist K.set_value(_model.optimizer.lr, 1e50) # force NaNs train_model(_model, iterations=20) rnn_heatmap(_model, 1) data = get_rnn_weights(_model, 1) rnn_heatmap(_model, 1, input_data=x, labels=y, data=data) os.environ["TF_KERAS"] = '0' get_rnn_weights(_model, 1, concat_gates=False) del _model assert True cprint("\n<< MISC TESTS PASSED >>\n", 'green')
def _test_prefetched_data(model): weights = get_rnn_weights(model, 1) rnn_histogram(model, 1, data=weights) rnn_heatmap(model, 1, data=weights)
def viz_weights_grads(model, layer_idx=1): x, y = make_data(K.int_shape(model.input), model.layers[2].units) kws = dict(layer_idx=layer_idx, input_data=x, labels=y) rnn_histogram(model, mode='grads', bins=400, **kws) print('\n') rnn_heatmap(model, mode='grads', cmap=None, absolute_value=True, **kws) def viz_prefetched_data(model, data, layer_idx=1): rnn_histogram(model, layer_idx=layer_idx, data=data) rnn_heatmap(model, layer_idx=layer_idx, data=data) ############################################################################### units = 64 batch_shape = (32, 100, 16) model = make_model(LSTM, batch_shape, units) train_model(model, 50) viz_outs(model, 1) viz_outs_grads(model, 1) viz_outs_grads_last(model, 2) viz_weights(model, 1) viz_weights_grads(model, 1) data = get_rnn_weights(model, layer_idx=1) viz_prefetched_data(model, data, 1)
def viz_weights_grads(model, idx=1): x, y, _ = make_data(K.int_shape(model.input), model.layers[2].units) kws = dict(_id=idx, input_data=x, labels=y) rnn_histogram(model, mode='grads', bins=400, **kws) print('\n') rnn_heatmap(model, mode='grads', cmap=None, absolute_value=True, **kws) def viz_prefetched_data(model, data, idx=1): rnn_histogram(model, idx, data=data) rnn_heatmap(model, idx, data=data) ############################################################################### units = 64 batch_shape = (32, 100, 16) model = make_model(LSTM, batch_shape, units) train_model(model, 50) viz_outs(model, 1) viz_outs_grads(model, 1) viz_outs_grads_last(model, 2) viz_weights(model, 1) viz_weights_grads(model, 1) data = get_rnn_weights(model, 1) viz_prefetched_data(model, data, 1)