def infer_train_hist(model, input_data, layer=None, keep_borders=True, bins=100, xlims=None, fontsize=14, vline=None, w=1, h=1): """Histograms of flattened layer output values in inference and train mode. Useful for comparing effect of layers that behave differently in train vs infrence modes (Dropout, BatchNormalization, etc) on model prediction (or intermediate activations). Arguments: model: models.Model / models.Sequential (keras / tf.keras) The model. input_data: np.ndarray / list[np.ndarray] Data to feed to `model` to fetch outputs. List of arrays for multi-input networks. layer: layers.Layer / None Layer whose outputs to fetch; defaults to last layer (output). keep_borders: bool Whether to keep the plots' bounding box. bins: int Number of histogram bins; kwarg to `plt.hist()` xlims: tuple[float, float] / None Histogram x limits. Defaults to min/max of flattened data per plot. fontsize: int Title font size. vline: float / None x-coordinate of vertical line to draw (e.g. predict threshold). w, h: float Scale figure width & height, respectively. """ layer = layer or model.layers[-1] outs = [get_outputs(model, '', input_data, layer, learning_phase=0), get_outputs(model, '', input_data, layer, learning_phase=1)] fig, axes = plt.subplots(2, 1, sharex=True, sharey=True, figsize=(13 * w, 6 * h)) for i, (ax, out) in enumerate(zip(axes.flat, outs)): out = np.asarray(out).ravel() ax.hist(out, bins=bins) mode = "ON" if i == 0 else "OFF" ax.set_title("Train mode " + mode, weight='bold', fontsize=fontsize) if not keep_borders: ax.box(on=None) if vline is not None: ax.axvline(vline, color='r', linewidth=2) _xlims = xlims or (out.min(), out.max()) ax.set_xlim(*_xlims) scalefig(fig) plt.show()
def test_multi_io(): def _make_multi_io_model(): ipt1 = Input((40, 8)) ipt2 = Input((40, 16)) ipts = concatenate([ipt1, ipt2]) out1 = GRU(6, return_sequences=True)(ipts) out2 = GRU(12, return_sequences=True)(ipts) model = Model([ipt1, ipt2], [out1, out2]) model.compile('adam', 'mse') return model def _make_multi_io_data(): x1 = np.random.randn(8, 40, 8) x2 = np.random.randn(8, 40, 16) y1 = np.random.randn(8, 40, 6) y2 = np.random.randn(8, 40, 12) return x1, x2, y1, y2 model = _make_multi_io_model() x1, x2, y1, y2 = _make_multi_io_data() grads = get_gradients(model, '*', [x1, x2], [y1, y2]) outs = get_outputs(model, '*', [x1, x2]) assert outs[0].shape == grads[0].shape == (8, 40, 24) assert outs[1].shape == grads[1].shape == (8, 40, 6) assert outs[2].shape == grads[2].shape == (8, 40, 12) cprint("\n<< MULTI_IO TESTS PASSED >>\n", 'green')
def test_errors(): # test Exception cases units = 6 batch_shape = (8, 100, 2 * units) reset_seeds(reset_graph_with_backend=K) model = make_model(GRU, batch_shape, activation='relu', recurrent_dropout=0.3, IMPORTS=IMPORTS) x, y, sw = make_data(batch_shape, units) model.train_on_batch(x, y, sw) grads = get_gradients(model, 1, x, y) grads_4D = np.expand_dims(grads, -1) from see_rnn.inspect_gen import get_layer, _make_grads_fn pass_on_error(features_0D, grads) pass_on_error(features_0D, grads_4D) pass_on_error(features_1D, grads_4D) pass_on_error(features_2D, grads_4D) pass_on_error(features_2D, grads) pass_on_error(get_gradients, model, 1, x, y, mode='cactus') pass_on_error(get_gradients, model, 1, x, y, layer=model.layers[1]) pass_on_error(_make_grads_fn, model, model.layers[1], mode='banana') pass_on_error(features_hist, grads[:, :4, :3], po='tato') pass_on_error(features_hist_v2, grads[:, :4, :3], po='tato') pass_on_error(get_layer, model) pass_on_error(get_layer, model, 'capsule') pass_on_error(rnn_heatmap, model, 1, input_data=x, labels=y, mode='coffee') pass_on_error(rnn_heatmap, model, 1, co='vid') pass_on_error(rnn_heatmap, model, 1, norm=(0, 1, 2)) pass_on_error(rnn_heatmap, model, 1, mode='grads') pass_on_error(rnn_histogram, model, 1, norm=None) pass_on_error(rnn_heatmap, model, layer_index=9001) pass_on_error(features_0D, grads, cake='lie') pass_on_error(features_1D, grads, pup='not just any') pass_on_error(features_2D, grads, true=False) outs = list(get_outputs(model, 1, x, as_dict=True).values()) pass_on_error(rnn_histogram, model, 1, data=outs) pass_on_error(rnn_histogram, model, 1, data=[1]) pass_on_error(rnn_histogram, model, 1, data=[[1]]) pass_on_error(features_hist, grads, co='vid') pass_on_error(features_0D, grads, configs={'x': {}}) pass_on_error(features_1D, grads, configs={'x': {}}) pass_on_error(features_2D, grads, configs={'x': {}}) pass_on_error(features_hist, grads, configs={'x': {}}) pass_on_error(features_hist_v2, grads, configs={'x': {}}) cprint("\n<< EXCEPTION TESTS PASSED >>\n", 'green') assert True
def _get_data(model, _id, mode, input_data, labels, omit_names, kw): if mode == 'weights': data = get_weights(model, _id, omit_names, as_dict=True) elif 'gradients' in mode: if mode in ('gradients', 'gradients:outputs'): data = get_gradients(model, _id, input_data, labels, mode='outputs', as_dict=True) else: data = get_gradients(model, _id, input_data, labels, mode='weights', as_dict=True) elif mode == 'outputs': data = get_outputs(model, _id, input_data, as_dict=True) data_flat = [x.ravel() for x in data.values()] return data_flat, list(data)
def _test_outputs(model): x, *_ = make_data(K.int_shape(model.input), model.layers[2].units) outs = get_outputs(model, 1, x) features_1D(outs[:1], show_y_zero=True) features_1D(outs[0]) features_2D(outs)
def test_misc(): # test miscellaneous functionalities units = 6 batch_shape = (8, 100, 2 * units) reset_seeds(reset_graph_with_backend=K) model = make_model(GRU, batch_shape, activation='relu', recurrent_dropout=0.3, IMPORTS=IMPORTS) x, y, sw = make_data(batch_shape, units) model.train_on_batch(x, y, sw) weights_norm(model, 'gru', omit_names='bias', verbose=1) weights_norm(model, ['gru', 1, (1, 1)], norm_fn=np.abs) stats = weights_norm(model, 'gru') weights_norm(model, 'gru', _dict=stats) grads = get_gradients(model, 1, x, y) get_gradients(model, 1, x, y, as_dict=True) get_gradients(model, ['gru', 1], x, y) get_outputs(model, ['gru', 1], x) features_1D(grads, subplot_samples=True, tight=True, borderwidth=2, share_xy=False) with tempdir() as dirpath: features_0D(grads[0], savepath=os.path.join(dirpath, 'img.png')) with tempdir() as dirpath: features_1D(grads[0], subplot_samples=True, annotations=[1, 'pi'], savepath=os.path.join(dirpath, 'img.png')) features_2D(grads.T, n_rows=1.5, tight=True, borderwidth=2) with tempdir() as dirpath: features_2D(grads.T[:, :, 0], norm='auto', savepath=os.path.join(dirpath, 'img.png')) with tempdir() as dirpath: features_hist(grads, show_borders=False, borderwidth=1, annotations=[0], show_xy_ticks=[0, 0], share_xy=(1, 1), title="grads", savepath=os.path.join(dirpath, 'img.png')) with tempdir() as dirpath: features_hist_v2(list(grads[:, :4, :3]), colnames=list('abcd'), show_borders=False, xlims=(-.01, .01), ylim=100, borderwidth=1, show_xy_ticks=[0, 0], side_annot='row', share_xy=True, title="Grads", savepath=os.path.join(dirpath, 'img.png')) features_hist(grads, center_zero=True, xlims=(-1, 1), share_xy=0) features_hist_v2(list(grads[:, :4, :3]), center_zero=True, xlims=(-1, 1), share_xy=(False, False)) with tempdir() as dirpath: rnn_histogram(model, 1, show_xy_ticks=[0, 0], equate_axes=2, savepath=os.path.join(dirpath, 'img.png')) rnn_histogram(model, 1, equate_axes=False, configs={ 'tight': dict(left=0, right=1), 'plot': dict(color='red'), 'title': dict(fontsize=14), }) rnn_heatmap(model, 1, cmap=None, normalize=True, show_borders=False) rnn_heatmap(model, 1, cmap=None, norm='auto', absolute_value=True) rnn_heatmap(model, 1, norm=None) with tempdir() as dirpath: rnn_heatmap(model, 1, norm=(-.004, .004), savepath=os.path.join(dirpath, 'img.png')) hist_clipped(grads, peaks_to_clip=2) _, ax = plt.subplots(1, 1) hist_clipped(grads, peaks_to_clip=2, ax=ax, annot_kw=dict(fontsize=15)) get_full_name(model, 'gru') get_full_name(model, 1) pass_on_error(get_full_name, model, 'croc') get_weights(model, 'gru', as_dict=False) get_weights(model, 'gru', as_dict=True) get_weights(model, 'gru/bias') get_weights(model, ['gru', 1, (1, 1)]) pass_on_error(get_weights, model, 'gru/goo') get_weights(model, '*') get_gradients(model, '*', x, y) get_outputs(model, '*', x) from see_rnn.utils import _filter_duplicates_by_keys keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4]) assert keys == ['a', 'b', 'c'] assert data == [1, 2, 4] keys, data = _filter_duplicates_by_keys(list('abbc'), [1, 2, 3, 4], [5, 6, 7, 8]) assert keys == ['a', 'b', 'c'] assert data[0] == [1, 2, 4] and data[1] == [5, 6, 8] from see_rnn.inspect_gen import get_layer, detect_nans get_layer(model, 'gru') get_rnn_weights(model, 1, concat_gates=False, as_tensors=True) rnn_heatmap(model, 1, input_data=x, labels=y, mode='weights') _test_prefetched_data(model) # test NaN/Inf detection nan_txt = detect_nans(np.array([1] * 9999 + [np.nan])).replace('\n', ' ') print(nan_txt) # case: print as quantity data = np.array([np.nan, np.inf, -np.inf, 0]) print(detect_nans(data, include_inf=True)) print(detect_nans(data, include_inf=False)) data = np.array([np.inf, 0]) print(detect_nans(data, include_inf=True)) detect_nans(np.array([0])) K.set_value(model.optimizer.lr, 1e12) train_model(model, iterations=10) rnn_histogram(model, 1) rnn_heatmap(model, 1) del model reset_seeds(reset_graph_with_backend=K) # test SimpleRNN & other _model = make_model(SimpleRNN, batch_shape, units=128, use_bias=False, IMPORTS=IMPORTS) train_model(_model, iterations=1) # TF2-Keras-Graph bug workaround rnn_histogram(_model, 1) # test _pretty_hist K.set_value(_model.optimizer.lr, 1e50) # force NaNs train_model(_model, iterations=20) rnn_heatmap(_model, 1) data = get_rnn_weights(_model, 1) rnn_heatmap(_model, 1, input_data=x, labels=y, data=data) os.environ["TF_KERAS"] = '0' get_rnn_weights(_model, 1, concat_gates=False) del _model assert True cprint("\n<< MISC TESTS PASSED >>\n", 'green')
def viz_outs(model, idx=1): x, y, _ = make_data(K.int_shape(model.input), model.layers[2].units) outs = get_outputs(model, idx, x) features_1D(outs[:1], n_rows=8, show_borders=False) features_2D(outs, n_rows=8, norm=(-1, 1))