def _forward(self, steps, outputs, features, batch, return_seq): activation = 'tanh' inpt = np.random.uniform(size=(batch, features)) inpt_keras, _ = data_to_timesteps(inpt, steps=steps) assert inpt_keras.shape == (batch, steps, features) # weights init kernel = np.random.uniform(low=-1, high=1, size=(features, outputs)) recurrent_kernel = np.random.uniform(low=-1, high=1, size=(outputs, outputs)) bias = np.random.uniform(low=-1, high=1, size=(outputs, )) # create keras model inp = Input(shape=inpt_keras.shape[1:]) rnn = SimpleRNN(units=outputs, activation=activation, return_sequences=return_seq)(inp) model = Model(inputs=inp, outputs=rnn) # set weights for the keras model model.set_weights([kernel, recurrent_kernel, bias]) # create NumPyNet layer layer = SimpleRNN_layer(outputs=outputs, steps=steps, input_shape=(batch, 1, 1, features), activation=activation, return_sequence=return_seq) # set NumPyNet weights layer.load_weights( np.concatenate( [bias.ravel(), kernel.ravel(), recurrent_kernel.ravel()])) # FORWARD # forward for keras forward_out_keras = model.predict(inpt_keras) # forward NumPyNet layer.forward(inpt) forward_out_numpynet = layer.output.reshape(forward_out_keras.shape) assert np.allclose(forward_out_numpynet, forward_out_keras, atol=1e-4, rtol=1e-3)
def test_forward (self): batch = 11 timesteps = 5 features = 3 outputs = 5 np.random.seed(123) data = np.random.uniform(size=(batch, features)) inpt_keras, _ = data_to_timesteps(data, timesteps) assert inpt_keras.shape == (batch - timesteps, timesteps, features) weights = [np.random.uniform(size=(features, outputs)), np.random.uniform(size=(outputs,outputs))] bias = [np.zeros(shape=(outputs,), dtype=float), np.zeros(shape=outputs, dtype=float)] # assign same weights to all the kernel in keras as for NumPyNet keras_weights1 = np.concatenate([weights[0] for i in range(4)], axis=1) keras_weights2 = np.concatenate([weights[1] for i in range(4)], axis=1) keras_bias = np.concatenate([bias[0] for i in range(4)]) for i in range(4): np.testing.assert_allclose(keras_weights1[:,outputs*i:outputs*(i+1)], weights[0], rtol=1e-5, atol=1e-8) for i in range(4): np.testing.assert_allclose(keras_weights2[:,outputs*i:outputs*(i+1)], weights[1], rtol=1e-5, atol=1e-8) inp = Input(shape=(inpt_keras.shape[1:])) lstm = LSTM(units=outputs, implementation=1, use_bias=False)(inp) model = Model(inputs=[inp], outputs=[lstm]) model.set_weights([keras_weights1, keras_weights2]) inpt_numpynet = data.reshape(batch, 1, 1, features) layer = LSTM_layer(outputs=outputs, steps=timesteps, weights=weights, bias=bias, input_shape=inpt_numpynet.shape) np.testing.assert_allclose(layer.uf.weights, model.get_weights()[0][:, :outputs], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.ui.weights, model.get_weights()[0][:, outputs:2*outputs], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.ug.weights, model.get_weights()[0][:, 2*outputs:3*outputs], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.uo.weights, model.get_weights()[0][:, 3*outputs:4*outputs], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.wf.weights, model.get_weights()[1][:, :outputs], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.wi.weights, model.get_weights()[1][:, outputs:2*outputs], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.wg.weights, model.get_weights()[1][:, 2*outputs:3*outputs], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.wo.weights, model.get_weights()[1][:, 3*outputs:4*outputs], rtol=1e-5, atol=1e-8) forward_out_keras = model.predict(inpt_keras) layer.forward(inpt=inpt_numpynet) forward_out_numpynet = layer.output.reshape(batch, outputs)
def _forward(self): outputs = 30 steps = 1 features = 10 batch = 16 data = np.random.uniform(size=(batch, features)) weights = [ np.random.uniform(size=(features, outputs)), np.random.uniform(size=(features, outputs)), np.random.uniform(size=(features, outputs)), np.random.uniform(size=(outputs, outputs)), np.random.uniform(size=(outputs, outputs)), np.random.uniform(size=(outputs, outputs)) ] bias = [np.zeros(shape=(outputs, )), np.zeros(shape=outputs)] # assign same weights to all the kernel in keras as for NumPyNet keras_weights1 = np.concatenate([weights[i] for i in range(3)], axis=1) keras_weights2 = np.concatenate([weights[i] for i in range(3, 6)], axis=1) keras_bias = np.concatenate([bias[0] for i in range(4)]) inpt_keras, _ = data_to_timesteps(data, steps) assert inpt_keras.shape == (batch - steps, steps, features) inp = Input(shape=(steps, features)) gru = GRU(units=outputs, use_bias=False)(inp) model = Model(inputs=inp, outputs=gru) model.set_weights([keras_weights1, keras_weights2]) layer = GRU_layer(outputs=outputs, steps=steps, weights=weights, bias=[0, 0, 0]) layer.forward(data) forward_out_keras = model.predict(inpt_keras) forward_out_numpynet = layer.output np.allclose(forward_out_keras, forward_out_numpynet) forward_out_keras forward_out_numpynet
def test_forward(self, steps, outputs, features, batch, return_seq): activation = 'tanh' inpt = np.random.uniform(size=(batch, features)) inpt_keras, _ = data_to_timesteps(inpt, steps=steps) assert inpt_keras.shape == (batch - steps, steps, features) # INITIALIZE model, layer = TestSimpleRNNLayer.initialize_step( steps, outputs, features, batch, return_seq, activation, inpt_keras) # FORWARD forward_out_numpynet, forward_out_keras = TestSimpleRNNLayer.forward_step( model, layer, inpt_keras, inpt) assert np.allclose(forward_out_numpynet, forward_out_keras, atol=1e-4, rtol=1e-3)
__email__ = ['*****@*****.**', '*****@*****.**'] np.random.seed(42) if __name__ == '__main__': Npoints = 1000 train_size = 800 time = np.arange(0, Npoints) noisy_signal = np.sin(0.02 * time) + 2 * np.random.rand(Npoints) steps = 4 window_size=steps X, _ = data_to_timesteps(noisy_signal, steps=steps) y = np.concatenate([X[1:, 0, :], X[-1:, 0, :]], axis=0) # Reshape the data according to a 4D tensor num_samples, size, _ = X.shape if size != steps: raise ValueError('Something went wrong with the stride trick!') if X.max() > noisy_signal.max() or X.min() < noisy_signal.min(): raise ValueError('Something went wrong with the stride trick!') X = X.reshape(num_samples, 1, 1, size) X_train, X_test = X[:train_size, ...], X[train_size:train_size+180, ...] y_train, y_test = y[:train_size, ...], y[train_size:train_size+180, ...]
def test_backprop(self, outputs, steps, features, batch, return_seq): activation = 'tanh' return_seq = False with tf.Graph().as_default(): inpt = np.random.uniform(size=(batch, features)) inpt_keras, _ = data_to_timesteps(inpt, steps=steps) # INITIALIZE model, layer = TestSimpleRNNLayer.initialize_step( steps, outputs, features, batch, return_seq, activation, inpt_keras) # FORWARD TestSimpleRNNLayer.forward_step(model, layer, inpt_keras, inpt) lr = .1 momentum = .9 decay = 0. layer.optimizer = Momentum(lr=lr, momentum=momentum, decay=decay) model.optimizer = SGD(learning_rate=lr, momentum=momentum, nesterov=False, decay=decay) # BACKWARD print('outputs ' + str(outputs)) print('steps ' + str(steps)) print('features ' + str(features)) print('batch ' + str(batch)) print('return_seq ' + str(return_seq)) layer.delta = np.ones(shape=layer.out_shape, dtype=float) a, b, c = layer.X.shape delta = np.ones(shape=(b, a, c), dtype=float) layer.backward(delta=delta, copy=True) tf.compat.v1.disable_eager_execution() # Compute the gradient of output w.r.t the weights gradient = K.gradients(model.output, model.trainable_weights) # Define a function to evaluate the gradient func = K.function( model.inputs + model.trainable_weights + model.outputs, gradient) updates = func([inpt_keras]) weights_update_keras = updates[0] recurrent_weights_update_keras = updates[1] bias_update_keras = updates[2] np.testing.assert_allclose(layer.bias_update, bias_update_keras, atol=1e-4, rtol=1e-4) np.testing.assert_allclose(layer.weights_update, weights_update_keras, atol=1e-4, rtol=1e-4) np.testing.assert_allclose(layer.recurrent_weights_update, recurrent_weights_update_keras, atol=1e-4, rtol=1e-4)
def _backward(self, steps, outputs, features, batch, return_seq): return_seq = False # fixed to "many_to_one" for now activation = 'tanh' inpt = np.random.uniform(size=(batch, features)) inpt_keras, _ = data_to_timesteps(inpt, steps=steps) assert inpt_keras.shape == (batch, steps, features) # weights init kernel = np.random.uniform(low=-1, high=1, size=(features, outputs)) recurrent_kernel = np.random.uniform(low=-1, high=1, size=(outputs, outputs)) bias = np.random.uniform(low=-1, high=1, size=(outputs, )) # create keras model inp = Input(shape=inpt_keras.shape[1:]) rnn = SimpleRNN(units=outputs, activation=activation, return_sequences=return_seq)(inp) model = Model(inputs=inp, outputs=rnn) # set weights for the keras model model.set_weights([kernel, recurrent_kernel, bias]) # create NumPyNet layer layer = SimpleRNN_layer(outputs=outputs, steps=steps, input_shape=(batch, 1, 1, features), activation=activation, return_sequence=return_seq) # set NumPyNet weights layer.load_weights( np.concatenate( [bias.ravel(), kernel.ravel(), recurrent_kernel.ravel()])) np.testing.assert_allclose(layer.weights, model.get_weights()[0], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.recurrent_weights, model.get_weights()[1], rtol=1e-5, atol=1e-8) np.testing.assert_allclose(layer.bias, model.get_weights()[2], rtol=1e-5, atol=1e-8) # FORWARD # forward for keras forward_out_keras = model.predict(inpt_keras) # forward NumPyNet layer.forward(inpt) forward_out_numpynet = layer.output.reshape(forward_out_keras.shape) np.testing.assert_allclose(forward_out_numpynet, forward_out_keras, atol=1e-4, rtol=1e-3) # BACKWARD # Compute the gradient of output w.r.t input gradient1 = K.gradients(model.output, [model.input]) gradient2 = K.gradients(model.output, model.trainable_weights) # Define a function to evaluate the gradient func1 = K.function(model.inputs + [model.output], gradient1) func2 = K.function( model.inputs + model.trainable_weights + model.outputs, gradient2) # Compute delta for Keras delta_keras = func1([inpt_keras])[0] updates = func2([inpt_keras]) weights_update_keras = updates[0] recurrent_weights_update_keras = updates[1] bias_update_keras = updates[2] # backward pass for NumPyNet delta = np.zeros(shape=inpt_keras.shape, dtype=float) layer.delta = np.ones(shape=layer.output.shape, dtype=float) layer.backward(inpt, delta, copy=True) np.testing.assert_allclose(layer.bias_update, bias_update_keras, atol=1e-8, rtol=1e-5) np.testing.assert_allclose(layer.weights_update, weights_update_keras, atol=1e-8, rtol=1e-5) np.testing.assert_allclose(delta, delta_keras, atol=1e-8, rtol=1e-5) np.testing.assert_allclose(layer.recurrent_weights_update, recurrent_weights_update_keras, atol=1e-8, rtol=1e-5)