def test_rnn_fprop(sequence_length, input_size, hidden_size, batch_size, return_sequence, weight_initializer, bias_initializer, init_state, extra_axes, backward, transformer_factory): assert batch_size == 1, "the recurrent reference implementation only support batch size 1" # Get input placeholder and numpy array input_placeholder, input_value = make_placeholder(input_size, sequence_length, batch_size, extra_axes=extra_axes) # Construct network weights and initial state, if desired W_in, W_rec, b, init_state, init_state_value = make_weights( input_placeholder, hidden_size, weight_initializer, bias_initializer, init_state) # Compute reference numpy RNN rnn_ref = RefRecurrent(input_size, hidden_size, return_sequence=return_sequence) rnn_ref.set_weights(W_in.reshape(rnn_ref.Wxh.shape), W_rec, b.reshape(rnn_ref.bh.shape)) # Compute reference numpy RNN input_shape = (input_size, sequence_length, batch_size) h_ref_list = rnn_ref.fprop_only(input_value.reshape(input_shape).transpose( [1, 0, 2]), init_states=init_state_value, backward=backward) # Generate ngraph RNN rnn_ng = Recurrent(hidden_size, init=W_in, init_inner=W_rec, activation=Tanh(), reset_cells=True, return_sequence=return_sequence, backward=backward) # fprop ngraph RNN out_ng = rnn_ng(input_placeholder, init_state=init_state) with ExecutorFactory() as ex: # Create computation and execute if init_state is not None: fprop_neon_fun = ex.executor(out_ng, input_placeholder, init_state) fprop_neon = fprop_neon_fun(input_value, init_state_value) else: fprop_neon_fun = ex.executor(out_ng, input_placeholder) fprop_neon = fprop_neon_fun(input_value) # Compare output with reference implementation if return_sequence is True: fprop_neon = fprop_neon[:, :, 0] ng.testing.assert_allclose(fprop_neon, h_ref_list, rtol=fprop_rtol, atol=fprop_atol)
def test_rnn_deriv_ref(sequence_length, input_size, hidden_size, batch_size, return_sequence, weight_initializer, bias_initializer, transformer_factory): assert batch_size == 1, "the recurrent reference implementation only support batch size 1" assert return_sequence is True, "the reference rnn only supports sequences for deriv" # Get input placeholder and numpy array input_placeholder, input_value = make_placeholder(input_size, sequence_length, batch_size) # Construct network weights and initial state, if desired W_in, W_rec, b, init_state, init_state_value = make_weights(input_placeholder, hidden_size, weight_initializer, bias_initializer) # Compute reference numpy RNN rnn_ref = RefRecurrent(input_size, hidden_size, return_sequence=return_sequence) rnn_ref.set_weights(W_in, W_rec, b.reshape(rnn_ref.bh.shape)) # Prepare deltas for gradient check output_shape = (hidden_size, sequence_length, batch_size) # generate random deltas tensor deltas = np.random.randn(*output_shape) # the reference code expects these shapes: # input_shape: (seq_len, input_size, batch_size) # output_shape: (seq_len, hidden_size, batch_size) dW_in, dW_rec, db = rnn_ref.lossFun(input_value.transpose([1, 0, 2]), deltas.copy().transpose([1, 0, 2]), init_states=init_state_value)[:3] # Generate ngraph RNN rnn_ng = Recurrent(hidden_size, init=W_in, init_inner=W_rec, activation=Tanh(), reset_cells=True, return_sequence=return_sequence) # fprop ngraph RNN out_ng = rnn_ng.train_outputs(input_placeholder) deltas_constant = ng.constant(deltas, axes=out_ng.axes) params = [(rnn_ng.W_input, W_in), (rnn_ng.W_recur, W_rec), (rnn_ng.b, b)] with ExecutorFactory() as ex: # Create derivative computations and execute param_updates = list() for px, _ in params: update = ng.deriv(out_ng, px, error=deltas_constant) param_updates.append(ex.executor(update, input_placeholder)) for update_fun, ref_val in zip(param_updates, [dW_in, dW_rec, db]): ng.testing.assert_allclose(update_fun(input_value), ref_val.squeeze(), rtol=bprop_rtol, atol=bprop_atol)
def check_rnn(seq_len, input_size, hidden_size, batch_size, init_func, inp_moms=[0.0, 1.0]): # init_func is the initializer for the model params # inp_moms is the [ mean, std dev] of the random input input_shape = (input_size, seq_len * batch_size) output_shape = (hidden_size, seq_len * batch_size) NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size # ======== create models ======== # neon RNN rnn = Recurrent(hidden_size, init_func, activation=Tanh()) # reference numpy RNN rnn_ref = RefRecurrent(input_size, hidden_size) Wxh = rnn_ref.Wxh Whh = rnn_ref.Whh bh = rnn_ref.bh # ========= generate data ================= # generate random input tensor inp = np.random.rand(*input_shape) * inp_moms[1] + inp_moms[0] inpa = rnn.be.array(inp) # generate random deltas tensor deltas = np.random.randn(*output_shape) # the reference code expects these shapes: # input_shape: (seq_len, input_size, batch_size) # output_shape: (seq_len, hidden_size, batch_size) inp_ref = inp.copy().T.reshape(seq_len, batch_size, input_size).swapaxes(1, 2) deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size).swapaxes(1, 2) # ========= running models ========== # run neon fprop rnn.configure((input_size, seq_len)) rnn.prev_layer = True rnn.allocate() dtree = DeltasTree() rnn.allocate_deltas(dtree) dtree.allocate_buffers() rnn.set_deltas(dtree) rnn.fprop(inpa) # weights are only initialized after doing fprop, so now # make ref weights and biases the same with neon model Wxh[:] = rnn.W_input.get() Whh[:] = rnn.W_recur.get() bh[:] = rnn.b.get() (dWxh_ref, dWhh_ref, db_ref, h_ref_list, dh_ref_list, d_out_ref) = rnn_ref.lossFun(inp_ref, deltas_ref) # now test the bprop rnn.bprop(rnn.be.array(deltas)) # grab the delta W from gradient buffer dWxh_neon = rnn.dW_input.get() dWhh_neon = rnn.dW_recur.get() db_neon = rnn.db.get() # comparing outputs neon_logger.display('====Verifying hidden states====') assert allclose_with_out(rnn.outputs.get(), h_ref_list, rtol=0.0, atol=1.0e-5) neon_logger.display('fprop is verified') neon_logger.display('====Verifying update on W and b ====') neon_logger.display('dWxh') assert allclose_with_out(dWxh_neon, dWxh_ref, rtol=0.0, atol=1.0e-5) neon_logger.display('dWhh') assert allclose_with_out(dWhh_neon, dWhh_ref, rtol=0.0, atol=1.0e-5) neon_logger.display('====Verifying update on bias====') neon_logger.display('db') assert allclose_with_out(db_neon, db_ref, rtol=0.0, atol=1.0e-5) neon_logger.display('bprop is verified') return
def check_rnn(seq_len, input_size, hidden_size, batch_size, init_func, return_seq=True): # init_func is the initializer for the model params assert batch_size == 1, "the recurrent reference implementation only support batch size 1" # ========== neon model ========== Cin = ng.make_axis(input_size) REC = ng.make_axis(seq_len, recurrent=True) N = ng.make_axis(batch_size, batch=True) H = ng.make_axis(hidden_size) ax_s = ng.make_axes([H, N]) ex = ExecutorFactory() np.random.seed(0) rnn_ng = Recurrent(hidden_size, init_func, activation=Tanh(), reset_cells=True, return_sequence=return_seq) inp_ng = ng.placeholder([Cin, REC, N]) init_state_ng = ng.placeholder(ax_s) # fprop graph out_ng = rnn_ng.train_outputs(inp_ng, init_state=init_state_ng) out_ng.input = True rnn_W_input = rnn_ng.W_input rnn_W_input.input = True rnn_W_recur = rnn_ng.W_recur rnn_W_recur.input = True rnn_b = rnn_ng.b rnn_b.input = True fprop_neon_fun = ex.executor(out_ng, inp_ng, init_state_ng) dWrecur_s_fun = ex.derivative(out_ng, rnn_W_recur, inp_ng, rnn_W_input, rnn_b) dWrecur_n_fun = ex.numeric_derivative(out_ng, rnn_W_recur, delta, inp_ng, rnn_W_input, rnn_b) dWinput_s_fun = ex.derivative(out_ng, rnn_W_input, inp_ng, rnn_W_recur, rnn_b) dWinput_n_fun = ex.numeric_derivative(out_ng, rnn_W_input, delta, inp_ng, rnn_W_recur, rnn_b) dWb_s_fun = ex.derivative(out_ng, rnn_b, inp_ng, rnn_W_input, rnn_W_recur) dWb_n_fun = ex.numeric_derivative(out_ng, rnn_b, delta, inp_ng, rnn_W_input, rnn_W_recur) # fprop on random inputs input_value = rng.uniform(-1, 1, inp_ng.axes) init_state_value = rng.uniform(-1, 1, init_state_ng.axes) fprop_neon = fprop_neon_fun(input_value, init_state_value).copy() # after the rnn graph has been executed, can get the W values. Get copies so # shared values don't confuse derivatives Wxh_neon = rnn_ng.W_input.value.get(None).copy() Whh_neon = rnn_ng.W_recur.value.get(None).copy() bh_neon = rnn_ng.b.value.get(None).copy() # bprop derivs dWrecur_s = dWrecur_s_fun(Whh_neon, input_value, Wxh_neon, bh_neon) dWrecur_n = dWrecur_n_fun(Whh_neon, input_value, Wxh_neon, bh_neon) np.testing.assert_allclose(dWrecur_s, dWrecur_n, rtol=rtol, atol=atol) dWb_s = dWb_s_fun(bh_neon, input_value, Wxh_neon, Whh_neon) dWb_n = dWb_n_fun(bh_neon, input_value, Wxh_neon, Whh_neon) np.testing.assert_allclose(dWb_s, dWb_n, rtol=rtol, atol=atol) dWinput_s = dWinput_s_fun(Wxh_neon, input_value, Whh_neon, bh_neon) dWinput_n = dWinput_n_fun(Wxh_neon, input_value, Whh_neon, bh_neon) np.testing.assert_allclose(dWinput_s, dWinput_n, rtol=rtol, atol=atol) # ========= reference model ========== output_shape = (hidden_size, seq_len * batch_size) # generate random deltas tensor deltas = np.random.randn(*output_shape) # the reference code expects these shapes: # input_shape: (seq_len, input_size, batch_size) # output_shape: (seq_len, hidden_size, batch_size) deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size).swapaxes(1, 2) inp_ref = input_value.transpose([1, 0, 2]) # reference numpy RNN rnn_ref = RefRecurrent(input_size, hidden_size) rnn_ref.Wxh[:] = Wxh_neon rnn_ref.Whh[:] = Whh_neon rnn_ref.bh[:] = bh_neon.reshape(rnn_ref.bh.shape) (dWxh_ref, dWhh_ref, db_ref, h_ref_list, dh_ref_list, d_out_ref) = rnn_ref.lossFun(inp_ref, deltas_ref, init_states=init_state_value) # comparing outputs if return_seq is False: h_ref_list = h_ref_list[:, -1].reshape(-1, 1) else: fprop_neon = fprop_neon[:, :, 0] np.testing.assert_allclose(fprop_neon, h_ref_list, rtol=0.0, atol=1.0e-5) return