def check_lstm(seq_len, input_size, hidden_size, batch_size, init_func, inp_moms=[0.0, 1.0]): # init_func is the initializer for the model params # inp_moms is the [ mean, std dev] of the random input input_shape = (input_size, seq_len * batch_size) hidden_shape = (hidden_size, seq_len * batch_size) NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size # neon LSTM lstm = LSTM(hidden_size, init_func, activation=Tanh(), gate_activation=Logistic()) inp = np.random.rand(*input_shape) * inp_moms[1] + inp_moms[0] inpa = lstm.be.array(inp) # import pdb; pdb.set_trace() # run neon fprop lstm.fprop(inpa) # reference numpy LSTM lstm_ref = RefLSTM() WLSTM = lstm_ref.init(input_size, hidden_size) # make ref weights and biases with neon model WLSTM[0, :] = lstm.b.get().T WLSTM[1:input_size + 1, :] = lstm.W_input.get().T WLSTM[input_size + 1:] = lstm.W_recur.get().T # transpose input X and do fprop inp_ref = inp.copy().T.reshape(seq_len, batch_size, input_size) (Hout_ref, cprev, hprev, batch_cache) = lstm_ref.forward(inp_ref, WLSTM) # the output needs transpose as well Hout_ref = Hout_ref.reshape(seq_len * batch_size, hidden_size).T IFOGf_ref = batch_cache['IFOGf'].reshape(seq_len * batch_size, hidden_size * 4).T Ct_ref = batch_cache['Ct'].reshape(seq_len * batch_size, hidden_size).T # compare results print '====Verifying IFOG====' allclose_with_out(lstm.ifog_buffer.get(), IFOGf_ref, rtol=0.0, atol=1.0e-5) print '====Verifying cell states====' allclose_with_out(lstm.c_act_buffer.get(), Ct_ref, rtol=0.0, atol=1.0e-5) print '====Verifying hidden states====' allclose_with_out(lstm.h_buffer.get(), Hout_ref, rtol=0.0, atol=1.0e-5) print 'fprop is verified' # now test the bprop # generate random deltas tensor deltas = np.random.randn(*hidden_shape) lstm.bprop(lstm.be.array(deltas)) # grab the delta W from gradient buffer dWinput_neon = lstm.dW_input.get() dWrecur_neon = lstm.dW_recur.get() db_neon = lstm.db.get() # import pdb; pdb.set_trace() deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size) (dX_ref, dWLSTM_ref, dc0_ref, dh0_ref) = lstm_ref.backward(deltas_ref, batch_cache) dWrecur_ref = dWLSTM_ref[-hidden_size:, :] dWinput_ref = dWLSTM_ref[1:input_size + 1, :] db_ref = dWLSTM_ref[0, :] dX_ref = dX_ref.reshape(seq_len * batch_size, input_size).T # compare results print 'Making sure neon LSTM match numpy LSTM in bprop' print '====Verifying update on W_recur====' assert allclose_with_out(dWrecur_neon, dWrecur_ref.T, rtol=0.0, atol=1.0e-5) print '====Verifying update on W_input====' assert allclose_with_out(dWinput_neon, dWinput_ref.T, rtol=0.0, atol=1.0e-5) print '====Verifying update on bias====' assert allclose_with_out(db_neon.flatten(), db_ref, rtol=0.0, atol=1.0e-5) print '====Verifying output delta====' assert allclose_with_out(lstm.out_deltas_buffer.get(), dX_ref, rtol=0.0, atol=1.0e-5) print 'bprop is verified' return
def check_lstm(seq_len, input_size, hidden_size, batch_size, init_func, inp_moms=[0.0, 1.0]): # init_func is the initializer for the model params # inp_moms is the [ mean, std dev] of the random input input_shape = (input_size, seq_len * batch_size) hidden_shape = (hidden_size, seq_len * batch_size) NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size # neon LSTM lstm = LSTM(hidden_size, init_func, activation=Tanh(), gate_activation=Logistic()) inp = np.random.rand(*input_shape)*inp_moms[1] + inp_moms[0] inpa = lstm.be.array(inp) # run neon fprop lstm.configure((input_size, seq_len)) lstm.prev_layer = True # Hack to force allocating a delta buffer lstm.allocate() lstm.set_deltas([lstm.be.iobuf(lstm.in_shape)]) lstm.fprop(inpa) # reference numpy LSTM lstm_ref = RefLSTM() WLSTM = lstm_ref.init(input_size, hidden_size) # make ref weights and biases with neon model WLSTM[0, :] = lstm.b.get().T WLSTM[1:input_size+1, :] = lstm.W_input.get().T WLSTM[input_size+1:] = lstm.W_recur.get().T # transpose input X and do fprop inp_ref = inp.copy().T.reshape(seq_len, batch_size, input_size) (Hout_ref, cprev, hprev, batch_cache) = lstm_ref.forward(inp_ref, WLSTM) # the output needs transpose as well Hout_ref = Hout_ref.reshape(seq_len * batch_size, hidden_size).T IFOGf_ref = batch_cache['IFOGf'].reshape(seq_len * batch_size, hidden_size * 4).T Ct_ref = batch_cache['Ct'].reshape(seq_len * batch_size, hidden_size).T # compare results print '====Verifying IFOG====' allclose_with_out(lstm.ifog_buffer.get(), IFOGf_ref, rtol=0.0, atol=1.0e-5) print '====Verifying cell states====' allclose_with_out(lstm.c_act_buffer.get(), Ct_ref, rtol=0.0, atol=1.0e-5) print '====Verifying hidden states====' allclose_with_out(lstm.outputs.get(), Hout_ref, rtol=0.0, atol=1.0e-5) print 'fprop is verified' # now test the bprop # generate random deltas tensor deltas = np.random.randn(*hidden_shape) lstm.bprop(lstm.be.array(deltas)) # grab the delta W from gradient buffer dWinput_neon = lstm.dW_input.get() dWrecur_neon = lstm.dW_recur.get() db_neon = lstm.db.get() deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size) (dX_ref, dWLSTM_ref, dc0_ref, dh0_ref) = lstm_ref.backward(deltas_ref, batch_cache) dWrecur_ref = dWLSTM_ref[-hidden_size:, :] dWinput_ref = dWLSTM_ref[1:input_size+1, :] db_ref = dWLSTM_ref[0, :] dX_ref = dX_ref.reshape(seq_len * batch_size, input_size).T # compare results print 'Making sure neon LSTM match numpy LSTM in bprop' print '====Verifying update on W_recur====' assert allclose_with_out(dWrecur_neon, dWrecur_ref.T, rtol=0.0, atol=1.0e-5) print '====Verifying update on W_input====' assert allclose_with_out(dWinput_neon, dWinput_ref.T, rtol=0.0, atol=1.0e-5) print '====Verifying update on bias====' assert allclose_with_out(db_neon.flatten(), db_ref, rtol=0.0, atol=1.0e-5) print '====Verifying output delta====' assert allclose_with_out(lstm.out_deltas_buffer.get(), dX_ref, rtol=0.0, atol=1.0e-5) print 'bprop is verified' return
def gradient_check_ref(seq_len, input_size, hidden_size, batch_size, epsilon=1.0e-5, dtypeu=np.float64, threshold=1e-4): # this is a check of the reference code itself # estimates the gradients by adding perturbations # to the input and the weights and compares to # the values calculated in bprop # generate sparse random input matrix NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size input_shape = (seq_len, input_size, batch_size) # hidden_shape = (seq_len, hidden_size, batch_size) (inp_bl, nz_inds) = sparse_rand(input_shape, frac=1.0 / input_shape[1]) inp_bl = np.random.randn(*input_shape) # convert input matrix from neon to ref code format inp_bl = inp_bl.swapaxes(1, 2).astype(dtypeu) # generate reference LSTM lstm_ref = RefLSTM() WLSTM = lstm_ref.init(input_size, hidden_size).astype(dtypeu) # init parameters as done for neon WLSTM = np.random.randn(*WLSTM.shape) (Hout, cprev, hprev, cache) = lstm_ref.forward(inp_bl, WLSTM) # scale Hout by random matrix... rand_scale = np.random.random(Hout.shape) * 2.0 - 1.0 rand_scale = dtypeu(rand_scale) # line below would be the loss function # loss_bl = np.sum(rand_scale * Hout) # run bprop, input deltas is rand_scale (dX_bl, dWLSTM_bl, dc0, dh0) = lstm_ref.backward(rand_scale, cache) grads_est = np.zeros(dX_bl.shape) inp_pert = inp_bl.copy() for pert_ind in range(inp_bl.size): save_val = inp_pert.flat[pert_ind] # add/subtract perturbations to input inp_pert.flat[pert_ind] = save_val + epsilon # and run fprop on perturbed input (Hout_pos, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM) inp_pert.flat[pert_ind] = save_val - epsilon (Hout_neg, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM) # calculate the loss on outputs loss_pos = np.sum(rand_scale * Hout_pos) loss_neg = np.sum(rand_scale * Hout_neg) grads_est.flat[pert_ind] = 0.5 * (loss_pos - loss_neg) / epsilon # reset input inp_pert.flat[pert_ind] = save_val # assert that gradient estimates within rel threshold of # bprop calculated deltas assert allclose_with_out(grads_est, dX_bl, rtol=threshold, atol=0.0) return
def gradient_check_ref(seq_len, input_size, hidden_size, batch_size, epsilon=1.0e-5, dtypeu=np.float64, threshold=1e-4): # this is a check of the reference code itself # estimates the gradients by adding perturbations # to the input and the weights and compares to # the values calculated in bprop # generate sparse random input matrix NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size input_shape = (seq_len, input_size, batch_size) # hidden_shape = (seq_len, hidden_size, batch_size) (inp_bl, nz_inds) = sparse_rand(input_shape, frac=1.0/input_shape[1]) inp_bl = np.random.randn(*input_shape) # convert input matrix from neon to ref code format inp_bl = inp_bl.swapaxes(1, 2).astype(dtypeu) # generate reference LSTM lstm_ref = RefLSTM() WLSTM = lstm_ref.init(input_size, hidden_size).astype(dtypeu) # init parameters as done for neon WLSTM = np.random.randn(*WLSTM.shape) (Hout, cprev, hprev, cache) = lstm_ref.forward(inp_bl, WLSTM) # scale Hout by random matrix... rand_scale = np.random.random(Hout.shape)*2.0 - 1.0 rand_scale = dtypeu(rand_scale) # line below would be the loss function # loss_bl = np.sum(rand_scale * Hout) # run bprop, input deltas is rand_scale (dX_bl, dWLSTM_bl, dc0, dh0) = lstm_ref.backward(rand_scale, cache) grads_est = np.zeros(dX_bl.shape) inp_pert = inp_bl.copy() for pert_ind in range(inp_bl.size): save_val = inp_pert.flat[pert_ind] # add/subtract perturbations to input inp_pert.flat[pert_ind] = save_val + epsilon # and run fprop on perturbed input (Hout_pos, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM) inp_pert.flat[pert_ind] = save_val - epsilon (Hout_neg, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM) # calculate the loss on outputs loss_pos = np.sum(rand_scale*Hout_pos) loss_neg = np.sum(rand_scale*Hout_neg) grads_est.flat[pert_ind] = 0.5*(loss_pos-loss_neg)/epsilon # reset input inp_pert.flat[pert_ind] = save_val # assert that gradient estimates within rel threshold of # bprop calculated deltas assert allclose_with_out(grads_est, dX_bl, rtol=threshold, atol=0.0) return