def check_lstm(seq_len,
input_size,
hidden_size,
batch_size,
init_func,
inp_moms=[0.0, 1.0]):
# init_func is the initializer for the model params
# inp_moms is the [ mean, std dev] of the random input
input_shape = (input_size, seq_len * batch_size)
hidden_shape = (hidden_size, seq_len * batch_size)
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
# neon LSTM
lstm = LSTM(hidden_size,
init_func,
activation=Tanh(),
gate_activation=Logistic())
inp = np.random.rand(*input_shape) * inp_moms[1] + inp_moms[0]
inpa = lstm.be.array(inp)
# import pdb; pdb.set_trace()
# run neon fprop
lstm.fprop(inpa)
# reference numpy LSTM
lstm_ref = RefLSTM()
WLSTM = lstm_ref.init(input_size, hidden_size)
# make ref weights and biases with neon model
WLSTM[0, :] = lstm.b.get().T
WLSTM[1:input_size + 1, :] = lstm.W_input.get().T
WLSTM[input_size + 1:] = lstm.W_recur.get().T
# transpose input X and do fprop
inp_ref = inp.copy().T.reshape(seq_len, batch_size, input_size)
(Hout_ref, cprev, hprev, batch_cache) = lstm_ref.forward(inp_ref, WLSTM)
# the output needs transpose as well
Hout_ref = Hout_ref.reshape(seq_len * batch_size, hidden_size).T
IFOGf_ref = batch_cache['IFOGf'].reshape(seq_len * batch_size,
hidden_size * 4).T
Ct_ref = batch_cache['Ct'].reshape(seq_len * batch_size, hidden_size).T
# compare results
print '====Verifying IFOG===='
allclose_with_out(lstm.ifog_buffer.get(), IFOGf_ref, rtol=0.0, atol=1.0e-5)
print '====Verifying cell states===='
allclose_with_out(lstm.c_act_buffer.get(), Ct_ref, rtol=0.0, atol=1.0e-5)
print '====Verifying hidden states===='
allclose_with_out(lstm.h_buffer.get(), Hout_ref, rtol=0.0, atol=1.0e-5)
print 'fprop is verified'
# now test the bprop
# generate random deltas tensor
deltas = np.random.randn(*hidden_shape)
lstm.bprop(lstm.be.array(deltas))
# grab the delta W from gradient buffer
dWinput_neon = lstm.dW_input.get()
dWrecur_neon = lstm.dW_recur.get()
db_neon = lstm.db.get()
# import pdb; pdb.set_trace()
deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size)
(dX_ref, dWLSTM_ref, dc0_ref,
dh0_ref) = lstm_ref.backward(deltas_ref, batch_cache)
dWrecur_ref = dWLSTM_ref[-hidden_size:, :]
dWinput_ref = dWLSTM_ref[1:input_size + 1, :]
db_ref = dWLSTM_ref[0, :]
dX_ref = dX_ref.reshape(seq_len * batch_size, input_size).T
# compare results
print 'Making sure neon LSTM match numpy LSTM in bprop'
print '====Verifying update on W_recur===='
assert allclose_with_out(dWrecur_neon,
dWrecur_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on W_input===='
assert allclose_with_out(dWinput_neon,
dWinput_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on bias===='
assert allclose_with_out(db_neon.flatten(), db_ref, rtol=0.0, atol=1.0e-5)
print '====Verifying output delta===='
assert allclose_with_out(lstm.out_deltas_buffer.get(),
dX_ref,
rtol=0.0,
atol=1.0e-5)
print 'bprop is verified'
return
def check_lstm(seq_len, input_size, hidden_size,
batch_size, init_func, inp_moms=[0.0, 1.0]):
# init_func is the initializer for the model params
# inp_moms is the [ mean, std dev] of the random input
input_shape = (input_size, seq_len * batch_size)
hidden_shape = (hidden_size, seq_len * batch_size)
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
# neon LSTM
lstm = LSTM(hidden_size,
init_func,
activation=Tanh(),
gate_activation=Logistic())
inp = np.random.rand(*input_shape)*inp_moms[1] + inp_moms[0]
inpa = lstm.be.array(inp)
# run neon fprop
lstm.configure((input_size, seq_len))
lstm.prev_layer = True # Hack to force allocating a delta buffer
lstm.allocate()
lstm.set_deltas([lstm.be.iobuf(lstm.in_shape)])
lstm.fprop(inpa)
# reference numpy LSTM
lstm_ref = RefLSTM()
WLSTM = lstm_ref.init(input_size, hidden_size)
# make ref weights and biases with neon model
WLSTM[0, :] = lstm.b.get().T
WLSTM[1:input_size+1, :] = lstm.W_input.get().T
WLSTM[input_size+1:] = lstm.W_recur.get().T
# transpose input X and do fprop
inp_ref = inp.copy().T.reshape(seq_len, batch_size, input_size)
(Hout_ref, cprev, hprev, batch_cache) = lstm_ref.forward(inp_ref,
WLSTM)
# the output needs transpose as well
Hout_ref = Hout_ref.reshape(seq_len * batch_size, hidden_size).T
IFOGf_ref = batch_cache['IFOGf'].reshape(seq_len * batch_size, hidden_size * 4).T
Ct_ref = batch_cache['Ct'].reshape(seq_len * batch_size, hidden_size).T
# compare results
print '====Verifying IFOG===='
allclose_with_out(lstm.ifog_buffer.get(),
IFOGf_ref,
rtol=0.0,
atol=1.0e-5)
print '====Verifying cell states===='
allclose_with_out(lstm.c_act_buffer.get(),
Ct_ref,
rtol=0.0,
atol=1.0e-5)
print '====Verifying hidden states===='
allclose_with_out(lstm.outputs.get(),
Hout_ref,
rtol=0.0,
atol=1.0e-5)
print 'fprop is verified'
# now test the bprop
# generate random deltas tensor
deltas = np.random.randn(*hidden_shape)
lstm.bprop(lstm.be.array(deltas))
# grab the delta W from gradient buffer
dWinput_neon = lstm.dW_input.get()
dWrecur_neon = lstm.dW_recur.get()
db_neon = lstm.db.get()
deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size)
(dX_ref, dWLSTM_ref, dc0_ref, dh0_ref) = lstm_ref.backward(deltas_ref,
batch_cache)
dWrecur_ref = dWLSTM_ref[-hidden_size:, :]
dWinput_ref = dWLSTM_ref[1:input_size+1, :]
db_ref = dWLSTM_ref[0, :]
dX_ref = dX_ref.reshape(seq_len * batch_size, input_size).T
# compare results
print 'Making sure neon LSTM match numpy LSTM in bprop'
print '====Verifying update on W_recur===='
assert allclose_with_out(dWrecur_neon,
dWrecur_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on W_input===='
assert allclose_with_out(dWinput_neon,
dWinput_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on bias===='
assert allclose_with_out(db_neon.flatten(),
db_ref,
rtol=0.0,
atol=1.0e-5)
print '====Verifying output delta===='
assert allclose_with_out(lstm.out_deltas_buffer.get(),
dX_ref,
rtol=0.0,
atol=1.0e-5)
print 'bprop is verified'
return
def gradient_check_ref(seq_len,
input_size,
hidden_size,
batch_size,
epsilon=1.0e-5,
dtypeu=np.float64,
threshold=1e-4):
# this is a check of the reference code itself
# estimates the gradients by adding perturbations
# to the input and the weights and compares to
# the values calculated in bprop
# generate sparse random input matrix
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
input_shape = (seq_len, input_size, batch_size)
# hidden_shape = (seq_len, hidden_size, batch_size)
(inp_bl, nz_inds) = sparse_rand(input_shape, frac=1.0 / input_shape[1])
inp_bl = np.random.randn(*input_shape)
# convert input matrix from neon to ref code format
inp_bl = inp_bl.swapaxes(1, 2).astype(dtypeu)
# generate reference LSTM
lstm_ref = RefLSTM()
WLSTM = lstm_ref.init(input_size, hidden_size).astype(dtypeu)
# init parameters as done for neon
WLSTM = np.random.randn(*WLSTM.shape)
(Hout, cprev, hprev, cache) = lstm_ref.forward(inp_bl, WLSTM)
# scale Hout by random matrix...
rand_scale = np.random.random(Hout.shape) * 2.0 - 1.0
rand_scale = dtypeu(rand_scale)
# line below would be the loss function
# loss_bl = np.sum(rand_scale * Hout)
# run bprop, input deltas is rand_scale
(dX_bl, dWLSTM_bl, dc0, dh0) = lstm_ref.backward(rand_scale, cache)
grads_est = np.zeros(dX_bl.shape)
inp_pert = inp_bl.copy()
for pert_ind in range(inp_bl.size):
save_val = inp_pert.flat[pert_ind]
# add/subtract perturbations to input
inp_pert.flat[pert_ind] = save_val + epsilon
# and run fprop on perturbed input
(Hout_pos, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM)
inp_pert.flat[pert_ind] = save_val - epsilon
(Hout_neg, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM)
# calculate the loss on outputs
loss_pos = np.sum(rand_scale * Hout_pos)
loss_neg = np.sum(rand_scale * Hout_neg)
grads_est.flat[pert_ind] = 0.5 * (loss_pos - loss_neg) / epsilon
# reset input
inp_pert.flat[pert_ind] = save_val
# assert that gradient estimates within rel threshold of
# bprop calculated deltas
assert allclose_with_out(grads_est, dX_bl, rtol=threshold, atol=0.0)
return
def gradient_check_ref(seq_len, input_size, hidden_size, batch_size,
epsilon=1.0e-5, dtypeu=np.float64, threshold=1e-4):
# this is a check of the reference code itself
# estimates the gradients by adding perturbations
# to the input and the weights and compares to
# the values calculated in bprop
# generate sparse random input matrix
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
input_shape = (seq_len, input_size, batch_size)
# hidden_shape = (seq_len, hidden_size, batch_size)
(inp_bl, nz_inds) = sparse_rand(input_shape, frac=1.0/input_shape[1])
inp_bl = np.random.randn(*input_shape)
# convert input matrix from neon to ref code format
inp_bl = inp_bl.swapaxes(1, 2).astype(dtypeu)
# generate reference LSTM
lstm_ref = RefLSTM()
WLSTM = lstm_ref.init(input_size, hidden_size).astype(dtypeu)
# init parameters as done for neon
WLSTM = np.random.randn(*WLSTM.shape)
(Hout, cprev, hprev, cache) = lstm_ref.forward(inp_bl, WLSTM)
# scale Hout by random matrix...
rand_scale = np.random.random(Hout.shape)*2.0 - 1.0
rand_scale = dtypeu(rand_scale)
# line below would be the loss function
# loss_bl = np.sum(rand_scale * Hout)
# run bprop, input deltas is rand_scale
(dX_bl, dWLSTM_bl, dc0, dh0) = lstm_ref.backward(rand_scale, cache)
grads_est = np.zeros(dX_bl.shape)
inp_pert = inp_bl.copy()
for pert_ind in range(inp_bl.size):
save_val = inp_pert.flat[pert_ind]
# add/subtract perturbations to input
inp_pert.flat[pert_ind] = save_val + epsilon
# and run fprop on perturbed input
(Hout_pos, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM)
inp_pert.flat[pert_ind] = save_val - epsilon
(Hout_neg, cprev, hprev, cache) = lstm_ref.forward(inp_pert, WLSTM)
# calculate the loss on outputs
loss_pos = np.sum(rand_scale*Hout_pos)
loss_neg = np.sum(rand_scale*Hout_neg)
grads_est.flat[pert_ind] = 0.5*(loss_pos-loss_neg)/epsilon
# reset input
inp_pert.flat[pert_ind] = save_val
# assert that gradient estimates within rel threshold of
# bprop calculated deltas
assert allclose_with_out(grads_est, dX_bl, rtol=threshold, atol=0.0)
return
