def gradient_calc(seq_len, input_size, hidden_size, batch_size,
epsilon=None, rand_scale=None, inp_bl=None):
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
input_shape = (input_size, seq_len * batch_size)
# generate input if one is not given
if inp_bl is None:
inp_bl = np.random.randn(*input_shape)
# neon lstm instance
lstm = LSTM(hidden_size, Gaussian(), Tanh(), Logistic())
inpa = lstm.be.array(np.copy(inp_bl))
# run fprop on the baseline input
lstm.configure((input_size, seq_len))
lstm.prev_layer = True # Hack to force allocating a delta buffer
lstm.allocate()
lstm.set_deltas([lstm.be.iobuf(lstm.in_shape)])
out_bl = lstm.fprop(inpa).get()
# random scaling/hash to generate fake loss
if rand_scale is None:
rand_scale = np.random.random(out_bl.shape) * 2.0 - 1.0
# loss function would be:
# loss_bl = np.sum(rand_scale * out_bl)
# run back prop with rand_scale as the errors
# use copy to avoid any interactions
deltas_neon = lstm.bprop(lstm.be.array(np.copy(rand_scale))).get()
# add a perturbation to each input element
grads_est = np.zeros(inpa.shape)
inp_pert = inp_bl.copy()
for pert_ind in range(inpa.size):
save_val = inp_pert.flat[pert_ind]
inp_pert.flat[pert_ind] = save_val + epsilon
reset_lstm(lstm)
lstm.allocate()
out_pos = lstm.fprop(lstm.be.array(inp_pert)).get()
inp_pert.flat[pert_ind] = save_val - epsilon
reset_lstm(lstm)
lstm.allocate()
out_neg = lstm.fprop(lstm.be.array(inp_pert)).get()
# calculate the loss with perturbations
loss_pos = np.sum(rand_scale*out_pos)
loss_neg = np.sum(rand_scale*out_neg)
# compute the gradient estimate
grad = 0.5*(loss_pos-loss_neg)/epsilon
grads_est.flat[pert_ind] = grad
# reset the perturbed input element
inp_pert.flat[pert_ind] = save_val
del lstm
return (grads_est, deltas_neon)
def test_biLSTM_fprop_rnn(backend_default, fargs):
# basic sanity check with 0 weights random inputs
seq_len, input_size, hidden_size, batch_size = fargs
in_shape = (input_size, seq_len)
out_shape = (hidden_size, seq_len)
NervanaObject.be.bsz = batch_size
# setup the bi-directional rnn
init_glorot = GlorotUniform()
bilstm = BiLSTM(hidden_size, gate_activation=Logistic(),
activation=Tanh(), init=init_glorot, reset_cells=True)
bilstm.configure(in_shape)
bilstm.prev_layer = True
bilstm.allocate()
# setup the bi-directional rnn
init_glorot = GlorotUniform()
rnn = LSTM(hidden_size, gate_activation=Logistic(),
activation=Tanh(), init=init_glorot, reset_cells=True)
rnn.configure(in_shape)
rnn.prev_layer = True
rnn.allocate()
# same weight for bi-rnn backward and rnn weights
nout = hidden_size
bilstm.W_input_b[:] = bilstm.W_input_f
bilstm.W_recur_b[:] = bilstm.W_recur_f
bilstm.b_b[:] = bilstm.b_f
bilstm.dW[:] = 0
rnn.W_input[:] = bilstm.W_input_f
rnn.W_recur[:] = bilstm.W_recur_f
rnn.b[:] = bilstm.b_f
rnn.dW[:] = 0
# inputs - random and flipped left-to-right inputs
lr = np.random.random((input_size, seq_len * batch_size))
lr_rev = list(reversed(get_steps(lr.copy(), in_shape)))
rl = con(lr_rev, axis=1)
inp_lr = bilstm.be.array(lr)
inp_rl = bilstm.be.array(rl)
inp_rnn = rnn.be.array(lr)
# outputs
out_lr = bilstm.fprop(inp_lr).get().copy()
bilstm.h_buffer[:] = 0
out_rl = bilstm.fprop(inp_rl).get()
out_rnn = rnn.fprop(inp_rnn).get().copy()
# views
out_lr_f_s = get_steps(out_lr[:nout], out_shape)
out_lr_b_s = get_steps(out_lr[nout:], out_shape)
out_rl_f_s = get_steps(out_rl[:nout], out_shape)
out_rl_b_s = get_steps(out_rl[nout:], out_shape)
out_rnn_s = get_steps(out_rnn, out_shape)
# asserts for fprop
for x_rnn, x_f, x_b, y_f, y_b in zip(out_rnn_s, out_lr_f_s, out_lr_b_s,
reversed(out_rl_f_s), reversed(out_rl_b_s)):
assert allclose_with_out(x_f, y_b, rtol=0.0, atol=1.0e-5)
assert allclose_with_out(x_b, y_f, rtol=0.0, atol=1.0e-5)
assert allclose_with_out(x_rnn, x_f, rtol=0.0, atol=1.0e-5)
assert allclose_with_out(x_rnn, y_b, rtol=0.0, atol=1.0e-5)
def test_biLSTM_fprop_rnn(backend_default, fargs):
# basic sanity check with 0 weights random inputs
seq_len, input_size, hidden_size, batch_size = fargs
in_shape = (input_size, seq_len)
out_shape = (hidden_size, seq_len)
NervanaObject.be.bsz = batch_size
# setup the bi-directional rnn
init_glorot = GlorotUniform()
bilstm = BiLSTM(hidden_size,
gate_activation=Logistic(),
activation=Tanh(),
init=init_glorot,
reset_cells=True)
bilstm.configure(in_shape)
bilstm.prev_layer = True
bilstm.allocate()
# setup the bi-directional rnn
init_glorot = GlorotUniform()
rnn = LSTM(hidden_size,
gate_activation=Logistic(),
activation=Tanh(),
init=init_glorot,
reset_cells=True)
rnn.configure(in_shape)
rnn.prev_layer = True
rnn.allocate()
# same weight for bi-rnn backward and rnn weights
nout = hidden_size
bilstm.W_input_b[:] = bilstm.W_input_f
bilstm.W_recur_b[:] = bilstm.W_recur_f
bilstm.b_b[:] = bilstm.b_f
bilstm.dW[:] = 0
rnn.W_input[:] = bilstm.W_input_f
rnn.W_recur[:] = bilstm.W_recur_f
rnn.b[:] = bilstm.b_f
rnn.dW[:] = 0
# inputs - random and flipped left-to-right inputs
lr = np.random.random((input_size, seq_len * batch_size))
lr_rev = list(reversed(get_steps(lr.copy(), in_shape)))
rl = con(lr_rev, axis=1)
inp_lr = bilstm.be.array(lr)
inp_rl = bilstm.be.array(rl)
inp_rnn = rnn.be.array(lr)
# outputs
out_lr = bilstm.fprop(inp_lr).get().copy()
bilstm.h_buffer[:] = 0
out_rl = bilstm.fprop(inp_rl).get()
out_rnn = rnn.fprop(inp_rnn).get().copy()
# views
out_lr_f_s = get_steps(out_lr[:nout], out_shape)
out_lr_b_s = get_steps(out_lr[nout:], out_shape)
out_rl_f_s = get_steps(out_rl[:nout], out_shape)
out_rl_b_s = get_steps(out_rl[nout:], out_shape)
out_rnn_s = get_steps(out_rnn, out_shape)
# asserts for fprop
for x_rnn, x_f, x_b, y_f, y_b in zip(out_rnn_s, out_lr_f_s, out_lr_b_s,
reversed(out_rl_f_s),
reversed(out_rl_b_s)):
assert allclose_with_out(x_f, y_b, rtol=0.0, atol=1.0e-5)
assert allclose_with_out(x_b, y_f, rtol=0.0, atol=1.0e-5)
assert allclose_with_out(x_rnn, x_f, rtol=0.0, atol=1.0e-5)
assert allclose_with_out(x_rnn, y_b, rtol=0.0, atol=1.0e-5)
def check_lstm(seq_len, input_size, hidden_size,
batch_size, init_func, inp_moms=[0.0, 1.0]):
# init_func is the initializer for the model params
# inp_moms is the [ mean, std dev] of the random input
input_shape = (input_size, seq_len * batch_size)
hidden_shape = (hidden_size, seq_len * batch_size)
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
# neon LSTM
lstm = LSTM(hidden_size,
init_func,
activation=Tanh(),
gate_activation=Logistic())
inp = np.random.rand(*input_shape)*inp_moms[1] + inp_moms[0]
inpa = lstm.be.array(inp)
# run neon fprop
lstm.configure((input_size, seq_len))
lstm.prev_layer = True # Hack to force allocating a delta buffer
lstm.allocate()
lstm.set_deltas([lstm.be.iobuf(lstm.in_shape)])
lstm.fprop(inpa)
# reference numpy LSTM
lstm_ref = RefLSTM()
WLSTM = lstm_ref.init(input_size, hidden_size)
# make ref weights and biases with neon model
WLSTM[0, :] = lstm.b.get().T
WLSTM[1:input_size+1, :] = lstm.W_input.get().T
WLSTM[input_size+1:] = lstm.W_recur.get().T
# transpose input X and do fprop
inp_ref = inp.copy().T.reshape(seq_len, batch_size, input_size)
(Hout_ref, cprev, hprev, batch_cache) = lstm_ref.forward(inp_ref,
WLSTM)
# the output needs transpose as well
Hout_ref = Hout_ref.reshape(seq_len * batch_size, hidden_size).T
IFOGf_ref = batch_cache['IFOGf'].reshape(seq_len * batch_size, hidden_size * 4).T
Ct_ref = batch_cache['Ct'].reshape(seq_len * batch_size, hidden_size).T
# compare results
print '====Verifying IFOG===='
allclose_with_out(lstm.ifog_buffer.get(),
IFOGf_ref,
rtol=0.0,
atol=1.0e-5)
print '====Verifying cell states===='
allclose_with_out(lstm.c_act_buffer.get(),
Ct_ref,
rtol=0.0,
atol=1.0e-5)
print '====Verifying hidden states===='
allclose_with_out(lstm.outputs.get(),
Hout_ref,
rtol=0.0,
atol=1.0e-5)
print 'fprop is verified'
# now test the bprop
# generate random deltas tensor
deltas = np.random.randn(*hidden_shape)
lstm.bprop(lstm.be.array(deltas))
# grab the delta W from gradient buffer
dWinput_neon = lstm.dW_input.get()
dWrecur_neon = lstm.dW_recur.get()
db_neon = lstm.db.get()
deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size)
(dX_ref, dWLSTM_ref, dc0_ref, dh0_ref) = lstm_ref.backward(deltas_ref,
batch_cache)
dWrecur_ref = dWLSTM_ref[-hidden_size:, :]
dWinput_ref = dWLSTM_ref[1:input_size+1, :]
db_ref = dWLSTM_ref[0, :]
dX_ref = dX_ref.reshape(seq_len * batch_size, input_size).T
# compare results
print 'Making sure neon LSTM match numpy LSTM in bprop'
print '====Verifying update on W_recur===='
assert allclose_with_out(dWrecur_neon,
dWrecur_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on W_input===='
assert allclose_with_out(dWinput_neon,
dWinput_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on bias===='
assert allclose_with_out(db_neon.flatten(),
db_ref,
rtol=0.0,
atol=1.0e-5)
print '====Verifying output delta===='
assert allclose_with_out(lstm.out_deltas_buffer.get(),
dX_ref,
rtol=0.0,
atol=1.0e-5)
print 'bprop is verified'
return
def check_lstm(seq_len,
input_size,
hidden_size,
batch_size,
init_func,
inp_moms=[0.0, 1.0]):
# init_func is the initializer for the model params
# inp_moms is the [ mean, std dev] of the random input
input_shape = (input_size, seq_len * batch_size)
hidden_shape = (hidden_size, seq_len * batch_size)
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
# neon LSTM
lstm = LSTM(hidden_size,
init_func,
activation=Tanh(),
gate_activation=Logistic())
inp = np.random.rand(*input_shape) * inp_moms[1] + inp_moms[0]
inpa = lstm.be.array(inp)
# run neon fprop
lstm.configure((input_size, seq_len))
lstm.allocate()
lstm.fprop(inpa)
# reference numpy LSTM
lstm_ref = RefLSTM()
WLSTM = lstm_ref.init(input_size, hidden_size)
# make ref weights and biases with neon model
WLSTM[0, :] = lstm.b.get().T
WLSTM[1:input_size + 1, :] = lstm.W_input.get().T
WLSTM[input_size + 1:] = lstm.W_recur.get().T
# transpose input X and do fprop
inp_ref = inp.copy().T.reshape(seq_len, batch_size, input_size)
(Hout_ref, cprev, hprev, batch_cache) = lstm_ref.forward(inp_ref, WLSTM)
# the output needs transpose as well
Hout_ref = Hout_ref.reshape(seq_len * batch_size, hidden_size).T
IFOGf_ref = batch_cache['IFOGf'].reshape(seq_len * batch_size,
hidden_size * 4).T
Ct_ref = batch_cache['Ct'].reshape(seq_len * batch_size, hidden_size).T
# compare results
print '====Verifying IFOG===='
allclose_with_out(lstm.ifog_buffer.get(), IFOGf_ref, rtol=0.0, atol=1.0e-5)
print '====Verifying cell states===='
allclose_with_out(lstm.c_act_buffer.get(), Ct_ref, rtol=0.0, atol=1.0e-5)
print '====Verifying hidden states===='
allclose_with_out(lstm.h_buffer.get(), Hout_ref, rtol=0.0, atol=1.0e-5)
print 'fprop is verified'
# now test the bprop
# generate random deltas tensor
deltas = np.random.randn(*hidden_shape)
lstm.bprop(lstm.be.array(deltas))
# grab the delta W from gradient buffer
dWinput_neon = lstm.dW_input.get()
dWrecur_neon = lstm.dW_recur.get()
db_neon = lstm.db.get()
# import pdb; pdb.set_trace()
deltas_ref = deltas.copy().T.reshape(seq_len, batch_size, hidden_size)
(dX_ref, dWLSTM_ref, dc0_ref,
dh0_ref) = lstm_ref.backward(deltas_ref, batch_cache)
dWrecur_ref = dWLSTM_ref[-hidden_size:, :]
dWinput_ref = dWLSTM_ref[1:input_size + 1, :]
db_ref = dWLSTM_ref[0, :]
dX_ref = dX_ref.reshape(seq_len * batch_size, input_size).T
# compare results
print 'Making sure neon LSTM match numpy LSTM in bprop'
print '====Verifying update on W_recur===='
assert allclose_with_out(dWrecur_neon,
dWrecur_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on W_input===='
assert allclose_with_out(dWinput_neon,
dWinput_ref.T,
rtol=0.0,
atol=1.0e-5)
print '====Verifying update on bias===='
assert allclose_with_out(db_neon.flatten(), db_ref, rtol=0.0, atol=1.0e-5)
print '====Verifying output delta===='
assert allclose_with_out(lstm.out_deltas_buffer.get(),
dX_ref,
rtol=0.0,
atol=1.0e-5)
print 'bprop is verified'
return
def gradient_calc(seq_len,
input_size,
hidden_size,
batch_size,
epsilon=None,
rand_scale=None,
inp_bl=None):
NervanaObject.be.bsz = NervanaObject.be.batch_size = batch_size
input_shape = (input_size, seq_len * batch_size)
# generate input if one is not given
if inp_bl is None:
inp_bl = np.random.randn(*input_shape)
# neon lstm instance
lstm = LSTM(hidden_size, Gaussian(), Tanh(), Logistic())
inpa = lstm.be.array(np.copy(inp_bl))
# run fprop on the baseline input
lstm.configure((input_size, seq_len))
lstm.allocate()
out_bl = lstm.fprop(inpa).get()
# random scaling/hash to generate fake loss
if rand_scale is None:
rand_scale = np.random.random(out_bl.shape) * 2.0 - 1.0
# loss function would be:
# loss_bl = np.sum(rand_scale * out_bl)
# run back prop with rand_scale as the errors
# use copy to avoid any interactions
deltas_neon = lstm.bprop(lstm.be.array(np.copy(rand_scale))).get()
# add a perturbation to each input element
grads_est = np.zeros(inpa.shape)
inp_pert = inp_bl.copy()
for pert_ind in range(inpa.size):
save_val = inp_pert.flat[pert_ind]
inp_pert.flat[pert_ind] = save_val + epsilon
reset_lstm(lstm)
lstm.allocate()
out_pos = lstm.fprop(lstm.be.array(inp_pert)).get()
inp_pert.flat[pert_ind] = save_val - epsilon
reset_lstm(lstm)
lstm.allocate()
out_neg = lstm.fprop(lstm.be.array(inp_pert)).get()
# calculate the loss with perturbations
loss_pos = np.sum(rand_scale * out_pos)
loss_neg = np.sum(rand_scale * out_neg)
# compute the gradient estimate
grad = 0.5 * (loss_pos - loss_neg) / epsilon
grads_est.flat[pert_ind] = grad
# reset the perturbed input element
inp_pert.flat[pert_ind] = save_val
del lstm
return (grads_est, deltas_neon)