def test_biRNN_bprop(backend_default, fargs, deltas_buffer): # basic sanity check with 0 weights random inputs seq_len, input_size, hidden_size, batch_size = fargs in_shape = (input_size, seq_len) NervanaObject.be.bsz = batch_size # setup the bi-directional rnn init_glorot = GlorotUniform() birnn = BiRNN(hidden_size, activation=Rectlinclip(slope=0), init=init_glorot) birnn.configure(in_shape) birnn.prev_layer = True birnn.allocate() birnn.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() birnn.set_deltas(deltas_buffer) # same weight for bi-rnn backward and rnn weights birnn.W_input_b[:] = birnn.W_input_f birnn.W_recur_b[:] = birnn.W_recur_f birnn.b_b[:] = birnn.b_f birnn.dW[:] = 0 # same weight for bi-directional rnn init_glorot = GlorotUniform() rnn = Recurrent(hidden_size, activation=Rectlinclip(slope=0), init=init_glorot) rnn.configure(in_shape) rnn.prev_layer = True rnn.allocate() rnn.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() rnn.set_deltas(deltas_buffer) # inputs and views lr = np.random.random((input_size, seq_len * batch_size)) lr_rev = list(reversed(get_steps(lr.copy(), in_shape))) rl = con(lr_rev, axis=1) # allocate gpu buffers inp_lr = birnn.be.array(lr) inp_rl = birnn.be.array(rl) # outputs out_lr_g = birnn.fprop(inp_lr) del_lr = birnn.bprop(out_lr_g).get().copy() birnn.h_buffer[:] = 0 out_rl_g = birnn.fprop(inp_rl) del_rl = birnn.bprop(out_rl_g).get().copy() del_lr_s = get_steps(del_lr, in_shape) del_rl_s = get_steps(del_rl, in_shape) for (x, y) in zip(del_lr_s, reversed(del_rl_s)): assert np.allclose(x, y, rtol=0.0, atol=1.0e-5)
def test_glorot(backend, args): be = NervanaObject.be shape_1 = (1, 2) shape_2 = (1000, 10000) Wdev_1 = be.empty(shape_1) Wdev_2 = be.empty(shape_2) glorot_init = GlorotUniform() glorot_init.fill(Wdev_1) glorot_init.fill(Wdev_2) Whost_1 = Wdev_1.get() Whost_2 = Wdev_2.get() mean_1 = np.mean(Whost_1) mean_2 = np.mean(Whost_2) assert np.abs(mean_1) > np.abs(mean_2) return
def test_lookuptable_ones_error(backend_default, basic_linargs): nin, nout, batch_size, vocab_size = basic_linargs NervanaObject.be.bsz = batch_size dtypeu = np.float32 init_glorot = GlorotUniform() layer = LookupTable( vocab_size=vocab_size, embedding_dim=nout, init=init_glorot) inp = np.random.random_integers(0, vocab_size - 1, size=nin * batch_size) layer.configure(nin) layer.allocate() layer.prev_layer = True # Hack to force delta buffer allocation layer.set_deltas([layer.be.iobuf(nin)]) inputs = layer.be.array(inp.reshape((nin, batch_size))) out = layer.fprop(inputs).get() W = layer.W.get() for i in range(nin * batch_size): assert np.all(W[inp[i]].T == out[:, i]) err = dtypeu(np.ones((nout, nin * batch_size))) layer.bprop(layer.be.array(err)).get() dw = layer.dW.get() unqidx, count = np.unique(inp, return_counts=True) dw_exp = np.zeros((1, nout)) for wrd_id, cnt in zip(unqidx, count): dw_exp = err[:, 0] * cnt assert np.all(dw_exp == dw[wrd_id, :]) return
def test_lookuptable_zeros_error(backend_default, basic_linargs): # basic sanity check with 0 weights random inputs nin, nout, batch_size, vocab_size = basic_linargs NervanaObject.be.bsz = batch_size dtypeu = np.float32 init_glorot = GlorotUniform() layer = LookupTable( vocab_size=vocab_size, embedding_dim=nout, init=init_glorot) inp = np.random.random_integers(0, vocab_size - 1, size=nin * batch_size) layer.configure(nin) layer.allocate() layer.prev_layer = True # Hack to force delta buffer allocation layer.set_deltas([layer.be.iobuf(nin)]) inputs = layer.be.array(inp.reshape((nin, batch_size))) out = layer.fprop(inputs).get() W = layer.W.get() for i in range(nin * batch_size): assert np.all(W[inp[i]].T == out[:, i]) err = dtypeu(np.zeros((nout, nin * batch_size))) layer.bprop(layer.be.array(err)).get() dw = layer.dW.get() assert np.min(dw) == 0.0 and np.max(dw) == 0.0 return
def test_biLSTM_bprop(backend_default, fargs): # basic sanity check with 0 weights random inputs seq_len, input_size, hidden_size, batch_size = fargs in_shape = (input_size, seq_len) out_shape = (hidden_size, seq_len) NervanaObject.be.bsz = batch_size # setup the bi-directional rnn init_glorot = GlorotUniform() bilstm = BiLSTM(hidden_size, gate_activation=Logistic(), activation=Tanh(), init=init_glorot, reset_cells=True) bilstm.configure(in_shape) bilstm.prev_layer = True bilstm.allocate() bilstm.set_deltas([bilstm.be.iobuf(bilstm.in_shape)]) # same weight for bi-rnn backward and rnn weights nout = hidden_size bilstm.W_input_b[:] = bilstm.W_input_f bilstm.W_recur_b[:] = bilstm.W_recur_f bilstm.b_b[:] = bilstm.b_f bilstm.dW[:] = 0 # inputs and views lr = np.random.random((input_size, seq_len * batch_size)) lr_rev = list(reversed(get_steps(lr.copy(), in_shape))) rl = con(lr_rev, axis=1) # allocate gpu buffers inp_lr = bilstm.be.array(lr) inp_rl = bilstm.be.array(rl) # outputs out_lr_g = bilstm.fprop(inp_lr) out_lr = out_lr_g.get().copy() del_lr = bilstm.bprop(out_lr_g).get().copy() bilstm.h_buffer[:] = 0 out_rl_g = bilstm.fprop(inp_rl) out_rl = out_rl_g.get().copy() del_rl = bilstm.bprop(out_rl_g).get().copy() # views out_lr_f_s = get_steps(out_lr[:nout], out_shape) out_lr_b_s = get_steps(out_lr[nout:], out_shape) out_rl_f_s = get_steps(out_rl[:nout], out_shape) out_rl_b_s = get_steps(out_rl[nout:], out_shape) # asserts for x_f, x_b, y_f, y_b in zip(out_lr_f_s, out_lr_b_s, reversed(out_rl_f_s), reversed(out_rl_b_s)): assert np.allclose(x_f, y_b, rtol=0.0, atol=1.0e-5) assert np.allclose(x_b, y_f, rtol=0.0, atol=1.0e-5) del_lr_s = get_steps(del_lr, in_shape) del_rl_s = get_steps(del_rl, in_shape) for (x, y) in zip(del_lr_s, reversed(del_rl_s)): assert np.allclose(x, y, rtol=0.0, atol=1.0e-5)
def test_biLSTM_fprop(backend_default, fargs): # basic sanity check with 0 weights random inputs seq_len, input_size, hidden_size, batch_size = fargs in_shape = (input_size, seq_len) out_shape = (hidden_size, seq_len) NervanaObject.be.bsz = batch_size # setup the bi-directional rnn init_glorot = GlorotUniform() bilstm = BiLSTM(hidden_size, gate_activation=Logistic(), init=init_glorot, activation=Tanh(), reset_cells=True) bilstm.configure(in_shape) bilstm.prev_layer = True bilstm.allocate() # same weight nout = hidden_size bilstm.W_input_b[:] = bilstm.W_input_f bilstm.W_recur_b[:] = bilstm.W_recur_f bilstm.b_b[:] = bilstm.b_f bilstm.dW[:] = 0 # inputs - random and flipped left-to-right inputs lr = np.random.random((input_size, seq_len * batch_size)) lr_rev = list(reversed(get_steps(lr.copy(), in_shape))) rl = con(lr_rev, axis=1) inp_lr = bilstm.be.array(lr) inp_rl = bilstm.be.array(rl) # outputs out_lr = bilstm.fprop(inp_lr).get().copy() bilstm.h_buffer[:] = 0 out_rl = bilstm.fprop(inp_rl).get().copy() # views out_lr_f_s = get_steps(out_lr[:nout], out_shape) out_lr_b_s = get_steps(out_lr[nout:], out_shape) out_rl_f_s = get_steps(out_rl[:nout], out_shape) out_rl_b_s = get_steps(out_rl[nout:], out_shape) # asserts for x_f, x_b, y_f, y_b in zip(out_lr_f_s, out_lr_b_s, reversed(out_rl_f_s), reversed(out_rl_b_s)): assert allclose_with_out(x_f, y_b, rtol=0.0, atol=1.0e-5) assert allclose_with_out(x_b, y_f, rtol=0.0, atol=1.0e-5)
def test_biRNN_fprop(backend_default, fargs): # basic sanity check with 0 weights random inputs seq_len, input_size, hidden_size, batch_size = fargs in_shape = (input_size, seq_len) out_shape = (hidden_size, seq_len) NervanaObject.be.bsz = batch_size # setup the bi-directional rnn init_glorot = GlorotUniform() birnn = BiRNN(hidden_size, activation=Rectlinclip(slope=0), init=init_glorot) birnn.configure(in_shape) birnn.prev_layer = True birnn.allocate() birnn.set_deltas([birnn.be.iobuf(birnn.in_shape)]) # same weight nout = hidden_size birnn.W_input_b[:] = birnn.W_input_f birnn.W_recur_b[:] = birnn.W_recur_f birnn.b_b[:] = birnn.b_f birnn.dW[:] = 0 # inputs - random and flipped left-to-right inputs lr = np.random.random((input_size, seq_len * batch_size)) lr_rev = list(reversed(get_steps(lr.copy(), in_shape))) rl = con(lr_rev, axis=1) inp_lr = birnn.be.array(lr) inp_rl = birnn.be.array(rl) # outputs out_lr = birnn.fprop(inp_lr).get().copy() birnn.h_buffer[:] = 0 out_rl = birnn.fprop(inp_rl).get().copy() # views out_lr_f_s = get_steps(out_lr[:nout], out_shape) out_lr_b_s = get_steps(out_lr[nout:], out_shape) out_rl_f_s = get_steps(out_rl[:nout], out_shape) out_rl_b_s = get_steps(out_rl[nout:], out_shape) # asserts for x_f, x_b, y_f, y_b in zip(out_lr_f_s, out_lr_b_s, reversed(out_rl_f_s), reversed(out_rl_b_s)): assert np.allclose(x_f, y_b, rtol=0.0, atol=1.0e-5) assert np.allclose(x_b, y_f, rtol=0.0, atol=1.0e-5)
def test_lookuptable_rand_error(backend_default, basic_linargs, deltas_buffer): nin, nout, batch_size, vocab_size = basic_linargs NervanaObject.be.bsz = batch_size dtypeu = np.float32 init_glorot = GlorotUniform() layer = LookupTable(vocab_size=vocab_size, embedding_dim=nout, init=init_glorot) inp = np.random.random_integers(0, vocab_size - 1, size=nin * batch_size) layer.configure(nin) layer.allocate() layer.prev_layer = True # Hack to force delta buffer allocation layer.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() layer.set_deltas(deltas_buffer) inputs = layer.be.array(inp.reshape((nin, batch_size))) out = layer.fprop(inputs).get() W = layer.W.get() for i in range(nin * batch_size): assert np.all(W[inp[i]].T == out[:, i]) err = dtypeu(np.random.random((nout, nin * batch_size))) layer.bprop(layer.be.array(err)).get() dw = layer.dW.get() unqidx, count = np.unique(inp, return_counts=True) dw_exp = np.zeros((1, nout)) for wrd_id, cnt in zip(unqidx, count): dw_exp[:] = 0 cnt_exp = 0 for i, w_id in enumerate(inp): if w_id == wrd_id: dw_exp[:] = dw_exp[:] + err[:, i] cnt_exp += 1 assert np.allclose(dw[wrd_id, :], dw_exp, atol=0, rtol=1e-4) assert np.allclose(dw_exp, dw[wrd_id, :], atol=0, rtol=1e-4) assert cnt == cnt_exp return
def test_bibn(backend_default, fargs): seq_len, input_size, hidden_size, batch_size = fargs in_shape = (input_size, seq_len) NervanaObject.be.bsz = batch_size # setup the bi-directional rnn init_glorot = GlorotUniform() birnn = BiBNRNN(hidden_size, activation=Logistic(), init=init_glorot) birnn.configure(in_shape) birnn.prev_layer = True birnn.allocate() birnn.set_deltas([birnn.be.iobuf(birnn.in_shape)]) # test fprop # set the ff buffer inp_np = np.random.random(birnn.h_ff_buffer.shape) inp_be = birnn.be.array(inp_np) birnn.h_ff_buffer[:] = inp_np # compare the bn output with calling the backend bn xsum = birnn.be.zeros_like(birnn.xmean) xvar = birnn.be.zeros_like(birnn.xvar) gmean = birnn.be.zeros_like(birnn.gmean) gvar = birnn.be.zeros_like(birnn.gvar) gamma = birnn.be.ones(birnn.gamma.shape) beta = birnn.be.zeros_like(birnn.beta) grad_gamma = birnn.be.zeros_like(gamma) grad_beta = birnn.be.zeros_like(beta) out_ref = birnn.be.zeros_like(birnn.h_ff_buffer) xsum[:] = birnn.be.sum(birnn.h_ff_buffer, axis=1) birnn.be.compound_fprop_bn(birnn.h_ff_buffer, xsum, xvar, gmean, gvar, gamma, beta, out_ref, birnn.eps, birnn.rho, accumbeta=0, relu=False) # call the bibnrnn layer fprop_bn out_bn = birnn._fprop_bn(birnn.h_ff_buffer, inference=False) assert allclose_with_out(out_bn.get(), out_ref.get(), rtol=0.0, atol=1.0e-5) # test bprop err_np = np.random.random(birnn.h_ff_buffer.shape) err_be = birnn.be.array(err_np) err_out_ref = birnn.be.empty_like(err_be) birnn.be.compound_bprop_bn(err_out_ref, grad_gamma, grad_beta, err_be, inp_be, xsum, xvar, gamma, birnn.eps) err_out_bn = birnn._bprop_bn(err_be, out_bn) assert allclose_with_out(err_out_bn.get(), err_out_ref.get(), rtol=0.0, atol=1.0e-5)