def test_backward(self): """Test gradient computation for inputs and all layer's parameters.""" linear = LinearLayer(n_in=10, n_out=5) check = check_finite_differences(linear.forward, linear.backward, gen_input_fn=lambda: (np.random.randn(30, 10), ), aux_only=True) self.assertTrue(check) inp = (np.random.randn(50, 30, 10), ) checker = TestParamGradInLayer(linear, 'W', layer_input=inp) check = check_finite_differences( checker.forward, checker.backward, gen_input_fn=lambda: (np.random.randn(*linear.params['W'].shape), ), aux_only=True) self.assertTrue(check) checker = TestParamGradInLayer(linear, 'b', layer_input=inp) check = check_finite_differences( checker.forward, checker.backward, gen_input_fn=lambda: (np.random.randn(*linear.params['b'].shape), ), aux_only=True) self.assertTrue(check)
def test_backward(self): np.random.seed(9) def gen(): x = np.random.randn(5, 3, 10) h0 = np.random.randn(3, 14) c0 = np.random.randn(3, 14) return ( x, h0, c0, ) lstm = LSTM(n_in=10, n_out=14) check = check_finite_differences(lstm.forward, lstm.backward, gen_input_fn=gen, aux_only=True, test_inputs=( 0, 1, 2, )) self.assertTrue(check) params_shape = lstm.params['WLSTM'].shape checker = TestParamGradInLayer(lstm, 'WLSTM', layer_input=gen()) check = check_finite_differences(checker.forward, checker.backward, gen_input_fn=lambda: (np.random.randn(*params_shape), ), aux_only=True) self.assertTrue(check)
def test_backward(self): self.assertTrue( check_finite_differences( SeqLoss.forward, SeqLoss.backward, gen_input_fn=lambda: (np.random.dirichlet([1, 1], ( 10, )), [np.random.binomial(1, 0.5) for i in range(10)]), aux_only=True))
def test_backward(self): softmax = Softmax() self.assertTrue( check_finite_differences(softmax.forward, softmax.backward, gen_input_fn=lambda: (np.random.randn(7, 3), ), aux_only=True))
def test_backward(self): calc = DataCalc(max_num=5, n_words=50) db = DB(calc.get_db(), calc.get_vocab()) emb = OneHot(n_tokens=len(db.vocab)) nton = NTON(n_tokens=len(db.vocab), db=db, emb=emb, n_cells=5) nton.print_step = lambda *args, **kwargs: None #nton.max_gen = ((dec_sym, ), _) = emb.forward(([db.vocab['[EOS]']], )) def gen_input(): ((E, ), _) = emb.forward( (np.random.randint(1, len(db.vocab), (5, )), )) return ( E, dec_sym[0], ) check = check_finite_differences(nton.forward, nton.backward, gen_input_fn=gen_input, aux_only=True) self.assertTrue(check) # ['att__Wh', 'att__Wy', 'att__w', 'in_rnn__WLSTM', 'out_rnn__WLSTM', # 'out_rnn_clf__00__W', 'out_rnn_clf__00__b', 'switch__00__W', 'switch__00__b'] #for param_name in ['out_rnn_clf__00__W', 'out_rnn_clf__00__b']: #nton.params.names(): for param_name in ['switch__00__W' ]: #, 'switch__00__b']: #nton.params.names(): params_shape = nton.params[param_name].shape checker = TestParamGradInLayer(nton, param_name, layer_input=gen_input()) check = check_finite_differences( checker.forward, checker.backward, gen_input_fn=lambda: (np.random.randn(*params_shape), ), aux_only=True, test_outputs=(0, ), n_times=100) self.assertTrue(check, msg='Failed check for: %s' % param_name)
def test_backward(self): self.assertTrue( check_finite_differences( Sigmoid.forward, Sigmoid.backward, gen_input_fn=lambda: (np.random.randn(7, 3), ), aux_only=True ) )
def test_backward(self): emb = Embeddings(n_tokens=10, n_dims=100) inp = (np.array([0, 1, 9]), ) checker = TestParamGradInLayer(emb, 'W', layer_input=inp) check = check_finite_differences( checker.forward, checker.backward, gen_input_fn=lambda: (np.random.randn(*emb.params['W'].shape), ), aux_only=True) self.assertTrue(check)
def test_backward(self): seq = Sequential([ LinearLayer(n_in=5, n_out=2, init_w=Eye(), init_b=Constant(0.0)), Softmax() ]) check = check_finite_differences( fwd_fn=seq.forward, bwd_fn=seq.backward, gen_input_fn=lambda: (np.random.randn(3, 5), ), aux_only=True ) self.assertTrue(check)
def test_backward(self): att = Attention(n_hidden=5) def gen_input(): h_out = np.random.randn(11, 5) g_t = np.random.randn(5) emb_in = np.random.randn(11, 13) # Input emb size 13. return (h_out, g_t, emb_in, ) check = check_finite_differences( att.forward, att.backward, gen_input_fn=gen_input, test_inputs=(0, 1, 2), aux_only=True ) self.assertTrue(check)
def test_backward(self): def gen_input(): p1 = np.random.randn(1) in1 = np.random.randn(100) in2 = np.random.randn(100) return ( p1, in1, in2, ) check = check_finite_differences(Switch.forward, Switch.backward, gen_input_fn=gen_input, test_inputs=(0, 1, 2), aux_only=True) self.assertTrue(check)
def test_backward_gen(self): calc = DataCalc(max_num=5, n_words=50) db = DB(calc.get_db(), calc.get_vocab()) n_words = len(db.vocab) emb = OneHot(n_tokens=len(db.vocab)) nton = NTON(n_tokens=len(db.vocab), db=db, emb=emb, n_cells=5) nton.print_step = lambda *args, **kwargs: None shapes = [(n_words, ), (nton.n_cells, ), (nton.n_cells, ), (6, nton.n_cells), (6, n_words)] check = check_finite_differences(nton.forward_gen_step, nton.backward_gen_step, gen_input_fn=lambda: tuple( np.random.randn(*shp) for shp in shapes), aux_only=True, n_times=100) self.assertTrue(check)
def test_forward_backward(self): vocab = Vocab() for i in range(19): vocab.add(str(i)) #print str(i), '->', vocab.add(str(i)), vocab[str(i)] content = [] for i in range(10): for y in range(10): content.append((str(i), str(y), str(i + y))) db = DB2(content, vocab) for e1, r, r2 in [(1, 1, 2), (1, 2, 3), (1, 5, 6), (5, 1, 6)]: ((y, ), aux) = db.forward( (db.get_vector(str(e1)), db.get_vector(str(r)))) dy = np.random.randn(*y.shape) (de1, dr) = db.backward(aux, (dy, )) def gen_input(): (e1, r, e2) = random.choice(db.content) v_e1 = db.get_vector(str(e1)) v_e1 += np.random.randn(*v_e1.shape) v_r = db.get_vector(str(r)) v_r += np.random.randn(*v_r.shape) return ( v_e1, v_r, ) check = check_finite_differences(db.forward, db.backward, gen_input_fn=gen_input, aux_only=True) self.assertTrue(check)