def test_graph_logreg(seed): rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4], need_grad=True) w1 = nn.Variable([12, 5], need_grad=True) w2 = nn.Variable([12, 5], need_grad=True) b1 = nn.Variable([5], need_grad=True) b2 = nn.Variable([5], need_grad=True) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) w1.d = rng.randn(*w1.shape) w2.d = rng.randn(*w2.shape) b1.d = rng.randn(*b1.shape) b2.d = rng.randn(*b2.shape) t.d = rng.randint(0, 5, size=t.shape) nn.set_default_context(nn.Context()) # Forwardprop by definintion z1 = F.affine(x, w1, b1, 1) z2 = F.affine(x, w2, b2, 1) l1 = F.softmax_cross_entropy(z1, t, 1) L1 = F.mean(l1) l2 = F.softmax_cross_entropy(z2, t, 1) L2 = F.mean(l2) nn.forward_all([L1, L2]) # Backprop for z1 # Diff should be initialized since they are always accumulated x.g = 0 w1.g = 0 b1.g = 0 L1.backward(clear_buffer=True) inputs = [x, w1, b1] from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L1, inputs, 1e-3, False) assert_allclose(ngrad, agrad, atol=1e-2) # Backprop for z2 # Diff should be initialized since they are always accumulated x.g = 0 w2.g = 0 b2.g = 0 L2.backward(clear_buffer=True) inputs = [x, w2, b2] from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L2, inputs, 1e-3, False) assert_allclose(ngrad, agrad, atol=1e-2)
def test_graph_logreg(seed): rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4], need_grad=True) w = nn.Variable([12, 5], need_grad=True) b = nn.Variable([5], need_grad=True) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) w.d = rng.randn(*w.shape) b.d = rng.randn(*b.shape) t.d = rng.randint(0, 5, size=t.shape) nn.set_default_context(nn.Context()) # Forwardprop by definition with nn.auto_forward(): z = F.affine(x, w, b, 1) l = F.softmax_cross_entropy(z, t, 1) L = F.mean(l) # Backprop # Diff should be initialized since they are always accumulated x.g = 0 w.g = 0 b.g = 0 L.backward(clear_buffer=True) x.g = rng.randn(*x.shape) inputs = [x, w, b] from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L, inputs, 1e-3) assert_allclose(ngrad, agrad, atol=1e-2)
def test_graph_logreg(seed): rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4], need_grad=True) w = nn.Variable([12, 5], need_grad=True) b = nn.Variable([5], need_grad=True) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) w.d = rng.randn(*w.shape) b.d = rng.randn(*b.shape) t.d = rng.randint(0, 5, size=t.shape) nn.set_default_context(nn.Context()) # Forwardprop by definintion with nn.auto_forward(): z = F.affine(x, w, b, 1) l = F.softmax_cross_entropy(z, t, 1) L = F.mean(l) # Backprop # Diff should be initialized since they are always accumulated x.g = 0 w.g = 0 b.g = 0 L.backward(clear_buffer=True) x.g = rng.randn(*x.shape) inputs = [x, w, b] from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L, inputs, 1e-3) assert np.allclose(ngrad, agrad, atol=1e-2)
def test_graph_model(model, seed): np.random.seed(313) rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4, 4], need_grad=True) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) t.d = rng.randint(0, 5, size=t.shape) nn.set_default_context(nn.Context()) # Forwardprop by definition nn.clear_parameters() if model == "mlp": with nn.parameter_scope('fc1'): z = PF.affine(x, 3) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) elif model == "recurrent": with nn.parameter_scope('fc1'): z = PF.affine(x, 8) z2 = F.relu(z, inplace=True) h = z2 for _ in range(2): with nn.parameter_scope('fc2'): h = PF.affine(h, 8) h = F.relu(h, inplace=True) with nn.parameter_scope('fc3'): z3 = PF.affine(h, 5) elif model == "convolution": with nn.parameter_scope('conv1'): z = PF.convolution(x, 3, (2, 2)) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) else: raise ValueError() l = F.softmax_cross_entropy(z3, t, 1) L = F.mean(l) # Forwardprop L.forward(clear_no_need_grad=True) # Backprop # Diff should be initialized since they are always accumulated x.grad.zero() L.backward(clear_buffer=True) x.g = rng.randn(*x.shape) parameters = nn.get_parameters() for param in parameters.values(): param.grad.zero() inputs = [x] + list(parameters.values()) from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L, inputs, 1e-3) assert_allclose(ngrad, agrad, atol=1.05e-2)
def test_graph_model(model, seed): np.random.seed(313) rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4, 4], need_grad=True) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) t.d = rng.randint(0, 5, size=t.shape) nn.set_default_context(nn.Context()) # Forwardprop by definintion nn.clear_parameters() if model == "mlp": with nn.parameter_scope('fc1'): z = PF.affine(x, 3) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) elif model == "recurrent": with nn.parameter_scope('fc1'): z = PF.affine(x, 3) z2 = F.relu(z, inplace=True) h = z2 for _ in range(2): with nn.parameter_scope('fc2'): h = PF.affine(h, 3) h = F.relu(h, inplace=True) with nn.parameter_scope('fc3'): z3 = PF.affine(h, 5) elif model == "convolution": with nn.parameter_scope('conv1'): z = PF.convolution(x, 3, (2, 2)) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) else: raise ValueError() l = F.softmax_cross_entropy(z3, t, 1) L = F.mean(l) # Forwardprop L.forward(clear_no_need_grad=True) # Backprop # Diff should be initialized since they are always accumulated x.grad.zero() L.backward(clear_buffer=True) x.g = rng.randn(*x.shape) parameters = nn.get_parameters() for param in parameters.values(): param.grad.zero() inputs = [x] + list(parameters.values()) from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L, inputs, 1e-3) assert np.allclose(ngrad, agrad, atol=1.05e-2)