def compare_optimizer_variable_select(opt_ng, opt_ref): # Set up data placeholders C = ng.make_axis(20) N = ng.make_axis(32, name='N') data = ng.placeholder([C, N]) target = ng.placeholder([N]) # params to be updated using optimizer to be tested np_W1 = np.random.rand(C.length) np_W2 = np.random.rand(C.length) W1 = ng.variable([C], initial_value=np_W1) W2 = ng.variable([C], initial_value=np_W2) # Set up op graph cost = ng.sum(target - ng.dot(W1, data) - ng.dot(W2, data), out_axis=()) updated_weights = ng.sequential([opt_ng(cost, variables=[W1]), W1]) # Set up the computation and run the "train" loop with ExecutorFactory() as ex: opt_ng_comp = ex.transformer.computation([updated_weights, W2], data, target) mock_dataset = data_generator(20, C.length, N.length) for x, y in mock_dataset: [ng_W1, ng_W2] = opt_ng_comp(x, y) # updated weights for ngraph optimizer np_W1 = opt_ref(x, np_W1) # updated weights for reference optimizer ng.testing.assert_allclose(np_W1, ng_W1, rtol=1e-3) ng.testing.assert_allclose(np_W2, ng_W2, rtol=1e-3)
def test_flat_tensor_dot_tensor(): """ Ensure that a flattened argument axis is not unflattend in the result. """ H = ng.make_axis(2) W = ng.make_axis(7) C = ng.make_axis(3) K = ng.make_axis(11) axes_a = ng.make_axes([H, W, C]) a = ng.constant(np.ones(axes_a.lengths), axes=axes_a) flat_a = ng.flatten_at(a, 2) axes_b = ng.make_axes([C, K]) b = ng.constant(np.ones(axes_b.lengths), axes=axes_b) result = ng.dot(b, flat_a) with ExecutorFactory() as factory: result_fun = factory.executor(result) result_val = result_fun() result_correct = np.ones_like(result_val) * C.length ng.testing.assert_allclose(result_val, result_correct)
def test_weight_clipping(w_clip, optimizer): opt_ng = optimizer(0.1, weight_clip_value=w_clip) # Set up data placeholders C = ng.make_axis(20) N = ng.make_axis(32, name='N') data = ng.placeholder([C, N]) target = ng.placeholder([N]) # params to be updated using optimizer to be tested # make sure initial values are higher than clip values np_W = 10 * w_clip * (2 * np.random.rand(C.length) - 1) W = ng.variable([C], initial_value=np_W) # double check generated initial W value assert np.max(np_W) > w_clip assert np.min(np_W) < -w_clip # Set up op graph cost = ng.sum(target - ng.dot(W, data), out_axis=()) updated_weights = ng.sequential([opt_ng(cost), W]) epsilon = w_clip * 1e-3 # Set up the computation and run the "train" loop with ExecutorFactory() as ex: opt_ng_comp = ex.transformer.computation(updated_weights, data, target) mock_dataset = data_generator(20, C.length, N.length) for x, y in mock_dataset: ng_W = opt_ng_comp(x, y) # updated weights for ngraph optimizer assert np.max(ng_W) < w_clip + epsilon assert np.min(ng_W) > -w_clip - epsilon
def test_flatten_deriv_simplified(): """ Test derivative with dot and flatten """ ax_N = ng.make_axis(length=3) ax_Y = ng.make_axis(length=2) x = ng.placeholder(ng.make_axes([ax_N])) w = ng.constant([5, 2], axes=ng.make_axes([ax_Y])) logits = ng.dot(x, w) cost = ng.sum(logits, reduction_axes=logits.axes) delta = 0.001 u = rng.uniform(.1, 5.0, x.axes) check_derivative(cost, x, delta, u, atol=1e-2, rtol=1e-2)
def test_cputensor_dot(): Y = ng.make_axis(length=2) M = ng.make_axis(length=1) N = ng.make_axis(length=3) np_a = np.array([[1, 2, 3]], dtype=np.float32) np_b = np.array([[1, 2], [2, 3], [3, 4]], dtype=np.float32) np_c = np.dot(np_a, np_b) a = ng.constant(np_a, [M, N]).named('a') b = ng.constant(np_b, [N, Y]).named('b') c = ng.dot(a, b) with executor(c) as ex: result = ex() assert np.array_equal(result, np_c)
def test_dot(): H = ng.make_axis(length=1) W = ng.make_axis(length=4) np_a = np.array([[1, 2, 3, 4]], dtype=np.float32) np_b = np.array(3, dtype=np.float32) a = ng.constant(np_a, [H, W]) b = ng.constant(np_b, []) c = ng.dot(a, b) with executor(c) as _dot: _dot_val = _dot() # compute reference _dot_val_ref = np.dot(np_a, np_b) # this checks the dot product between scalar and vector, this is equivalent to # elementwise multiplication between scalar and vector assert np.allclose(_dot_val, _dot_val_ref)
def test_cputensor_mlp(): """TODO.""" D = ng.make_axis(length=3) H = ng.make_axis(length=2) N = ng.make_axis(length=1) np_x = np.array([[1, 2, 3]], dtype=np.float32) np_w = np.array([[1, 1], [1, 1], [1, 1]], dtype=np.float32) np_b = np.array([1, 2], dtype=np.float32) np_c = np.dot(np_x, np_w) + np_b x = ng.constant(np_x, [N, D]) w = ng.constant(np_w, [D, H]) b = ng.constant(np_b, [H]) wx = ng.dot(x, w) c = wx + b with executor(c) as ex: result = ex() assert np.array_equal(result, np_c)
def test_logreg(): # xs: (C, N), y: (N,) xs = np.array([[0.52, 0.88, 0.52, 0.74], [1.12, -1.08, 0.06, -2.49], [0.77, 0.15, -1.3, 1.39]]) ys = np.array([1, 1, 0, 1]) max_iter = 10 alpha = 0.1 thetas = np.array([0., 0., 0.]) np_logreg = NumpyLogreg(xs, ys, thetas) C, N = ng.make_axis(length=3), ng.make_axis(length=4) # input tensors xs_v = ng.placeholder((C, N)) ys_v = ng.placeholder([N]) alpha_v = ng.placeholder(()) thetas_var = ng.variable([C], initial_value=thetas) # define ops ys_pred = ng.sigmoid(ng.dot(thetas_var, xs_v)) log_likelihoods = ng.log(ys_pred) * ys_v + ng.log(1 - ys_pred) * (1 - ys_v) loss = -ng.sum(log_likelihoods, reduction_axes=[N]) grad_comp = ng.deriv(loss, thetas_var) weight_update = ng.sequential( [ng.assign(thetas_var, thetas_var - alpha_v * grad_comp), thetas_var]) # transformer with ExecutorFactory() as ex: train_eval_func = ex.executor([grad_comp, loss, weight_update], xs_v, ys_v, alpha_v) # evaluate for i in range(max_iter): grad_np, loss_np, thetas_np = np_logreg.optimize(alpha) grad_ng, loss_ng, thetas_ng = train_eval_func(xs, ys, alpha) ng.testing.assert_allclose(loss_np, loss_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False) ng.testing.assert_allclose(grad_np, grad_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False) ng.testing.assert_allclose(thetas_np, thetas_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False)
def test_evaluation_twice(): """Test executing a computation graph twice on a one layer MLP.""" C = ng.make_axis(length=2) D = ng.make_axis(length=2) W = ng.make_axis(length=1) x = ng.constant(np.array([[1, 2], [3, 4]], dtype='float32'), ng.make_axes([C, D])) hidden1_weights = ng.constant(np.array([[1], [1]], dtype='float32'), ng.make_axes([C, W])) hidden1_biases = ng.constant(np.array([[2], [2]], dtype='float32'), ng.make_axes([D, W])) hidden1 = ng.dot(hidden1_weights, x) + hidden1_biases with executor(hidden1) as comp: result_1 = comp() result_2 = comp() assert np.array_equal(result_1, result_2)
def test_sum(num_units, sequence_length, batch_size): """ This tests for a non-deterministic error that arose in ng.sum following a dot product using the gpu transformer. """ shape = (num_units, sequence_length, batch_size) np_inp = np.random.uniform(-1, 1, shape) # Use an identity weight matrix on top of it np_w = np.eye(shape[0]) # Create ngraph versions inp = ng.constant(np_inp) reduction_axes = inp.axes[:-2] other_axes = inp.axes[-2:] new_axis = ng.make_axis(length=shape[0]) w_axes = ng.make_axes(new_axis) | reduction_axes w = ng.constant(np_w, axes=w_axes) # Reshape to do similar dot in numpy inp_reshape = np.reshape( np_inp, (np.prod(reduction_axes.lengths), np.prod(other_axes.lengths))) w_reshape = np.reshape(np_w, (new_axis.length, inp_reshape.shape[0])) # Reduce dimensions with identity weight matrix np_x = np.dot(w_reshape, inp_reshape) x = ng.dot(w, inp) # Sum over all but the first axis output_axes = ng.make_axes(x.axes[0]) y = ng.sum(x, out_axes=output_axes) np_y = np.sum(np_x, axis=1) with executor([y, x]) as f: y_val, x_val = f() assert_allclose(x_val.ravel(), np_x.ravel(), atol=1e-1) assert_allclose(y_val, np_y, atol=1e-1)
def test_tensor_dot_tensor(): """TODO.""" C = ng.make_axis().named('C') D = ng.make_axis().named('D') H = ng.make_axis().named('H') N = ng.make_axis().named('N') tests = [ { 'tensor1': [[1, 2], [4, 5], [3, 4]], 'tensor1_axes': (C, D), 'tensor2': [2, 5], 'tensor2_axes': (D,), 'expected_output': [12, 33, 26], 'axes_lengths': {C: 3, D: 2} }, { 'tensor1': [[1, 4, 3], [2, 5, 4]], 'tensor1_axes': (D, C), 'tensor2': [2, 5], 'tensor2_axes': (D,), 'expected_output': [12, 33, 26], 'axes_lengths': {C: 3, D: 2} }, { 'tensor1': [[[1, 4], [2, 5]], [[7, 12], [13, 2]]], 'tensor1_axes': (N, D, C), 'tensor2': [[[3, 6], [7, 2]], [[9, 8], [10, 4]]], 'tensor2_axes': (H, D, C), 'expected_output': [[51, 81], [188, 297]], 'axes_lengths': {N: 2, D: 2, C: 2, H: 2} }, { 'tensor1': [1, 2], 'tensor1_axes': (C,), 'tensor2': [7, 11, 13], 'tensor2_axes': (D,), 'expected_output': [[7, 11, 13], [14, 22, 26]], 'axes_lengths': {C: 2, D: 3} }, { 'tensor1': [[1, 4], [6, 2]], 'tensor1_axes': (C, D), 'tensor2': [[1, 4], [6, 2]], 'tensor2_axes': (C, D), 'expected_output': 57, 'axes_lengths': {C: 2, D: 2} } ] for test in tests: # set up axis for axis, length in test['axes_lengths'].items(): axis.length = length # set up tensors tensor1 = ng.placeholder(test['tensor1_axes']) value1 = np.array(test['tensor1'], dtype=np.float32) tensor2 = ng.placeholder(test['tensor2_axes']) value2 = np.array( test['tensor2'], dtype=np.float32 ) # compute outputs expected_output = np.array(test['expected_output'], dtype=np.float32) dot = ng.dot(tensor1, tensor2) with executor(dot, tensor1, tensor2) as evaluated_fun: # assert outputs are equal evaluated = evaluated_fun(value1, value2) np.testing.assert_equal(evaluated, expected_output)
def test_tensor_dot_tensor(): """TODO.""" C = ng.make_axis().named('C') D = ng.make_axis().named('D') H = ng.make_axis().named('H') N = ng.make_axis().named('N') tests = [ { 'tensor1': [[1, 2], [4, 5], [3, 4]], 'tensor1_axes': (C, D), 'tensor2': [2, 5], 'tensor2_axes': (D,), 'expected_output': [12, 33, 26], 'axes_lengths': {C: 3, D: 2} }, { 'tensor1': [[1, 4, 3], [2, 5, 4]], 'tensor1_axes': (D, C), 'tensor2': [2, 5], 'tensor2_axes': (D,), 'expected_output': [12, 33, 26], 'axes_lengths': {C: 3, D: 2} }, { 'tensor1': [[[1, 4], [2, 5]], [[7, 12], [13, 2]]], 'tensor1_axes': (N, D, C), 'tensor2': [[[3, 6], [7, 2]], [[9, 8], [10, 4]]], 'tensor2_axes': (H, D, C), 'expected_output': [[51, 81], [188, 297]], 'axes_lengths': {N: 2, D: 2, C: 2, H: 2} }, { 'tensor1': [1, 2], 'tensor1_axes': (C,), 'tensor2': [7, 11, 13], 'tensor2_axes': (D,), 'expected_output': [[7, 11, 13], [14, 22, 26]], 'axes_lengths': {C: 2, D: 3} }, { 'tensor1': [[1, 4], [6, 2]], 'tensor1_axes': (C, D), 'tensor2': [[1, 4], [6, 2]], 'tensor2_axes': (C, D), 'expected_output': 57, 'axes_lengths': {C: 2, D: 2} } ] for test in tests: # set up axis for axis, length in test['axes_lengths'].items(): axis.length = length # set up tensors tensor1 = ng.placeholder(test['tensor1_axes']) value1 = np.array(test['tensor1'], dtype=np.float32) tensor2 = ng.placeholder(test['tensor2_axes']) value2 = np.array( test['tensor2'], dtype=np.float32 ) # compute outputs expected_output = np.array(test['expected_output'], dtype=np.float32) with ExecutorFactory() as ex: dot = ng.dot(tensor1, tensor2) evaluated_fun = ex.executor(dot, tensor1, tensor2) deriv1_fun_num = ex.numeric_derivative(dot, tensor1, 1e-3, tensor2) deriv1_fun_sym = ex.derivative(dot, tensor1, tensor2) deriv2_fun_num = ex.numeric_derivative(dot, tensor2, 1e-3, tensor1) deriv2_fun_sym = ex.derivative(dot, tensor2, tensor1) # assert outputs are equal evaluated = evaluated_fun(value1, value2) np.testing.assert_equal(evaluated, expected_output) # assert derivative wrt to both tensors is the same when computed # symbolically by ngraph and numerically deriv1_val_sym = deriv1_fun_sym(value1, value2) deriv2_val_sym = deriv2_fun_sym(value2, value1) deriv1_val_num = deriv1_fun_num(value1, value2) deriv2_val_num = deriv2_fun_num(value2, value1) ng.testing.assert_allclose(deriv1_val_num, deriv1_val_sym, rtol=1e-2, atol=1e-2) ng.testing.assert_allclose(deriv2_val_num, deriv2_val_sym, rtol=1e-2, atol=1e-2)
def test_dot_sum_backprop(): delta = 1e-3 rtol = atol = 1e-2 C = ng.make_axis(length=2).named('C') N = ng.make_axis(length=3, name='N') x_axes = ng.make_axes((C, N)) y_axes = ng.make_axes((C,)) x_np = np.random.random(x_axes.lengths).astype('float32') y_np = np.random.random(y_axes.lengths).astype('float32') x_np[...] = [[1.0, 0.0, 1.0], [2.0, 0.0, 3.0]] y_np[...] = [-1.0, 1.0] x = ng.placeholder(x_axes).named('x') y = ng.placeholder(y_axes).named('y') d = ng.dot(x, y) s = ng.sum(d, out_axes=()) with ExecutorFactory() as ex: s_fun = ex.executor(s, x, y) d_fun = ex.executor(d, x, y) dd_dx_fun_num = ex.numeric_derivative(d, x, delta, y) dd_dx_fun_sym = ex.derivative(d, x, y) dd_dy_fun_num = ex.numeric_derivative(d, y, delta, x) dd_dy_fun_sym = ex.derivative(d, y, x) ds_dx_fun_num = ex.numeric_derivative(s, x, delta, y) ds_dx_fun_sym = ex.derivative(s, x, y) ds_dy_fun_num = ex.numeric_derivative(s, y, delta, x) ds_dy_fun_sym = ex.derivative(s, y, x) # assert outputs are equal d_np = x_np.T.dot(y_np) d_val = d_fun(x_np, y_np) ng.testing.assert_allclose(d_np, d_val, rtol=rtol, atol=atol) s_np = np.sum(d_np) s_val = s_fun(x_np, y_np) ng.testing.assert_allclose(s_val, s_np, rtol=rtol, atol=atol) # assert derivative wrt to both tensors is the same when computed # symbolically by ngraph and numerically dd_dx_val_sym = dd_dx_fun_sym(x_np, y_np) dd_dy_val_sym = dd_dy_fun_sym(y_np, x_np) ds_dx_val_sym = ds_dx_fun_sym(x_np, y_np) ds_dy_val_sym = ds_dy_fun_sym(y_np, x_np) dd_dx_val_num = dd_dx_fun_num(x_np, y_np) dd_dy_val_num = dd_dy_fun_num(y_np, x_np) ds_dx_val_num = ds_dx_fun_num(x_np, y_np) ds_dy_val_num = ds_dy_fun_num(y_np, x_np) ng.testing.assert_allclose(dd_dx_val_num, dd_dx_val_sym, rtol=rtol, atol=atol) ng.testing.assert_allclose(dd_dy_val_num, dd_dy_val_sym, rtol=rtol, atol=atol) ng.testing.assert_allclose(ds_dx_val_num, ds_dx_val_sym, rtol=rtol, atol=atol) ng.testing.assert_allclose(ds_dy_val_num, ds_dy_val_sym, rtol=rtol, atol=atol)