def test_exit_condition(transformer_factory): bsz = 16 class_num = 10 # Limiting maximum absolute value for tensors elements to 7.9. # # There is used np.random.randn function to fill tensors with random values. It can give any # value as a result however values above 5 are highly improbable and would appear very rarely. # Limit 7.9 would almost never modify the tested tensor but would prevent from random # failures from time to time when the test is run in continuous environment. # This limit is approximate upper bound of range [4, 8). Numbers from this region can be # expressed by flexpoint number of the same dec. # Why not 15.9 that is approximate limit of [8, 16) range ? # Numbers above 8 are highly improbable and if appear from time to time can cause random # failures due to reduced accuracy of all numbers in tensor. Most numbers in normal # distribution are close to 0. is_flex = is_flex_factory(transformer_factory) clip_val = 7.9 if is_flex else 0 N, Y = ng.make_axis(bsz), ng.make_axis(class_num) y_val = rng.randn_abs_clip(ng.make_axes([N, Y]), clip_max=clip_val) y = ng.constant(y_val, ng.make_axes([N, Y])) likelihood = ng.log(ng.softmax(y, normalization_axes=y.axes[1])) with ExecutorFactory() as ex: comp = ex.executor(likelihood) val1 = comp() val2 = comp() ng.testing.assert_allclose(val1, val2, atol=0, rtol=0)
def test_unary_op_(ng_func, np_func): H = ng.make_axis().named('H') W = ng.make_axis().named('W') tests = [ { 'tensor1': [[1, 2, 3, 4], [5, 6, 7, 8]], 'tensor1_axes': (H, W), 'axes_lengths': {H: 2, W: 4} }] for test in tests: # set up tensors for axis, length in test['axes_lengths'].items(): axis.length = length tensor1 = ng.placeholder(test['tensor1_axes']) value1 = np.array(test['tensor1'], dtype=np.float32) _ng_func = ng_func(tensor1) with ExecutorFactory() as ex: _ng_computation = ex.executor(_ng_func, tensor1) _ng_val = _ng_computation(value1) _ng_ref = np_func(value1) assert np.allclose(_ng_val, _ng_ref, rtol=0, atol=2)
def test_stack(): W = ng.make_axis(length=4) H = ng.make_axis(length=5) I = ng.make_axis(length=3) axes = ng.make_axes([W, H]) rng = RandomTensorGenerator(0, np.float32) a_v = [rng.uniform(0, 1, axes) for i in range(I.length)] for pos in range(len(axes) + 1): a = [ng.placeholder(axes, initial_value=p) for p in a_v] s = ng.stack(a, I, pos) with ExecutorFactory() as ex: num_funs = [ ex.numeric_derivative(s, p, delta, *(np for np in a if np is not p)) for p in a ] sym_funs = [ ex.derivative(s, p, *(np for np in a if np is not p)) for p in a ] for n_fun, s_fun, a_i in zip(num_funs, sym_funs, a_v): na_is = list(na_i for na_i in a_v if na_i is not a_i) d_n = n_fun(a_i, *na_is) d_s = s_fun(a_i, *na_is) ng.testing.assert_allclose(d_n, d_s, rtol=rtol, atol=atol)
def test_flat_tensor_dot_tensor(): """ Ensure that a flattened argument axis is not unflattend in the result. """ H = ng.make_axis(2) W = ng.make_axis(7) C = ng.make_axis(3) K = ng.make_axis(11) axes_a = ng.make_axes([H, W, C]) a = ng.constant(np.ones(axes_a.lengths), axes=axes_a) flat_a = ng.flatten_at(a, 2) axes_b = ng.make_axes([C, K]) b = ng.constant(np.ones(axes_b.lengths), axes=axes_b) result = ng.dot(b, flat_a) with ExecutorFactory() as factory: result_fun = factory.executor(result) result_val = result_fun() result_correct = np.ones_like(result_val) * C.length ng.testing.assert_allclose(result_val, result_correct)
def test_binary_op(ng_func, np_func): H = ng.make_axis().named('H') W = ng.make_axis().named('W') tests = [ { 'tensor1': [[1, 2, 3, 4], [5, 6, 7, 8]], 'tensor1_axes': (H, W), 'tensor2': [[10, 2, 3, 40], [15, 6, 9, 8]], 'tensor2_axes': (H, W), 'axes_lengths': {H: 2, W: 4} }] for test in tests: # set up tensors for axis, length in test['axes_lengths'].items(): axis.length = length tensor1 = ng.placeholder(test['tensor1_axes']) value1 = np.array(test['tensor1'], dtype=np.float32) tensor2 = ng.placeholder(test['tensor2_axes']) value2 = np.array( test['tensor2'], dtype=np.float32 ) _ng_func = ng_func(tensor1, tensor2) with ExecutorFactory() as ex: _ng_computation = ex.executor(_ng_func, tensor1, tensor2) _ng_val = _ng_computation(value1, value2) _ng_ref = np_func(value1, value2) np.testing.assert_equal(_ng_val, _ng_ref)
def test_variable_init(C): w_init = np.random.rand(C.length) W = ng.variable(ng.make_axes([C]), initial_value=w_init) with ExecutorFactory() as ex: result = ex.executor(W)() ng.testing.assert_allclose(result, w_init)
def test_dropout_train(nin, batch_size, keep): # set inputs N = ng.make_axis(batch_size, name='N') F = ng.make_axis(nin, name='F') inp = ng.placeholder([F, N]) layer = Dropout(keep=keep) fprop = layer(inp) # create data x = np.random.uniform(size=(nin, batch_size)) # evaluate with ExecutorFactory() as ex: comp = ex.executor([fprop, layer.mask], inp) out, mask = comp(x) numpy_out = x * mask[:, None] ng.testing.assert_allclose(out, numpy_out, atol=atol, rtol=rtol) if keep < 1.0: out1, mask1 = out.copy(), mask.copy() out2, mask2 = comp(x) assert (out1 != out2).any() assert (mask1 != mask2).any()
def test_initial_value(): # Test work-around for issue #1138 w = [3, 4, 5] x = ng.constant(w) y = ng.variable([ng.make_axis(length=len(w))], initial_value=x) with ExecutorFactory() as ex: result = ex.executor(y)() ng.testing.assert_allclose(result, np.asarray(w, dtype=np.float32))
def test_fill_state(): with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.fill([N], -1) f = ex.executor(x) x_val = f() assert np.allclose(-1, x_val)
def test_sign(): x_np = np.array([-1.2, 2.3, 0.0, 1.2]) N = ng.make_axis(len(x_np)) x = ng.variable([N]) y = ng.sign(x) y_np = np.sign(x_np) with ExecutorFactory() as ex: y_val = ex.executor(y, x)(x_np) assert np.allclose(y_val, y_np)
def test_modify_state(): with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.variable([N], initial_value=x_np).named('x') val = ng.sequential([ng.assign(x, x + x), x]) f = ex.executor(val) x_val = f() assert np.allclose(x_np + x_np, x_val)
def test_sequential(N): x = ng.variable([N], initial_value=0) x0 = x + x x1 = x + x p = ng.sequential([x0, ng.assign(x, 2), x1, x0]) with ExecutorFactory() as ex: x0_val, x1_val, p_val = ex.executor([x0, x1, p])() assert x0_val == 0 assert x1_val == 4 assert p_val == 0
def baseline_value(self, x): ''' Use defined ngraph constructed computation to evaluate activation on inputs x ''' X = ng.placeholder([ng.make_axis(), ng.make_axis(name='N')]) X.axes.set_shape(x.shape) with ExecutorFactory() as ex: activation_function = ex.executor(self.neon_activation(X), X) return activation_function(x)
def test_write_state(): """ This reads back a tensor set from an argument. No code is generated. """ with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.persistent_tensor([N]).named('x') f = ex.executor(x, x) x_val = f(x_np) assert np.allclose(x_np, x_val)
def test_read_state(): """ This just reads back a tensor. No code is generated. """ with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.variable([N], initial_value=x_np).named('x') f = ex.executor(x) x_val = f() assert np.allclose(x_np, x_val)
def test_scalar_broadcast(): """ Test broadcasting a scalar into a tensor """ with ExecutorFactory() as ex: x_axes = ng.make_axes() broadcast_axes = ng.make_axes([ng.make_axis(2), ng.make_axis(3)]) x = ng.constant(1., axes=x_axes) z = ng.broadcast(x, axes=broadcast_axes) z_comp = ex.executor(z) assert np.array_equal(z_comp(), np.ones(broadcast_axes.lengths))
def test_sequential_side(M): x1_np = 2 x2_np = 3 b_np = 1 x_np = np.array([1, 2, 3], dtype=np.float32) x = ng.variable([M], initial_value=x_np) x1 = ng.persistent_tensor(axes=(), initial_value=x1_np) x2 = ng.persistent_tensor(axes=(), initial_value=x2_np) x1_vo = ng.value_of(x1) x2_vo = ng.value_of(x2) b = ng.persistent_tensor(axes=(), initial_value=b_np) y = ng.sequential([ x1_vo, x2_vo, ng.assign(x1, ng.sum(x, out_axes=()) + x1 * b + (1 - b)), ng.assign(x2, ng.mean(x, out_axes=()) + x2 * b + (1 - b)), x * 2 ]) with ExecutorFactory() as ex: main_effect = ex.executor((y, x1_vo, x2_vo, x1, x2)) current_values = ex.executor((x1, x2)) # Run main path #1 y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect( ) y_np = x_np * 2 assert np.allclose(y_val, y_np) assert np.allclose(x1_init_val, x1_np) assert np.allclose(x2_init_val, x2_np) x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np) x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np) assert np.allclose(x1_final_val, x1_np) assert np.allclose(x2_final_val, x2_np) x1_val, x2_val = current_values() assert np.allclose(x1_val, x1_np) assert np.allclose(x2_val, x2_np) # Run main path #2 (Should be the same as before) y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect( ) y_np = x_np * 2 assert np.allclose(y_val, y_np) assert np.allclose(x1_init_val, x1_np) assert np.allclose(x2_init_val, x2_np) x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np) x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np) assert np.allclose(x1_final_val, x1_np) assert np.allclose(x2_final_val, x2_np)
def test_use_state(): """ Uses the value of a tensor in a computation. """ with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.variable([N], initial_value=x_np).named('x') xx = x + x f = ex.executor(xx) xx_val = f() assert np.allclose(x_np + x_np, xx_val)
def baseline_derivative(self, x): X = ng.placeholder([ng.make_axis(), ng.make_axis(name='N')]) X.axes.set_shape(x.shape) with ExecutorFactory() as ex: activation_derivative = ex.derivative(self.neon_activation(X), X) # hack to get derivatives result = activation_derivative(x) result = result.ravel()[0:result.size:(x.size + 1)] result = result.reshape(x.shape) return result
def test_linear_zeros(input_placeholder, output_size): # basic sanity check with 0 weights random inputs x = np.random.random(input_placeholder.axes.lengths) layer = Linear(nout=output_size, init=UniformInit(0.0, 0.0)) with ExecutorFactory() as ex: if ex.transformer.transformer_name == 'hetr': pytest.xfail("hetr fork-safe issue on mac") comp = ex.executor(layer(input_placeholder), input_placeholder) output_values = comp(x) assert np.min(output_values) == 0.0 and np.max(output_values) == 0.0
def test_concatenate(concatenate_variables): x_list, np_list, pos = concatenate_variables with ExecutorFactory() as ex: v = ng.concat_along_axis(x_list, x_list[0].axes[pos]) d = ng.deriv(v, x_list[0], error=ng.constant(np.ones(v.axes.lengths), axes=v.axes)) f = ex.executor([v, d]) e_v, e_d = f() np_v = np.concatenate(np_list, axis=pos) ng.testing.assert_allclose(e_v.copy(), np_v) ng.testing.assert_allclose(e_d.copy(), np.ones(x_list[0].axes.lengths))
def test_concatenate(): with ExecutorFactory() as ex: A = ng.make_axis(name='A', length=3) B = ng.make_axis(name='B', length=4) np_shape = (A.length, B.length) x0_np = -np.ones(np_shape) x1_np = np.ones(np_shape) x0_ng = ng.persistent_tensor([A, B], initial_value=x0_np).named('x0') x1_ng = ng.persistent_tensor([A, B], initial_value=x1_np).named('x1') j_np = np.concatenate([x0_np, x1_np], axis=0) j_ng = ng.concat_along_axis([x0_ng, x1_ng], A) f = ex.executor(j_ng) j_val = f() ng.testing.assert_allclose(j_val, j_np)
def test_sequential_reduce(M): x = ng.variable([M], initial_value=1) x0 = x + x x1 = ng.sum(x0, out_axes=()) x2 = ng.sum(x0, out_axes=()) + x0 p = ng.sequential([x0, x1, x2]) with ExecutorFactory() as ex: x0_val, x1_val, x2_val, p_val, x_val = ex.executor([x0, x1, x2, p, x])() x0_np = x_val + x_val x1_np = np.sum(x0_np) x2_np = x1_np + x0_np assert np.allclose(x0_val, x0_np) assert np.allclose(x1_val, x1_np) assert np.allclose(x2_val, x2_np) assert np.allclose(p_val, x2_np)
def test_slice_deriv(): C = ng.make_axis(length=2) D = ng.make_axis(length=3) x_np = np.array([[10, 20, 30], [1, 2, 3]], dtype='float32') x = ng.placeholder([C, D]).named('x') x_slice = x[0, :] + x[1, :] with ExecutorFactory() as ex: sym_deriv_fun = ex.derivative(x_slice, x) val_ng = sym_deriv_fun(x_np) val_np = np.zeros((D.length, C.length, D.length)) for i in range(D.length): for j in range(C.length): val_np[i, j, i] = 1 ng.testing.assert_allclose(val_ng, val_np)
def test_linear_ones(input_size, input_placeholder, output_size): # basic sanity check with all ones on the inputs and weights, check that # each row in output is the sum of the weights for that output this check # will confirm that the correct number of operations is being run x = np.ones(input_placeholder.axes.lengths) layer = Linear(nout=output_size, init=UniformInit(1.0, 1.0)) with ExecutorFactory() as ex: if ex.transformer.transformer_name == 'hetr': pytest.xfail("hetr fork-safe issue on mac") out = layer(input_placeholder) comp = ex.executor([out, layer.W], input_placeholder) output_values, w = comp(x) ng.testing.assert_allclose(np.ones(out.axes.lengths) * input_size, output_values, atol=0.0, rtol=0.0)
def test_concat_different_axis_lengths(): ax1 = ng.make_axis(length=3, name="concat") ax2 = ng.make_axis(length=2, name="concat") ax3 = ng.make_axis(length=10, name="other") x = ng.placeholder(axes=[ax1, ax3]) y = ng.placeholder(axes=[ax2, ax3]) np_x = np.zeros(x.axes.lengths) np_y = np.zeros(y.axes.lengths) # ax1 and ax2 have same name, so this should work v = ng.concat_along_axis([x, y], ax1) with ExecutorFactory() as ex: f = ex.executor(v, x, y) e_v = f(np_x, np_y) np_v = np.concatenate([np_x, np_y], axis=0) ng.testing.assert_allclose(e_v.copy(), np_v)
def test_idempotent_axes_c(): """ Test test axes transformations with autodiff, case c, with broadcast, slice, cast and dim-shuffle """ with ExecutorFactory() as ex: axes = ng.make_axes([ng.make_axis(3), ng.make_axis(1)]) result_axes = [ng.make_axis(length=axis.length) for axis in axes] # variable w = ng.variable(axes, initial_value=np.ones((3, 1))) # broadcast l / r, introducing dummy length 1 axes l = ng.broadcast(w, axes) r = ng.broadcast(w, axes) # slice axes_slice = [slice(None, None, None), slice(None, None, None)] l_sliced = ng.tensor_slice(l, axes_slice) r_sliced = ng.tensor_slice(r, axes_slice) # cast r r_sliced_casted = ng.cast_axes(r_sliced, axes) # perform add result = ng.add(l_sliced, r_sliced_casted) # cast / dimshuffle result = ng.cast_axes(result, result_axes) result = ng.axes_with_order(result, result_axes) # cost and grad cost = ng.sum(result, reduction_axes=result.axes) grad = ng.deriv(cost, w) grad_comp = ex.executor(grad) cost_comp = ex.executor(cost) cost_comp_ng = cost_comp() grad_comp_ng = grad_comp() grad_comp_np = np.ones((3, 1)) * 2. assert cost_comp_ng == 6.0 assert np.array_equal(grad_comp_ng, grad_comp_np)
def test_logreg(): # xs: (C, N), y: (N,) xs = np.array([[0.52, 0.88, 0.52, 0.74], [1.12, -1.08, 0.06, -2.49], [0.77, 0.15, -1.3, 1.39]]) ys = np.array([1, 1, 0, 1]) max_iter = 10 alpha = 0.1 thetas = np.array([0., 0., 0.]) np_logreg = NumpyLogreg(xs, ys, thetas) C, N = ng.make_axis(length=3), ng.make_axis(length=4) # input tensors xs_v = ng.placeholder((C, N)) ys_v = ng.placeholder([N]) alpha_v = ng.placeholder(()) thetas_var = ng.variable([C], initial_value=thetas) # define ops ys_pred = ng.sigmoid(ng.dot(thetas_var, xs_v)) log_likelihoods = ng.log(ys_pred) * ys_v + ng.log(1 - ys_pred) * (1 - ys_v) loss = -ng.sum(log_likelihoods, reduction_axes=[N]) grad_comp = ng.deriv(loss, thetas_var) weight_update = ng.sequential( [ng.assign(thetas_var, thetas_var - alpha_v * grad_comp), thetas_var]) # transformer with ExecutorFactory() as ex: train_eval_func = ex.executor([grad_comp, loss, weight_update], xs_v, ys_v, alpha_v) # evaluate for i in range(max_iter): grad_np, loss_np, thetas_np = np_logreg.optimize(alpha) grad_ng, loss_ng, thetas_ng = train_eval_func(xs, ys, alpha) ng.testing.assert_allclose(loss_np, loss_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False) ng.testing.assert_allclose(grad_np, grad_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False) ng.testing.assert_allclose(thetas_np, thetas_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False)
def test_specific_slice_deriv(): # with ExecutorFactory() as ex: A = ng.make_axis(name='A', length=3) B = ng.make_axis(name='B', length=4) np_shape = (A.length, B.length) x_np = np.empty(np_shape, dtype=np.float32) for i in range(A.length): for j in range(B.length): x_np[i, j] = 10 * i + j x_ng = ng.persistent_tensor([A, B], initial_value=x_np) for i in range(A.length): for j in range(B.length): slice = ng.tensor_slice(x_ng, (i, j)) dslice_dx = ng.deriv(slice, x_ng) dslice_dx_fun = ex.executor(dslice_dx) dslice_dx_val = dslice_dx_fun() dslice_dx_np = np.zeros_like(x_np) dslice_dx_np[i, j] = 1 ng.testing.assert_allclose(dslice_dx_val, dslice_dx_np)
def test_squared_L2(): H = ng.make_axis(2) W = ng.make_axis(3) N = ng.make_axis(5, name='N') axes = ng.make_axes([H, W, N]) a = ng.constant(np.ones(axes.lengths), axes=axes) with ExecutorFactory() as factory: l2_samples_fun = factory.executor(ng.squared_L2(a)) l2_samples_val = np.ones([N.length]) * H.length * W.length l2_all_fun = factory.executor(ng.squared_L2(a, out_axes=[])) l2_all_val = np.ones([]) * W.length * H.length * N.length l2_W_fun = factory.executor(ng.squared_L2(a, reduction_axes=[H, N])) l2_W_val = np.ones([W.length]) * H.length * N.length l2_samples_result = l2_samples_fun() l2_all_result = l2_all_fun() l2_W_result = l2_W_fun() ng.testing.assert_allclose(l2_samples_val, l2_samples_result) ng.testing.assert_allclose(l2_all_val, l2_all_result) ng.testing.assert_allclose(l2_W_val, l2_W_result)