def test_constant_add(): """TODO.""" a = ng.constant(1) b = ng.constant(2) c = a + b with executor(c) as ex: result = ex() assert result == 3
def __call__(self, *args, **kwargs): if len(self.ops) == 0: self.beta_1 = ng.constant(self.beta_1, dtype=np.float32) self.beta_2 = ng.constant(self.beta_2, dtype=np.float32) self.t = ng.persistent_tensor(axes=(), initial_value=0) self.t = ng.sequential([ng.assign(self.t, self.t + 1), self.t]) self.ell = self.lrate * ng.sqrt(1 - self.beta_2 ** self.t) / (1 - self.beta_1 ** self.t) return super(Adam, self).__call__(*args, **kwargs)
def test_constant_init(): """TODO.""" a = ng.constant(5) with executor(a) as ex: result = ex() assert (result == 5) nparray = np.array(range(5)) a = ng.constant(nparray) with executor(a) as ex: result = ex() ng.testing.assert_allclose(result, nparray)
def test_cputensor_add_constant(): """TODO.""" M = ng.make_axis(length=1) N = ng.make_axis(length=3) np_a = np.array([[1, 2, 3]], dtype=np.float32) np_c = np.add(np_a, 2) a = ng.constant(np_a, [M, N]) b = ng.constant(2) c = ng.add(a, b) with executor(c) as ex: result = ex() assert np.array_equal(result, np_c)
def test_exit_condition(transformer_factory): bsz = 16 class_num = 10 # Limiting maximum absolute value for tensors elements to 7.9. # # There is used np.random.randn function to fill tensors with random values. It can give any # value as a result however values above 5 are highly improbable and would appear very rarely. # Limit 7.9 would almost never modify the tested tensor but would prevent from random # failures from time to time when the test is run in continuous environment. # This limit is approximate upper bound of range [4, 8). Numbers from this region can be # expressed by flexpoint number of the same dec. # Why not 15.9 that is approximate limit of [8, 16) range ? # Numbers above 8 are highly improbable and if appear from time to time can cause random # failures due to reduced accuracy of all numbers in tensor. Most numbers in normal # distribution are close to 0. is_flex = is_flex_factory(transformer_factory) clip_val = 7.9 if is_flex else 0 N, Y = ng.make_axis(bsz), ng.make_axis(class_num) y_val = rng.randn_abs_clip(ng.make_axes([N, Y]), clip_max=clip_val) y = ng.constant(y_val, ng.make_axes([N, Y])) likelihood = ng.log(ng.softmax(y, normalization_axes=y.axes[1])) with ExecutorFactory() as ex: comp = ex.executor(likelihood) val1 = comp() val2 = comp() ng.testing.assert_allclose(val1, val2, atol=0, rtol=0)
def test_4d_chained(transformer_factory, input_axes): # Limiting maximum absolute value for tensors elements to 7.9. # See description in function test_exit_condition above # Limitting minimum absolute value for tensors being input to reciprocal operation to 1/7.9 # # This is consequence of the above and flexpoint accuracy. # Numbers very small have poor absolute accuracy. When reciprocal of them is calculated the # results becomes very large and has even worse accuracy. When small numbers would be accepted # as an input to reciprocal in the test the absolute maximum value of the result is undefined # and so absolute tolerance. # To have possibility to set atol in the test and test could pass with it minimum element of # the tensor that is input to reciprocal operation has to be limited. is_flex = is_flex_factory(transformer_factory) clip_val_max = 7.9 if is_flex else 0 clip_val_min = 1.0 / 7.9 if is_flex else 0 x_val = rng.randn_abs_clip(input_axes, clip_min=clip_val_min, clip_max=clip_val_max) y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val_max) x = ng.constant(x_val, input_axes) y = ng.constant(y_val, input_axes) im = ng.reciprocal(x) out = ng.sum(ng.add(im, y), reduction_axes=input_axes[0]) with executor(out) as ex: graph_val = ex() np_val = np.sum(np.add(np.reciprocal(x_val), y_val), 0) # atol_multiplier = 15 * x_val.shape[0] # # x_val.shape[0] is number elements added together in operation # ng.sum(X, reduction_axes=input_axes[0]) # # 15 is calculated the following way: # # Input tensor has values from the range 1/7.9 - 7.9 # For DEC=12 absolute error is equal to 0.5*2^-12 = 0.000122 # 1/7.9 = 0.126582 with this error becomes 0.126704 # Reciprocal of 1/7.9 is 7.9 # Reciprocal of 1/7.9 + err = 7.892389 # Absolute difference is 0.007611 # It is 15.2 times larger then atol limit 5e-4 from Argon transformer ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4, atol_multiplier=15 * x_val.shape[0])
def test_cputensor_fusion(): """TODO.""" M = ng.make_axis(length=1) N = ng.make_axis(length=3) np_a = np.array([[1, 2, 3]], dtype=np.float32) np_b = np.array([[3, 2, 1]], dtype=np.float32) np_d = np.multiply(np_b, np.add(np_a, 2)) a = ng.constant(np_a, [M, N]) b = ng.constant(np_b, [M, N]) c = ng.constant(2) d = ng.multiply(b, ng.add(a, c)) with executor(d) as ex: result = ex() assert np.array_equal(result, np_d)
def test_cputensor_dot(): Y = ng.make_axis(length=2) M = ng.make_axis(length=1) N = ng.make_axis(length=3) np_a = np.array([[1, 2, 3]], dtype=np.float32) np_b = np.array([[1, 2], [2, 3], [3, 4]], dtype=np.float32) np_c = np.dot(np_a, np_b) a = ng.constant(np_a, [M, N]).named('a') b = ng.constant(np_b, [N, Y]).named('b') c = ng.dot(a, b) with executor(c) as ex: result = ex() assert np.array_equal(result, np_c)
def test_initial_value(): # Test work-around for issue #1138 w = [3, 4, 5] x = ng.constant(w) y = ng.variable([ng.make_axis(length=len(w))], initial_value=x) with ExecutorFactory() as ex: result = ex.executor(y)() ng.testing.assert_allclose(result, np.asarray(w, dtype=np.float32))
def test_dot(): H = ng.make_axis(length=1) W = ng.make_axis(length=4) np_a = np.array([[1, 2, 3, 4]], dtype=np.float32) np_b = np.array(3, dtype=np.float32) a = ng.constant(np_a, [H, W]) b = ng.constant(np_b, []) c = ng.dot(a, b) with executor(c) as _dot: _dot_val = _dot() # compute reference _dot_val_ref = np.dot(np_a, np_b) # this checks the dot product between scalar and vector, this is equivalent to # elementwise multiplication between scalar and vector assert np.allclose(_dot_val, _dot_val_ref)
def test_cputensor_mlp(): """TODO.""" D = ng.make_axis(length=3) H = ng.make_axis(length=2) N = ng.make_axis(length=1) np_x = np.array([[1, 2, 3]], dtype=np.float32) np_w = np.array([[1, 1], [1, 1], [1, 1]], dtype=np.float32) np_b = np.array([1, 2], dtype=np.float32) np_c = np.dot(np_x, np_w) + np_b x = ng.constant(np_x, [N, D]) w = ng.constant(np_w, [D, H]) b = ng.constant(np_b, [H]) wx = ng.dot(x, w) c = wx + b with executor(c) as ex: result = ex() assert np.array_equal(result, np_c)
def test_scalar_broadcast(): """ Test broadcasting a scalar into a tensor """ with ExecutorFactory() as ex: x_axes = ng.make_axes() broadcast_axes = ng.make_axes([ng.make_axis(2), ng.make_axis(3)]) x = ng.constant(1., axes=x_axes) z = ng.broadcast(x, axes=broadcast_axes) z_comp = ex.executor(z) assert np.array_equal(z_comp(), np.ones(broadcast_axes.lengths))
def test_evaluation_twice(): """Test executing a computation graph twice on a one layer MLP.""" C = ng.make_axis(length=2) D = ng.make_axis(length=2) W = ng.make_axis(length=1) x = ng.constant(np.array([[1, 2], [3, 4]], dtype='float32'), ng.make_axes([C, D])) hidden1_weights = ng.constant(np.array([[1], [1]], dtype='float32'), ng.make_axes([C, W])) hidden1_biases = ng.constant(np.array([[2], [2]], dtype='float32'), ng.make_axes([D, W])) hidden1 = ng.dot(hidden1_weights, x) + hidden1_biases with executor(hidden1) as comp: result_1 = comp() result_2 = comp() assert np.array_equal(result_1, result_2)
def test_4d_elementwise(transformer_factory, input_axes): # Limiting maximum absolute value for tensors elements to 7.9. # See description in function test_exit_condition above is_flex = is_flex_factory(transformer_factory) clip_val = 7.9 if is_flex else 0 x_val = rng.randn_abs_clip(input_axes, clip_max=clip_val) y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val) x = ng.constant(x_val, input_axes) y = ng.constant(y_val, input_axes) out = ng.add(x, y) with executor(out) as ex: graph_val = ex() np_val = np.add(x_val, y_val) ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4)
def test_concatenate(concatenate_variables): x_list, np_list, pos = concatenate_variables with ExecutorFactory() as ex: v = ng.concat_along_axis(x_list, x_list[0].axes[pos]) d = ng.deriv(v, x_list[0], error=ng.constant(np.ones(v.axes.lengths), axes=v.axes)) f = ex.executor([v, d]) e_v, e_d = f() np_v = np.concatenate(np_list, axis=pos) ng.testing.assert_allclose(e_v.copy(), np_v) ng.testing.assert_allclose(e_d.copy(), np.ones(x_list[0].axes.lengths))
def test_cputensor_add(): """TODO.""" Y = ng.make_axis(length=2) M = ng.make_axis(length=2) N = ng.make_axis(length=2) a = ng.constant(np.array([3, 5], dtype=np.float32), [Y]) b = ng.constant(np.array([3, 5], dtype=np.float32), [Y]) c = a + b with executor(c) as ex: result = ex() assert np.array_equal(result, [6, 10]) np_a = np.array([[1, 2], [3, 4]], dtype=np.float32) np_b = np.array([[1, 2], [3, 4]], dtype=np.float32) np_c = np_a + np_b a = ng.constant(np_a, [M, N]) b = ng.constant(np_b, [M, N]) c = a + b with executor(c) as ex: result = ex() assert np.array_equal(result, np_c)
def concatenate_variables(request): num_vars, num_axes, concat_pos = request.param common_axes = [ng.make_axis(length=2) for _ in range(num_axes - 1)] x_list = list() np_list = list() ax = ng.make_axis(length=np.random.randint(3, 10)) axes = ng.make_axes(common_axes[:concat_pos] + [ax] + common_axes[concat_pos:]) for _ in range(num_vars): var = np.random.uniform(0, 1, axes.full_lengths) np_list.append(var) x_list.append(ng.constant(var, axes=axes)) return x_list, np_list, concat_pos
def test_broadcast(): M = ng.make_axis(length=1) N = ng.make_axis(length=4) np_a = np.array([[1, 2, 3, 4]], dtype=np.float32) np_c = np.add(np_a, 2) a = ng.constant(np_a, [M, N]) c = ng.add(a, 2) with executor(c) as _add: result = _add() assert np.allclose(result, np_c)
def test_sum(num_units, sequence_length, batch_size): """ This tests for a non-deterministic error that arose in ng.sum following a dot product using the gpu transformer. """ shape = (num_units, sequence_length, batch_size) np_inp = np.random.uniform(-1, 1, shape) # Use an identity weight matrix on top of it np_w = np.eye(shape[0]) # Create ngraph versions inp = ng.constant(np_inp) reduction_axes = inp.axes[:-2] other_axes = inp.axes[-2:] new_axis = ng.make_axis(length=shape[0]) w_axes = ng.make_axes(new_axis) | reduction_axes w = ng.constant(np_w, axes=w_axes) # Reshape to do similar dot in numpy inp_reshape = np.reshape( np_inp, (np.prod(reduction_axes.lengths), np.prod(other_axes.lengths))) w_reshape = np.reshape(np_w, (new_axis.length, inp_reshape.shape[0])) # Reduce dimensions with identity weight matrix np_x = np.dot(w_reshape, inp_reshape) x = ng.dot(w, inp) # Sum over all but the first axis output_axes = ng.make_axes(x.axes[0]) y = ng.sum(x, out_axes=output_axes) np_y = np.sum(np_x, axis=1) with executor([y, x]) as f: y_val, x_val = f() assert_allclose(x_val.ravel(), np_x.ravel(), atol=1e-1) assert_allclose(y_val, np_y, atol=1e-1)
def test_flatten_deriv_simplified(): """ Test derivative with dot and flatten """ ax_N = ng.make_axis(length=3) ax_Y = ng.make_axis(length=2) x = ng.placeholder(ng.make_axes([ax_N])) w = ng.constant([5, 2], axes=ng.make_axes([ax_Y])) logits = ng.dot(x, w) cost = ng.sum(logits, reduction_axes=logits.axes) delta = 0.001 u = rng.uniform(.1, 5.0, x.axes) check_derivative(cost, x, delta, u, atol=1e-2, rtol=1e-2)
def ngraph_l2_norm(np_array): """ TODO. Arguments: np_array: TODO Returns: TODO """ axes = () for i, l in enumerate(np_array.shape): axes |= (ng.make_axis(length=l).named('axis%s' % i),) np_tensor = ng.constant(np_array, axes) var = ng.variable(axes, initial_value=np_tensor) with executor(ng.sqrt(ng.squared_L2(var))) as ex: return ex()
def test_4d_reduction(transformer_factory, input_axes): # Limiting maximum absolute value for tensors elements to 7.9. # See description in function test_exit_condition above is_flex = is_flex_factory(transformer_factory) clip_val = 7.9 if is_flex else 0 x_val = rng.randn_abs_clip(input_axes, clip_max=clip_val) x = ng.constant(x_val, input_axes) out1 = ng.sum(x, reduction_axes=input_axes[1]) out2 = ng.sum(x, reduction_axes=input_axes[3]) with executor([out1, out2]) as ex: graph_val1, graph_val2 = ex() np_val1 = np.sum(x_val, 1) np_val2 = np.sum(x_val, 3) ng.testing.assert_allclose(graph_val1, np_val1, rtol=1e-4, atol_multiplier=x_val.shape[1]) ng.testing.assert_allclose(graph_val2, np_val2, rtol=1e-4, atol_multiplier=x_val.shape[3])
def test_squared_L2(): H = ng.make_axis(2) W = ng.make_axis(3) N = ng.make_axis(5, name='N') axes = ng.make_axes([H, W, N]) a = ng.constant(np.ones(axes.lengths), axes=axes) with ExecutorFactory() as factory: l2_samples_fun = factory.executor(ng.squared_L2(a)) l2_samples_val = np.ones([N.length]) * H.length * W.length l2_all_fun = factory.executor(ng.squared_L2(a, out_axes=[])) l2_all_val = np.ones([]) * W.length * H.length * N.length l2_W_fun = factory.executor(ng.squared_L2(a, reduction_axes=[H, N])) l2_W_val = np.ones([W.length]) * H.length * N.length l2_samples_result = l2_samples_fun() l2_all_result = l2_all_fun() l2_W_result = l2_W_fun() ng.testing.assert_allclose(l2_samples_val, l2_samples_result) ng.testing.assert_allclose(l2_all_val, l2_all_result) ng.testing.assert_allclose(l2_W_val, l2_W_result)
def test_metadata_capture(): layer = Dummy() x = ng.constant(2) ret = layer.configure(x) assert ret.metadata['layer_type'] == 'convolution'
def cifar_mean_subtract(x): bgr_mean = ng.constant( const=np.array([104., 119., 127.]), axes=[x.axes.channel_axis()]) return (x - bgr_mean) / 255.
def __init__(self, params): lr_policy.__init__(self, params['name'], params['base_lr']) self.gamma = ng.constant(axes=(), const=params['gamma']) self.step = ng.constant(axes=(), const=params['step'], dtype=uint_dtype)
def i1k_mean_subtract(x): bgr_mean = ng.constant( axes=[x.axes.channel_axis()], const=np.array([127.0, 119.0, 104.0])) return (x - bgr_mean)
def __init__(self, params): lr_policy.__init__(self, params['name'], params['base_lr']) self.max_iter = ng.constant(axes=(), const=params['max_iter']) self.power = ng.constant(axes=(), const=params['power'])
def __init__(self, name, base_lr): self.name = name self.base_lr = ng.constant(axes=(), const=base_lr)
def __init__(self, params): lr_policy.__init__(self, params['name'], params['base_lr']) self.gamma = ng.constant(axes=(), const=params['gamma']) self.step_size = ng.constant(axes=(), const=params['step_size'])