def test_unpack_axis_times_transpose_unpack_axis(output_rank, x_input_shape, x_data, y_input_shape, y_data): #test free axis times from unpack batch x = C.input_variable(x_input_shape) y = C.input_variable(y_input_shape) xx = C.unpack_batch(x) yy = C.unpack_batch(y) yyy = C.transpose(yy, range(len(yy.shape))[::-1]) t = C.times(xx, yyy, output_rank=output_rank) cntk_result = t.eval({x: x_data, y: y_data}) np_result = np.tensordot(x_data, np.transpose(y_data), axes = len(x_data.shape) - output_rank) np.testing.assert_allclose(np_result, cntk_result)
def test_convert_dynamic_axis(): #test fix batch size batch_size = 4 a = C.parameter(shape=(batch_size, 2, 3), init=1) dynamic_a = C.to_batch(a) assert len(dynamic_a.dynamic_axes) == 1 assert dynamic_a.shape == (2, 3) x = C.input_variable((2, 3)) y = x * dynamic_a #test grad data = np.arange(batch_size * 2 * 3).reshape(batch_size, 2, 3).astype('f') assert np.array_equal(y.grad({x: data}, [a]), data) const_a = C.unpack_batch(y) assert len(const_a.dynamic_axes) == 0 assert const_a.shape == (C.FreeDimension, 2, 3) f = C.assign(a, const_a) f.eval({x: data}) assert np.array_equal(a.value, data) #test reshape with batch axis x = C.input_variable((2, 3)) const_x = C.unpack_batch(x) assert len(const_x.dynamic_axes) == 0 assert const_x.shape == (C.FreeDimension, 2, 3) const_y = C.reshape(const_x, (-1, 3)) assert const_y.shape == (C.FreeDimension, 3) y = C.to_batch(const_y) assert len(y.dynamic_axes) == 1 assert y.shape == (3, ) z = y * 2 expected = data.reshape((8, 3)) * 2 assert np.array_equal(z.eval({x: data}), expected) #test inferred dimension x = C.input_variable((C.InferredDimension, 3)) const_x = C.unpack_batch(x) assert len(const_x.dynamic_axes) == 0 assert const_x.shape == (C.FreeDimension, C.InferredDimension, 3) const_y = const_x * 2 y = C.to_batch(const_y) assert len(y.dynamic_axes) == 1 assert y.shape == (C.InferredDimension, 3)
def test_convert_dynamic_axis(): #test fix batch size batch_size = 4 a = C.parameter(shape=(batch_size, 2, 3), init=1) dynamic_a = C.to_batch(a) assert len(dynamic_a.dynamic_axes) == 1 assert dynamic_a.shape == (2, 3) x = C.input_variable((2, 3)) y = x * dynamic_a #test grad data = np.arange(batch_size * 2 * 3).reshape(batch_size, 2, 3).astype('f') assert np.array_equal(y.grad({x:data}, [a]), data) const_a = C.unpack_batch(y) assert len(const_a.dynamic_axes) == 0 assert const_a.shape == (C.FreeDimension, 2, 3) f = C.assign(a, const_a) f.eval({x:data}) assert np.array_equal(a.value, data) #test reshape with batch axis x = C.input_variable((2,3)) const_x = C.unpack_batch(x) assert len(const_x.dynamic_axes) == 0 assert const_x.shape == (C.FreeDimension, 2, 3) const_y = C.reshape(const_x, (-1, 3)) assert const_y.shape == (C.FreeDimension, 3) y = C.to_batch(const_y) assert len(y.dynamic_axes) == 1 assert y.shape == (3,) z = y * 2 expected = data.reshape((8, 3)) * 2 assert np.array_equal(z.eval({x:data}), expected) #test inferred dimension x = C.input_variable((C.InferredDimension, 3)) const_x = C.unpack_batch(x) assert len(const_x.dynamic_axes) == 0 assert const_x.shape == (C.FreeDimension, C.InferredDimension, 3) const_y = const_x * 2 y = C.to_batch(const_y) assert len(y.dynamic_axes) == 1 assert y.shape == (C.InferredDimension, 3)
def test_unpack_axis_times_transpose_unpack_axis(output_rank, x_input_shape, x_data, y_input_shape, y_data): #test free axis times from unpack batch x = C.input_variable(x_input_shape) y = C.input_variable(y_input_shape) xx = C.unpack_batch(x) yy = C.unpack_batch(y) yyy = C.transpose(yy, range(len(yy.shape))[::-1]) t = C.times(xx, yyy, output_rank=output_rank) cntk_result = t.eval({x: x_data, y: y_data}) np_result = np.tensordot(x_data, np.transpose(y_data), axes=len(x_data.shape) - output_rank) np.testing.assert_allclose(np_result, cntk_result)
def __cntk_cov2__(m): m = C.reshape(m, -1) m = C.unpack_batch(m) m = C.transpose(m, [1, 0]) count = C.reduce_sum(C.reduce_mean(C.ones_like(m), axis=0)) fact = 1.0 / (count - 1) m -= C.reduce_mean(m, axis=1) mt = C.transpose(m, [1, 0]) return fact * C.squeeze(m @ mt)
def multivariate_kl_divergence(input_layer): _dim = input_layer.shape[0] out_value = C.unpack_batch(input_layer) _mu1 = C.transpose(C.reduce_mean(out_value, axis=0), [1, 0]) _sigma1 = C.cov2(input_layer) _mu2 = C.zeros_like(_mu1) _sigma2 = C.Constant(np.eye(_dim)) _sigma2_inv = _sigma2 # identity matrix return 0.5 * (C.log(C.det(_sigma2) / C.det(_sigma1)) - _dim + C.trace(_sigma2_inv @ _sigma1) + C.transpose( (_mu2 - _mu1), [1, 0]) @ _sigma2_inv @ (_mu2 - _mu1))
def test_data_resize(): batch_size = 8 w = C.parameter(shape=(3, 2), name='w1') x = C.input_variable(shape=[3], name='x') y = C.softmax(C.times(x, w)) y = C.unpack_batch(y) y = C.reshape(y, [batch_size * 2]) loss = C.reduce_mean(-C.log(y)) learning_rate = 0.01 lr_schedule = C.learning_rate_schedule(learning_rate, C.UnitType.minibatch) learner = C.sgd(y.parameters, lr_schedule, gradient_clipping_threshold_per_sample=1.0) trainer = C.Trainer(y, (loss), [learner]) features = np.random.randn(batch_size, 3) trainer.train_minibatch({x: features})
def test_eye_like(operand, sparse_output, device_id, precision): np_eye_like = lambda matrix: np.eye(matrix.shape[0], matrix.shape[1], dtype=np.float32) operand = AA(operand).astype(np.float32) expected = np_eye_like(operand) expected_grad = np.zeros_like(operand).reshape(expected.shape) my_eval = (lambda f, arg: f.eval(arg).todense()) if sparse_output else (lambda f, arg: f.eval(arg)) from .. import eye_like import cntk as C #testing with direct numpy input y = C.eye_like(operand, sparse_output=sparse_output) actual = y.eval().todense() if sparse_output else y.eval() np.testing.assert_almost_equal(actual, expected) #testing through input_variable #test load and save: import tempfile import os x = C.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected) os.remove(tempdir) cntk_eye_like = C.eye_like(C.unpack_batch(x), sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test2') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected) os.remove(tempdir) cntk_eye_like = C.eye_like(C.transpose(C.unpack_batch(x), (1,0)), sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected.transpose()) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test3') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected.transpose()) os.remove(tempdir) #test expecting exception with sequence axis with pytest.raises(Exception) as info: #no sequence axis is allowed x = C.sequence.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) with pytest.raises(Exception) as info: #no more than 2 axes is allowed (including any dynamic axes) x = C.input_variable((3, 3), dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) with pytest.raises(Exception) as info: #no less than 2 axes is allowed (including any dynamic axes) x = C.input_variable((), dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
def crossentropy(y, t): prob = C.squeeze(C.reduce_sum(y * t, axis=0), 0) return -C.reduce_mean(C.unpack_batch(C.log(prob)))
def test_eye_like(operand, sparse_output, device_id, precision): np_eye_like = lambda matrix: np.eye( matrix.shape[0], matrix.shape[1], dtype=np.float32) operand = AA(operand).astype(np.float32) expected = np_eye_like(operand) expected_grad = np.zeros_like(operand).reshape(expected.shape) my_eval = (lambda f, arg: f.eval(arg).todense()) if sparse_output else ( lambda f, arg: f.eval(arg)) from .. import eye_like import cntk as C #testing with direct numpy input y = C.eye_like(operand, sparse_output=sparse_output) actual = y.eval().todense() if sparse_output else y.eval() np.testing.assert_almost_equal(actual, expected) #testing through input_variable #test load and save: import tempfile import os x = C.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal( my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected) os.remove(tempdir) cntk_eye_like = C.eye_like(C.unpack_batch(x), sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test2') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal( my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected) os.remove(tempdir) cntk_eye_like = C.eye_like(C.transpose(C.unpack_batch(x), (1, 0)), sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected.transpose()) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test3') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal( my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected.transpose()) os.remove(tempdir) #test expecting exception with sequence axis with pytest.raises(Exception) as info: #no sequence axis is allowed x = C.sequence.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) with pytest.raises(Exception) as info: #no more than 2 axes is allowed (including any dynamic axes) x = C.input_variable((3, 3), dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) with pytest.raises(Exception) as info: #no less than 2 axes is allowed (including any dynamic axes) x = C.input_variable((), dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
def test_eye_like(operand, sparse_output, device_id, precision): np_eye_like = lambda matrix: np.eye( matrix.shape[0], matrix.shape[1], dtype=np.float32) operand = AA(operand).astype(np.float32) expected = np_eye_like(operand) expected_grad = np.zeros_like(operand).reshape(expected.shape) my_eval = (lambda f, arg: f.eval(arg).todense()) if sparse_output else ( lambda f, arg: f.eval(arg)) from .. import eye_like import cntk as C #testing with direct numpy input y = C.eye_like(operand, sparse_output=sparse_output) actual = y.eval().todense() if sparse_output else y.eval() np.testing.assert_almost_equal(actual, expected) #testing through input_variable #test load and save: import tempfile import os x = C.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal( my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected) os.remove(tempdir) cntk_eye_like = C.eye_like(C.unpack_batch(x), sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test2') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal( my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected) os.remove(tempdir) cntk_eye_like = C.eye_like(C.transpose(C.unpack_batch(x), (1, 0)), sparse_output=sparse_output) actual = my_eval(cntk_eye_like, {x: operand}) grad = cntk_eye_like.grad({x: operand}) np.testing.assert_almost_equal(actual, expected.transpose()) np.testing.assert_almost_equal(grad, expected_grad) tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test3') cntk_eye_like.save(tempdir) cntk_eye_like2 = C.load_model(tempdir) np.testing.assert_almost_equal( my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected.transpose()) os.remove(tempdir) #test pass through gradients #test direct input: no gradients pass through to inputs data = operand op = lambda x: eye_like( x, sparse_output=False ) #sparse are not supported for some of the following basic operations w = C.parameter(x.shape, init=np.ones(x.shape).astype(np.float32) * 3.0) expected_x_backward = np.zeros_like(data) expected_w_backward = np.zeros_like(w) op_func = op(x) grad = op_func.grad({x: data}, [x]) np.testing.assert_almost_equal(grad, expected_x_backward) # test inputs through sub-expressions: no gradients pass through to inputs (e.g. x, w) of the subexpressoin (e.g. x * w here) op_func = op(x * w) grad = op_func.grad({x: data}, [w, x]) np.testing.assert_almost_equal(grad[x], expected_x_backward) np.testing.assert_almost_equal(grad[w], expected_w_backward) # testing inputs through shared sub-expressions: no gradients pass through reduce arg ops to inputs (e.g. x, w) of the subexpressoin # (e.g. x * w here), therefore the gradients will depend on how the shared expressions participate in other experssions: shared_exp = x * w op_func = op(shared_exp) + x + w + shared_exp ref_op_func = x + w + shared_exp grad = op_func.grad({x: data}, [w, x]) ref_grad = ref_op_func.grad({x: data}, [w, x]) np.testing.assert_almost_equal(grad[x], ref_grad[x]) np.testing.assert_almost_equal(grad[w], ref_grad[w]) #test expecting exception with sequence axis with pytest.raises(Exception) as info: #no sequence axis is allowed x = C.sequence.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) with pytest.raises(Exception) as info: #no more than 2 axes is allowed (including any dynamic axes) x = C.input_variable((3, 3), dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output) with pytest.raises(Exception) as info: #no less than 2 axes is allowed (including any dynamic axes) x = C.input_variable((), dtype=np.float32, needs_gradient=True) cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)