def test_unpack_axis_times_transpose_unpack_axis(output_rank, x_input_shape, x_data, y_input_shape, y_data):
#test free axis times from unpack batch
x = C.input_variable(x_input_shape)
y = C.input_variable(y_input_shape)
xx = C.unpack_batch(x)
yy = C.unpack_batch(y)
yyy = C.transpose(yy, range(len(yy.shape))[::-1])
t = C.times(xx, yyy, output_rank=output_rank)
cntk_result = t.eval({x: x_data, y: y_data})
np_result = np.tensordot(x_data, np.transpose(y_data), axes = len(x_data.shape) - output_rank)
np.testing.assert_allclose(np_result, cntk_result)
def test_convert_dynamic_axis():
#test fix batch size
batch_size = 4
a = C.parameter(shape=(batch_size, 2, 3), init=1)
dynamic_a = C.to_batch(a)
assert len(dynamic_a.dynamic_axes) == 1
assert dynamic_a.shape == (2, 3)
x = C.input_variable((2, 3))
y = x * dynamic_a
#test grad
data = np.arange(batch_size * 2 * 3).reshape(batch_size, 2, 3).astype('f')
assert np.array_equal(y.grad({x: data}, [a]), data)
const_a = C.unpack_batch(y)
assert len(const_a.dynamic_axes) == 0
assert const_a.shape == (C.FreeDimension, 2, 3)
f = C.assign(a, const_a)
f.eval({x: data})
assert np.array_equal(a.value, data)
#test reshape with batch axis
x = C.input_variable((2, 3))
const_x = C.unpack_batch(x)
assert len(const_x.dynamic_axes) == 0
assert const_x.shape == (C.FreeDimension, 2, 3)
const_y = C.reshape(const_x, (-1, 3))
assert const_y.shape == (C.FreeDimension, 3)
y = C.to_batch(const_y)
assert len(y.dynamic_axes) == 1
assert y.shape == (3, )
z = y * 2
expected = data.reshape((8, 3)) * 2
assert np.array_equal(z.eval({x: data}), expected)
#test inferred dimension
x = C.input_variable((C.InferredDimension, 3))
const_x = C.unpack_batch(x)
assert len(const_x.dynamic_axes) == 0
assert const_x.shape == (C.FreeDimension, C.InferredDimension, 3)
const_y = const_x * 2
y = C.to_batch(const_y)
assert len(y.dynamic_axes) == 1
assert y.shape == (C.InferredDimension, 3)
def test_convert_dynamic_axis():
#test fix batch size
batch_size = 4
a = C.parameter(shape=(batch_size, 2, 3), init=1)
dynamic_a = C.to_batch(a)
assert len(dynamic_a.dynamic_axes) == 1
assert dynamic_a.shape == (2, 3)
x = C.input_variable((2, 3))
y = x * dynamic_a
#test grad
data = np.arange(batch_size * 2 * 3).reshape(batch_size, 2, 3).astype('f')
assert np.array_equal(y.grad({x:data}, [a]), data)
const_a = C.unpack_batch(y)
assert len(const_a.dynamic_axes) == 0
assert const_a.shape == (C.FreeDimension, 2, 3)
f = C.assign(a, const_a)
f.eval({x:data})
assert np.array_equal(a.value, data)
#test reshape with batch axis
x = C.input_variable((2,3))
const_x = C.unpack_batch(x)
assert len(const_x.dynamic_axes) == 0
assert const_x.shape == (C.FreeDimension, 2, 3)
const_y = C.reshape(const_x, (-1, 3))
assert const_y.shape == (C.FreeDimension, 3)
y = C.to_batch(const_y)
assert len(y.dynamic_axes) == 1
assert y.shape == (3,)
z = y * 2
expected = data.reshape((8, 3)) * 2
assert np.array_equal(z.eval({x:data}), expected)
#test inferred dimension
x = C.input_variable((C.InferredDimension, 3))
const_x = C.unpack_batch(x)
assert len(const_x.dynamic_axes) == 0
assert const_x.shape == (C.FreeDimension, C.InferredDimension, 3)
const_y = const_x * 2
y = C.to_batch(const_y)
assert len(y.dynamic_axes) == 1
assert y.shape == (C.InferredDimension, 3)
def test_unpack_axis_times_transpose_unpack_axis(output_rank, x_input_shape,
x_data, y_input_shape,
y_data):
#test free axis times from unpack batch
x = C.input_variable(x_input_shape)
y = C.input_variable(y_input_shape)
xx = C.unpack_batch(x)
yy = C.unpack_batch(y)
yyy = C.transpose(yy, range(len(yy.shape))[::-1])
t = C.times(xx, yyy, output_rank=output_rank)
cntk_result = t.eval({x: x_data, y: y_data})
np_result = np.tensordot(x_data,
np.transpose(y_data),
axes=len(x_data.shape) - output_rank)
np.testing.assert_allclose(np_result, cntk_result)
def __cntk_cov2__(m):
m = C.reshape(m, -1)
m = C.unpack_batch(m)
m = C.transpose(m, [1, 0])
count = C.reduce_sum(C.reduce_mean(C.ones_like(m), axis=0))
fact = 1.0 / (count - 1)
m -= C.reduce_mean(m, axis=1)
mt = C.transpose(m, [1, 0])
return fact * C.squeeze(m @ mt)
def multivariate_kl_divergence(input_layer):
_dim = input_layer.shape[0]
out_value = C.unpack_batch(input_layer)
_mu1 = C.transpose(C.reduce_mean(out_value, axis=0), [1, 0])
_sigma1 = C.cov2(input_layer)
_mu2 = C.zeros_like(_mu1)
_sigma2 = C.Constant(np.eye(_dim))
_sigma2_inv = _sigma2 # identity matrix
return 0.5 * (C.log(C.det(_sigma2) / C.det(_sigma1)) - _dim +
C.trace(_sigma2_inv @ _sigma1) + C.transpose(
(_mu2 - _mu1), [1, 0]) @ _sigma2_inv @ (_mu2 - _mu1))
def test_data_resize():
batch_size = 8
w = C.parameter(shape=(3, 2), name='w1')
x = C.input_variable(shape=[3], name='x')
y = C.softmax(C.times(x, w))
y = C.unpack_batch(y)
y = C.reshape(y, [batch_size * 2])
loss = C.reduce_mean(-C.log(y))
learning_rate = 0.01
lr_schedule = C.learning_rate_schedule(learning_rate, C.UnitType.minibatch)
learner = C.sgd(y.parameters, lr_schedule, gradient_clipping_threshold_per_sample=1.0)
trainer = C.Trainer(y, (loss), [learner])
features = np.random.randn(batch_size, 3)
trainer.train_minibatch({x: features})
def test_data_resize():
batch_size = 8
w = C.parameter(shape=(3, 2), name='w1')
x = C.input_variable(shape=[3], name='x')
y = C.softmax(C.times(x, w))
y = C.unpack_batch(y)
y = C.reshape(y, [batch_size * 2])
loss = C.reduce_mean(-C.log(y))
learning_rate = 0.01
lr_schedule = C.learning_rate_schedule(learning_rate, C.UnitType.minibatch)
learner = C.sgd(y.parameters, lr_schedule, gradient_clipping_threshold_per_sample=1.0)
trainer = C.Trainer(y, (loss), [learner])
features = np.random.randn(batch_size, 3)
trainer.train_minibatch({x: features})
def test_eye_like(operand, sparse_output, device_id, precision):
np_eye_like = lambda matrix: np.eye(matrix.shape[0], matrix.shape[1], dtype=np.float32)
operand = AA(operand).astype(np.float32)
expected = np_eye_like(operand)
expected_grad = np.zeros_like(operand).reshape(expected.shape)
my_eval = (lambda f, arg: f.eval(arg).todense()) if sparse_output else (lambda f, arg: f.eval(arg))
from .. import eye_like
import cntk as C
#testing with direct numpy input
y = C.eye_like(operand, sparse_output=sparse_output)
actual = y.eval().todense() if sparse_output else y.eval()
np.testing.assert_almost_equal(actual, expected)
#testing through input_variable
#test load and save:
import tempfile
import os
x = C.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.unpack_batch(x), sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test2')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.transpose(C.unpack_batch(x), (1,0)), sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected.transpose())
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test3')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected.transpose())
os.remove(tempdir)
#test expecting exception with sequence axis
with pytest.raises(Exception) as info:
#no sequence axis is allowed
x = C.sequence.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no more than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((3, 3), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no less than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
def crossentropy(y, t):
prob = C.squeeze(C.reduce_sum(y * t, axis=0), 0)
return -C.reduce_mean(C.unpack_batch(C.log(prob)))
def test_eye_like(operand, sparse_output, device_id, precision):
np_eye_like = lambda matrix: np.eye(
matrix.shape[0], matrix.shape[1], dtype=np.float32)
operand = AA(operand).astype(np.float32)
expected = np_eye_like(operand)
expected_grad = np.zeros_like(operand).reshape(expected.shape)
my_eval = (lambda f, arg: f.eval(arg).todense()) if sparse_output else (
lambda f, arg: f.eval(arg))
from .. import eye_like
import cntk as C
#testing with direct numpy input
y = C.eye_like(operand, sparse_output=sparse_output)
actual = y.eval().todense() if sparse_output else y.eval()
np.testing.assert_almost_equal(actual, expected)
#testing through input_variable
#test load and save:
import tempfile
import os
x = C.input_variable(operand.shape[1:],
dtype=np.float32,
needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(
my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}),
expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.unpack_batch(x), sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test2')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(
my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}),
expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.transpose(C.unpack_batch(x), (1, 0)),
sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected.transpose())
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test3')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(
my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}),
expected.transpose())
os.remove(tempdir)
#test expecting exception with sequence axis
with pytest.raises(Exception) as info:
#no sequence axis is allowed
x = C.sequence.input_variable(operand.shape[1:],
dtype=np.float32,
needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no more than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((3, 3), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no less than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
def test_eye_like(operand, sparse_output, device_id, precision):
np_eye_like = lambda matrix: np.eye(
matrix.shape[0], matrix.shape[1], dtype=np.float32)
operand = AA(operand).astype(np.float32)
expected = np_eye_like(operand)
expected_grad = np.zeros_like(operand).reshape(expected.shape)
my_eval = (lambda f, arg: f.eval(arg).todense()) if sparse_output else (
lambda f, arg: f.eval(arg))
from .. import eye_like
import cntk as C
#testing with direct numpy input
y = C.eye_like(operand, sparse_output=sparse_output)
actual = y.eval().todense() if sparse_output else y.eval()
np.testing.assert_almost_equal(actual, expected)
#testing through input_variable
#test load and save:
import tempfile
import os
x = C.input_variable(operand.shape[1:],
dtype=np.float32,
needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(
my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}),
expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.unpack_batch(x), sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test2')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(
my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}),
expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.transpose(C.unpack_batch(x), (1, 0)),
sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected.transpose())
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test3')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(
my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}),
expected.transpose())
os.remove(tempdir)
#test pass through gradients
#test direct input: no gradients pass through to inputs
data = operand
op = lambda x: eye_like(
x, sparse_output=False
) #sparse are not supported for some of the following basic operations
w = C.parameter(x.shape, init=np.ones(x.shape).astype(np.float32) * 3.0)
expected_x_backward = np.zeros_like(data)
expected_w_backward = np.zeros_like(w)
op_func = op(x)
grad = op_func.grad({x: data}, [x])
np.testing.assert_almost_equal(grad, expected_x_backward)
# test inputs through sub-expressions: no gradients pass through to inputs (e.g. x, w) of the subexpressoin (e.g. x * w here)
op_func = op(x * w)
grad = op_func.grad({x: data}, [w, x])
np.testing.assert_almost_equal(grad[x], expected_x_backward)
np.testing.assert_almost_equal(grad[w], expected_w_backward)
# testing inputs through shared sub-expressions: no gradients pass through reduce arg ops to inputs (e.g. x, w) of the subexpressoin
# (e.g. x * w here), therefore the gradients will depend on how the shared expressions participate in other experssions:
shared_exp = x * w
op_func = op(shared_exp) + x + w + shared_exp
ref_op_func = x + w + shared_exp
grad = op_func.grad({x: data}, [w, x])
ref_grad = ref_op_func.grad({x: data}, [w, x])
np.testing.assert_almost_equal(grad[x], ref_grad[x])
np.testing.assert_almost_equal(grad[w], ref_grad[w])
#test expecting exception with sequence axis
with pytest.raises(Exception) as info:
#no sequence axis is allowed
x = C.sequence.input_variable(operand.shape[1:],
dtype=np.float32,
needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no more than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((3, 3), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no less than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
