def test_op_broadcast_as(device_id, precision):
from .. import sequence
a_data = [AA([1], dtype=PRECISION_TO_TYPE[precision]),
AA([2], dtype=PRECISION_TO_TYPE[precision]),
AA([3], dtype=PRECISION_TO_TYPE[precision])]
b_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])]
a = I(shape=(1,),
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='a',
dynamic_axes=[Axis.default_batch_axis()])
b = I(shape=(1,),
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='b')
broadcast_a_as_b = sequence.broadcast_as(a, b)
res = broadcast_a_as_b.eval({a: a_data, b: b_data})
assert np.array_equal(res[0], np.asarray([[1.]]))
assert np.array_equal(res[1], np.asarray([[2.], [2.]]))
assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_op_gather_derived_dynamic_axes_equivalence(device_id, precision):
from .. import sequence
input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision])
input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision])
a = sequence.input(shape=input_data1.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='a')
b = sequence.input(shape=input_data2.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='b')
a_last = sequence.gather(a, sequence.is_last(a), new_sequence_axis_typeinfo=(0, 1))
b_last = sequence.gather(b, sequence.is_last(b), new_sequence_axis_typeinfo=(0, 1))
z = a_last + b_last
# create batch
input_data1.shape = (1, 1) + input_data1.shape
input_data2.shape = (1, 1) + input_data2.shape
res = z.eval({a: input_data1, b: input_data2})
expected_forward = [[3.]]
assert np.array_equal(res, expected_forward)
def test_op_gather_derived_dynamic_axes_equivalence(device_id, precision):
from .. import sequence
input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision])
input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision])
a = I(shape=input_data1.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='a')
b = I(shape=input_data2.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='b')
a_last = sequence.gather(a, sequence.is_last(a), new_sequence_axis_typeinfo=(0, 1))
b_last = sequence.gather(b, sequence.is_last(b), new_sequence_axis_typeinfo=(0, 1))
z = a_last + b_last
# create batch
input_data1.shape = (1, 1) + input_data1.shape
input_data2.shape = (1, 1) + input_data2.shape
res = z.eval({a: input_data1, b: input_data2})
expected_forward = [[3.]]
assert np.array_equal(res, expected_forward)
def test_sanitize_dtype_cntk():
for dtype in ['float', 'float32', np.float32, int]:
assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Float, dtype
for dtype in [float, 'float64', np.float64]:
assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Double, dtype
for dtype in ['int8', np.int8]:
assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Int8, dtype
def test_splice(shape1, shape2):
a = C.input_variable(shape=shape1,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
b = C.input_variable(shape=shape2,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='b')
# create batch
input_data1.shape = (1,) + input_data1.shape
input_data2.shape = (1,) + input_data2.shape
# splice using the operator
root_op = C.splice(a, b, axis=axis, name='splice_ab')
forward_input = {a: input_data1, b: input_data2}
# Backward pass test
# ==================
# The gradient of the splice operator is all ones in the shape of the input
def grad_splice(x):
return np.ones_like(x)
expected_forward = [expected_result]
expected_backward = {
a: grad_splice(np.asarray(input_data1)),
b: grad_splice(np.asarray(input_data2))
}
unittest_helper(root_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_broadcast_as(device_id, precision):
a_data = [
AA([1], dtype=PRECISION_TO_TYPE[precision]),
AA([2], dtype=PRECISION_TO_TYPE[precision]),
AA([3], dtype=PRECISION_TO_TYPE[precision])
]
b_data = [
AA([[2]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])
]
a = C.input_variable(shape=(1, ),
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
name='a')
b = C.sequence.input_variable(shape=(1, ),
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
name='b')
broadcast_a_as_b = C.sequence.broadcast_as(a, b)
res = broadcast_a_as_b.eval({a: a_data, b: b_data})
assert np.array_equal(res[0], np.asarray([[1.]]))
assert np.array_equal(res[1], np.asarray([[2.], [2.]]))
assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_op_gather_dynamic_axes_equivalence(device_id, precision):
input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision])
input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision])
a = C.sequence.input_variable(shape=input_data1.shape,
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
name='a')
b = C.sequence.input_variable(shape=input_data2.shape,
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
name='b')
is_last_a = C.sequence.is_last(a)
a_last = C.sequence.gather(a, is_last_a)
b_last = C.sequence.gather(b, is_last_a)
z = a_last + b_last
# create batch
input_data1.shape = (1, 1) + input_data1.shape
input_data2.shape = (1, 1) + input_data2.shape
res = z.eval({a: input_data1, b: input_data2})
expected_forward = [[[3.]]]
assert np.array_equal(res, expected_forward)
def test_op_broadcast_as(device_id, precision):
from .. import sequence
a_data = [
AA([1], dtype=PRECISION_TO_TYPE[precision]),
AA([2], dtype=PRECISION_TO_TYPE[precision]),
AA([3], dtype=PRECISION_TO_TYPE[precision])
]
b_data = [
AA([[2]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])
]
a = I(shape=(1, ),
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='a',
dynamic_axes=[Axis.default_batch_axis()])
b = I(shape=(1, ),
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='b')
broadcast_a_as_b = sequence.broadcast_as(a, b)
res = broadcast_a_as_b.eval({a: a_data, b: b_data})
assert np.array_equal(res[0], np.asarray([[1.]]))
assert np.array_equal(res[1], np.asarray([[2.], [2.]]))
assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_splice(shape1, shape2):
a = C.input_variable(shape=shape1,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
b = C.input_variable(shape=shape2,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='b')
# create batch
input_data1.shape = (1,) + input_data1.shape
input_data2.shape = (1,) + input_data2.shape
# splice using the operator
root_op = C.splice(a, b, axis=axis, name='splice_ab')
forward_input = {a: input_data1, b: input_data2}
# Backward pass test
# ==================
# The gradient of the splice operator is all ones in the shape of the input
def grad_splice(x):
return np.ones_like(x)
expected_forward = [expected_result]
expected_backward = {
a: grad_splice(np.asarray(input_data1)),
b: grad_splice(np.asarray(input_data2))
}
unittest_helper(root_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_reduce_batch_sequence_static_axes_together(input_data,
dynamic_axes,
static_axes, device_id,
precision):
from cntk import Axis
dt = PRECISION_TO_TYPE[precision]
data = AA(input_data, dtype=dt)
if dynamic_axes == [C.Axis.default_batch_axis()]:
#Reduction along the batch axis on input sequence is currently unsupported, so only batch axis input is tested
v = C.input_variable(data.shape[1:],
dtype=sanitize_dtype_cntk(dt),
needs_gradient=True)
numpy_axis_offset = 1
ignore_max_min = True
else:
v = C.sequence.input_variable(data.shape[2:],
dtype=sanitize_dtype_cntk(dt),
needs_gradient=True)
numpy_axis_offset = 2
ignore_max_min = False
for op, fwd, bwd in reduce_batch_sequence_static_ops(dt):
cntk_axes = tuple(dynamic_axes + static_axes)
numpy_axes = tuple([
0 if a == C.Axis.default_batch_axis() else 1 for a in dynamic_axes
] + [ax + numpy_axis_offset for ax in static_axes])
_test_reduce_ops(v, data, op, cntk_axes, numpy_axes, fwd, bwd)
def _test_binary_op(precision, device_id, op_func, left_operand, right_operand,
expected_forward, expected_backward_all, wrap_batch_seq=True, op_param_dict={}, batch_size_greater_than_one=False):
dt = PRECISION_TO_TYPE[precision]
dev = cntk_device(device_id)
left_value = AA(left_operand, dtype=dt)
right_value = AA(right_operand, dtype=dt)
left_operand_shape = left_value.shape[1:] if batch_size_greater_than_one else left_value.shape
right_operand_shape = right_value.shape[1:] if batch_size_greater_than_one else right_value.shape
a = C.input_variable(shape=left_operand_shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='a')
b = C.input_variable(shape=right_operand_shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='b')
const_a = constant(left_value, device=dev)
const_b = constant(right_value, device=dev)
if (type(op_func) == str):
input_op_constant = eval('a %s const_b' % op_func)
constant_op_input = eval('const_a %s b' % op_func)
input_op_input = eval('a %s b' % op_func)
else:
input_op_constant = op_func(a, const_b, **op_param_dict)
constant_op_input = op_func(const_a, b, **op_param_dict)
input_op_input = op_func(a, b, **op_param_dict)
# create batch by wrapping the data point into a batch of one sample
if wrap_batch_seq and not batch_size_greater_than_one:
left_value.shape = (1,) + left_value.shape
right_value.shape = (1,) + right_value.shape
forward_input = {a: left_value, b: right_value}
expected_backward = {a: expected_backward_all[
'left_arg'], b: expected_backward_all['right_arg'], }
unittest_helper(input_op_input,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
if not batch_size_greater_than_one:
forward_input = {a: left_value}
expected_backward = {a: expected_backward_all['left_arg'], }
unittest_helper(input_op_constant,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
forward_input = {b: right_value}
expected_backward = {b: expected_backward_all['right_arg'], }
unittest_helper(constant_op_input,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_sanitize_dtype_cntk():
for dtype in ['float', 'float32', np.float32, int]:
assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Float, dtype
for dtype in [float, 'float64', np.float64]:
assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Double, dtype
for dtype in ['int8', np.int8]:
assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Int8, dtype
for dtype in ['int16', np.int16]:
assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Int16, dtype
def _test_binary_op(precision, device_id, op_func, left_operand, right_operand,
expected_forward, expected_backward_all, wrap_batch_seq=True, op_param_dict={}):
dt = PRECISION_TO_TYPE[precision]
dev = cntk_device(device_id)
left_value = AA(left_operand, dtype=dt)
right_value = AA(right_operand, dtype=dt)
a = input(shape=left_value.shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='a')
b = input(shape=right_value.shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='b')
const_a = constant(left_value, device=dev)
const_b = constant(right_value, device=dev)
if (type(op_func) == str):
input_op_constant = eval('a %s const_b' % op_func)
constant_op_input = eval('const_a %s b' % op_func)
input_op_input = eval('a %s b' % op_func)
else:
input_op_constant = op_func(a, const_b, **op_param_dict)
constant_op_input = op_func(const_a, b, **op_param_dict)
input_op_input = op_func(a, b, **op_param_dict)
# create batch by wrapping the data point into a batch of one sample
if wrap_batch_seq:
left_value.shape = (1,) + left_value.shape
right_value.shape = (1,) + right_value.shape
forward_input = {a: left_value, b: right_value}
expected_backward = {a: expected_backward_all[
'left_arg'], b: expected_backward_all['right_arg'], }
unittest_helper(input_op_input,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
forward_input = {a: left_value}
expected_backward = {a: expected_backward_all['left_arg'], }
unittest_helper(input_op_constant,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
forward_input = {b: right_value}
expected_backward = {b: expected_backward_all['right_arg'], }
unittest_helper(constant_op_input,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_splice(input_data1, input_data2, axis, expected_result, device_id,
precision):
# FIXME This test currently fails in C++ with
# RuntimeError: Node 'splice_ab' (RowStack operation): Attempted to
# type-cast node to struct Microsoft::MSR::CNTK::INumInputs, which is not
# possible.
input_data1 = AA(input_data1, dtype=PRECISION_TO_TYPE[precision])
input_data2 = AA(input_data2, dtype=PRECISION_TO_TYPE[precision])
a = C.input_variable(shape=input_data1.shape,
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
b = C.input_variable(shape=input_data2.shape,
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='b')
# create batch
input_data1.shape = (1, ) + input_data1.shape
input_data2.shape = (1, ) + input_data2.shape
# splice using the operator
root_op = C.splice(a, b, axis=axis, name='splice_ab')
forward_input = {a: input_data1, b: input_data2}
# Backward pass test
# ==================
# The gradient of the splice operator is all ones in the shape of the input
def grad_splice(x):
return np.ones_like(x)
expected_forward = [expected_result]
expected_backward = {
a: grad_splice(np.asarray(input_data1)),
b: grad_splice(np.asarray(input_data2))
}
unittest_helper(root_op,
forward_input,
expected_forward,
expected_backward,
device_id=device_id,
precision=precision)
def test_op_reshape_gradient_accumulation(device_id, precision):
from .. import reshape
input_shape = (2,3)
output_shape = (3,2)
expected_output_shape = (3,2)
num_tensor_elements = np.multiply.reduce(input_shape)
input_tensor = np.arange(
num_tensor_elements, dtype=PRECISION_TO_TYPE[precision])
input_reshaped = input_tensor.reshape(expected_output_shape)
a = C.input_variable(shape=input_tensor.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
a_reshaped1 = reshape(a, output_shape)
a_reshaped2 = reshape(a, output_shape)
input_op = a_reshaped1 + a_reshaped2
resulting_multiplicative_factor = 2
expected_forward = [input_reshaped * resulting_multiplicative_factor]
# create batch
input_tensor.shape = (1,) + input_tensor.shape
expected_backward = {a: np.full(input_tensor.shape, resulting_multiplicative_factor, dtype=PRECISION_TO_TYPE[precision])}
forward_input = {a: input_tensor}
unittest_helper(input_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_pooling_ceil(input_size, pooling_window, strides, result, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(1, total_size + 1, 1, dtype=dt)
input_operand = x.reshape(input_size)
a = C.input_variable(shape=input_operand.shape[1:], dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a')
result_array = np.asarray(result, dtype=dt)
max_elements = result_array.reshape(result_array.size).tolist()
# place 1.0s where maximum elements are
backward = np.zeros_like(input_operand)
for element in max_elements:
backward += np.asarray(input_operand == element)
from cntk import pooling
input_op = pooling(a, MAX_POOLING, pooling_window, strides, ceil_out_dim=True)
forward_input = {a: input_operand}
expected_forward = AA(result)
expected_backward = {a: backward}
unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id,
precision=precision)
def test_op_average_pooling_include_pad(input_size, pooling_window, strides,
result, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
total_size = np.prod(input_size)
x = np.arange(1, total_size + 1, 1, dtype=dt)
input_operand = x.reshape(input_size)
a = C.input_variable(shape=input_operand.shape[1:],
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='a')
backward = (1 / np.prod(pooling_window)) * np.ones_like(input_operand)
from cntk import pooling
input_op = pooling(a,
AVG_POOLING,
pooling_window,
strides,
auto_padding=[True],
include_pad=True)
forward_input = {a: input_operand}
expected_forward = AA(result)
expected_backward = {a: backward}
unittest_helper(input_op,
forward_input,
expected_forward,
expected_backward,
device_id=device_id,
precision=precision)
def test_convolution_transpose(input_size, conv_size, result, device_id,
precision):
dt = PRECISION_TO_TYPE[precision]
dev = cntk_device(device_id)
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(total_size, dtype=dt)
input_operand = x.reshape(input_size)
a = C.input_variable(shape=input_operand.shape[1:],
dtype=sanitize_dtype_cntk(precision),
needs_gradient=False,
name='a')
# do the same for convolution kernel
total_size = np.prod(conv_size)
y = np.arange(total_size, dtype=dt)
conv_map = constant(value=y.reshape(conv_size), device=dev)
from cntk import convolution_transpose
input_op = convolution_transpose(conv_map, a, auto_padding=[False])
forward_input = {a: input_operand}
expected_forward = AA(result)
unittest_helper(input_op,
forward_input,
expected_forward,
None,
device_id=device_id,
precision=precision)
def test_asym_convolution(input_size, conv_size, result, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
dev = cntk_device(device_id)
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(total_size, dtype=dt)
input_operand = x.reshape(input_size)
a = C.input_variable(shape=input_operand.shape[1:],
dtype=sanitize_dtype_cntk(precision),
needs_gradient=False,
name='a')
# do the same for convolution kernel
total_size = np.prod(conv_size)
y = np.arange(total_size, dtype=dt)
conv_map = constant(value=y.reshape(conv_size), device=dev)
from cntk import convolution
input_op = convolution(conv_map, a, auto_padding=[True])
forward_input = {a: input_operand}
expected_forward = AA(result)
unittest_helper(input_op, forward_input, expected_forward,
None, device_id=device_id, precision=precision)
def test_op_reshape_multiple_free_dimensions(input_shape, replacement_shape, expected_output_shape, device_id, precision):
dev = cntk_device(device_id)
from cntk.internal import sanitize_dtype_cntk
from .. import reshape, element_times
num_tensor_elements = np.multiply.reduce(input_shape)
input_tensor = np.arange(
num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape)
input_reshaped = input_tensor.reshape(expected_output_shape)
a = C.input_variable(shape=tuple([C.FreeDimension]*len(input_tensor.shape)),
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
a_reshaped = reshape(a, replacement_shape)
const_input_reshaped = constant(input_reshaped, device=dev)
input_op = element_times(a_reshaped, const_input_reshaped)
expected_forward = [input_reshaped**2]
expected_backward = {a: input_tensor}
# create batch
input_tensor.shape = (1,) + input_tensor.shape
forward_input = {a: input_tensor}
unittest_helper(input_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_slice_sequence(input_data, slice_params, expected_result,
device_id, precision):
input_data = AA(input_data, dtype=PRECISION_TO_TYPE[precision])
t = Axis.new_unique_dynamic_axis('t')
sample_shape = input_data.shape[1:]
a = C.sequence.input_variable(shape=sample_shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
sequence_axis=t,
name='a')
result = C.sequence.slice(a,
begin_index=slice_params[0],
end_index=slice_params[1])
def grad_slice(x, beg_index, end_index):
res = np.zeros_like(x)
res[beg_index:end_index] = 1
return res
expected_forward = AA([expected_result],
dtype=PRECISION_TO_TYPE[precision])
expected_backward = {
a: [grad_slice(np.asarray(input_data), *slice_params)]
}
# create batch
input_data.shape = (1,) + input_data.shape
forward_input = {a: input_data}
unittest_helper(result,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def __init__(self,
shape=None,
init=None,
dtype=None,
device=None,
name=''):
if not device:
device = use_default_device()
if dtype is not None:
if isinstance(init, np.ndarray) and dtype != init.dtype:
init = np.array(init, dtype=dtype)
else:
if np.isscalar(init) and not shape:
shape = ()
if isinstance(init, np.ndarray):
dtype = init.dtype
else:
dtype = np.float32
if init is None:
init = 0
if isinstance(init, (np.ndarray, list, float, int)):
ndav = sanitize_value(shape, init, dtype, device)
super(Parameter, self).__init__(ndav, name)
else:
shape = sanitize_shape(shape)
cntk_dtype = sanitize_dtype_cntk(dtype)
super(Parameter, self).__init__(shape, cntk_dtype, init, device,
name)
def __init__(self, name, stream_id, storage_format, dtype, shape):
super(StreamInformation, self).__init__()
self.m_name = name
self.m_id = stream_id
self.m_storage_format = StreamInformation._storage[storage_format]
self.m_element_type = sanitize_dtype_cntk(dtype)
self.m_sample_layout = cntk_py.NDShape(shape)
def test_op_reduce_over_batch_axis(input_data, device_id, precision):
from .. import reduce_sum, reduce_max, reduce_min, reduce_mean, reduce_log_sum_exp, reduce_prod
from cntk import Axis
dt = PRECISION_TO_TYPE[precision]
data = AA(input_data, dtype=dt)
a = C.input_variable(shape=data.shape[1:],
dtype=sanitize_dtype_cntk(dt),
needs_gradient=True,
name='a')
ops = [(reduce_sum, lambda x: np.sum(x, axis=0, keepdims=False),
lambda x, f: np.ones_like(x)),
(reduce_max, lambda x: np.amax(x, axis=0, keepdims=False),
lambda x, f: min_max_bwd(x, f, dt)),
(reduce_min, lambda x: np.amin(x, axis=0, keepdims=False),
lambda x, f: min_max_bwd(x, f, dt)),
(reduce_mean, lambda x: np.mean(x, axis=0, keepdims=False),
lambda x, f: np.ones_like(x) / x.shape[0]),
(reduce_log_sum_exp,
lambda x: np.log(np.sum(np.exp(x), axis=0, keepdims=False)),
lambda x, f: np.exp(x - f)),
(reduce_prod, lambda x: np.prod(x, axis=0, keepdims=False),
lambda x, f: f / x)]
for op, fwd, bwd in ops:
input_op = op(a, axis=Axis.default_batch_axis())
expected_forward = fwd(data)
expected_backward = bwd(data, expected_forward)
binding = {a: data}
actual_backward = input_op.grad(binding)
actual_forward = input_op.eval(binding)
assert np.allclose(actual_forward, expected_forward)
for ab, eb in zip(actual_backward, expected_backward):
assert np.allclose(ab, eb)
def test_op_sequence_reduce_sum(device_id, precision):
from .. import sequence
a = sequence.input(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a')
sequence_sum_a_plus_sequence_sum_a = sequence.reduce_sum(a) + sequence.reduce_sum(a)
a_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])]
actual_grad = sequence_sum_a_plus_sequence_sum_a.grad({a: a_data}, [a])
assert np.array_equal(actual_grad[0], np.asarray([[2.]]))
assert np.array_equal(actual_grad[1], np.asarray([[2.], [2.]]))
assert np.array_equal(actual_grad[2], np.asarray([[2.], [2.], [2.]]))
res = sequence_sum_a_plus_sequence_sum_a.eval({a: a_data})
assert np.array_equal(res[0], np.asarray([4.]))
assert np.array_equal(res[1], np.asarray([10.]))
assert np.array_equal(res[2], np.asarray([18.]))
# Verify that calling sequence reduction on a placeholder with known
# shape but unknown dynamic axes does not result in a problem
p = C.placeholder(shape=(1,))
r = sequence.reduce_sum(p)
r.replace_placeholder(a)
res = r.eval({a: a_data})
assert np.array_equal(res[0], np.asarray([2.]))
assert np.array_equal(res[1], np.asarray([5.]))
assert np.array_equal(res[2], np.asarray([9.]))
def test_op_reduce_over_batch_axis(input_data, device_id, precision):
from .. import reduce_sum, reduce_max, reduce_min, reduce_mean, reduce_log_sum_exp, reduce_prod
from cntk import Axis
dt = PRECISION_TO_TYPE[precision]
data = AA(input_data, dtype=dt)
a = C.input_variable(shape=data.shape[1:],
dtype=sanitize_dtype_cntk(dt),
needs_gradient=True,
name='a')
ops = [
(reduce_sum, lambda x:np.sum(x, axis=0, keepdims=False), lambda x,f:np.ones_like(x)),
(reduce_max, lambda x:np.amax(x, axis=0, keepdims=False), lambda x,f:min_max_bwd(x,f, dt)),
(reduce_min, lambda x:np.amin(x, axis=0, keepdims=False), lambda x,f:min_max_bwd(x,f, dt)),
(reduce_mean, lambda x:np.mean(x, axis=0, keepdims=False), lambda x,f:np.ones_like(x)/x.shape[0]),
(reduce_log_sum_exp, lambda x:np.log(np.sum(np.exp(x), axis=0, keepdims=False)), lambda x,f:np.exp(x-f)),
(reduce_prod, lambda x:np.prod(x, axis=0, keepdims=False), lambda x,f:f / x)
]
for op,fwd,bwd in ops:
input_op = op(a, axis=Axis.default_batch_axis())
expected_forward = fwd(data)
expected_backward = bwd(data, expected_forward)
binding = {a: data}
actual_backward = input_op.grad(binding)
actual_forward = input_op.eval(binding)
assert np.allclose(actual_forward, expected_forward)
for ab,eb in zip (actual_backward, expected_backward):
assert np.allclose(ab, eb)
def test_op_pooling_geometry(input_size, pooling_window, strides, padding,
result, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(total_size, dtype=dt)
input_operand = x.reshape(input_size)
a = I(shape=input_operand.shape[2:],
dtype=sanitize_dtype_cntk(precision),
needs_gradient=False,
name='a')
from cntk import pooling
input_op = pooling(a,
MAX_POOLING,
pooling_window,
strides,
auto_padding=padding)
forward_input = {a: input_operand}
expected_forward = AA([result])
unittest_helper(input_op,
forward_input,
expected_forward,
None,
device_id=device_id,
precision=precision)
def _test_unary_op(precision,
device_id,
op_func,
value,
expected_forward,
expected_backward_all,
op_param_dict={}):
value = AA(value, dtype=PRECISION_TO_TYPE[precision])
a = I(shape=value.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
# create batch
value.shape = (1, 1) + value.shape
if (type(op_func) == str):
input_op = eval('%s a' % op_func)
else:
input_op = op_func(a, **op_param_dict)
forward_input = {a: value}
expected_backward = {
a: expected_backward_all['arg'],
} if expected_backward_all is not None else None
unittest_helper(input_op,
forward_input,
expected_forward,
expected_backward,
device_id=device_id,
precision=precision)
def __init__(self,
value=None,
shape=None,
dtype=None,
device=None,
name=''):
if not device:
device = use_default_device()
if (np.isscalar(value) or isinstance(value, np.ndarray)) and not shape:
shape = ()
if dtype is not None:
if isinstance(value, np.ndarray) and dtype != value.dtype:
value = np.array(value, dtype=dtype)
else:
if isinstance(value, np.ndarray):
dtype = value.dtype
else:
dtype = np.float32
if np.isscalar(value):
super(Constant, self).__init__(sanitize_shape(shape),
sanitize_dtype_cntk(dtype), value,
device, name)
else:
ndav = sanitize_value(shape, value, dtype, device)
super(Constant, self).__init__(ndav, name)
def test_op_reshape(input_shape, output_shape, expected_output_shape, device_id, precision):
# Reshaping is just moving the input values to different indexes of the result tensor.
# If we compute the gradients on the unmodified tensor, reshape would get 1 for all inputs
# For testing the gradients we want to have different gradients for each input index otherwise we can't
# test if they get wrongly permuted during test. To this end we multiply
# the reshaping result with itself.
dev = cntk_device(device_id)
from .. import reshape, element_times
num_tensor_elements = np.multiply.reduce(input_shape)
input_tensor = np.arange(
num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape)
input_reshaped = input_tensor.reshape(expected_output_shape)
a = C.input_variable(shape=input_tensor.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
a_reshaped = reshape(a, output_shape)
const_input_reshaped = constant(input_reshaped, device=dev)
input_op = element_times(a_reshaped, const_input_reshaped)
expected_forward = [input_reshaped**2]
expected_backward = {a: input_tensor}
# create batch
input_tensor.shape = (1,) + input_tensor.shape
forward_input = {a: input_tensor}
unittest_helper(input_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_sequence_reduce_sum(device_id, precision):
a = C.sequence.input_variable(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a')
sequence_sum_a_plus_sequence_sum_a = C.sequence.reduce_sum(a) + C.sequence.reduce_sum(a)
a_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])]
actual_grad = sequence_sum_a_plus_sequence_sum_a.grad({a: a_data}, [a])
assert np.array_equal(actual_grad[0], np.asarray([[2.]]))
assert np.array_equal(actual_grad[1], np.asarray([[2.], [2.]]))
assert np.array_equal(actual_grad[2], np.asarray([[2.], [2.], [2.]]))
res = sequence_sum_a_plus_sequence_sum_a.eval({a: a_data})
assert np.array_equal(res[0], np.asarray([4.]))
assert np.array_equal(res[1], np.asarray([10.]))
assert np.array_equal(res[2], np.asarray([18.]))
# Verify that calling sequence reduction on a placeholder with known
# shape but unknown dynamic axes does not result in a problem
p = C.placeholder(shape=(1,))
r = C.sequence.reduce_sum(p)
r.replace_placeholder(a)
res = r.eval({a: a_data})
assert np.array_equal(res[0], np.asarray([2.]))
assert np.array_equal(res[1], np.asarray([5.]))
assert np.array_equal(res[2], np.asarray([9.]))
def test_op_pooling_ceil(input_size, pooling_window, strides, result, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(1, total_size + 1, 1, dtype=dt)
input_operand = x.reshape(input_size)
a = C.input_variable(shape=input_operand.shape[1:], dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a')
result_array = np.asarray(result, dtype=dt)
max_elements = result_array.reshape(result_array.size).tolist()
# place 1.0s where maximum elements are
backward = np.zeros_like(input_operand)
for element in max_elements:
backward += np.asarray(input_operand == element)
from cntk import pooling
input_op = pooling(a, MAX_POOLING, pooling_window, strides, ceil_out_dim=True)
forward_input = {a: input_operand}
expected_forward = AA(result)
expected_backward = {a: backward}
unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id,
precision=precision)
def test_op_avg_pooling(input_size, pooling_window, strides, result, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(1, total_size + 1, 1, dtype=dt)
input_operand = x.reshape(input_size)
a = C.sequence.input_variable(shape=input_operand.shape[2:],
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='a')
backward = (1 / np.prod(pooling_window)) * np.ones_like(input_operand)
from cntk import pooling
input_op = pooling(a, AVG_POOLING, pooling_window, strides, auto_padding=[True])
forward_input = {a: input_operand}
expected_forward = AA([result])
expected_backward = {a: backward}
unittest_helper(input_op, forward_input, expected_forward,
expected_backward, device_id=device_id, precision=precision)
def test_depth_to_space(image_shape, num_channels, block_size, device_id,
precision):
dev = cntk_device(device_id)
from cntk.internal import sanitize_dtype_cntk
input_val = np.array(np.reshape(range(num_channels), (num_channels, 1, 1)),
dtype=PRECISION_TO_TYPE[precision])
input_val = np.tile(input_val, (1, ) + image_shape)
img = C.input_variable(
(num_channels, ) + image_shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]))
# Result from depth_to_space node.
depth_to_space_op = C.depth_to_space(img, block_size)
output_test = depth_to_space_op.eval({img: input_val})
# Reference result from simulating depth_to_space with other CNTK ops.
h, w = image_shape
reshape_node = C.reshape(img, (block_size, block_size, num_channels //
(block_size**2), h, w))
transpose_node = C.transpose(reshape_node, [2, 3, 0, 4, 1])
depth_to_space_sim_op = C.reshape(
transpose_node,
(num_channels // (block_size**2), h * block_size, w * block_size))
output_ref = depth_to_space_sim_op.eval({img: input_val})
assert np.array_equal(output_test, output_ref)
def test_op_pooling_geometry(input_size, pooling_window, strides, padding, result, use_input_shape_with_inferred_dimension, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(total_size, dtype=dt)
input_operand = x.reshape(input_size)
pool_input_shape = input_operand.shape[1:]
if use_input_shape_with_inferred_dimension:
pool_input_shape = tuple(-1 for x in pool_input_shape)
a = C.input_variable(shape=pool_input_shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=False,
name='a')
from cntk import pooling
input_op = pooling(a, MAX_POOLING, pooling_window, strides, auto_padding=padding)
forward_input = {a: input_operand}
expected_forward = AA(result)
unittest_helper(input_op, forward_input, expected_forward,
None, device_id=device_id, precision=precision)
def test_op_dropout(shape, dropout_rate, device_id, precision):
from cntk import dropout, input
count = 10
resulted_non_zeros = 0
# As the dropout node is stochastic, we run it a couple times and aggregate
# over the results to get more stable tests.
for i in range(count):
value = np.ones(shape=shape, dtype=PRECISION_TO_TYPE[precision])
a = input(shape=value.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
dropout_node = dropout(a, dropout_rate=dropout_rate)
value.shape = (1, ) + value.shape
forward_input = {a: value}
forward, backward = cntk_eval(dropout_node,
forward_input,
precision,
cntk_device(device_id),
backward_pass=True)
resulted_non_zeros += np.count_nonzero(forward[dropout_node.output])
resulted_non_zeros /= count
num_elements = np.multiply.reduce(shape)
expected_non_zeros = num_elements * (1 - dropout_rate)
max_off = 0.2 * num_elements
assert (abs(resulted_non_zeros - expected_non_zeros) < max_off)
def test_op_dropout_with_explicit_seed(device_id, precision):
from cntk import combine, dropout, input
value = np.ones(shape=(10, 10), dtype=PRECISION_TO_TYPE[precision])
a = input(shape=value.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
seed = 123
dropout_nodes = [
dropout(a, dropout_rate=0.5, seed=seed),
dropout(a, dropout_rate=0.5, seed=seed),
dropout(a, dropout_rate=0.5, seed=seed + 1),
dropout(a, dropout_rate=0.5)
]
value.shape = (1, 1) + value.shape
forward_input = {a: value}
results = []
for node in dropout_nodes:
forward, backward = cntk_eval(node,
forward_input,
precision,
cntk_device(device_id),
backward_pass=True)
results.append(forward[node.output])
assert np.allclose(results[0], results[1])
assert not np.allclose(results[0], results[2])
assert not np.allclose(results[0], results[3])
def test_op_reshape_multiple_free_dimensions(input_shape, replacement_shape, expected_output_shape, device_id, precision):
dev = cntk_device(device_id)
from cntk.internal import sanitize_dtype_cntk
from .. import reshape, element_times
num_tensor_elements = np.multiply.reduce(input_shape)
input_tensor = np.arange(
num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape)
input_reshaped = input_tensor.reshape(expected_output_shape)
a = C.input_variable(shape=tuple([C.FreeDimension]*len(input_tensor.shape)),
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
a_reshaped = reshape(a, replacement_shape)
const_input_reshaped = constant(input_reshaped, device=dev)
input_op = element_times(a_reshaped, const_input_reshaped)
expected_forward = [input_reshaped**2]
expected_backward = {a: input_tensor}
# create batch
input_tensor.shape = (1,) + input_tensor.shape
forward_input = {a: input_tensor}
unittest_helper(input_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_avg_pooling(input_size, pooling_window, strides, result, device_id,
precision):
dt = PRECISION_TO_TYPE[precision]
# fill input operand with a sequence 1,2,3,... til total size and then
# resize to input_size
total_size = np.prod(input_size)
x = np.arange(1, total_size + 1, 1, dtype=dt)
input_operand = x.reshape(input_size)
a = C.sequence.input_variable(shape=input_operand.shape[2:],
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='a')
backward = (1 / np.prod(pooling_window)) * np.ones_like(input_operand)
from cntk import pooling
input_op = pooling(a,
AVG_POOLING,
pooling_window,
strides,
auto_padding=[True])
forward_input = {a: input_operand}
expected_forward = AA([result])
expected_backward = {a: backward}
unittest_helper(input_op,
forward_input,
expected_forward,
expected_backward,
device_id=device_id,
precision=precision)
def test_op_reshape(input_shape, output_shape, expected_output_shape, device_id, precision):
# Reshaping is just moving the input values to different indexes of the result tensor.
# If we compute the gradients on the unmodified tensor, reshape would get 1 for all inputs
# For testing the gradients we want to have different gradients for each input index otherwise we can't
# test if they get wrongly permuted during test. To this end we multiply
# the reshaping result with itself.
dev = cntk_device(device_id)
from .. import reshape, element_times
num_tensor_elements = np.multiply.reduce(input_shape)
input_tensor = np.arange(
num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape)
input_reshaped = input_tensor.reshape(expected_output_shape)
a = C.input_variable(shape=input_tensor.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
a_reshaped = reshape(a, output_shape)
const_input_reshaped = constant(input_reshaped, device=dev)
input_op = element_times(a_reshaped, const_input_reshaped)
expected_forward = [input_reshaped**2]
expected_backward = {a: input_tensor}
# create batch
input_tensor.shape = (1,) + input_tensor.shape
forward_input = {a: input_tensor}
unittest_helper(input_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_slice_sequence(input_data, slice_params, expected_result,
device_id, precision):
input_data = AA(input_data, dtype=PRECISION_TO_TYPE[precision])
t = Axis.new_unique_dynamic_axis('t')
sample_shape = input_data.shape[1:]
a = C.sequence.input_variable(shape=sample_shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
sequence_axis=t,
name='a')
result = C.sequence.slice(a,
begin_index=slice_params[0],
end_index=slice_params[1])
def grad_slice(x, beg_index, end_index):
res = np.zeros_like(x)
res[beg_index:end_index] = 1
return res
expected_forward = AA([expected_result],
dtype=PRECISION_TO_TYPE[precision])
expected_backward = {
a: [grad_slice(np.asarray(input_data), *slice_params)]
}
# create batch
input_data.shape = (1,) + input_data.shape
forward_input = {a: input_data}
unittest_helper(result,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_reshape_gradient_accumulation(device_id, precision):
from .. import reshape
input_shape = (2,3)
output_shape = (3,2)
expected_output_shape = (3,2)
num_tensor_elements = np.multiply.reduce(input_shape)
input_tensor = np.arange(
num_tensor_elements, dtype=PRECISION_TO_TYPE[precision])
input_reshaped = input_tensor.reshape(expected_output_shape)
a = C.input_variable(shape=input_tensor.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
a_reshaped1 = reshape(a, output_shape)
a_reshaped2 = reshape(a, output_shape)
input_op = a_reshaped1 + a_reshaped2
resulting_multiplicative_factor = 2
expected_forward = [input_reshaped * resulting_multiplicative_factor]
# create batch
input_tensor.shape = (1,) + input_tensor.shape
expected_backward = {a: np.full(input_tensor.shape, resulting_multiplicative_factor, dtype=PRECISION_TO_TYPE[precision])}
forward_input = {a: input_tensor}
unittest_helper(input_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def test_op_as_block(input_shape, output_shape, expected_output_shape,
device_id, precision):
# We test using reshape as the operation that is encapsulated in a block
dev = cntk_device(device_id)
from cntk.internal import sanitize_dtype_cntk
from .. import reshape, element_times, as_block
num_tensor_elements = np.multiply.reduce(input_shape)
input_tensor = np.arange(
num_tensor_elements,
dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape)
input_reshaped = input_tensor.reshape(expected_output_shape)
a_placeholder = C.placeholder()
a_reshaped = reshape(a_placeholder, output_shape)
const_input_reshaped = constant(input_reshaped, device=dev)
block_composite = element_times(a_reshaped,
const_input_reshaped,
name='element_times_inside_block')
a = C.input_variable(shape=input_tensor.shape,
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
input_op = as_block(block_composite, [(a_placeholder, a)],
'reshape_test_op',
block_instance_name='reshape_test_op')
# Test some basic methods related to blocks
assert input_op.is_composite
block_primitive = input_op.root_function.find_by_name('reshape_test_op')
assert block_primitive.name == 'reshape_test_op'
assert block_primitive.is_primitive
assert block_primitive.is_block
element_times_inside_block = block_primitive.block_root.find_by_name(
'element_times_inside_block')
assert element_times_inside_block.name == 'element_times_inside_block'
assert element_times_inside_block.is_primitive
block_arguments_map = block_primitive.block_arguments_mapping
assert len(block_arguments_map) == 1
expected_forward = [input_reshaped**2]
expected_backward = {a: input_tensor}
# create batch
input_tensor.shape = (1, ) + input_tensor.shape
forward_input = {a: input_tensor}
unittest_helper(input_op,
forward_input,
expected_forward,
expected_backward,
device_id=device_id,
precision=precision)
def __init__(self, shape, data_type, device=None):
from cntk.internal import sanitize_shape, sanitize_dtype_cntk
shape = sanitize_shape(shape)
data_type = sanitize_dtype_cntk(data_type)
if device is None:
device = use_default_device()
super(NDArrayView, self).__init__(data_type, cntk_py.StorageFormat_Dense, shape,
device)
def __init__(self, shape, data_type, device=None):
from cntk.internal import sanitize_shape, sanitize_dtype_cntk
shape = sanitize_shape(shape)
data_type = sanitize_dtype_cntk(data_type)
if device is None:
device = use_default_device()
super(NDArrayView, self).__init__(data_type, cntk_py.StorageFormat_Dense, shape,
device)
def __init__(self, name, stream_id, storage_format, dtype,
shape):
super(StreamInformation, self).__init__()
self.m_name = name
self.m_id = stream_id
self.m_storage_format = StreamInformation._storage[storage_format]
self.m_element_type = sanitize_dtype_cntk(dtype)
self.m_sample_layout = cntk_py.NDShape(shape)
def test_op_plus_var_sequences_input_input(left_batch, right_batch, device_id,
precision):
from .. import plus, sequence
assert len(left_batch) == len(right_batch)
expected_forward = [
AA(left_batch[i]) + AA(right_batch[i]) for i in range(len(left_batch))
]
expected_backward = {
'left': _ones_like(left_batch, PRECISION_TO_TYPE[precision]),
'right': _ones_like(right_batch, PRECISION_TO_TYPE[precision])
}
left_value = [
AA(sample, dtype=PRECISION_TO_TYPE[precision]) for sample in left_batch
]
left_shape = left_value[0][0].shape
right_value = [
AA(sample, dtype=PRECISION_TO_TYPE[precision])
for sample in right_batch
]
right_shape = right_value[0][0].shape
a = sequence.input_variable(shape=left_shape,
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
b = sequence.input_variable(shape=right_shape,
dtype=sanitize_dtype_cntk(
PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='b')
input_op_input = plus(a, b)
forward_input = {a: left_value, b: right_value}
backward_input = {a: None, b: None}
expected_backward = {
a: expected_backward['left'],
b: expected_backward['right'],
}
unittest_helper(input_op_input, forward_input, expected_forward,
expected_backward, device_id, precision)
def test_op_broadcast_as(device_id, precision):
a_data = [AA([1], dtype=PRECISION_TO_TYPE[precision]),
AA([2], dtype=PRECISION_TO_TYPE[precision]),
AA([3], dtype=PRECISION_TO_TYPE[precision])]
b_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]),
AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])]
a = C.input_variable(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='a')
b = C.sequence.input_variable(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='b')
broadcast_a_as_b = C.sequence.broadcast_as(a, b)
res = broadcast_a_as_b.eval({a: a_data, b: b_data})
assert np.array_equal(res[0], np.asarray([[1.]]))
assert np.array_equal(res[1], np.asarray([[2.], [2.]]))
assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_op_splice(input_data1, input_data2, axis, expected_result, device_id, precision):
# FIXME This test currently fails in C++ with
# RuntimeError: Node 'splice_ab' (RowStack operation): Attempted to
# type-cast node to struct Microsoft::MSR::CNTK::INumInputs, which is not
# possible.
input_data1 = AA(input_data1, dtype=PRECISION_TO_TYPE[precision])
input_data2 = AA(input_data2, dtype=PRECISION_TO_TYPE[precision])
a = C.input_variable(shape=input_data1.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
b = C.input_variable(shape=input_data2.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='b')
# create batch
input_data1.shape = (1,) + input_data1.shape
input_data2.shape = (1,) + input_data2.shape
# splice using the operator
root_op = C.splice(a, b, axis=axis, name='splice_ab')
forward_input = {a: input_data1, b: input_data2}
# Backward pass test
# ==================
# The gradient of the splice operator is all ones in the shape of the input
def grad_splice(x):
return np.ones_like(x)
expected_forward = [expected_result]
expected_backward = {
a: grad_splice(np.asarray(input_data1)),
b: grad_splice(np.asarray(input_data2))
}
unittest_helper(root_op,
forward_input, expected_forward, expected_backward,
device_id=device_id, precision=precision)
def future_value(x, initial_state=None, time_step=1, name=''):
'''
This function returns the future value w.r.t. ``x``. It is most often used when
creating RNNs. The resulting tensor has the same shape as the input but is
the next logical sample. The ``time_step`` parameter is the number of steps
to look into the future and is 1 by default. If there is no future value (i.e.
the current sample is the last one in the tensor) then the ``initial_state``
value is returned.
The initial state can be a constant (scalar or tensor), a learnable tensor
or input data (which has a batch dimension, as needed for sequence-to-sequence models).
Example:
>>> x = C.sequence.input_variable(shape=(3,2))
>>> # Create one sequence with 4 tensors of shape (3, 2)
>>> x0 = np.reshape(np.arange(24,dtype=np.float32),(1,4,3,2))
>>> y = C.sequence.future_value(x) # using initial state of 0 by default
>>> y.eval({x:x0})
[array([[[ 6., 7.],
[ 8., 9.],
[ 10., 11.]],
<BLANKLINE>
[[ 12., 13.],
[ 14., 15.],
[ 16., 17.]],
<BLANKLINE>
[[ 18., 19.],
[ 20., 21.],
[ 22., 23.]],
<BLANKLINE>
[[ 0., 0.],
[ 0., 0.],
[ 0., 0.]]], dtype=float32)]
Args:
x: the tensor (or its name) from which the future value is obtained.
initial_state: tensor or scalar representing the initial value to be used when the input tensor is shifted in time.
time_step (int): the number of time steps to look into the future (default 1)
name (str, optional): the name of the Function instance in the network
Returns:
:class:`~cntk.ops.functions.Function`
'''
from cntk.internal import sanitize_dtype_cntk
from ...cntk_py import Constant
from cntk.cntk_py import future_value
if initial_state is None:
initial_state = Constant.scalar(sanitize_dtype_cntk(x.dtype), 0.0)
else:
initial_state = sanitize_input(initial_state)
x = sanitize_input(x)
return future_value(x, initial_state, time_step, name)
def test_op_mean_variance_normalization(input_operand, use_stats_across_channels, do_variance_scaling, epsilon, output_ref, device_id, precision):
dt_precision = PRECISION_TO_TYPE[precision]
input_ref = AA(input_operand, dtype=dt_precision)
a = C.input_variable(shape=input_ref.shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=False,
name='a')
norm_op = C.mean_variance_normalization(a, epsilon=epsilon, use_stats_across_channels=use_stats_across_channels, do_variance_scaling=do_variance_scaling)
output_test = norm_op.eval({a:input_ref}, device=cntk_device(device_id))
assert np.allclose(output_test, output_ref, atol=1e-4)
def __init__(self, shape=None, dtype=None, needs_gradient=False, is_sparse=False,
dynamic_axes=[cntk_py.Axis.default_batch_axis(), cntk_py.Axis.default_dynamic_axis()], name=''):
shape = sanitize_shape(shape)
if dtype is None:
dtype = np.float32
dtype = sanitize_dtype_cntk(dtype)
dynamic_axes = sanitize_dynamic_axes(dynamic_axes)
super(Variable, self).__init__(shape, is_sparse, dtype, needs_gradient, name, dynamic_axes)
def test_depth_to_space(image_shape, num_channels, block_size, output_ref, device_id, precision):
dev = cntk_device(device_id)
from cntk.internal import sanitize_dtype_cntk
input_val = np.array(np.reshape(range(num_channels), (num_channels, 1, 1)), dtype=PRECISION_TO_TYPE[precision])
input_val = np.tile(input_val, (1,) + image_shape)
img = C.input_variable((num_channels,) + image_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]))
depth_to_space_op = C.depth_to_space(img, block_size)
output_test = depth_to_space_op.eval({ img : input_val })
assert np.array_equal(output_test, output_ref)
def __init__(self, name, stream_id, storage_format, dtype,
shape):
super(StreamInformation, self).__init__()
self.m_name = name
self.m_id = stream_id
self.m_storage_format = StreamInformation._storage[storage_format]
self.m_element_type = sanitize_dtype_cntk(dtype)
# raw NDShape is column based, so we need to reverse dimensions.
self.m_sample_layout = cntk_py.NDShape(list(reversed(shape)))
self.sample_shape = shape
self.storage_format = storage_format
def test_space_to_depth(image_shape, num_channels, block_size, device_id, precision):
dev = cntk_device(device_id)
from cntk.internal import sanitize_dtype_cntk
input_val = np.random.randint(low=0, high=100, size=(num_channels,) + image_shape).astype(PRECISION_TO_TYPE[precision])
img = C.input_variable((num_channels,) + image_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]))
depth_to_space_op = C.depth_to_space(img, block_size)
space_to_depth_op = C.space_to_depth(depth_to_space_op, block_size)
output_val = np.squeeze(space_to_depth_op.eval({ img : input_val }), 0)
assert np.array_equal(output_val, input_val)
def test_op_plus_var_sequences_input_input(left_batch, right_batch, device_id, precision):
from .. import plus, sequence
assert len(left_batch) == len(right_batch)
expected_forward = [AA(left_batch[i]) + AA(right_batch[i])
for i in range(len(left_batch))]
expected_backward = {
'left': _ones_like(left_batch, PRECISION_TO_TYPE[precision]),
'right': _ones_like(right_batch, PRECISION_TO_TYPE[precision])
}
left_value = [AA(sample, dtype=PRECISION_TO_TYPE[precision])
for sample in left_batch]
left_shape = left_value[0][0].shape
right_value = [AA(sample, dtype=PRECISION_TO_TYPE[precision])
for sample in right_batch]
right_shape = right_value[0][0].shape
a = sequence.input_variable(shape=left_shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='a')
b = sequence.input_variable(shape=right_shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
needs_gradient=True,
name='b')
input_op_input = plus(a, b)
forward_input = {a: left_value, b: right_value}
backward_input = {a: None, b: None}
expected_backward = {
a: expected_backward['left'],
b: expected_backward['right'], }
unittest_helper(input_op_input,
forward_input, expected_forward,
expected_backward,
device_id, precision)
def test_op_gather_dynamic_axes_equivalence(device_id, precision):
input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision])
input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision])
a = C.sequence.input_variable(shape=input_data1.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='a')
b = C.sequence.input_variable(shape=input_data2.shape,
dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]),
name='b')
is_last_a = C.sequence.is_last(a)
a_last = C.sequence.gather(a, is_last_a)
b_last = C.sequence.gather(b, is_last_a)
z = a_last + b_last
# create batch
input_data1.shape = (1, 1) + input_data1.shape
input_data2.shape = (1, 1) + input_data2.shape
res = z.eval({a: input_data1, b: input_data2})
expected_forward = [[[3.]]]
assert np.array_equal(res, expected_forward)
def test_op_combine(left_operand, right_operand, operations, expected_results, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
from .. import combine
left_value = AA(left_operand, dtype=dt)
right_value = AA(right_operand, dtype=dt)
a = C.input_variable(shape=left_value.shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='a')
b = C.input_variable(shape=right_value.shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='b')
left_value.shape = (1, 1) + left_value.shape
right_value.shape = (1, 1) + right_value.shape
forward_input = {a: left_value, b: right_value}
combine_list = []
for op in operations:
combine_list.append(op(a,b))
combine_node = combine(combine_list)
expected_forward_results = [np.asarray([[i]], dtype=dt) for i in expected_results]
forward_results, _ = cntk_eval(combine_node, forward_input, precision,
cntk_device(device_id))
results = list(forward_results.values())
assert compare_lists_of_np_arrays(results, expected_forward_results)
def test_op_maxroipooling(input_map, input_rois, expected_fwd, expected_bkwd, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
# AA == as numpy array
conv_input = AA(input_map, dtype=dt)
roi_input = AA(input_rois, dtype=dt)
exp_fwd_value = AA(expected_fwd, dtype=dt)
exp_bkwd_value = AA(expected_bkwd, dtype=dt)
# adding batch, sequence and roi axis
exp_fwd_value.shape = (1,1) + exp_fwd_value.shape
exp_bkwd_value.shape = (1,) + exp_bkwd_value.shape
# I == define cntk input variables
a = C.input_variable(shape=conv_input.shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=True,
name='a')
b = C.input_variable(shape=roi_input.shape,
dtype=sanitize_dtype_cntk(precision),
needs_gradient=False,
name='b')
# adding batch and sequence axis
conv_input.shape = (1,) + conv_input.shape
roi_input.shape = (1,) + roi_input.shape
from cntk import roipooling
input_op = roipooling(a, b, C.MAX_POOLING, (3,3), 1.)
forward_input = {a: conv_input, b: roi_input}
expected_backward = {a: exp_bkwd_value}
unittest_helper(input_op,
forward_input, exp_fwd_value, expected_backward,
device_id=device_id, precision=precision)
def test_op_reduce_batch_sequence_static_axes_together(input_data, dynamic_axes, static_axes, device_id, precision):
from cntk import Axis
dt = PRECISION_TO_TYPE[precision]
data = AA(input_data, dtype=dt)
if dynamic_axes == [C.Axis.default_batch_axis()]:
#Reduction along the batch axis on input sequence is currently unsupported, so only batch axis input is tested
v = C.input_variable(data.shape[1:],
dtype=sanitize_dtype_cntk(dt),
needs_gradient=True)
numpy_axis_offset = 1
ignore_max_min = True
else:
v = C.sequence.input_variable(data.shape[2:],
dtype=sanitize_dtype_cntk(dt),
needs_gradient=True)
numpy_axis_offset = 2
ignore_max_min = False
for op, fwd, bwd in reduce_batch_sequence_static_ops(dt):
cntk_axes = tuple(dynamic_axes + static_axes)
numpy_axes = tuple([0 if a == C.Axis.default_batch_axis() else 1 for a in dynamic_axes] + [ax + numpy_axis_offset for ax in static_axes])
_test_reduce_ops(v, data, op, cntk_axes, numpy_axes, fwd, bwd)
