def test_op_broadcast_as(device_id, precision): from .. import sequence a_data = [AA([1], dtype=PRECISION_TO_TYPE[precision]), AA([2], dtype=PRECISION_TO_TYPE[precision]), AA([3], dtype=PRECISION_TO_TYPE[precision])] b_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])] a = I(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='a', dynamic_axes=[Axis.default_batch_axis()]) b = I(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='b') broadcast_a_as_b = sequence.broadcast_as(a, b) res = broadcast_a_as_b.eval({a: a_data, b: b_data}) assert np.array_equal(res[0], np.asarray([[1.]])) assert np.array_equal(res[1], np.asarray([[2.], [2.]])) assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_op_gather_derived_dynamic_axes_equivalence(device_id, precision): from .. import sequence input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision]) input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision]) a = sequence.input(shape=input_data1.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='a') b = sequence.input(shape=input_data2.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='b') a_last = sequence.gather(a, sequence.is_last(a), new_sequence_axis_typeinfo=(0, 1)) b_last = sequence.gather(b, sequence.is_last(b), new_sequence_axis_typeinfo=(0, 1)) z = a_last + b_last # create batch input_data1.shape = (1, 1) + input_data1.shape input_data2.shape = (1, 1) + input_data2.shape res = z.eval({a: input_data1, b: input_data2}) expected_forward = [[3.]] assert np.array_equal(res, expected_forward)
def test_op_gather_derived_dynamic_axes_equivalence(device_id, precision): from .. import sequence input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision]) input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision]) a = I(shape=input_data1.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='a') b = I(shape=input_data2.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='b') a_last = sequence.gather(a, sequence.is_last(a), new_sequence_axis_typeinfo=(0, 1)) b_last = sequence.gather(b, sequence.is_last(b), new_sequence_axis_typeinfo=(0, 1)) z = a_last + b_last # create batch input_data1.shape = (1, 1) + input_data1.shape input_data2.shape = (1, 1) + input_data2.shape res = z.eval({a: input_data1, b: input_data2}) expected_forward = [[3.]] assert np.array_equal(res, expected_forward)
def test_sanitize_dtype_cntk(): for dtype in ['float', 'float32', np.float32, int]: assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Float, dtype for dtype in [float, 'float64', np.float64]: assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Double, dtype for dtype in ['int8', np.int8]: assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Int8, dtype
def test_splice(shape1, shape2): a = C.input_variable(shape=shape1, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') b = C.input_variable(shape=shape2, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='b') # create batch input_data1.shape = (1,) + input_data1.shape input_data2.shape = (1,) + input_data2.shape # splice using the operator root_op = C.splice(a, b, axis=axis, name='splice_ab') forward_input = {a: input_data1, b: input_data2} # Backward pass test # ================== # The gradient of the splice operator is all ones in the shape of the input def grad_splice(x): return np.ones_like(x) expected_forward = [expected_result] expected_backward = { a: grad_splice(np.asarray(input_data1)), b: grad_splice(np.asarray(input_data2)) } unittest_helper(root_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_op_broadcast_as(device_id, precision): a_data = [ AA([1], dtype=PRECISION_TO_TYPE[precision]), AA([2], dtype=PRECISION_TO_TYPE[precision]), AA([3], dtype=PRECISION_TO_TYPE[precision]) ] b_data = [ AA([[2]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision]) ] a = C.input_variable(shape=(1, ), dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), name='a') b = C.sequence.input_variable(shape=(1, ), dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), name='b') broadcast_a_as_b = C.sequence.broadcast_as(a, b) res = broadcast_a_as_b.eval({a: a_data, b: b_data}) assert np.array_equal(res[0], np.asarray([[1.]])) assert np.array_equal(res[1], np.asarray([[2.], [2.]])) assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_op_gather_dynamic_axes_equivalence(device_id, precision): input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision]) input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision]) a = C.sequence.input_variable(shape=input_data1.shape, dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), name='a') b = C.sequence.input_variable(shape=input_data2.shape, dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), name='b') is_last_a = C.sequence.is_last(a) a_last = C.sequence.gather(a, is_last_a) b_last = C.sequence.gather(b, is_last_a) z = a_last + b_last # create batch input_data1.shape = (1, 1) + input_data1.shape input_data2.shape = (1, 1) + input_data2.shape res = z.eval({a: input_data1, b: input_data2}) expected_forward = [[[3.]]] assert np.array_equal(res, expected_forward)
def test_op_broadcast_as(device_id, precision): from .. import sequence a_data = [ AA([1], dtype=PRECISION_TO_TYPE[precision]), AA([2], dtype=PRECISION_TO_TYPE[precision]), AA([3], dtype=PRECISION_TO_TYPE[precision]) ] b_data = [ AA([[2]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision]) ] a = I(shape=(1, ), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='a', dynamic_axes=[Axis.default_batch_axis()]) b = I(shape=(1, ), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='b') broadcast_a_as_b = sequence.broadcast_as(a, b) res = broadcast_a_as_b.eval({a: a_data, b: b_data}) assert np.array_equal(res[0], np.asarray([[1.]])) assert np.array_equal(res[1], np.asarray([[2.], [2.]])) assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_op_reduce_batch_sequence_static_axes_together(input_data, dynamic_axes, static_axes, device_id, precision): from cntk import Axis dt = PRECISION_TO_TYPE[precision] data = AA(input_data, dtype=dt) if dynamic_axes == [C.Axis.default_batch_axis()]: #Reduction along the batch axis on input sequence is currently unsupported, so only batch axis input is tested v = C.input_variable(data.shape[1:], dtype=sanitize_dtype_cntk(dt), needs_gradient=True) numpy_axis_offset = 1 ignore_max_min = True else: v = C.sequence.input_variable(data.shape[2:], dtype=sanitize_dtype_cntk(dt), needs_gradient=True) numpy_axis_offset = 2 ignore_max_min = False for op, fwd, bwd in reduce_batch_sequence_static_ops(dt): cntk_axes = tuple(dynamic_axes + static_axes) numpy_axes = tuple([ 0 if a == C.Axis.default_batch_axis() else 1 for a in dynamic_axes ] + [ax + numpy_axis_offset for ax in static_axes]) _test_reduce_ops(v, data, op, cntk_axes, numpy_axes, fwd, bwd)
def _test_binary_op(precision, device_id, op_func, left_operand, right_operand, expected_forward, expected_backward_all, wrap_batch_seq=True, op_param_dict={}, batch_size_greater_than_one=False): dt = PRECISION_TO_TYPE[precision] dev = cntk_device(device_id) left_value = AA(left_operand, dtype=dt) right_value = AA(right_operand, dtype=dt) left_operand_shape = left_value.shape[1:] if batch_size_greater_than_one else left_value.shape right_operand_shape = right_value.shape[1:] if batch_size_greater_than_one else right_value.shape a = C.input_variable(shape=left_operand_shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a') b = C.input_variable(shape=right_operand_shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='b') const_a = constant(left_value, device=dev) const_b = constant(right_value, device=dev) if (type(op_func) == str): input_op_constant = eval('a %s const_b' % op_func) constant_op_input = eval('const_a %s b' % op_func) input_op_input = eval('a %s b' % op_func) else: input_op_constant = op_func(a, const_b, **op_param_dict) constant_op_input = op_func(const_a, b, **op_param_dict) input_op_input = op_func(a, b, **op_param_dict) # create batch by wrapping the data point into a batch of one sample if wrap_batch_seq and not batch_size_greater_than_one: left_value.shape = (1,) + left_value.shape right_value.shape = (1,) + right_value.shape forward_input = {a: left_value, b: right_value} expected_backward = {a: expected_backward_all[ 'left_arg'], b: expected_backward_all['right_arg'], } unittest_helper(input_op_input, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision) if not batch_size_greater_than_one: forward_input = {a: left_value} expected_backward = {a: expected_backward_all['left_arg'], } unittest_helper(input_op_constant, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision) forward_input = {b: right_value} expected_backward = {b: expected_backward_all['right_arg'], } unittest_helper(constant_op_input, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_sanitize_dtype_cntk(): for dtype in ['float', 'float32', np.float32, int]: assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Float, dtype for dtype in [float, 'float64', np.float64]: assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Double, dtype for dtype in ['int8', np.int8]: assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Int8, dtype for dtype in ['int16', np.int16]: assert sanitize_dtype_cntk(dtype) == C.cntk_py.DataType_Int16, dtype
def _test_binary_op(precision, device_id, op_func, left_operand, right_operand, expected_forward, expected_backward_all, wrap_batch_seq=True, op_param_dict={}): dt = PRECISION_TO_TYPE[precision] dev = cntk_device(device_id) left_value = AA(left_operand, dtype=dt) right_value = AA(right_operand, dtype=dt) a = input(shape=left_value.shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a') b = input(shape=right_value.shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='b') const_a = constant(left_value, device=dev) const_b = constant(right_value, device=dev) if (type(op_func) == str): input_op_constant = eval('a %s const_b' % op_func) constant_op_input = eval('const_a %s b' % op_func) input_op_input = eval('a %s b' % op_func) else: input_op_constant = op_func(a, const_b, **op_param_dict) constant_op_input = op_func(const_a, b, **op_param_dict) input_op_input = op_func(a, b, **op_param_dict) # create batch by wrapping the data point into a batch of one sample if wrap_batch_seq: left_value.shape = (1,) + left_value.shape right_value.shape = (1,) + right_value.shape forward_input = {a: left_value, b: right_value} expected_backward = {a: expected_backward_all[ 'left_arg'], b: expected_backward_all['right_arg'], } unittest_helper(input_op_input, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision) forward_input = {a: left_value} expected_backward = {a: expected_backward_all['left_arg'], } unittest_helper(input_op_constant, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision) forward_input = {b: right_value} expected_backward = {b: expected_backward_all['right_arg'], } unittest_helper(constant_op_input, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_op_splice(input_data1, input_data2, axis, expected_result, device_id, precision): # FIXME This test currently fails in C++ with # RuntimeError: Node 'splice_ab' (RowStack operation): Attempted to # type-cast node to struct Microsoft::MSR::CNTK::INumInputs, which is not # possible. input_data1 = AA(input_data1, dtype=PRECISION_TO_TYPE[precision]) input_data2 = AA(input_data2, dtype=PRECISION_TO_TYPE[precision]) a = C.input_variable(shape=input_data1.shape, dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') b = C.input_variable(shape=input_data2.shape, dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), needs_gradient=True, name='b') # create batch input_data1.shape = (1, ) + input_data1.shape input_data2.shape = (1, ) + input_data2.shape # splice using the operator root_op = C.splice(a, b, axis=axis, name='splice_ab') forward_input = {a: input_data1, b: input_data2} # Backward pass test # ================== # The gradient of the splice operator is all ones in the shape of the input def grad_splice(x): return np.ones_like(x) expected_forward = [expected_result] expected_backward = { a: grad_splice(np.asarray(input_data1)), b: grad_splice(np.asarray(input_data2)) } unittest_helper(root_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_op_reshape_gradient_accumulation(device_id, precision): from .. import reshape input_shape = (2,3) output_shape = (3,2) expected_output_shape = (3,2) num_tensor_elements = np.multiply.reduce(input_shape) input_tensor = np.arange( num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]) input_reshaped = input_tensor.reshape(expected_output_shape) a = C.input_variable(shape=input_tensor.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') a_reshaped1 = reshape(a, output_shape) a_reshaped2 = reshape(a, output_shape) input_op = a_reshaped1 + a_reshaped2 resulting_multiplicative_factor = 2 expected_forward = [input_reshaped * resulting_multiplicative_factor] # create batch input_tensor.shape = (1,) + input_tensor.shape expected_backward = {a: np.full(input_tensor.shape, resulting_multiplicative_factor, dtype=PRECISION_TO_TYPE[precision])} forward_input = {a: input_tensor} unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_op_pooling_ceil(input_size, pooling_window, strides, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] # fill input operand with a sequence 1,2,3,... til total size and then # resize to input_size total_size = np.prod(input_size) x = np.arange(1, total_size + 1, 1, dtype=dt) input_operand = x.reshape(input_size) a = C.input_variable(shape=input_operand.shape[1:], dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a') result_array = np.asarray(result, dtype=dt) max_elements = result_array.reshape(result_array.size).tolist() # place 1.0s where maximum elements are backward = np.zeros_like(input_operand) for element in max_elements: backward += np.asarray(input_operand == element) from cntk import pooling input_op = pooling(a, MAX_POOLING, pooling_window, strides, ceil_out_dim=True) forward_input = {a: input_operand} expected_forward = AA(result) expected_backward = {a: backward} unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_op_average_pooling_include_pad(input_size, pooling_window, strides, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] total_size = np.prod(input_size) x = np.arange(1, total_size + 1, 1, dtype=dt) input_operand = x.reshape(input_size) a = C.input_variable(shape=input_operand.shape[1:], dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a') backward = (1 / np.prod(pooling_window)) * np.ones_like(input_operand) from cntk import pooling input_op = pooling(a, AVG_POOLING, pooling_window, strides, auto_padding=[True], include_pad=True) forward_input = {a: input_operand} expected_forward = AA(result) expected_backward = {a: backward} unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_convolution_transpose(input_size, conv_size, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] dev = cntk_device(device_id) # fill input operand with a sequence 1,2,3,... til total size and then # resize to input_size total_size = np.prod(input_size) x = np.arange(total_size, dtype=dt) input_operand = x.reshape(input_size) a = C.input_variable(shape=input_operand.shape[1:], dtype=sanitize_dtype_cntk(precision), needs_gradient=False, name='a') # do the same for convolution kernel total_size = np.prod(conv_size) y = np.arange(total_size, dtype=dt) conv_map = constant(value=y.reshape(conv_size), device=dev) from cntk import convolution_transpose input_op = convolution_transpose(conv_map, a, auto_padding=[False]) forward_input = {a: input_operand} expected_forward = AA(result) unittest_helper(input_op, forward_input, expected_forward, None, device_id=device_id, precision=precision)
def test_asym_convolution(input_size, conv_size, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] dev = cntk_device(device_id) # fill input operand with a sequence 1,2,3,... til total size and then # resize to input_size total_size = np.prod(input_size) x = np.arange(total_size, dtype=dt) input_operand = x.reshape(input_size) a = C.input_variable(shape=input_operand.shape[1:], dtype=sanitize_dtype_cntk(precision), needs_gradient=False, name='a') # do the same for convolution kernel total_size = np.prod(conv_size) y = np.arange(total_size, dtype=dt) conv_map = constant(value=y.reshape(conv_size), device=dev) from cntk import convolution input_op = convolution(conv_map, a, auto_padding=[True]) forward_input = {a: input_operand} expected_forward = AA(result) unittest_helper(input_op, forward_input, expected_forward, None, device_id=device_id, precision=precision)
def test_op_reshape_multiple_free_dimensions(input_shape, replacement_shape, expected_output_shape, device_id, precision): dev = cntk_device(device_id) from cntk.internal import sanitize_dtype_cntk from .. import reshape, element_times num_tensor_elements = np.multiply.reduce(input_shape) input_tensor = np.arange( num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape) input_reshaped = input_tensor.reshape(expected_output_shape) a = C.input_variable(shape=tuple([C.FreeDimension]*len(input_tensor.shape)), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') a_reshaped = reshape(a, replacement_shape) const_input_reshaped = constant(input_reshaped, device=dev) input_op = element_times(a_reshaped, const_input_reshaped) expected_forward = [input_reshaped**2] expected_backward = {a: input_tensor} # create batch input_tensor.shape = (1,) + input_tensor.shape forward_input = {a: input_tensor} unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_op_slice_sequence(input_data, slice_params, expected_result, device_id, precision): input_data = AA(input_data, dtype=PRECISION_TO_TYPE[precision]) t = Axis.new_unique_dynamic_axis('t') sample_shape = input_data.shape[1:] a = C.sequence.input_variable(shape=sample_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, sequence_axis=t, name='a') result = C.sequence.slice(a, begin_index=slice_params[0], end_index=slice_params[1]) def grad_slice(x, beg_index, end_index): res = np.zeros_like(x) res[beg_index:end_index] = 1 return res expected_forward = AA([expected_result], dtype=PRECISION_TO_TYPE[precision]) expected_backward = { a: [grad_slice(np.asarray(input_data), *slice_params)] } # create batch input_data.shape = (1,) + input_data.shape forward_input = {a: input_data} unittest_helper(result, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def __init__(self, shape=None, init=None, dtype=None, device=None, name=''): if not device: device = use_default_device() if dtype is not None: if isinstance(init, np.ndarray) and dtype != init.dtype: init = np.array(init, dtype=dtype) else: if np.isscalar(init) and not shape: shape = () if isinstance(init, np.ndarray): dtype = init.dtype else: dtype = np.float32 if init is None: init = 0 if isinstance(init, (np.ndarray, list, float, int)): ndav = sanitize_value(shape, init, dtype, device) super(Parameter, self).__init__(ndav, name) else: shape = sanitize_shape(shape) cntk_dtype = sanitize_dtype_cntk(dtype) super(Parameter, self).__init__(shape, cntk_dtype, init, device, name)
def __init__(self, name, stream_id, storage_format, dtype, shape): super(StreamInformation, self).__init__() self.m_name = name self.m_id = stream_id self.m_storage_format = StreamInformation._storage[storage_format] self.m_element_type = sanitize_dtype_cntk(dtype) self.m_sample_layout = cntk_py.NDShape(shape)
def test_op_reduce_over_batch_axis(input_data, device_id, precision): from .. import reduce_sum, reduce_max, reduce_min, reduce_mean, reduce_log_sum_exp, reduce_prod from cntk import Axis dt = PRECISION_TO_TYPE[precision] data = AA(input_data, dtype=dt) a = C.input_variable(shape=data.shape[1:], dtype=sanitize_dtype_cntk(dt), needs_gradient=True, name='a') ops = [(reduce_sum, lambda x: np.sum(x, axis=0, keepdims=False), lambda x, f: np.ones_like(x)), (reduce_max, lambda x: np.amax(x, axis=0, keepdims=False), lambda x, f: min_max_bwd(x, f, dt)), (reduce_min, lambda x: np.amin(x, axis=0, keepdims=False), lambda x, f: min_max_bwd(x, f, dt)), (reduce_mean, lambda x: np.mean(x, axis=0, keepdims=False), lambda x, f: np.ones_like(x) / x.shape[0]), (reduce_log_sum_exp, lambda x: np.log(np.sum(np.exp(x), axis=0, keepdims=False)), lambda x, f: np.exp(x - f)), (reduce_prod, lambda x: np.prod(x, axis=0, keepdims=False), lambda x, f: f / x)] for op, fwd, bwd in ops: input_op = op(a, axis=Axis.default_batch_axis()) expected_forward = fwd(data) expected_backward = bwd(data, expected_forward) binding = {a: data} actual_backward = input_op.grad(binding) actual_forward = input_op.eval(binding) assert np.allclose(actual_forward, expected_forward) for ab, eb in zip(actual_backward, expected_backward): assert np.allclose(ab, eb)
def test_op_sequence_reduce_sum(device_id, precision): from .. import sequence a = sequence.input(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') sequence_sum_a_plus_sequence_sum_a = sequence.reduce_sum(a) + sequence.reduce_sum(a) a_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])] actual_grad = sequence_sum_a_plus_sequence_sum_a.grad({a: a_data}, [a]) assert np.array_equal(actual_grad[0], np.asarray([[2.]])) assert np.array_equal(actual_grad[1], np.asarray([[2.], [2.]])) assert np.array_equal(actual_grad[2], np.asarray([[2.], [2.], [2.]])) res = sequence_sum_a_plus_sequence_sum_a.eval({a: a_data}) assert np.array_equal(res[0], np.asarray([4.])) assert np.array_equal(res[1], np.asarray([10.])) assert np.array_equal(res[2], np.asarray([18.])) # Verify that calling sequence reduction on a placeholder with known # shape but unknown dynamic axes does not result in a problem p = C.placeholder(shape=(1,)) r = sequence.reduce_sum(p) r.replace_placeholder(a) res = r.eval({a: a_data}) assert np.array_equal(res[0], np.asarray([2.])) assert np.array_equal(res[1], np.asarray([5.])) assert np.array_equal(res[2], np.asarray([9.]))
def test_op_reduce_over_batch_axis(input_data, device_id, precision): from .. import reduce_sum, reduce_max, reduce_min, reduce_mean, reduce_log_sum_exp, reduce_prod from cntk import Axis dt = PRECISION_TO_TYPE[precision] data = AA(input_data, dtype=dt) a = C.input_variable(shape=data.shape[1:], dtype=sanitize_dtype_cntk(dt), needs_gradient=True, name='a') ops = [ (reduce_sum, lambda x:np.sum(x, axis=0, keepdims=False), lambda x,f:np.ones_like(x)), (reduce_max, lambda x:np.amax(x, axis=0, keepdims=False), lambda x,f:min_max_bwd(x,f, dt)), (reduce_min, lambda x:np.amin(x, axis=0, keepdims=False), lambda x,f:min_max_bwd(x,f, dt)), (reduce_mean, lambda x:np.mean(x, axis=0, keepdims=False), lambda x,f:np.ones_like(x)/x.shape[0]), (reduce_log_sum_exp, lambda x:np.log(np.sum(np.exp(x), axis=0, keepdims=False)), lambda x,f:np.exp(x-f)), (reduce_prod, lambda x:np.prod(x, axis=0, keepdims=False), lambda x,f:f / x) ] for op,fwd,bwd in ops: input_op = op(a, axis=Axis.default_batch_axis()) expected_forward = fwd(data) expected_backward = bwd(data, expected_forward) binding = {a: data} actual_backward = input_op.grad(binding) actual_forward = input_op.eval(binding) assert np.allclose(actual_forward, expected_forward) for ab,eb in zip (actual_backward, expected_backward): assert np.allclose(ab, eb)
def test_op_pooling_geometry(input_size, pooling_window, strides, padding, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] # fill input operand with a sequence 1,2,3,... til total size and then # resize to input_size total_size = np.prod(input_size) x = np.arange(total_size, dtype=dt) input_operand = x.reshape(input_size) a = I(shape=input_operand.shape[2:], dtype=sanitize_dtype_cntk(precision), needs_gradient=False, name='a') from cntk import pooling input_op = pooling(a, MAX_POOLING, pooling_window, strides, auto_padding=padding) forward_input = {a: input_operand} expected_forward = AA([result]) unittest_helper(input_op, forward_input, expected_forward, None, device_id=device_id, precision=precision)
def _test_unary_op(precision, device_id, op_func, value, expected_forward, expected_backward_all, op_param_dict={}): value = AA(value, dtype=PRECISION_TO_TYPE[precision]) a = I(shape=value.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') # create batch value.shape = (1, 1) + value.shape if (type(op_func) == str): input_op = eval('%s a' % op_func) else: input_op = op_func(a, **op_param_dict) forward_input = {a: value} expected_backward = { a: expected_backward_all['arg'], } if expected_backward_all is not None else None unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def __init__(self, value=None, shape=None, dtype=None, device=None, name=''): if not device: device = use_default_device() if (np.isscalar(value) or isinstance(value, np.ndarray)) and not shape: shape = () if dtype is not None: if isinstance(value, np.ndarray) and dtype != value.dtype: value = np.array(value, dtype=dtype) else: if isinstance(value, np.ndarray): dtype = value.dtype else: dtype = np.float32 if np.isscalar(value): super(Constant, self).__init__(sanitize_shape(shape), sanitize_dtype_cntk(dtype), value, device, name) else: ndav = sanitize_value(shape, value, dtype, device) super(Constant, self).__init__(ndav, name)
def test_op_reshape(input_shape, output_shape, expected_output_shape, device_id, precision): # Reshaping is just moving the input values to different indexes of the result tensor. # If we compute the gradients on the unmodified tensor, reshape would get 1 for all inputs # For testing the gradients we want to have different gradients for each input index otherwise we can't # test if they get wrongly permuted during test. To this end we multiply # the reshaping result with itself. dev = cntk_device(device_id) from .. import reshape, element_times num_tensor_elements = np.multiply.reduce(input_shape) input_tensor = np.arange( num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape) input_reshaped = input_tensor.reshape(expected_output_shape) a = C.input_variable(shape=input_tensor.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') a_reshaped = reshape(a, output_shape) const_input_reshaped = constant(input_reshaped, device=dev) input_op = element_times(a_reshaped, const_input_reshaped) expected_forward = [input_reshaped**2] expected_backward = {a: input_tensor} # create batch input_tensor.shape = (1,) + input_tensor.shape forward_input = {a: input_tensor} unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_op_sequence_reduce_sum(device_id, precision): a = C.sequence.input_variable(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') sequence_sum_a_plus_sequence_sum_a = C.sequence.reduce_sum(a) + C.sequence.reduce_sum(a) a_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])] actual_grad = sequence_sum_a_plus_sequence_sum_a.grad({a: a_data}, [a]) assert np.array_equal(actual_grad[0], np.asarray([[2.]])) assert np.array_equal(actual_grad[1], np.asarray([[2.], [2.]])) assert np.array_equal(actual_grad[2], np.asarray([[2.], [2.], [2.]])) res = sequence_sum_a_plus_sequence_sum_a.eval({a: a_data}) assert np.array_equal(res[0], np.asarray([4.])) assert np.array_equal(res[1], np.asarray([10.])) assert np.array_equal(res[2], np.asarray([18.])) # Verify that calling sequence reduction on a placeholder with known # shape but unknown dynamic axes does not result in a problem p = C.placeholder(shape=(1,)) r = C.sequence.reduce_sum(p) r.replace_placeholder(a) res = r.eval({a: a_data}) assert np.array_equal(res[0], np.asarray([2.])) assert np.array_equal(res[1], np.asarray([5.])) assert np.array_equal(res[2], np.asarray([9.]))
def test_op_avg_pooling(input_size, pooling_window, strides, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] # fill input operand with a sequence 1,2,3,... til total size and then # resize to input_size total_size = np.prod(input_size) x = np.arange(1, total_size + 1, 1, dtype=dt) input_operand = x.reshape(input_size) a = C.sequence.input_variable(shape=input_operand.shape[2:], dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a') backward = (1 / np.prod(pooling_window)) * np.ones_like(input_operand) from cntk import pooling input_op = pooling(a, AVG_POOLING, pooling_window, strides, auto_padding=[True]) forward_input = {a: input_operand} expected_forward = AA([result]) expected_backward = {a: backward} unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def test_depth_to_space(image_shape, num_channels, block_size, device_id, precision): dev = cntk_device(device_id) from cntk.internal import sanitize_dtype_cntk input_val = np.array(np.reshape(range(num_channels), (num_channels, 1, 1)), dtype=PRECISION_TO_TYPE[precision]) input_val = np.tile(input_val, (1, ) + image_shape) img = C.input_variable( (num_channels, ) + image_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision])) # Result from depth_to_space node. depth_to_space_op = C.depth_to_space(img, block_size) output_test = depth_to_space_op.eval({img: input_val}) # Reference result from simulating depth_to_space with other CNTK ops. h, w = image_shape reshape_node = C.reshape(img, (block_size, block_size, num_channels // (block_size**2), h, w)) transpose_node = C.transpose(reshape_node, [2, 3, 0, 4, 1]) depth_to_space_sim_op = C.reshape( transpose_node, (num_channels // (block_size**2), h * block_size, w * block_size)) output_ref = depth_to_space_sim_op.eval({img: input_val}) assert np.array_equal(output_test, output_ref)
def test_op_pooling_geometry(input_size, pooling_window, strides, padding, result, use_input_shape_with_inferred_dimension, device_id, precision): dt = PRECISION_TO_TYPE[precision] # fill input operand with a sequence 1,2,3,... til total size and then # resize to input_size total_size = np.prod(input_size) x = np.arange(total_size, dtype=dt) input_operand = x.reshape(input_size) pool_input_shape = input_operand.shape[1:] if use_input_shape_with_inferred_dimension: pool_input_shape = tuple(-1 for x in pool_input_shape) a = C.input_variable(shape=pool_input_shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=False, name='a') from cntk import pooling input_op = pooling(a, MAX_POOLING, pooling_window, strides, auto_padding=padding) forward_input = {a: input_operand} expected_forward = AA(result) unittest_helper(input_op, forward_input, expected_forward, None, device_id=device_id, precision=precision)
def test_op_dropout(shape, dropout_rate, device_id, precision): from cntk import dropout, input count = 10 resulted_non_zeros = 0 # As the dropout node is stochastic, we run it a couple times and aggregate # over the results to get more stable tests. for i in range(count): value = np.ones(shape=shape, dtype=PRECISION_TO_TYPE[precision]) a = input(shape=value.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') dropout_node = dropout(a, dropout_rate=dropout_rate) value.shape = (1, ) + value.shape forward_input = {a: value} forward, backward = cntk_eval(dropout_node, forward_input, precision, cntk_device(device_id), backward_pass=True) resulted_non_zeros += np.count_nonzero(forward[dropout_node.output]) resulted_non_zeros /= count num_elements = np.multiply.reduce(shape) expected_non_zeros = num_elements * (1 - dropout_rate) max_off = 0.2 * num_elements assert (abs(resulted_non_zeros - expected_non_zeros) < max_off)
def test_op_dropout_with_explicit_seed(device_id, precision): from cntk import combine, dropout, input value = np.ones(shape=(10, 10), dtype=PRECISION_TO_TYPE[precision]) a = input(shape=value.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') seed = 123 dropout_nodes = [ dropout(a, dropout_rate=0.5, seed=seed), dropout(a, dropout_rate=0.5, seed=seed), dropout(a, dropout_rate=0.5, seed=seed + 1), dropout(a, dropout_rate=0.5) ] value.shape = (1, 1) + value.shape forward_input = {a: value} results = [] for node in dropout_nodes: forward, backward = cntk_eval(node, forward_input, precision, cntk_device(device_id), backward_pass=True) results.append(forward[node.output]) assert np.allclose(results[0], results[1]) assert not np.allclose(results[0], results[2]) assert not np.allclose(results[0], results[3])
def test_op_as_block(input_shape, output_shape, expected_output_shape, device_id, precision): # We test using reshape as the operation that is encapsulated in a block dev = cntk_device(device_id) from cntk.internal import sanitize_dtype_cntk from .. import reshape, element_times, as_block num_tensor_elements = np.multiply.reduce(input_shape) input_tensor = np.arange( num_tensor_elements, dtype=PRECISION_TO_TYPE[precision]).reshape(input_shape) input_reshaped = input_tensor.reshape(expected_output_shape) a_placeholder = C.placeholder() a_reshaped = reshape(a_placeholder, output_shape) const_input_reshaped = constant(input_reshaped, device=dev) block_composite = element_times(a_reshaped, const_input_reshaped, name='element_times_inside_block') a = C.input_variable(shape=input_tensor.shape, dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') input_op = as_block(block_composite, [(a_placeholder, a)], 'reshape_test_op', block_instance_name='reshape_test_op') # Test some basic methods related to blocks assert input_op.is_composite block_primitive = input_op.root_function.find_by_name('reshape_test_op') assert block_primitive.name == 'reshape_test_op' assert block_primitive.is_primitive assert block_primitive.is_block element_times_inside_block = block_primitive.block_root.find_by_name( 'element_times_inside_block') assert element_times_inside_block.name == 'element_times_inside_block' assert element_times_inside_block.is_primitive block_arguments_map = block_primitive.block_arguments_mapping assert len(block_arguments_map) == 1 expected_forward = [input_reshaped**2] expected_backward = {a: input_tensor} # create batch input_tensor.shape = (1, ) + input_tensor.shape forward_input = {a: input_tensor} unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def __init__(self, shape, data_type, device=None): from cntk.internal import sanitize_shape, sanitize_dtype_cntk shape = sanitize_shape(shape) data_type = sanitize_dtype_cntk(data_type) if device is None: device = use_default_device() super(NDArrayView, self).__init__(data_type, cntk_py.StorageFormat_Dense, shape, device)
def test_op_plus_var_sequences_input_input(left_batch, right_batch, device_id, precision): from .. import plus, sequence assert len(left_batch) == len(right_batch) expected_forward = [ AA(left_batch[i]) + AA(right_batch[i]) for i in range(len(left_batch)) ] expected_backward = { 'left': _ones_like(left_batch, PRECISION_TO_TYPE[precision]), 'right': _ones_like(right_batch, PRECISION_TO_TYPE[precision]) } left_value = [ AA(sample, dtype=PRECISION_TO_TYPE[precision]) for sample in left_batch ] left_shape = left_value[0][0].shape right_value = [ AA(sample, dtype=PRECISION_TO_TYPE[precision]) for sample in right_batch ] right_shape = right_value[0][0].shape a = sequence.input_variable(shape=left_shape, dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') b = sequence.input_variable(shape=right_shape, dtype=sanitize_dtype_cntk( PRECISION_TO_TYPE[precision]), needs_gradient=True, name='b') input_op_input = plus(a, b) forward_input = {a: left_value, b: right_value} backward_input = {a: None, b: None} expected_backward = { a: expected_backward['left'], b: expected_backward['right'], } unittest_helper(input_op_input, forward_input, expected_forward, expected_backward, device_id, precision)
def test_op_broadcast_as(device_id, precision): a_data = [AA([1], dtype=PRECISION_TO_TYPE[precision]), AA([2], dtype=PRECISION_TO_TYPE[precision]), AA([3], dtype=PRECISION_TO_TYPE[precision])] b_data = [AA([[2]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3]], dtype=PRECISION_TO_TYPE[precision]), AA([[2], [3], [4]], dtype=PRECISION_TO_TYPE[precision])] a = C.input_variable(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='a') b = C.sequence.input_variable(shape=(1,), dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='b') broadcast_a_as_b = C.sequence.broadcast_as(a, b) res = broadcast_a_as_b.eval({a: a_data, b: b_data}) assert np.array_equal(res[0], np.asarray([[1.]])) assert np.array_equal(res[1], np.asarray([[2.], [2.]])) assert np.array_equal(res[2], np.asarray([[3.], [3.], [3.]]))
def test_op_splice(input_data1, input_data2, axis, expected_result, device_id, precision): # FIXME This test currently fails in C++ with # RuntimeError: Node 'splice_ab' (RowStack operation): Attempted to # type-cast node to struct Microsoft::MSR::CNTK::INumInputs, which is not # possible. input_data1 = AA(input_data1, dtype=PRECISION_TO_TYPE[precision]) input_data2 = AA(input_data2, dtype=PRECISION_TO_TYPE[precision]) a = C.input_variable(shape=input_data1.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') b = C.input_variable(shape=input_data2.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='b') # create batch input_data1.shape = (1,) + input_data1.shape input_data2.shape = (1,) + input_data2.shape # splice using the operator root_op = C.splice(a, b, axis=axis, name='splice_ab') forward_input = {a: input_data1, b: input_data2} # Backward pass test # ================== # The gradient of the splice operator is all ones in the shape of the input def grad_splice(x): return np.ones_like(x) expected_forward = [expected_result] expected_backward = { a: grad_splice(np.asarray(input_data1)), b: grad_splice(np.asarray(input_data2)) } unittest_helper(root_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision)
def future_value(x, initial_state=None, time_step=1, name=''): ''' This function returns the future value w.r.t. ``x``. It is most often used when creating RNNs. The resulting tensor has the same shape as the input but is the next logical sample. The ``time_step`` parameter is the number of steps to look into the future and is 1 by default. If there is no future value (i.e. the current sample is the last one in the tensor) then the ``initial_state`` value is returned. The initial state can be a constant (scalar or tensor), a learnable tensor or input data (which has a batch dimension, as needed for sequence-to-sequence models). Example: >>> x = C.sequence.input_variable(shape=(3,2)) >>> # Create one sequence with 4 tensors of shape (3, 2) >>> x0 = np.reshape(np.arange(24,dtype=np.float32),(1,4,3,2)) >>> y = C.sequence.future_value(x) # using initial state of 0 by default >>> y.eval({x:x0}) [array([[[ 6., 7.], [ 8., 9.], [ 10., 11.]], <BLANKLINE> [[ 12., 13.], [ 14., 15.], [ 16., 17.]], <BLANKLINE> [[ 18., 19.], [ 20., 21.], [ 22., 23.]], <BLANKLINE> [[ 0., 0.], [ 0., 0.], [ 0., 0.]]], dtype=float32)] Args: x: the tensor (or its name) from which the future value is obtained. initial_state: tensor or scalar representing the initial value to be used when the input tensor is shifted in time. time_step (int): the number of time steps to look into the future (default 1) name (str, optional): the name of the Function instance in the network Returns: :class:`~cntk.ops.functions.Function` ''' from cntk.internal import sanitize_dtype_cntk from ...cntk_py import Constant from cntk.cntk_py import future_value if initial_state is None: initial_state = Constant.scalar(sanitize_dtype_cntk(x.dtype), 0.0) else: initial_state = sanitize_input(initial_state) x = sanitize_input(x) return future_value(x, initial_state, time_step, name)
def test_op_mean_variance_normalization(input_operand, use_stats_across_channels, do_variance_scaling, epsilon, output_ref, device_id, precision): dt_precision = PRECISION_TO_TYPE[precision] input_ref = AA(input_operand, dtype=dt_precision) a = C.input_variable(shape=input_ref.shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=False, name='a') norm_op = C.mean_variance_normalization(a, epsilon=epsilon, use_stats_across_channels=use_stats_across_channels, do_variance_scaling=do_variance_scaling) output_test = norm_op.eval({a:input_ref}, device=cntk_device(device_id)) assert np.allclose(output_test, output_ref, atol=1e-4)
def __init__(self, shape=None, dtype=None, needs_gradient=False, is_sparse=False, dynamic_axes=[cntk_py.Axis.default_batch_axis(), cntk_py.Axis.default_dynamic_axis()], name=''): shape = sanitize_shape(shape) if dtype is None: dtype = np.float32 dtype = sanitize_dtype_cntk(dtype) dynamic_axes = sanitize_dynamic_axes(dynamic_axes) super(Variable, self).__init__(shape, is_sparse, dtype, needs_gradient, name, dynamic_axes)
def test_depth_to_space(image_shape, num_channels, block_size, output_ref, device_id, precision): dev = cntk_device(device_id) from cntk.internal import sanitize_dtype_cntk input_val = np.array(np.reshape(range(num_channels), (num_channels, 1, 1)), dtype=PRECISION_TO_TYPE[precision]) input_val = np.tile(input_val, (1,) + image_shape) img = C.input_variable((num_channels,) + image_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision])) depth_to_space_op = C.depth_to_space(img, block_size) output_test = depth_to_space_op.eval({ img : input_val }) assert np.array_equal(output_test, output_ref)
def __init__(self, name, stream_id, storage_format, dtype, shape): super(StreamInformation, self).__init__() self.m_name = name self.m_id = stream_id self.m_storage_format = StreamInformation._storage[storage_format] self.m_element_type = sanitize_dtype_cntk(dtype) # raw NDShape is column based, so we need to reverse dimensions. self.m_sample_layout = cntk_py.NDShape(list(reversed(shape))) self.sample_shape = shape self.storage_format = storage_format
def test_space_to_depth(image_shape, num_channels, block_size, device_id, precision): dev = cntk_device(device_id) from cntk.internal import sanitize_dtype_cntk input_val = np.random.randint(low=0, high=100, size=(num_channels,) + image_shape).astype(PRECISION_TO_TYPE[precision]) img = C.input_variable((num_channels,) + image_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision])) depth_to_space_op = C.depth_to_space(img, block_size) space_to_depth_op = C.space_to_depth(depth_to_space_op, block_size) output_val = np.squeeze(space_to_depth_op.eval({ img : input_val }), 0) assert np.array_equal(output_val, input_val)
def test_op_plus_var_sequences_input_input(left_batch, right_batch, device_id, precision): from .. import plus, sequence assert len(left_batch) == len(right_batch) expected_forward = [AA(left_batch[i]) + AA(right_batch[i]) for i in range(len(left_batch))] expected_backward = { 'left': _ones_like(left_batch, PRECISION_TO_TYPE[precision]), 'right': _ones_like(right_batch, PRECISION_TO_TYPE[precision]) } left_value = [AA(sample, dtype=PRECISION_TO_TYPE[precision]) for sample in left_batch] left_shape = left_value[0][0].shape right_value = [AA(sample, dtype=PRECISION_TO_TYPE[precision]) for sample in right_batch] right_shape = right_value[0][0].shape a = sequence.input_variable(shape=left_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='a') b = sequence.input_variable(shape=right_shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), needs_gradient=True, name='b') input_op_input = plus(a, b) forward_input = {a: left_value, b: right_value} backward_input = {a: None, b: None} expected_backward = { a: expected_backward['left'], b: expected_backward['right'], } unittest_helper(input_op_input, forward_input, expected_forward, expected_backward, device_id, precision)
def test_op_gather_dynamic_axes_equivalence(device_id, precision): input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision]) input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision]) a = C.sequence.input_variable(shape=input_data1.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='a') b = C.sequence.input_variable(shape=input_data2.shape, dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), name='b') is_last_a = C.sequence.is_last(a) a_last = C.sequence.gather(a, is_last_a) b_last = C.sequence.gather(b, is_last_a) z = a_last + b_last # create batch input_data1.shape = (1, 1) + input_data1.shape input_data2.shape = (1, 1) + input_data2.shape res = z.eval({a: input_data1, b: input_data2}) expected_forward = [[[3.]]] assert np.array_equal(res, expected_forward)
def test_op_combine(left_operand, right_operand, operations, expected_results, device_id, precision): dt = PRECISION_TO_TYPE[precision] from .. import combine left_value = AA(left_operand, dtype=dt) right_value = AA(right_operand, dtype=dt) a = C.input_variable(shape=left_value.shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a') b = C.input_variable(shape=right_value.shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='b') left_value.shape = (1, 1) + left_value.shape right_value.shape = (1, 1) + right_value.shape forward_input = {a: left_value, b: right_value} combine_list = [] for op in operations: combine_list.append(op(a,b)) combine_node = combine(combine_list) expected_forward_results = [np.asarray([[i]], dtype=dt) for i in expected_results] forward_results, _ = cntk_eval(combine_node, forward_input, precision, cntk_device(device_id)) results = list(forward_results.values()) assert compare_lists_of_np_arrays(results, expected_forward_results)
def test_op_maxroipooling(input_map, input_rois, expected_fwd, expected_bkwd, device_id, precision): dt = PRECISION_TO_TYPE[precision] # AA == as numpy array conv_input = AA(input_map, dtype=dt) roi_input = AA(input_rois, dtype=dt) exp_fwd_value = AA(expected_fwd, dtype=dt) exp_bkwd_value = AA(expected_bkwd, dtype=dt) # adding batch, sequence and roi axis exp_fwd_value.shape = (1,1) + exp_fwd_value.shape exp_bkwd_value.shape = (1,) + exp_bkwd_value.shape # I == define cntk input variables a = C.input_variable(shape=conv_input.shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=True, name='a') b = C.input_variable(shape=roi_input.shape, dtype=sanitize_dtype_cntk(precision), needs_gradient=False, name='b') # adding batch and sequence axis conv_input.shape = (1,) + conv_input.shape roi_input.shape = (1,) + roi_input.shape from cntk import roipooling input_op = roipooling(a, b, C.MAX_POOLING, (3,3), 1.) forward_input = {a: conv_input, b: roi_input} expected_backward = {a: exp_bkwd_value} unittest_helper(input_op, forward_input, exp_fwd_value, expected_backward, device_id=device_id, precision=precision)
def test_op_reduce_batch_sequence_static_axes_together(input_data, dynamic_axes, static_axes, device_id, precision): from cntk import Axis dt = PRECISION_TO_TYPE[precision] data = AA(input_data, dtype=dt) if dynamic_axes == [C.Axis.default_batch_axis()]: #Reduction along the batch axis on input sequence is currently unsupported, so only batch axis input is tested v = C.input_variable(data.shape[1:], dtype=sanitize_dtype_cntk(dt), needs_gradient=True) numpy_axis_offset = 1 ignore_max_min = True else: v = C.sequence.input_variable(data.shape[2:], dtype=sanitize_dtype_cntk(dt), needs_gradient=True) numpy_axis_offset = 2 ignore_max_min = False for op, fwd, bwd in reduce_batch_sequence_static_ops(dt): cntk_axes = tuple(dynamic_axes + static_axes) numpy_axes = tuple([0 if a == C.Axis.default_batch_axis() else 1 for a in dynamic_axes] + [ax + numpy_axis_offset for ax in static_axes]) _test_reduce_ops(v, data, op, cntk_axes, numpy_axes, fwd, bwd)