def test_Div(tmpdir): data0 = np.asarray([1., 1., 1., 1.], dtype=np.float32) data1 = np.asarray([0.5, 0.25, 0.125, 0.], dtype=np.float32) model = C.element_divide(data0, data1) verify_no_input(model, tmpdir, 'Div_0') x = C.input_variable(data0.shape) model = C.element_divide(x, data1) verify_one_input(model, data0, tmpdir, 'Div_1')
def test_Div(tmpdir): data0 = np.asarray([1., 1., 1., 1.], dtype=np.float32) data1 = np.asarray([0.5, 0.25, 0.125, 0.], dtype=np.float32) model = C.element_divide(data0, data1) verify_no_input(model, tmpdir, 'Div_0') x = C.input_variable(data0.shape) model = C.element_divide(x, data1) verify_one_input(model, data0, tmpdir, 'Div_1')
def squash(input): # ||Sj||^2 Sj_squared_norm = ct.reduce_sum(ct.square(input), axis=axis) # ||Sj||^2 / (1 + ||Sj||^2) * (Sj / ||Sj||) factor = ct.element_divide( ct.element_divide(Sj_squared_norm, ct.plus(1, Sj_squared_norm)), ct.sqrt(ct.plus(Sj_squared_norm, epsilon))) return factor * input
def element_divide(left, right, name=''): ''' The output of this operation is the element-wise division of the two input tensors. It supports broadcasting. In case of scalars its backward pass to left propagates :math:`1/right` times the received gradient, and the backward pass to right propagates. The operator (/) has been overloaded and can equally be used instead of element_divide(). :math:`(-left/right^2)` times the received gradient. Example: >>> C.eval(C.element_divide([1., 1., 1., 1.], [0.5, 0.25, 0.125, 0.])) [array([[ 2., 4., 8., 0.]])] >>> C.eval(C.element_divide([5., 10., 15., 30.], [2.])) [array([[ 2.5, 5. , 7.5, 15. ]])] Args: left: left side tensor right: right side tensor name (str): the name of the node in the network Returns: :class:`cntk.Function` ''' from cntk import element_divide left = sanitize_input(left, get_data_type(right)) right = sanitize_input(right, get_data_type(left)) return element_divide(left, right, name).output()
def multiFunc(self, arg1): # load or create the inputs we need multiIn = C.input(shape=arg1.shape, dynamic_axes = arg1.dynamic_axes) bit_map = C.constant(self.bit_map) max_bits = self.bit_map.max() shape = multiIn.shape reformed = C.reshape(multiIn, (-1,)) # lets compute the means we need # carry over represents the remaining value that needs to binarized. For a single bit, this is just the input. For more bits, # it is the difference between the previous bits approximation and the true value. carry_over = multiIn approx = C.element_times(multiIn, 0) # iterate through the maximum number of bits specified by the bit maps, basically compute each level of binarization for i in range(max_bits): # determine which values of the input should be binarized to i bits or more hot_vals = C.greater(bit_map, i) # select only the values which we need to binarize valid_vals = C.element_select(hot_vals, carry_over, 0) # compute mean on a per kernel basis, reshaping is done to allow for sum reduction along only axis 0 (the kernels) mean = C.element_divide(C.reduce_sum(C.reshape(C.abs(valid_vals), (valid_vals.shape[0], -1)), axis=1), C.reduce_sum(C.reshape(hot_vals, (hot_vals.shape[0], -1)), axis=1)) # reshape the mean to match the dimensionality of the input mean = C.reshape(mean, (mean.shape[0], mean.shape[1], 1, 1)) # binarize the carry over bits = C.greater(carry_over, 0) bits = C.element_select(bits, bits, -1) bits = C.element_select(hot_vals, bits, 0) # add in the equivalent binary representation to the approximation approx = C.plus(approx, C.element_times(mean, bits)) # compute the new carry over carry_over = C.plus(C.element_times(C.element_times(-1, bits), mean), carry_over) return approx, multiIn
def element_divide(left, right, name=''): ''' The output of this operation is the element-wise division of the two input tensors. It supports broadcasting. In case of scalars its backward pass to left propagates :math:`1/right` times the received gradient, and the backward pass to right propagates. The operator (/) has been overloaded and can equally be used instead of element_divide(). :math:`(-left/right^2)` times the received gradient. Example: >>> C.eval(C.element_divide([1., 1., 1., 1.], [0.5, 0.25, 0.125, 0.])) [array([[ 2., 4., 8., 0.]])] >>> C.eval(C.element_divide([5., 10., 15., 30.], [2.])) [array([[ 2.5, 5. , 7.5, 15. ]])] Args: left: left side tensor right: right side tensor name (str): the name of the node in the network Returns: :class:`cntk.Function` ''' from cntk import element_divide left = sanitize_input(left, get_data_type(right)) right = sanitize_input(right, get_data_type(left)) return element_divide(left, right, name).output()
def __local_response_normalization(self, k, n, alpha, beta, name=''): x = cntk.placeholder(name='lrn_arg') x2 = cntk.square(x) x2s = cntk.reshape(x2, (1, cntk.InferredDimension), 0, 1) W = cntk.constant(alpha / (2 * n + 1), (1, 2 * n + 1, 1, 1), name='W') y = cntk.convolution(W, x2s) b = cntk.reshape(y, cntk.InferredDimension, 0, 2) den = cntk.exp(beta * cntk.log(k + b)) apply_x = cntk.element_divide(x, den) return apply_x
def lrn(x, depth_radius, bias, alpha, beta, name=''): x2 = C.square(x) # reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed. x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1) W = C.constant(alpha/(2*depth_radius+1), shape=(1,2*depth_radius+1,1,1), dtype=dtype, name='W') # 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1 y = C.convolution (W, x2s) # reshape back to remove the fake singleton reduction dimension b = C.reshape(y, C.InferredDimension, 0, 2) den = C.exp(beta * C.log(bias + b)) return C.element_divide(x, den)
def lrn(x, depth_radius, bias, alpha, beta, name=''): x2 = C.square(x) # reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed. x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1) W = C.constant(alpha/(2*depth_radius+1), shape=(1,2*depth_radius+1,1,1), dtype=dtype, name='W') # 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1 y = C.convolution (W, x2s) # reshape back to remove the fake singleton reduction dimension b = C.reshape(y, C.InferredDimension, 0, 2) den = C.exp(beta * C.log(bias + b)) return C.element_divide(x, den)
def LocalResponseNormalization(k, n, alpha, beta, name=''): x = C.placeholder(name='lrn_arg') x2 = C.square(x) x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1) W = C.constant(alpha / (2 * n + 1), (1, 2 * n + 1, 1, 1), name='W') y = C.convolution(W, x2s) b = C.reshape(y, C.InferredDimension, 0, 2) den = C.exp(beta * C.log(k + b)) apply_x = C.element_divide(x, den) return apply_x
def LocalResponseNormalization(k, n, alpha, beta, name=''): x = C.placeholder(name='lrn_arg') x2 = C.square(x) # reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed. x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1) W = C.constant(alpha / (2 * n + 1), (1, 2 * n + 1, 1, 1), name='W') # 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1 y = C.convolution(W, x2s) # reshape back to remove the fake singleton reduction dimension b = C.reshape(y, C.InferredDimension, 0, 2) den = C.exp(beta * C.log(k + b)) apply_x = C.element_divide(x, den) return apply_x
def LocalResponseNormalization(k, n, alpha, beta, name=''): x = C.placeholder(name='lrn_arg') x2 = C.square(x) # reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed. x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1) W = C.constant(alpha/(2*n+1), (1,2*n+1,1,1), name='W') # 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1 y = C.convolution (W, x2s) # reshape back to remove the fake singleton reduction dimension b = C.reshape(y, C.InferredDimension, 0, 2) den = C.exp(beta * C.log(k + b)) apply_x = C.element_divide(x, den) return apply_x
def run_div_test(shape1, shape2, tmpdir): broadcast = 'no_broadcast' if (shape1 != shape2): broadcast = 'with_broadcast' data1 = np.random.rand(*shape1).astype(np.float32) data2 = np.random.rand(*shape2).astype(np.float32) x = C.input_variable(shape1) y = C.input_variable(shape2) model = C.element_divide(data1, data2) verify_no_input(model, tmpdir, 'Div_' + broadcast + '_d1d2') model = C.element_divide(x, data2) verify_one_input(model, data1, tmpdir, 'Div_' + broadcast + '_xd2') model = C.element_divide(data1, y) verify_one_input(model, data2, tmpdir, 'Div_' + broadcast + '_d1y') model = C.element_divide(x, y) verify_two_input(model, data1, data2, tmpdir, 'Div_' + broadcast + '_xy')
def run_div_test(shape1, shape2, tmpdir): broadcast = 'no_broadcast' if (shape1 != shape2): broadcast = 'with_broadcast' data1 = np.random.rand(*shape1).astype(np.float32) data2 = np.random.rand(*shape2).astype(np.float32) x = C.input_variable(shape1) y = C.input_variable(shape2) model = C.element_divide(data1, data2) verify_no_input(model, tmpdir, 'Div_' + broadcast + '_d1d2') model = C.element_divide(x, data2) verify_one_input(model, data1, tmpdir, 'Div_' + broadcast + '_xd2') model = C.element_divide(data1, y) verify_one_input(model, data2, tmpdir, 'Div_' + broadcast + '_d1y') model = C.element_divide(x, y) verify_two_input(model, data1, data2, tmpdir, 'Div_' + broadcast + '_xy')
def multiFunc(self, arg1): multiIn = C.input(shape=arg1.shape, dynamic_axes=arg1.dynamic_axes) bit_map = C.constant(self.bit_map) max_bits = self.bit_map.max() carry_over = multiIn approx = C.element_times(multiIn, 0) for i in range(max_bits): hot_vals = C.greater(bit_map, i) valid_vals = C.element_select(hot_vals, carry_over, 0) mean = C.element_divide(C.reduce_sum(C.abs(valid_vals)), C.reduce_sum(hot_vals)) bits = C.greater(carry_over, 0) bits = C.element_select(bits, bits, -1) bits = C.element_select(hot_vals, bits, 0) approx = C.plus(approx, C.element_times(mean, bits)) carry_over = C.plus( C.element_times(C.element_times(-1, bits), mean), carry_over) return approx, multiIn
def multiFunc(self, arg1): multiIn = C.input(shape=arg1.shape, dynamic_axes = arg1.dynamic_axes) bit_map = C.constant(self.bit_map) max_bits = self.bit_map.max() shape = multiIn.shape reformed = C.reshape(multiIn, (-1,)) carry_over = multiIn approx = C.element_times(multiIn, 0) for i in range(max_bits): hot_vals = C.greater(bit_map, i) valid_vals = C.element_select(hot_vals, carry_over, 0) mean = C.element_divide(C.reduce_sum(C.abs(valid_vals)), C.reduce_sum(hot_vals)) bits = C.greater(carry_over, 0) bits = C.element_select(bits, bits, -1) bits = C.element_select(hot_vals, bits, 0) approx = C.plus(approx, C.element_times(mean, bits)) carry_over = C.plus(C.element_times(C.element_times(-1, bits), mean), carry_over) return approx, multiIn
def multiFunc(self, arg1): # load or create the inputs we need multiIn = C.input(shape=arg1.shape, dynamic_axes=arg1.dynamic_axes) bit_map = C.constant(self.bit_map) max_bits = self.bit_map.max() shape = multiIn.shape reformed = C.reshape(multiIn, (-1, )) # lets compute the means we need # carry over represents the remaining value that needs to binarized. For a single bit, this is just the input. For more bits, # it is the difference between the previous bits approximation and the true value. carry_over = multiIn approx = C.element_times(multiIn, 0) # iterate through the maximum number of bits specified by the bit maps, basically compute each level of binarization for i in range(max_bits): # determine which values of the input should be binarized to i bits or more hot_vals = C.greater(bit_map, i) # select only the values which we need to binarize valid_vals = C.element_select(hot_vals, carry_over, 0) # compute mean on a per kernel basis, reshaping is done to allow for sum reduction along only axis 0 (the kernels) mean = C.element_divide( C.reduce_sum(C.reshape(C.abs(valid_vals), (valid_vals.shape[0], -1)), axis=1), C.reduce_sum(C.reshape(hot_vals, (hot_vals.shape[0], -1)), axis=1)) # reshape the mean to match the dimensionality of the input mean = C.reshape(mean, (mean.shape[0], mean.shape[1], 1, 1)) # binarize the carry over bits = C.greater(carry_over, 0) bits = C.element_select(bits, bits, -1) bits = C.element_select(hot_vals, bits, 0) # add in the equivalent binary representation to the approximation approx = C.plus(approx, C.element_times(mean, bits)) # compute the new carry over carry_over = C.plus( C.element_times(C.element_times(-1, bits), mean), carry_over) return approx, multiIn
# Import CNTK library import cntk import numpy as np ################################# #### Mathematical operations #### ################################# # Initial definition a = [1, 2, 3] b = [3, 2, 1] # Get the type of the variable print(type(a)) # Subtraction print(cntk.minus(a, b).eval()) # Additive print(cntk.plus(a, b).eval()) # Element-wise division print(cntk.element_divide(a, b).eval()) # Defining variable variable = cntk.input_variable((2), np.float32) print(variable)
def inner(x, y): quotient = C.element_divide(x, y) integers = C.floor(quotient) decimals = quotient - integers remaining_value = decimals * y return remaining_value
def get_model(f_dim, c_dim, l_dim, m_dim, num_stack_layers, super_res_class_weight, super_res_loss_weight, high_res_loss_weight): # Define the variables into which the minibatch data will be loaded. num_nlcd_classes, num_landcover_classes = c_dim _, block_size, _ = f_dim input_im = cntk.input_variable(f_dim, np.float32) lc = cntk.input_variable(l_dim, np.float32) lc_weight_map = cntk.input_variable((1, l_dim[1], l_dim[2]), np.float32) interval_center = cntk.input_variable(c_dim, np.float32) interval_radius = cntk.input_variable(c_dim, np.float32) mask = cntk.input_variable(m_dim, np.float32) # Create the model definition. c_map defines the number of filters trained # at layers of different depths in the model. num_stack_layers defines the # number of (modified) residual units per layer. # model = dense_fc_model( # input_tensor=input_im, # num_stack_layers=num_stack_layers, # c_map=[32, 32, 16, 16, 16], # num_classes=num_landcover_classes, # bs=block_size # ) model = cnn_model(input_tensor=input_im, num_stack_layers=num_stack_layers, c_map=[64, 32, 32, 32, 32], num_classes=num_landcover_classes, bs=block_size) # At this stage the model produces output for the whole region in the input # image, but we will focus only on the center of that region during # training. Here we drop the predictions at the edges. output = cntk.reshape(model, (num_landcover_classes, block_size, block_size)) probs = cntk.reshape(cntk.softmax(output, axis=0), (num_landcover_classes, block_size, block_size)) # Now we calculate the supre-res loss. Note that this loss function has the # potential to become negative since the variance is fractional. # Additionally, we need to make sure that when the nlcd mask[0, ...] # is always 1, which means that there's no nlcd label everywhere, # the supre_res_loss comes out as a constant. super_res_crit = 0 mask_size = cntk.reshape( cntk.reduce_sum(cntk.slice(mask, 0, 1, num_nlcd_classes)), (1, )) + 10.0 # Not considering nlcd class 0 for nlcd_id in range(1, num_nlcd_classes): c_mask = cntk.reshape(cntk.slice(mask, 0, nlcd_id, nlcd_id + 1), (1, block_size, block_size)) c_mask_size = cntk.reshape(cntk.reduce_sum(c_mask), (1, )) + 0.000001 c_interval_center = cntk.reshape( cntk.slice(interval_center, 0, nlcd_id, nlcd_id + 1), (num_landcover_classes, )) c_interval_radius = cntk.reshape( cntk.slice(interval_radius, 0, nlcd_id, nlcd_id + 1), (num_landcover_classes, )) # For each nlcd class, we have a landcover distribution: masked_probs = probs * c_mask # Mean mean of predicted distribution mean = cntk.reshape(cntk.reduce_sum(masked_probs, axis=(1, 2)), (num_landcover_classes, )) / c_mask_size # Mean var of predicted distribution var = cntk.reshape( cntk.reduce_sum(masked_probs * (1. - masked_probs), axis=(1, 2)), (num_landcover_classes, )) / c_mask_size c_super_res_crit = cntk.square(ddist(mean, c_interval_center, c_interval_radius)) / ( var / c_mask_size + c_interval_radius * c_interval_radius + 0.000001) \ + cntk.log(var + 0.03) super_res_crit += c_super_res_crit * c_mask_size / mask_size * super_res_class_weight[ nlcd_id] # Weight super_res loss according to the ratio of unlabeled LC pixels super_res_loss = cntk.reduce_sum(super_res_crit) * cntk.reduce_mean( cntk.slice(lc, 0, 0, 1)) log_probs = cntk.log(probs) high_res_crit = cntk.times([0.0, 1.0, 1.0, 1.0, 1.0], cntk.element_times( -cntk.element_times(log_probs, lc), lc_weight_map), output_rank=2) # Average across spatial dimensions # Sum over all landcover classes, only one of the landcover classes is non-zero #high_res_loss = cntk.reduce_mean(high_res_crit) print("probs", probs) print("lc", lc) print("lc_weight_map", lc_weight_map) print("cntk.element_times(probs, lc)", cntk.element_times(probs, lc)) iou_loss_i = cntk.element_times([0.0, 1.0, 1.0, 1.0, 1.0], cntk.reduce_sum(cntk.element_times( cntk.element_times(probs, lc), lc_weight_map), axis=(1, 2))) print("iou_loss_i", iou_loss_i) iou_loss_u = cntk.element_times([0.0, 1.0, 1.0, 1.0, 1.0], cntk.reduce_sum(cntk.minus( cntk.plus(probs, lc), cntk.element_times(probs, lc)), axis=(1, 2))) print("iou_loss_u", iou_loss_u) high_res_loss = 1.0 - ( (1 / 4.0) * cntk.reduce_mean(cntk.element_divide(iou_loss_i, iou_loss_u))) print("high_res_loss", high_res_loss) loss = super_res_loss_weight * super_res_loss + high_res_loss_weight * high_res_loss return input_im, lc, lc_weight_map, mask, interval_center, interval_radius, \ output, high_res_loss_weight * high_res_loss, loss
def inner(x, y): quotient = C.element_divide(x, y) integers = C.floor(quotient) return integers
import cntk print("Tensor A = [1,2,3]") print("Tensor B = [4,5,6]\n") print("A+B:") sum = cntk.plus([1, 2, 3], [4, 5, 6]).eval() print("{}\n".format(sum)) print("A-B:") minus = cntk.minus([1, 2, 3], [4, 5, 6]).eval() print("{}\n".format(minus)) print("A*B:") times = cntk.times([1, 3, 4], [4, 5, 6]).eval() print("{}\n".format(times)) print("A/B:") divide = cntk.element_divide([4, 32, 15], [2, 4, 5]).eval() print("{}\n".format(divide)) print("A^B:") pow = cntk.pow([1, 3, 4], [4, 2, 3]).eval() print("{}\n".format(pow)) print("Min(A,B):") min = cntk.element_min([1, 2, 3], [4, 5, 6], [2, 1, 0]).eval() print("{}\n".format(min)) print("Max(A,B):") max = cntk.element_max([1, 2, 3], [4, 5, 6], [2, 9, 0]).eval() print("{}\n".format(max))