def test_Div(tmpdir):
data0 = np.asarray([1., 1., 1., 1.], dtype=np.float32)
data1 = np.asarray([0.5, 0.25, 0.125, 0.], dtype=np.float32)
model = C.element_divide(data0, data1)
verify_no_input(model, tmpdir, 'Div_0')
x = C.input_variable(data0.shape)
model = C.element_divide(x, data1)
verify_one_input(model, data0, tmpdir, 'Div_1')
def test_Div(tmpdir):
data0 = np.asarray([1., 1., 1., 1.], dtype=np.float32)
data1 = np.asarray([0.5, 0.25, 0.125, 0.], dtype=np.float32)
model = C.element_divide(data0, data1)
verify_no_input(model, tmpdir, 'Div_0')
x = C.input_variable(data0.shape)
model = C.element_divide(x, data1)
verify_one_input(model, data0, tmpdir, 'Div_1')
def squash(input):
# ||Sj||^2
Sj_squared_norm = ct.reduce_sum(ct.square(input), axis=axis)
# ||Sj||^2 / (1 + ||Sj||^2) * (Sj / ||Sj||)
factor = ct.element_divide(
ct.element_divide(Sj_squared_norm, ct.plus(1, Sj_squared_norm)),
ct.sqrt(ct.plus(Sj_squared_norm, epsilon)))
return factor * input
def element_divide(left, right, name=''):
'''
The output of this operation is the element-wise division of the two input
tensors. It supports broadcasting. In case of scalars its backward pass to
left propagates :math:`1/right` times the received gradient, and the backward
pass to right propagates.
The operator (/) has been overloaded and can equally be used instead of element_divide().
:math:`(-left/right^2)` times the received gradient.
Example:
>>> C.eval(C.element_divide([1., 1., 1., 1.], [0.5, 0.25, 0.125, 0.]))
[array([[ 2., 4., 8., 0.]])]
>>> C.eval(C.element_divide([5., 10., 15., 30.], [2.]))
[array([[ 2.5, 5. , 7.5, 15. ]])]
Args:
left: left side tensor
right: right side tensor
name (str): the name of the node in the network
Returns:
:class:`cntk.Function`
'''
from cntk import element_divide
left = sanitize_input(left, get_data_type(right))
right = sanitize_input(right, get_data_type(left))
return element_divide(left, right, name).output()
def multiFunc(self, arg1):
# load or create the inputs we need
multiIn = C.input(shape=arg1.shape, dynamic_axes = arg1.dynamic_axes)
bit_map = C.constant(self.bit_map)
max_bits = self.bit_map.max()
shape = multiIn.shape
reformed = C.reshape(multiIn, (-1,))
# lets compute the means we need
# carry over represents the remaining value that needs to binarized. For a single bit, this is just the input. For more bits,
# it is the difference between the previous bits approximation and the true value.
carry_over = multiIn
approx = C.element_times(multiIn, 0)
# iterate through the maximum number of bits specified by the bit maps, basically compute each level of binarization
for i in range(max_bits):
# determine which values of the input should be binarized to i bits or more
hot_vals = C.greater(bit_map, i)
# select only the values which we need to binarize
valid_vals = C.element_select(hot_vals, carry_over, 0)
# compute mean on a per kernel basis, reshaping is done to allow for sum reduction along only axis 0 (the kernels)
mean = C.element_divide(C.reduce_sum(C.reshape(C.abs(valid_vals), (valid_vals.shape[0], -1)), axis=1), C.reduce_sum(C.reshape(hot_vals, (hot_vals.shape[0], -1)), axis=1))
# reshape the mean to match the dimensionality of the input
mean = C.reshape(mean, (mean.shape[0], mean.shape[1], 1, 1))
# binarize the carry over
bits = C.greater(carry_over, 0)
bits = C.element_select(bits, bits, -1)
bits = C.element_select(hot_vals, bits, 0)
# add in the equivalent binary representation to the approximation
approx = C.plus(approx, C.element_times(mean, bits))
# compute the new carry over
carry_over = C.plus(C.element_times(C.element_times(-1, bits), mean), carry_over)
return approx, multiIn
def element_divide(left, right, name=''):
'''
The output of this operation is the element-wise division of the two input
tensors. It supports broadcasting. In case of scalars its backward pass to
left propagates :math:`1/right` times the received gradient, and the backward
pass to right propagates.
The operator (/) has been overloaded and can equally be used instead of element_divide().
:math:`(-left/right^2)` times the received gradient.
Example:
>>> C.eval(C.element_divide([1., 1., 1., 1.], [0.5, 0.25, 0.125, 0.]))
[array([[ 2., 4., 8., 0.]])]
>>> C.eval(C.element_divide([5., 10., 15., 30.], [2.]))
[array([[ 2.5, 5. , 7.5, 15. ]])]
Args:
left: left side tensor
right: right side tensor
name (str): the name of the node in the network
Returns:
:class:`cntk.Function`
'''
from cntk import element_divide
left = sanitize_input(left, get_data_type(right))
right = sanitize_input(right, get_data_type(left))
return element_divide(left, right, name).output()
def __local_response_normalization(self, k, n, alpha, beta, name=''):
x = cntk.placeholder(name='lrn_arg')
x2 = cntk.square(x)
x2s = cntk.reshape(x2, (1, cntk.InferredDimension), 0, 1)
W = cntk.constant(alpha / (2 * n + 1), (1, 2 * n + 1, 1, 1), name='W')
y = cntk.convolution(W, x2s)
b = cntk.reshape(y, cntk.InferredDimension, 0, 2)
den = cntk.exp(beta * cntk.log(k + b))
apply_x = cntk.element_divide(x, den)
return apply_x
def lrn(x, depth_radius, bias, alpha, beta, name=''):
x2 = C.square(x)
# reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed.
x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1)
W = C.constant(alpha/(2*depth_radius+1), shape=(1,2*depth_radius+1,1,1), dtype=dtype, name='W')
# 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1
y = C.convolution (W, x2s)
# reshape back to remove the fake singleton reduction dimension
b = C.reshape(y, C.InferredDimension, 0, 2)
den = C.exp(beta * C.log(bias + b))
return C.element_divide(x, den)
def lrn(x, depth_radius, bias, alpha, beta, name=''):
x2 = C.square(x)
# reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed.
x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1)
W = C.constant(alpha/(2*depth_radius+1), shape=(1,2*depth_radius+1,1,1), dtype=dtype, name='W')
# 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1
y = C.convolution (W, x2s)
# reshape back to remove the fake singleton reduction dimension
b = C.reshape(y, C.InferredDimension, 0, 2)
den = C.exp(beta * C.log(bias + b))
return C.element_divide(x, den)
def LocalResponseNormalization(k, n, alpha, beta, name=''):
x = C.placeholder(name='lrn_arg')
x2 = C.square(x)
x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1)
W = C.constant(alpha / (2 * n + 1), (1, 2 * n + 1, 1, 1), name='W')
y = C.convolution(W, x2s)
b = C.reshape(y, C.InferredDimension, 0, 2)
den = C.exp(beta * C.log(k + b))
apply_x = C.element_divide(x, den)
return apply_x
def LocalResponseNormalization(k, n, alpha, beta, name=''):
x = C.placeholder(name='lrn_arg')
x2 = C.square(x)
# reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed.
x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1)
W = C.constant(alpha / (2 * n + 1), (1, 2 * n + 1, 1, 1), name='W')
# 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1
y = C.convolution(W, x2s)
# reshape back to remove the fake singleton reduction dimension
b = C.reshape(y, C.InferredDimension, 0, 2)
den = C.exp(beta * C.log(k + b))
apply_x = C.element_divide(x, den)
return apply_x
def LocalResponseNormalization(k, n, alpha, beta, name=''):
x = C.placeholder(name='lrn_arg')
x2 = C.square(x)
# reshape to insert a fake singleton reduction dimension after the 3th axis (channel axis). Note Python axis order and BrainScript are reversed.
x2s = C.reshape(x2, (1, C.InferredDimension), 0, 1)
W = C.constant(alpha/(2*n+1), (1,2*n+1,1,1), name='W')
# 3D convolution with a filter that has a non 1-size only in the 3rd axis, and does not reduce since the reduction dimension is fake and 1
y = C.convolution (W, x2s)
# reshape back to remove the fake singleton reduction dimension
b = C.reshape(y, C.InferredDimension, 0, 2)
den = C.exp(beta * C.log(k + b))
apply_x = C.element_divide(x, den)
return apply_x
def run_div_test(shape1, shape2, tmpdir):
broadcast = 'no_broadcast'
if (shape1 != shape2):
broadcast = 'with_broadcast'
data1 = np.random.rand(*shape1).astype(np.float32)
data2 = np.random.rand(*shape2).astype(np.float32)
x = C.input_variable(shape1)
y = C.input_variable(shape2)
model = C.element_divide(data1, data2)
verify_no_input(model, tmpdir, 'Div_' + broadcast + '_d1d2')
model = C.element_divide(x, data2)
verify_one_input(model, data1, tmpdir, 'Div_' + broadcast + '_xd2')
model = C.element_divide(data1, y)
verify_one_input(model, data2, tmpdir, 'Div_' + broadcast + '_d1y')
model = C.element_divide(x, y)
verify_two_input(model, data1, data2, tmpdir, 'Div_' + broadcast + '_xy')
def run_div_test(shape1, shape2, tmpdir):
broadcast = 'no_broadcast'
if (shape1 != shape2):
broadcast = 'with_broadcast'
data1 = np.random.rand(*shape1).astype(np.float32)
data2 = np.random.rand(*shape2).astype(np.float32)
x = C.input_variable(shape1)
y = C.input_variable(shape2)
model = C.element_divide(data1, data2)
verify_no_input(model, tmpdir, 'Div_' + broadcast + '_d1d2')
model = C.element_divide(x, data2)
verify_one_input(model, data1, tmpdir, 'Div_' + broadcast + '_xd2')
model = C.element_divide(data1, y)
verify_one_input(model, data2, tmpdir, 'Div_' + broadcast + '_d1y')
model = C.element_divide(x, y)
verify_two_input(model, data1, data2, tmpdir, 'Div_' + broadcast + '_xy')
def multiFunc(self, arg1):
multiIn = C.input(shape=arg1.shape, dynamic_axes=arg1.dynamic_axes)
bit_map = C.constant(self.bit_map)
max_bits = self.bit_map.max()
carry_over = multiIn
approx = C.element_times(multiIn, 0)
for i in range(max_bits):
hot_vals = C.greater(bit_map, i)
valid_vals = C.element_select(hot_vals, carry_over, 0)
mean = C.element_divide(C.reduce_sum(C.abs(valid_vals)),
C.reduce_sum(hot_vals))
bits = C.greater(carry_over, 0)
bits = C.element_select(bits, bits, -1)
bits = C.element_select(hot_vals, bits, 0)
approx = C.plus(approx, C.element_times(mean, bits))
carry_over = C.plus(
C.element_times(C.element_times(-1, bits), mean), carry_over)
return approx, multiIn
def multiFunc(self, arg1):
multiIn = C.input(shape=arg1.shape, dynamic_axes = arg1.dynamic_axes)
bit_map = C.constant(self.bit_map)
max_bits = self.bit_map.max()
shape = multiIn.shape
reformed = C.reshape(multiIn, (-1,))
carry_over = multiIn
approx = C.element_times(multiIn, 0)
for i in range(max_bits):
hot_vals = C.greater(bit_map, i)
valid_vals = C.element_select(hot_vals, carry_over, 0)
mean = C.element_divide(C.reduce_sum(C.abs(valid_vals)), C.reduce_sum(hot_vals))
bits = C.greater(carry_over, 0)
bits = C.element_select(bits, bits, -1)
bits = C.element_select(hot_vals, bits, 0)
approx = C.plus(approx, C.element_times(mean, bits))
carry_over = C.plus(C.element_times(C.element_times(-1, bits), mean), carry_over)
return approx, multiIn
def multiFunc(self, arg1):
# load or create the inputs we need
multiIn = C.input(shape=arg1.shape, dynamic_axes=arg1.dynamic_axes)
bit_map = C.constant(self.bit_map)
max_bits = self.bit_map.max()
shape = multiIn.shape
reformed = C.reshape(multiIn, (-1, ))
# lets compute the means we need
# carry over represents the remaining value that needs to binarized. For a single bit, this is just the input. For more bits,
# it is the difference between the previous bits approximation and the true value.
carry_over = multiIn
approx = C.element_times(multiIn, 0)
# iterate through the maximum number of bits specified by the bit maps, basically compute each level of binarization
for i in range(max_bits):
# determine which values of the input should be binarized to i bits or more
hot_vals = C.greater(bit_map, i)
# select only the values which we need to binarize
valid_vals = C.element_select(hot_vals, carry_over, 0)
# compute mean on a per kernel basis, reshaping is done to allow for sum reduction along only axis 0 (the kernels)
mean = C.element_divide(
C.reduce_sum(C.reshape(C.abs(valid_vals),
(valid_vals.shape[0], -1)),
axis=1),
C.reduce_sum(C.reshape(hot_vals, (hot_vals.shape[0], -1)),
axis=1))
# reshape the mean to match the dimensionality of the input
mean = C.reshape(mean, (mean.shape[0], mean.shape[1], 1, 1))
# binarize the carry over
bits = C.greater(carry_over, 0)
bits = C.element_select(bits, bits, -1)
bits = C.element_select(hot_vals, bits, 0)
# add in the equivalent binary representation to the approximation
approx = C.plus(approx, C.element_times(mean, bits))
# compute the new carry over
carry_over = C.plus(
C.element_times(C.element_times(-1, bits), mean), carry_over)
return approx, multiIn
# Import CNTK library import cntk import numpy as np ################################# #### Mathematical operations #### ################################# # Initial definition a = [1, 2, 3] b = [3, 2, 1] # Get the type of the variable print(type(a)) # Subtraction print(cntk.minus(a, b).eval()) # Additive print(cntk.plus(a, b).eval()) # Element-wise division print(cntk.element_divide(a, b).eval()) # Defining variable variable = cntk.input_variable((2), np.float32) print(variable)
def inner(x, y):
quotient = C.element_divide(x, y)
integers = C.floor(quotient)
decimals = quotient - integers
remaining_value = decimals * y
return remaining_value
def get_model(f_dim, c_dim, l_dim, m_dim, num_stack_layers,
super_res_class_weight, super_res_loss_weight,
high_res_loss_weight):
# Define the variables into which the minibatch data will be loaded.
num_nlcd_classes, num_landcover_classes = c_dim
_, block_size, _ = f_dim
input_im = cntk.input_variable(f_dim, np.float32)
lc = cntk.input_variable(l_dim, np.float32)
lc_weight_map = cntk.input_variable((1, l_dim[1], l_dim[2]), np.float32)
interval_center = cntk.input_variable(c_dim, np.float32)
interval_radius = cntk.input_variable(c_dim, np.float32)
mask = cntk.input_variable(m_dim, np.float32)
# Create the model definition. c_map defines the number of filters trained
# at layers of different depths in the model. num_stack_layers defines the
# number of (modified) residual units per layer.
# model = dense_fc_model(
# input_tensor=input_im,
# num_stack_layers=num_stack_layers,
# c_map=[32, 32, 16, 16, 16],
# num_classes=num_landcover_classes,
# bs=block_size
# )
model = cnn_model(input_tensor=input_im,
num_stack_layers=num_stack_layers,
c_map=[64, 32, 32, 32, 32],
num_classes=num_landcover_classes,
bs=block_size)
# At this stage the model produces output for the whole region in the input
# image, but we will focus only on the center of that region during
# training. Here we drop the predictions at the edges.
output = cntk.reshape(model,
(num_landcover_classes, block_size, block_size))
probs = cntk.reshape(cntk.softmax(output, axis=0),
(num_landcover_classes, block_size, block_size))
# Now we calculate the supre-res loss. Note that this loss function has the
# potential to become negative since the variance is fractional.
# Additionally, we need to make sure that when the nlcd mask[0, ...]
# is always 1, which means that there's no nlcd label everywhere,
# the supre_res_loss comes out as a constant.
super_res_crit = 0
mask_size = cntk.reshape(
cntk.reduce_sum(cntk.slice(mask, 0, 1, num_nlcd_classes)),
(1, )) + 10.0
# Not considering nlcd class 0
for nlcd_id in range(1, num_nlcd_classes):
c_mask = cntk.reshape(cntk.slice(mask, 0, nlcd_id, nlcd_id + 1),
(1, block_size, block_size))
c_mask_size = cntk.reshape(cntk.reduce_sum(c_mask), (1, )) + 0.000001
c_interval_center = cntk.reshape(
cntk.slice(interval_center, 0, nlcd_id, nlcd_id + 1),
(num_landcover_classes, ))
c_interval_radius = cntk.reshape(
cntk.slice(interval_radius, 0, nlcd_id, nlcd_id + 1),
(num_landcover_classes, ))
# For each nlcd class, we have a landcover distribution:
masked_probs = probs * c_mask
# Mean mean of predicted distribution
mean = cntk.reshape(cntk.reduce_sum(masked_probs, axis=(1, 2)),
(num_landcover_classes, )) / c_mask_size
# Mean var of predicted distribution
var = cntk.reshape(
cntk.reduce_sum(masked_probs * (1. - masked_probs), axis=(1, 2)),
(num_landcover_classes, )) / c_mask_size
c_super_res_crit = cntk.square(ddist(mean, c_interval_center, c_interval_radius)) / (
var / c_mask_size + c_interval_radius * c_interval_radius + 0.000001) \
+ cntk.log(var + 0.03)
super_res_crit += c_super_res_crit * c_mask_size / mask_size * super_res_class_weight[
nlcd_id]
# Weight super_res loss according to the ratio of unlabeled LC pixels
super_res_loss = cntk.reduce_sum(super_res_crit) * cntk.reduce_mean(
cntk.slice(lc, 0, 0, 1))
log_probs = cntk.log(probs)
high_res_crit = cntk.times([0.0, 1.0, 1.0, 1.0, 1.0],
cntk.element_times(
-cntk.element_times(log_probs, lc),
lc_weight_map),
output_rank=2)
# Average across spatial dimensions
# Sum over all landcover classes, only one of the landcover classes is non-zero
#high_res_loss = cntk.reduce_mean(high_res_crit)
print("probs", probs)
print("lc", lc)
print("lc_weight_map", lc_weight_map)
print("cntk.element_times(probs, lc)", cntk.element_times(probs, lc))
iou_loss_i = cntk.element_times([0.0, 1.0, 1.0, 1.0, 1.0],
cntk.reduce_sum(cntk.element_times(
cntk.element_times(probs, lc),
lc_weight_map),
axis=(1, 2)))
print("iou_loss_i", iou_loss_i)
iou_loss_u = cntk.element_times([0.0, 1.0, 1.0, 1.0, 1.0],
cntk.reduce_sum(cntk.minus(
cntk.plus(probs, lc),
cntk.element_times(probs, lc)),
axis=(1, 2)))
print("iou_loss_u", iou_loss_u)
high_res_loss = 1.0 - (
(1 / 4.0) *
cntk.reduce_mean(cntk.element_divide(iou_loss_i, iou_loss_u)))
print("high_res_loss", high_res_loss)
loss = super_res_loss_weight * super_res_loss + high_res_loss_weight * high_res_loss
return input_im, lc, lc_weight_map, mask, interval_center, interval_radius, \
output, high_res_loss_weight * high_res_loss, loss
def inner(x, y):
quotient = C.element_divide(x, y)
integers = C.floor(quotient)
return integers
import cntk
print("Tensor A = [1,2,3]")
print("Tensor B = [4,5,6]\n")
print("A+B:")
sum = cntk.plus([1, 2, 3], [4, 5, 6]).eval()
print("{}\n".format(sum))
print("A-B:")
minus = cntk.minus([1, 2, 3], [4, 5, 6]).eval()
print("{}\n".format(minus))
print("A*B:")
times = cntk.times([1, 3, 4], [4, 5, 6]).eval()
print("{}\n".format(times))
print("A/B:")
divide = cntk.element_divide([4, 32, 15], [2, 4, 5]).eval()
print("{}\n".format(divide))
print("A^B:")
pow = cntk.pow([1, 3, 4], [4, 2, 3]).eval()
print("{}\n".format(pow))
print("Min(A,B):")
min = cntk.element_min([1, 2, 3], [4, 5, 6], [2, 1, 0]).eval()
print("{}\n".format(min))
print("Max(A,B):")
max = cntk.element_max([1, 2, 3], [4, 5, 6], [2, 9, 0]).eval()
print("{}\n".format(max))
