def kldiv_loss_grad(pre_deriv, inputs, outputs):
"""
do backprop for kldiv loss
Args:
pre_deriv (tvm.tensor.Tensor): Gradient tensor for forward output.
inputs (tvm.tensor.Tensor): Forward input tensor.
outputs (tvm.tensor.Tensor): Forward output tensor.
Returns:
Gradient tensor for forward input.
"""
inputs_dtype = inputs.dtype
target_dtype = outputs.dtype
pre_deriv_dtype = pre_deriv.dtype
utils.ops_dtype_check([inputs_dtype, target_dtype, pre_deriv_dtype],
utils.DtypeForDavinci.ALL_FLOAT)
if get_const_tuple(outputs.shape) != get_const_tuple(inputs.shape):
raise RuntimeError("Please ensure inputs have the same size."
"", outputs.shape, inputs.shape)
inputs_dtype_old = inputs_dtype
if product_is_mini() and inputs_dtype == 'float32':
inputs = akg.topi.cast(inputs, "float16")
outputs = akg.topi.cast(outputs, "float16")
inputs_dtype = "float16"
cur_deriv = akg.topi.divide(outputs, inputs)
cur_deriv = akg.topi.multiply(cur_deriv, pre_deriv)
if product_is_mini() and inputs_dtype_old == 'float32':
cur_deriv = akg.topi.cast(cur_deriv, inputs_dtype_old)
return cur_deriv
def kldiv_loss(inputs, outputs, reduction='none'):
"""
Computes Kullback-Leibler divergence loss between outputs and inputs.
In default, loss = outputs*(log(outputs) - log(inputs)),
the way using to reduce loss is defined in reduction
Args:
inputs (tvm.tensor.Tensor): Tensor with type float16, float32
outputs (tvm.tensor.Tensor): Tensor with same type as inputs.
reduction (str): uses one of ['sum', 'mean', 'batchmean']
Returns:
Tensor with same type as input tensors.
"""
inputs_dtype = inputs.dtype
target_dtype = outputs.dtype
utils.ops_dtype_check([inputs_dtype, target_dtype],
utils.DtypeForDavinci.ALL_FLOAT)
if get_const_tuple(outputs.shape) != get_const_tuple(inputs.shape):
raise RuntimeError("Please ensure inputs have the same size.",
outputs.shape, inputs.shape)
inputs_dtype_old = inputs_dtype
if product_is_mini() and inputs_dtype == 'float32':
inputs = akg.topi.cast(inputs, "float16")
outputs = akg.topi.cast(outputs, "float16")
inputs_dtype = "float16"
log_inputs = akg.topi.log(inputs)
log_target = akg.topi.log(outputs)
loss = akg.topi.subtract(log_target, log_inputs)
loss = akg.topi.multiply(outputs, loss)
if reduction == 'sum':
loss = akg.topi.sum(loss)
if reduction == 'mean':
loss = akg.topi.sum(loss)
deno = 1.0
for num in inputs.shape:
deno = deno * num
deno = akg.topi.cast(deno, dtype=inputs_dtype)
loss = akg.topi.divide(loss, deno)
if reduction == 'batchmean':
reduce_axis = tuple(numpy.arange(1, len(inputs.shape)))
loss = akg.topi.sum(loss, axis=reduce_axis, keepdims=False)
deno = 1.0
for num in inputs.shape[1:]:
deno = deno * num
deno = akg.topi.cast(deno, dtype=inputs_dtype)
loss = akg.topi.divide(loss, deno)
if product_is_mini() and inputs_dtype_old == 'float32':
loss = akg.topi.cast(loss, inputs_dtype_old)
return loss
def l1_loss_grad(pre_deriv, inputs, target):
"""
do backprop for L1 loss (MAE)
"""
inputs_dtype = inputs.dtype
target_dtype = target.dtype
pre_deriv_dtype = pre_deriv.dtype
# check inputs data types
check_list = ["float16", "float32"]
if not inputs_dtype.lower() in check_list:
raise RuntimeError("inputs only support %s while dtype is %s" % (
",".join(check_list), inputs_dtype))
if not target_dtype.lower() in check_list:
raise RuntimeError("target only support %s while dtype is %s" % (
",".join(check_list), target_dtype))
if not pre_deriv_dtype.lower() in check_list:
raise RuntimeError("prev Derivative only support %s while dtype is %s" % (
",".join(check_list), pre_deriv_dtype))
if not get_const_tuple(target.shape) == get_const_tuple(inputs.shape):
raise RuntimeError(
"Please ensure inputs have the same size.", target.shape, prediction.shape)
inputs_dtype_old = inputs_dtype
if utils.product_is_mini() and inputs_dtype == 'float32':
inputs = akg.topi.cast(inputs, "float16")
target = akg.topi.cast(target, "float16")
inputs_dtype = "float16"
def grad_dsl(inputs, target, pre_deriv):
# do roadcast outside, cause tvm need shape check;if shape not fix how to check
#pre_deriv = akg.topi.broadcast_to(pre_deriv, inputs.shape)
coefficient = akg.tvm.const(-1.0, dtype=inputs_dtype)
res = akg.tvm.compute(inputs.shape,
lambda *i: akg.tvm.if_then_else(
inputs(*i) >= target(*i),
pre_deriv(*i), coefficient * pre_deriv(*i))
)
return res
cur_deriv = grad_dsl(inputs, target, pre_deriv)
if utils.product_is_mini() and inputs_dtype_old == 'float32':
cur_deriv = akg.topi.cast(cur_deriv, inputs_dtype_old)
return cur_deriv
def gen_data(dtype, shape):
# generate data
input = np.random.uniform(low=-1.0, high=1.0,
size=get_const_tuple(shape)).astype(dtype)
expect = np.ones_like(input)
output = np.full(shape, np.nan, dtype)
return input, expect, output
def gen_data(dtype, shape):
input_np = np.random.uniform(low=-1.0,
high=1.0,
size=get_const_tuple(shape)).astype(dtype)
head_np = np.random.uniform(low=-1.0, high=1.0, size=shape).astype(dtype)
expect = head_np * (input_np > 0)
output = np.full(expect.shape, np.nan, dtype)
return expect, head_np, input_np, output
def gen_data(dtype, shape):
input_np = np.random.uniform(low=-1.0,
high=1.0,
size=get_const_tuple(shape)).astype(dtype)
head_np = np.random.uniform(low=-1.0, high=1.0, size=shape).astype(dtype)
expect = np.exp(input_np) * head_np
return expect, head_np, input_np
def gen_data(c, dshape, dtype, h, n, pool_type, w):
# Result_Numpy
input = np.random.poisson(1, size=dshape).astype(dtype)
exp_output = np.mean(input, axis=(2, 3), keepdims=True)
print("-------------exp_output-------------: ", exp_output)
# inputs and output to hold the data
A = akg.tvm.placeholder((n, c, h, w), name='A', dtype="float16")
B = akg.topi.nn.global_pool(A, pool_type=pool_type)
output = np.zeros(get_const_tuple(B.shape), dtype=dtype)
args = [input, output]
return args, exp_output, input
def gen_data(dtype, shape, w_shape):
# input_data = random_gaussian(shape, miu=1, sigma=50.0).astype(dtype.lower())
input_data = np.random.uniform(low=-1.0,
high=1.0,
size=get_const_tuple(shape)).astype(dtype)
w_data = random_gaussian(w_shape, miu=1, sigma=2.0).astype(dtype.lower())
# expect = input_data * (input_data > 0) + input_data * (input_data < 0) * w_data[0]
if w_shape[0] == 1:
# pass
expect = input_data * (input_data > 0) + input_data * (input_data <
0) * w_data[0]
else:
w_reshape = w_data.reshape(1, w_shape[0], 1, 1)
w_broadcast = np.broadcast_to(w_reshape, shape)
expect = input_data * (input_data > 0) + input_data * (input_data <
0) * w_broadcast
# pass
# for j in range(shape[1]):
# for i in range(shape[0]):
# for k in range(shape[2]):
# for l in range(shape[3]):
# expect[i, j, k, l] = input_data[i, j, k, l] * (input_data[i, j, k, l] > 0) + input_data[i, j, k, l] * (input_data[i, j, k, l] < 0) * w_data[j]
return expect, input_data, w_data
def gen_data(dtype, shape):
input_np = np.random.uniform(low=-1.0,
high=1.0,
size=get_const_tuple(shape)).astype(dtype)
output_np = input_np * (input_np > 0)
return input_np, output_np