def test_op_cross_entropy_with_soft_max(output_vector, target_vector, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
o = AA(output_vector, dtype=dt)
t = AA(target_vector, dtype=dt)
ox = o - o.max() # subtract max to avoid overflow
exp_x = np.exp(ox)
s_max = exp_x / np.sum(exp_x) # softmax function
expected_forward = np.asarray(-np.sum(t * np.log(s_max, dtype=dt), dtype=dt))
expected_forward.shape = (1,1,1) + expected_forward.shape
s = np.sum(t, dtype=dt)
backward = np.subtract(s_max * s, t)
backward.shape = (1,) + backward.shape
expected_backward = {
'left_arg': backward,
'right_arg': [-1*o]
}
from cntk.losses import cross_entropy_with_softmax
_test_binary_op(precision, device_id, cross_entropy_with_softmax,
output_vector, target_vector,
expected_forward, expected_backward)
def test_op_cross_entropy_with_soft_max_and_axis(output_vector, target_vector, axis, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
x = AA(output_vector, dtype=dt)
t = AA(target_vector, dtype=dt)
expected_forward = []
expected_backward_left = []
expected_backward_right = []
for sample, target in zip(x, t):
ox = sample - sample.max() # subtract max to avoid overflow
exp_x = np.exp(ox)
s_max = exp_x / np.sum(exp_x) # softmax function
forward = np.asarray(-np.sum(target * np.log(s_max, dtype=dt), dtype=dt))
expected_forward.append(forward.tolist())
s = np.sum(target, dtype=dt)
backward = np.subtract(s_max * s, target)
expected_backward_left.append(backward.tolist())
expected_backward_right.append(-1*sample)
expected_forward = [np.reshape(AA(expected_forward, dtype=dt), (x.shape[0], 1))]
expected_backward_left = AA(expected_backward_left, dtype=dt)
expected_backward = {
'left_arg': [expected_backward_left],
'right_arg': [expected_backward_right]
}
from cntk.losses import cross_entropy_with_softmax
_test_binary_op(precision, device_id, cross_entropy_with_softmax,
output_vector, target_vector,
expected_forward, expected_backward, op_param_dict={'axis': axis})
def test_op_classification_error(output_vector, target_vector, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
o = AA(output_vector, dtype=dt)
t = AA(target_vector, dtype=dt)
different_position = np.argmax(t) != np.argmax(o)
expected_forward = [AA([[int(different_position)]], dtype=dt)]
zero_backward = np.zeros_like(t, dtype=dt)
left_backward = np.copy(zero_backward)
zero_backward[..., np.argmax(o)] = -1.
right_backward = zero_backward
expected_backward = {
'left_arg': [left_backward],
'right_arg': [right_backward]
}
from cntk.metrics import classification_error
_test_binary_op(precision, device_id, classification_error,
output_vector, target_vector,
expected_forward, expected_backward)
def test_op_classification_error_with_axis(output_vector, target_vector, axis, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
x = AA(output_vector, dtype=dt)
t = AA(target_vector, dtype=dt)
forward = []
expected_backward_left = []
expected_backward_right = []
for sample, target in zip(x, t):
different_position = np.argmax(target) != np.argmax(sample)
forward.append([int(different_position)])
zero_backward = np.zeros_like(target, dtype=dt)
expected_backward_left.append(zero_backward)
expected_backward_right.append(zero_backward)
forward = np.mean(forward)
expected_forward = AA([forward], dtype=dt)
expected_backward_left = AA([expected_backward_left], dtype=dt)
expected_backward_right = AA([expected_backward_right], dtype=dt)
expected_backward = {
'left_arg': expected_backward_left,
'right_arg': expected_backward_right
}
from cntk.metrics import classification_error
_test_binary_op(precision, device_id, classification_error,
output_vector, target_vector,
expected_forward, expected_backward, op_param_dict={'axis':axis})
def test_lambda_rank(grad, value, output, gain, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
score = AA(output, dtype=dt).reshape(-1, 1, 1)
gain = AA(gain, dtype=dt).reshape(-1, 1, 1)
group = np.ones_like(score).reshape(-1, 1, 1)
expected_value = AA(value, dtype=dt)
expected_grad = AA(grad, dtype=dt)
from cntk.losses import lambda_rank
g = C.input_variable((1, ))
s = C.input_variable((1, ), needs_gradient=True)
n = C.input_variable((1, ))
f = lambda_rank(s, n, g)
actual_grad, actual_value = f.grad({
s: score,
n: gain,
g: group
}, [s], [f.output])
assert np.allclose(actual_value, expected_value)
assert np.allclose(actual_grad, expected_grad)
def test_op_cross_entropy_with_soft_max(output_vector, target_vector, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
o = AA(output_vector, dtype=dt)
t = AA(target_vector, dtype=dt)
ox = o - o.max() # subtract max to avoid overflow
exp_x = np.exp(ox)
s_max = exp_x / np.sum(exp_x) # softmax function
expected_forward = np.asarray(-np.sum(t * np.log(s_max, dtype=dt), dtype=dt))
expected_forward.shape = (1,1,1) + expected_forward.shape
s = np.sum(t, dtype=dt)
backward = np.subtract(s_max * s, t)
backward.shape = (1,) + backward.shape
expected_backward = {
'left_arg': backward,
'right_arg': [-1*o]
}
from cntk.losses import cross_entropy_with_softmax
_test_binary_op(precision, device_id, cross_entropy_with_softmax,
output_vector, target_vector,
expected_forward, expected_backward)
def test_op_squared_error(output_vector, target_vector, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
o = AA(output_vector, dtype=dt)
t = AA(target_vector, dtype=dt)
expected_forward = AA([np.sum((t - o)**2)])
backward = 2 * np.subtract(o, t)
expected_backward = {'left_arg': [backward], 'right_arg': [-1 * backward]}
from cntk.losses import squared_error
_test_binary_op(precision, device_id, squared_error, output_vector,
target_vector, expected_forward, expected_backward)
def test_nce_loss(classes, xdim, batch, expected_value, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
from cntk.losses import nce_loss
import scipy
x = C.input_variable(xdim, needs_gradient=True)
y = C.input_variable(classes, is_sparse=True)
x0 = np.arange(batch * xdim, dtype=dt).reshape(
(batch, xdim)) / (batch * xdim)
data = np.ones(batch, dtype=dt)
indices = list(range(10, 10 * batch + 1, 10))
indptr = list(range(batch + 1))
y0 = scipy.sparse.csr_matrix((data, indices, indptr),
shape=(batch, classes))
q = np.arange(classes, dtype=dt) + 1
b = C.parameter((classes, 1), init=-np.log(classes))
W = C.parameter((classes, C.InferredDimension),
init=C.glorot_uniform(seed=98052))
loss = C.nce_loss(W, b, x, y, q, seed=98052)
v = loss.grad({x: x0, y: y0}, wrt=loss.parameters, as_numpy=False)
for key in v:
assert v[
key].is_sparse, "gradient of nce_loss with respect to %s is not sparse" % key
losses = np.zeros((100, batch))
for i in range(100):
losses[i, :] = loss.eval({x: x0, y: y0})
assert np.allclose(np.mean(losses, axis=0), AA(expected_value))
def test_ndcg(value, output, gain, device_id, precision):
dt = PRECISION_TO_TYPE[precision]
score = AA(output, dtype=dt).reshape(-1,1,1)
gain = AA(gain, dtype=dt).reshape(-1,1,1)
group = np.ones_like(score).reshape(-1,1,1)
expected_value = AA(value, dtype=dt)
from cntk.metrics import ndcg_at_1
g = input((1,))
s = input((1,))
n = input((1,))
f = ndcg_at_1(s, n, g)
actual_value = f.eval({s:score, n:gain, g:group})
assert np.allclose(actual_value, expected_value)