def forward_cpu(self, inputs):
self.retain_inputs((0, 1, 2))
x1, x2, gy = inputs
sqnorm = x1 ** 2 + x2 ** 2
gx1 = utils.force_array(x2 / sqnorm * gy)
gx2 = utils.force_array(-x1 / sqnorm * gy)
return gx1, gx2
def check_anchor_target_creator(
self, anchor_target_layer,
bbox, anchor, img_size):
xp = cuda.get_array_module(bbox)
loc, label = self.anchor_target_layer(
bbox, anchor, img_size)
# Test types
self.assertIsInstance(loc, xp.ndarray)
self.assertIsInstance(label, xp.ndarray)
# Test shapes
self.assertEqual(loc.shape, (self.n_anchor, 4))
self.assertEqual(label.shape, (self.n_anchor,))
# Test dtype
self.assertEqual(loc.dtype, np.float32)
self.assertEqual(label.dtype, np.int32)
# Test ratio of foreground and background labels
np.testing.assert_equal(
cuda.to_cpu(utils.force_array(xp.sum(label >= 0))),
self.n_sample)
n_pos = cuda.to_cpu(utils.force_array(xp.sum(label == 1)))
n_neg = cuda.to_cpu(utils.force_array(xp.sum(label == 0)))
self.assertLessEqual(
n_pos, self.n_sample * self.pos_ratio)
self.assertLessEqual(n_neg, self.n_sample - n_pos)
def forward_cpu(self, inputs):
gy, = inputs
gx1 = utils.force_array(numpy.where(self.cond, gy, gy.dtype.type(0)))
gx2 = utils.force_array(numpy.where(self.cond, gy.dtype.type(0), gy))
return (
utils.sum_to(gx1, self.x1_shape),
utils.sum_to(gx2, self.x2_shape))
def forward(self, inputs):
self.retain_inputs((0, 1))
xp = cuda.get_array_module(*inputs)
x, t = inputs
self.ignore_mask = (t != self.ignore_label)
# stable computation of the cross entropy.
loss = -(
self.ignore_mask *
(x * (t - (x >= 0)) - xp.log1p(xp.exp(-xp.abs(x)))))
if not self.reduce == 'mean':
return utils.force_array(loss.astype(x.dtype)),
if self.normalize:
count = xp.maximum(1, self.ignore_mask.sum())
else:
count = max(1, len(x))
self.count = count
# TODO(takagi): Fix to perform division in a specific dtype. See
# cupy/cupy#1534.
return utils.force_array(
xp.divide(xp.sum(loss), self.count), dtype=x.dtype),
def forward_cpu(self, inputs):
self.retain_inputs((0, 1, 2, 3))
p, x, y, gz = inputs
pg = p * gz
return (utils.force_array((x - y) * gz),
utils.force_array(pg),
utils.force_array(gz - pg))
def log_prob(self, x):
if isinstance(x, chainer.Variable):
x = x.data
x = x.astype(self.lam.dtype)
xp1 = (x + 1).astype(self.lam.dtype)
x, xp1 = utils.force_array(x), utils.force_array(xp1)
return x * exponential.log(self.lam) - lgamma.lgamma(xp1) - self.lam
def backward_cpu(self, inputs, grads):
p, x, y = inputs
g = grads[0]
pg = p * g
return (utils.force_array((x - y) * g),
utils.force_array(pg),
utils.force_array(g - pg))
def backward(self, inputs, grads):
x1, x2 = inputs
gy, = grads
gx = gy * 2 * self.difference
gx = utils.force_array(gx, dtype=x1.dtype)
gx_minus = utils.force_array(-gx, dtype=x1.dtype)
return gx, gx_minus
def forward_cpu(self, inputs):
self.retain_inputs((0, 1, 2))
x0, x1, gy = inputs
one = x1.dtype.type(1)
gx0 = utils.force_array(x1 * (x0 ** (x1 - one)) * gy)
gx1 = utils.force_array(numpy.log(x0) * self.y * gy)
return gx0, gx1
def forward(self, inputs):
x1, x2 = inputs
if self.check_on == 'forward_input':
self._check_contiguousness(x1)
self._check_contiguousness(x2)
self.retain_inputs((0, 1))
y1, y2 = _forward_correct(x1, x2)
return utils.force_array(y1), utils.force_array(y2)
def forward(self, inputs):
device = self.device
x1, x2 = inputs
if device.xp is chainerx:
fallback_device = device.fallback_device
assert isinstance(x1, fallback_device.supported_array_types)
assert isinstance(x2, fallback_device.supported_array_types)
self.retain_inputs((0, 1))
y1, y2 = _forward_correct(x1, x2)
return utils.force_array(y1), utils.force_array(y2)
def forward(self, inputs_and_grad_outputs):
device = self.device
x1, x2, gy1, gy2 = inputs_and_grad_outputs
if device.xp is chainerx:
fallback_device = device.fallback_device
assert isinstance(gy1, fallback_device.supported_array_types)
assert isinstance(gy2, fallback_device.supported_array_types)
self.retain_inputs((0, 1, 2, 3))
ggx1, ggx2 = _backward_correct(x1, x2, gy1, gy2)
return utils.force_array(ggx1), utils.force_array(ggx2)
def assert_allclose(x, y, atol=1e-5, rtol=1e-4, verbose=True):
"""Asserts if some corresponding element of x and y differs too much.
This function can handle both CPU and GPU arrays simultaneously.
Args:
x: Left-hand-side array.
y: Right-hand-side array.
atol (float): Absolute tolerance.
rtol (float): Relative tolerance.
verbose (bool): If ``True``, it outputs verbose messages on error.
"""
x = backend.CpuDevice().send(utils.force_array(x))
y = backend.CpuDevice().send(utils.force_array(y))
try:
numpy.testing.assert_allclose(
x, y, atol=atol, rtol=rtol, verbose=verbose)
except AssertionError as e:
f = six.StringIO()
f.write(str(e) + '\n\n')
f.write(
'assert_allclose failed: \n' +
' shape: {} {}\n'.format(x.shape, y.shape) +
' dtype: {} {}\n'.format(x.dtype, y.dtype))
if x.shape == y.shape:
xx = numpy.atleast_1d(x)
yy = numpy.atleast_1d(y)
err = numpy.abs(xx - yy)
tol_err = atol + rtol * numpy.abs(yy).astype(numpy.float64)
i = numpy.unravel_index(
numpy.argmax(err.astype(numpy.float64) - tol_err), err.shape)
if yy[i] == 0:
rel_err = 'inf'
else:
rel_err = err[i] / numpy.abs(yy[i])
f.write(
' i: {}\n'.format(i) +
' x[i]: {}\n'.format(xx[i]) +
' y[i]: {}\n'.format(yy[i]) +
' relative error[i]: {}\n'.format(rel_err) +
' absolute error[i]: {}\n'.format(err[i]))
opts = numpy.get_printoptions()
try:
numpy.set_printoptions(threshold=10000)
f.write('x: ' + numpy.array2string(x, prefix='x: ') + '\n')
f.write('y: ' + numpy.array2string(y, prefix='y: ') + '\n')
finally:
numpy.set_printoptions(**opts)
raise AssertionError(f.getvalue())
def setUp_configure(self):
from scipy import stats
self.dist = distributions.Cauchy
self.scipy_dist = stats.cauchy
self.test_targets = set(["batch_shape", "cdf", "entropy",
"event_shape", "icdf", "log_prob",
"support"])
loc = utils.force_array(
numpy.random.uniform(-1, 1, self.shape).astype(numpy.float32))
scale = utils.force_array(numpy.exp(
numpy.random.uniform(-1, 1, self.shape)).astype(numpy.float32))
self.params = {"loc": loc, "scale": scale}
self.scipy_params = {"loc": loc, "scale": scale}
def setUp_configure(self):
from scipy import stats
self.dist = distributions.Cauchy
self.scipy_dist = stats.cauchy
self.test_targets = set(['batch_shape', 'cdf', 'entropy',
'event_shape', 'icdf', 'log_prob',
'support'])
loc = utils.force_array(
numpy.random.uniform(-1, 1, self.shape).astype(numpy.float32))
scale = utils.force_array(numpy.exp(
numpy.random.uniform(-1, 1, self.shape)).astype(numpy.float32))
self.params = {'loc': loc, 'scale': scale}
self.scipy_params = {'loc': loc, 'scale': scale}
def forward_cpu(self, inputs):
self.retain_inputs((0, 1))
x, gy = inputs
gx = utils.force_array(numpy.sin(x))
numpy.negative(gx, out=gx)
gx *= gy
return gx,
def forward(self, inputs):
xp = cuda.get_array_module(inputs[0])
self.input_length = inputs[0]
label_length = inputs[1]
t = inputs[2]
xs = inputs[3:]
if chainer.is_debug():
# Batch size check.
assert len(xs[0]) == len(t)
assert len(xs[0]) == len(self.input_length)
assert len(xs[0]) == len(label_length)
# Length check.
assert len(xs) >= xp.max(self.input_length)
assert len(t[0]) >= xp.max(label_length)
self.path_length = 2 * label_length + 1
yseq_shape = (len(xs),) + xs[0].shape
self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp)
log_yseq = self.log_matrix(self.yseq, xp)
self.path = _label_to_path(t, self.blank_symbol, xp)
self.prob_trans = self.calc_trans(
log_yseq, self.input_length, t,
label_length, self.path, self.path_length, xp)
loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
if self.reduce == 'mean':
loss = utils.force_array(xp.mean(loss))
return loss,
def forward(self, inputs):
self.retain_inputs((0,))
x, = inputs
xp = cuda.get_array_module(x)
norm = (xp.sqrt(xp.sum(xp.square(x), axis=self.axis, keepdims=True))
+ x.dtype.type(self.eps))
return utils.force_array(x / norm),
def sign(x):
"""Elementwise sign function.
For a given input :math:`x`, this function returns :math:`sgn(x)`
defined as
.. math::
sgn(x) = \\left \\{ \\begin{array}{cc}
-1 & {\\rm if~x < 0} \\\\
0 & {\\rm if~x = 0} \\\\
1 & {\\rm if~x > 0} \\\\
\\end{array} \\right.
.. note::
The gradient of this function is ``None`` everywhere and therefore
unchains the computational graph.
Args:
x (~chainer.Variable): Input variable for which the sign is computed.
Returns:
~chainer.Variable: Output variable.
"""
if isinstance(x, chainer.variable.Variable):
x = x.array
xp = backend.get_array_module(x)
return chainer.as_variable(utils.force_array(xp.sign(x)))
def forward(self, inputs):
self.retain_inputs((0, 1))
sp, dn = inputs
c = _coo_matmul(sp, self.sp_row, self.sp_col, self.sp_shape,
self.sp_order, dn,
self.transa, self.transb, self.transc, self.dtype)
return utils.force_array(c, self.dtype),
def forward(self, inputs):
self.retain_inputs((0, 1))
xp = backend.get_array_module(*inputs)
x1, x2 = inputs
difference = x1 - x2
y = xp.square(difference)
return utils.force_array(y, dtype=x1.dtype),
def forward_cpu(self, inputs):
x = inputs[0]
half = x.dtype.type(0.5)
y = utils.force_array(numpy.tanh(x * half) * half + half)
self.retain_outputs((0,))
self._use_cudnn = False
return y,
def forward_cpu(self, x):
self.retain_outputs((0,))
try:
invx = utils.force_array(numpy.linalg.inv(x[0]))
except numpy.linalg.LinAlgError:
raise ValueError('Input has singular matrices.')
return invx,
def forward(self, inputs):
xp = backend.get_array_module(inputs[0])
self.input_length, label_length, t, xs = inputs
if self.zero_padding is None:
if xs.dtype == numpy.float16:
self.zero_padding = -10000.0
else:
self.zero_padding = -10000000000.0
if chainer.is_debug():
assert len(xs) >= xp.max(self.input_length)
assert t.shape[1] >= xp.max(label_length)
self.path_length = 2 * label_length + 1
self.yseq = _softmax(xs, xp)
log_yseq = self.log_matrix(self.yseq, xp)
self.path = _label_to_path(t, self.blank_symbol, xp)
self.prob_trans = self.calc_trans(
log_yseq, self.input_length, t,
label_length, self.path, self.path_length, xp)
loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
if self.reduce == 'mean':
loss = utils.force_array(xp.mean(loss))
return loss,
def forward_cpu(self, inputs):
x, = inputs
# y = log(1 + exp(beta * x)) / beta
bx = self.beta * x
y = (numpy.fmax(bx, 0) +
numpy.log1p(numpy.exp(-numpy.fabs(bx)))) * self.beta_inv
return utils.force_array(y, x.dtype),
def test_0dim_array(self):
x = utils.force_array(numpy.array(1, numpy.float32), dtype=self.dtype)
self.assertIsInstance(x, numpy.ndarray)
if self.dtype is None:
self.assertEqual(x.dtype, numpy.float32)
else:
self.assertEqual(x.dtype, self.dtype)
def forward(self, inputs):
self.retain_inputs((0, 1))
a, b = inputs
if self.a_axes is None or self.b_axes is None:
a_axes = [[], []] # 0:row axes, 1:col axes
b_axes = [[], []] # 0:row axes, 1:col axes
axes = self.axes
if isinstance(axes, collections.Sequence):
a_axes[1], b_axes[0] = axes
if numpy.isscalar(a_axes[1]):
a_axes[1] = a_axes[1],
if numpy.isscalar(b_axes[0]):
b_axes[0] = b_axes[0],
else:
a_axes[1] = six.moves.range(a.ndim - axes, a.ndim)
b_axes[0] = six.moves.range(axes)
a_range = six.moves.range(a.ndim)
a_axes[0] = [i for i in a_range if i not in a_axes[1]]
b_range = six.moves.range(b.ndim)
b_axes[1] = [i for i in b_range if i not in b_axes[0]]
self.a_axes = a_axes
self.b_axes = b_axes
c = _tensordot(a, b, self.a_axes, self.b_axes, self.c_axes)
if self.c_axes is None:
c_axes = [[], []] # 0:row axes, 1:col axes
c_row_ndim = len(self.a_axes[0])
c_col_ndim = len(self.b_axes[1])
c_axes[0] = six.moves.range(c_row_ndim)
c_axes[1] = six.moves.range(c_row_ndim, c_row_ndim + c_col_ndim)
self.c_axes = c_axes
return utils.force_array(c, self.dtype),
def forward_cpu(self, x):
if not available_cpu:
raise ImportError('SciPy is not available. Forward computation'
' of erfinv in CPU can not be done.' +
str(_import_error))
self.retain_outputs((0,))
return utils.force_array(special.erfinv(x[0]), dtype=x[0].dtype),
def forward(self, inputs):
self.retain_inputs((0, 1))
a, b = inputs
c = _coo_matmul_gradsp(a, b, self.sp_row, self.sp_col, self.sp_shape,
self.transa, self.transb, self.transc,
self.dtype)
return utils.force_array(c),
def backward(self, x, gy):
xp = cuda.get_array_module(*x)
gx = utils.force_array(xp.cos(x[0]))
xp.square(gx, out=gx)
xp.reciprocal(gx, out=gx)
gx *= gy[0]
return gx,
def forward_cpu(self, x):
self.retain_inputs((0,))
return utils.force_array(numpy.log(x[0])),
def backward(self, indexes, gy):
x = self.get_retained_inputs()[0]
return utils.force_array(gy[0] / x),
def forward_cpu(self, inputs):
self.retain_inputs((0, 1))
y, gy = inputs
one = y.dtype.type(1)
return utils.force_array(gy * (one - y * y)),
def backward_cpu(self, x, gy):
return utils.force_array(gy[0] * (x[0] > 0)),
def forward(self, inputs):
self.retain_inputs((0,))
x = inputs[0]
xp = cuda.get_array_module(x)
return utils.force_array(xp.log2(x)),
def forward(self, inputs):
x, t = inputs
xp = cuda.get_array_module(x)
loss = -xp.mean(t * xp.log2(x + 1e-6) +
(1 - t) * xp.log2((1 - x) + 1e-6))
return force_array(loss),
def forward_cpu(self, x):
self.retain_outputs((0, ))
return utils.force_array(numpy.expm1(x[0])),
def backward_cpu(self, x, gy):
gx = utils.force_array(numpy.cos(x[0]))
gx *= gy[0]
return gx,
def backward(self, retain_grad=False):
"""Runs error backpropagation (a.k.a. backprop) from this variable.
On backprop, :meth:`Function.backward` is called on each
:class:`Function` object appearing in the backward graph starting from
this variable. The backward graph is represented by backward references
from variables to their creators, and from functions to their inputs.
The backprop stops at all root variables. Some functions set ``None``
as gradients of some inputs, where further backprop does not take place
at such input variables.
This method uses :data:`grad` as the initial error array. User can
manually set a gradient array before calling this method. If
:data:`data` contains only one element (i.e., it is scalar) and
:data:`grad` is ``None``, then this method automatically complements
1.0 as the initial error. This is useful on starting backprop from
some scalar loss value.
Args:
retain_grad (bool): If ``True``, the gradient arrays of all
intermediate variables are kept. Otherwise, :data:`grad` of the
intermediate variables are set to ``None`` on appropriate
timing, which may reduce the maximum memory consumption.
In most cases of training some models, the purpose of backprop
is to compute gradients of parameters, not of variables, so it
is recommended to set this flag ``False``.
"""
if self.creator is None:
return
initial_device = None
if cuda.available and isinstance(self.data, cuda.cupy.ndarray):
try:
initial_device = cuda.Device()
except cuda.cupy.cuda.runtime.CUDARuntimeError as e:
if e.status != 38: # cudaErrorNoDevice
raise
is_debug = chainer.is_debug()
cand_funcs = []
seen_set = set()
seen_vars = set()
need_copy = set()
# Initialize error by 1, if this is a loss variable
if self.data.size == 1 and self.grad is None:
with cuda.get_device(self.data) as device:
if device is cuda.DummyDevice:
self.grad = numpy.ones_like(self.data)
else:
self.grad = cuda.cupy.ones_like(self.data)
def add_cand(cand):
if cand not in seen_set:
# Negate since heapq is min-heap
heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand))
seen_set.add(cand)
add_cand(self.creator)
while cand_funcs:
_, _, func = heapq.heappop(cand_funcs)
outputs = [y() for y in func.outputs] # access via weak ref
in_data = tuple([x.data for x in func.inputs])
out_grad = tuple([None if y is None else y.grad for y in outputs])
hooks = chainer.get_function_hooks()
if func._n_local_function_hooks != 0:
hooks = collections.OrderedDict(hooks)
hooks.update(func.local_function_hooks)
cuda.get_device(*(in_data + out_grad)).use()
for hook in six.itervalues(hooks):
hook.backward_preprocess(func, in_data, out_grad)
gxs = func.backward(in_data, out_grad)
assert len(gxs) == len(in_data)
for hook in six.itervalues(hooks):
hook.backward_postprocess(func, in_data, out_grad)
if is_debug:
for gx in gxs:
if gx is None:
continue
cuda.get_device(gx).use()
if cuda.get_array_module(gx).isnan(gx).any():
msg = 'NaN is detected on backward computation'
raise RuntimeError(msg)
if not retain_grad:
for y in outputs:
if y is not None and y is not self:
y.grad = None
for x, gx in zip(func.inputs, gxs):
if gx is None:
continue
_check_grad_type(func, x, gx)
# Accumulate the gradient to x. It is a bit tricky to handle
# branches and parameter gradient accumulation correctly.
id_x = id(x)
if x.creator is None: # leaf
if x._grad is None:
x.grad = gx
need_copy.add(id_x)
else:
cuda.get_device(gx).use()
if id_x in need_copy:
x.grad = utils.force_array(x.grad + gx) # copy
need_copy.remove(id_x)
else:
x._grad += gx
else: # not a leaf
add_cand(x.creator)
if id_x not in seen_vars: # 1st visit
x.grad = gx
seen_vars.add(id_x)
need_copy.add(id_x)
else:
cuda.get_device(gx).use()
if id_x in need_copy: # 2nd visit
x._grad = utils.force_array(gx + x._grad) # copied
need_copy.remove(id_x)
else: # 3rd or later visit
x._grad += gx
del gxs # to reduce memory usage
if initial_device is not None:
initial_device.use()
def forward_cpu(self, x):
return utils.force_array(numpy.maximum(x[0], 0, dtype=x[0].dtype)),
def forward(self, x):
xp = cuda.get_array_module(*x)
return utils.force_array(xp.floor(x[0]), x[0].dtype),
def backward_cpu(self, x, gy):
gx = utils.force_array(numpy.sin(x[0]))
numpy.negative(gx, out=gx)
gx *= gy[0]
return gx,
def forward_cpu(self, x):
self.invx = utils.force_array(numpy.linalg.inv(x[0]))
return self.invx,
def forward(self, x):
xp = cuda.get_array_module(*x)
return utils.force_array(xp.log10(x[0])),
def forward_cpu(self, x):
zero = utils.force_type(x[0].dtype, 0)
return utils.force_array(numpy.maximum(zero, x[0])),
def forward(self, x):
self.retain_outputs((0, ))
xp = backend.get_array_module(*x)
return utils.force_array(xp.sqrt(x[0], dtype=x[0].dtype)),
def forward_cpu(self, x):
y = utils.force_array(numpy.tanh(x[0]))
self.retain_outputs((0, ))
self._use_cudnn = False
return y,
def forward_gpu(self, inputs):
self.retain_outputs((0, ))
x, = inputs
out = cuda.cupyx.rsqrt(x, dtype=x.dtype)
return utils.force_array(out),
