def __call__(self, *xs):
"""Applies broadcasted elementwise product.
Args:
xs (list of Variables): Input variables whose length should
be one if the link has a learnable weight parameter, otherwise
should be two.
"""
axis = self.axis
# Case of only one argument where W is a learnt parameter.
if hasattr(self, 'W'):
if chainer.is_debug():
assert len(xs) == 1
x, = xs
W = self.W
z = scale.scale(x, W, axis)
# Case of two arguments where W is given as an argument.
else:
if chainer.is_debug():
assert len(xs) == 2
x, y = xs
z = scale.scale(x, y, axis)
# Forward propagate bias term if given.
if hasattr(self, 'bias'):
return self.bias(z)
else:
return z
def multi_node_mean(self, array_a, array_b):
# The name is allreduce but actually a mean
# Sigma(a, all-procs)/n -> b or
# Sigma(b, all-procs)/n -> b if array_a is None
if chainer.is_debug():
self.check_ready_to_allreduce(array_a, array_b)
is_float16 = array_b.dtype == numpy.float16
if array_a is None:
buffer_a = mpi4py.MPI.IN_PLACE
elif is_float16:
assert array_a.dtype == array_b.dtype
buffer_a = _memory_utility.array_to_buffer_object(
array_a.astype(numpy.float32))
else:
buffer_a = _memory_utility.array_to_buffer_object(array_a)
if is_float16:
array_b32 = array_b.astype(numpy.float32)
else:
array_b32 = array_b
buffer_b = _memory_utility.array_to_buffer_object(array_b32)
self.mpi_comm.Allreduce(buffer_a, buffer_b)
if is_float16:
xp = chainer.backend.get_array_module(array_b)
xp.copyto(array_b, array_b32.astype(numpy.float16), casting='no')
array_b *= 1.0 / self.mpi_comm.size
if chainer.is_debug():
self.ensure_all_finite(array_b)
def forward(self, inputs):
xp = backend.get_array_module(*inputs)
y, t = inputs
# numpy.bincount requires int32 on Windows
t = t.astype('i', copy=False)
if self.label_num is None:
label_num = xp.amax(t) + 1
else:
label_num = self.label_num
if chainer.is_debug():
assert (t < label_num).all()
mask = (t == self.ignore_label).ravel()
pred = xp.where(mask, label_num, y.argmax(axis=1).ravel())
true = xp.where(mask, label_num, t.ravel())
support = xp.bincount(true, minlength=label_num + 1)[:label_num]
relevant = xp.bincount(pred, minlength=label_num + 1)[:label_num]
tp_mask = xp.where(pred == true, true, label_num)
tp = xp.bincount(tp_mask, minlength=label_num + 1)[:label_num]
precision = tp / relevant
recall = tp / support
fbeta = _fbeta_score(precision, recall, self.beta)
return precision, recall, fbeta, support
def forward(self, inputs):
xp = cuda.get_array_module(inputs[0])
self.input_length = inputs[0]
label_length = inputs[1]
t = inputs[2]
xs = inputs[3:]
if chainer.is_debug():
# Batch size check.
assert len(xs[0]) == len(t)
assert len(xs[0]) == len(self.input_length)
assert len(xs[0]) == len(label_length)
# Length check.
assert len(xs) >= xp.max(self.input_length)
assert len(t[0]) >= xp.max(label_length)
self.path_length = 2 * label_length + 1
yseq_shape = (len(xs),) + xs[0].shape
self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp)
log_yseq = self.log_matrix(self.yseq, xp)
self.path = _label_to_path(t, self.blank_symbol, xp)
self.prob_trans = self.calc_trans(
log_yseq, self.input_length, t,
label_length, self.path, self.path_length, xp)
loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
if self.reduce == 'mean':
loss = utils.force_array(xp.mean(loss))
return loss,
def forward(self, inputs):
xp = backend.get_array_module(inputs[0])
self.input_length, label_length, t, xs = inputs
if self.zero_padding is None:
if xs.dtype == numpy.float16:
self.zero_padding = -10000.0
else:
self.zero_padding = -10000000000.0
if chainer.is_debug():
assert len(xs) >= xp.max(self.input_length)
assert t.shape[1] >= xp.max(label_length)
self.path_length = 2 * label_length + 1
self.yseq = _softmax(xs, xp)
log_yseq = self.log_matrix(self.yseq, xp)
self.path = _label_to_path(t, self.blank_symbol, xp)
self.prob_trans = self.calc_trans(
log_yseq, self.input_length, t,
label_length, self.path, self.path_length, xp)
loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
if self.reduce == 'mean':
loss = utils.force_array(xp.mean(loss))
return loss,
def forward_gpu(self, x):
self.retain_outputs((0,))
invx, info = _inv_gpu(x[0])
if chainer.is_debug():
if cuda.cupy.any(info != 0):
raise ValueError('Input has singular matrices.')
return invx,
def forward_gpu(self, inputs):
cupy = cuda.cupy
x, t = inputs
if chainer.is_debug():
self._check_input_values(x, t)
log_y = log_softmax._log_softmax(x, self.use_cudnn)
if self.cache_score:
self.y = cupy.exp(log_y)
if self.class_weight is not None:
shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
log_y *= cupy.broadcast_to(
self.class_weight.reshape(shape), x.shape)
if self.normalize:
coeff = cupy.maximum(1, (t != self.ignore_label).sum())
else:
coeff = max(1, len(t))
self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)
log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
ret = cuda.reduce(
'S t, raw T log_y, int32 n_channel, raw T coeff', 'T out',
't == -1 ? T(0) : log_y[_j * n_channel + t]',
'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'
)(t, log_y.reduced_view(), log_y.shape[-1], self._coeff)
return ret,
def forward_gpu(self, inputs):
cupy = cuda.cupy
x, t = inputs
if chainer.is_debug():
self._check_input_values(x, t)
log_y = softmax_log(x, self.use_cudnn)
if self.cache_score:
self.y = cupy.exp(log_y)
if getattr(self, "normalize", True):
coeff = cupy.maximum(1, (t != self.ignore_label).sum())
else:
coeff = max(1, len(t))
self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)
log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
ret = cuda.reduce(
"S t, raw T log_y, int32 n_channel, raw T coeff",
"T out",
"t == -1 ? T(0) : log_y[_j * n_channel + t]",
"a + b",
"out = a * -coeff[0]",
"0",
"crossent_fwd",
)(t, log_y.reduced_view(), log_y.shape[-1], self._coeff)
return (ret,)
def forward(self, inputs):
x, inds = inputs
if chainer.is_debug():
_check_indices(inds)
return self._permutate(x, inds, self.inv),
def setUp(self):
self.original_debug = chainer.is_debug()
chainer.set_debug(True)
self.one = numpy.array(1, numpy.float32)
self.f = chainer.FunctionNode()
self.return_value = tuple(None if x is None else chainer.Variable(x)
for x in self.return_data)
def forward_cpu(self, inputs):
x, t = inputs
if chainer.is_debug():
_check_input_values(x, t, self.ignore_label)
log_y = log_softmax._log_softmax(x)
if self.cache_score:
self.y = numpy.exp(log_y)
if self.class_weight is not None:
shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
log_y *= _broadcast_to(self.class_weight.reshape(shape), x.shape)
log_yd = numpy.rollaxis(log_y, 1)
log_yd = log_yd.reshape(len(log_yd), -1)
log_p = log_yd[numpy.maximum(t.ravel(), 0), numpy.arange(t.size)]
log_p *= (t.ravel() != self.ignore_label)
if self.reduce == 'mean':
# deal with the case where the SoftmaxCrossEntropy is
# unpickled from the old version
if self.normalize:
count = (t != self.ignore_label).sum()
else:
count = len(x)
self._coeff = 1.0 / max(count, 1)
y = log_p.sum(keepdims=True) * (-self._coeff)
return y.reshape(()),
else:
return -log_p.reshape(t.shape),
def forward(self, inputs):
x, W = inputs
if chainer.is_debug():
if not ((0 <= x).all() and (x < len(W)).all()):
msg = "Each `x` value need to satisfty `0 <= x < len(W)`"
raise ValueError(msg)
return (W.take(x, axis=0),)
def _double_backward_softmax_cross_entropy(x, t, normalize, class_weight,
ignore_label, reduce, is_chainerx):
if isinstance(t, variable.Variable):
t = t.data
F = chainer.functions
_check_class_weight_option(class_weight)
_check_reduce_option(reduce)
if chainer.is_debug():
_check_input_values(x, t, ignore_label)
loss = -chainer.functions.log_softmax(x)
if class_weight is not None:
shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
class_weight = F.broadcast_to(class_weight.reshape(shape), x.shape)
# TODO(niboshi): Remove this workaround after ChainerX supports
# type promotion.
if is_chainerx:
class_weight = F.cast(class_weight, x.dtype)
loss = loss * class_weight
in_use = (t != ignore_label).astype(x.dtype)
loss = F.rollaxis(loss, 1, loss.ndim)
loss = F.reshape(loss, (-1, loss.shape[-1]))
# Replace ignore_label value with one valid for F.select_item below.
t = t.clip(0, loss.shape[1] - 1)
loss = F.select_item(loss, t.ravel())
loss = F.reshape(loss, t.shape)
loss = loss * in_use
if reduce == 'mean':
reduc_dtype = _reduction_dtype(x.dtype)
if normalize:
# TODO(niboshi): Use in_use.sum(dtype=reduc_dtype) once chainerx
# supports dtype argument.
count = in_use.astype(reduc_dtype, copy=False).sum()
else:
count = len(x)
count = max(count, 1.)
if reduc_dtype == loss.dtype:
loss = F.sum(loss / count)
else:
# Sum in a promoted dtype
loss = F.cast(loss, reduc_dtype)
loss = F.sum(loss / count)
loss = F.cast(loss, x.dtype)
return loss
def forward_gpu(self, inputs):
class_weight = backend.from_chainerx(self.class_weight)
self.retain_inputs((0, 1))
cupy = cuda.cupy
x, t = inputs
if chainer.is_debug():
_check_input_values(x, t, self.ignore_label)
if x.size == 0:
y = cupy.zeros(t.shape, dtype=x.dtype)
if self.cache_score:
self.y = y
if self.reduce == 'mean':
return y.sum(),
else:
return y,
log_y = log_softmax._log_softmax(x)
if self.cache_score:
self.y = cupy.exp(log_y)
if class_weight is not None:
shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
log_y *= cupy.broadcast_to(class_weight.reshape(shape), x.shape)
if self.normalize:
coeff = cupy.maximum(1, (t != self.ignore_label).sum())
else:
coeff = max(1, len(t))
self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)
log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
if self.reduce == 'mean':
ret = cuda.reduce(
'S t, raw T log_y, int32 n_channel, raw T coeff, '
'S ignore_label',
'T out',
't == ignore_label ? T(0) : log_y[_j * n_channel + t]',
'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'
)(t, log_y.reduced_view(), log_y.shape[-1],
self._coeff, self.ignore_label)
else:
ret = cuda.elementwise(
'S t, raw T log_y, int32 n_channel, T ignore', 'T out',
'''
if (t == ignore) {
out = 0;
} else {
out = -log_y[i * n_channel + t];
}
''',
'softmax_crossent_no_reduce_fwd'
)(t, log_y.reduced_view(), log_y.shape[-1], self.ignore_label)
ret = ret.reshape(t.shape)
return ret,
def __init__(self, indices_or_sections, axis):
if not isinstance(
indices_or_sections,
six.integer_types + (collections.Iterable,)):
raise TypeError('indices_or_sections must be integer or 1-D array')
if (chainer.is_debug() and
isinstance(indices_or_sections, collections.Iterable)):
for p, n in six.moves.zip(
indices_or_sections, indices_or_sections[1:]):
if p > n:
raise ValueError('indices_or_sections must be sorted')
self.indices_or_sections = indices_or_sections
self.axis = axis
def __call__(self, *xs):
"""Applies broadcasted elementwise summation.
Args:
xs (list of ~chainer.Variable): Input variables whose length should
be one if the link has a learnable bias parameter, otherwise
should be two.
"""
axis = self.axis
# Case of only one argument where b is a learnt parameter.
if hasattr(self, 'b'):
if chainer.is_debug():
assert len(xs) == 1
x, = xs
b = self.b
return bias.bias(x, b, axis)
# Case of two arguments where b is given as an argument.
else:
if chainer.is_debug():
assert len(xs) == 2
x, y = xs
return bias.bias(x, y, axis)
def backprop_step(
func, target_input_indexes, grad_outputs, grad_inputs):
"""Accumulates gradients of a FunctionNode
This routine is used by :meth:`chainer.Variable.backward` and
:func:`chainer.grad`.
Args:
target_input_indexes (tuple of int): Sorted indices of the input
variables w.r.t. which the gradients are required. It is
guaranteed that this tuple contains at least one element.
grad_outputs (tuple of Variable): Gradients w.r.t. the output
variables. If the gradient w.r.t. an output variable is not
given, the corresponding element is ``None``.
grad_inputs (dict): References of radients w.r.t. the input variables.
"""
if chainer.is_debug():
assert isinstance(target_input_indexes, tuple)
assert target_input_indexes == tuple(sorted(target_input_indexes))
assert isinstance(grad_outputs, tuple)
if func.backward_accumulate.__code__ \
is not chainer.FunctionNode.backward_accumulate.__code__:
# backward_accumulate is overridden
grad_inputs_tuple = tuple([
_pop_or_none(grad_inputs[func.inputs[i]])
for i in target_input_indexes
])
gxs = func.backward_accumulate(
target_input_indexes, grad_outputs, grad_inputs_tuple)
else: # otherwise, backward should be overridden
gxs = func.backward(
target_input_indexes, grad_outputs)
len_gxs = len(gxs)
if len_gxs == len(func.inputs):
gxs = tuple([gxs[i] for i in target_input_indexes])
elif len_gxs != len(target_input_indexes):
raise ValueError(
'number of gradients returned by %s (%s) is incorrect.'
% (func._impl_name, func.label))
for i, gx in six.moves.zip(target_input_indexes, gxs):
if gx is not None:
grad_inputs[func.inputs[i]].append(gx)
if not func.lazy_grad_sum:
for gx in grad_inputs.values():
_reduce(gx)
def forward(self, inputs):
x, W = inputs
if chainer.is_debug():
if not ((0 <= x).all() and
(x < len(W)).all()):
msg = 'Each `x` value need to satisfy `0 <= x < len(W)`'
raise ValueError(msg)
if self.ignore_label is not None:
xp = cuda.get_array_module(*inputs)
mask = (x == self.ignore_label)
return xp.where(
mask[..., None], 0, W.take(xp.where(mask, 0, x), axis=0)),
return W.take(x, axis=0),
def __init__(self, slices):
if not isinstance(slices, collections.Iterable):
slices = tuple([slices])
if chainer.is_debug():
n_ellipses = 0
for s in slices:
if numpy.isscalar(s) or s is None or isinstance(s, slice):
pass
elif s is Ellipsis:
n_ellipses += 1
else:
raise ValueError("Only basic indexing is supported")
if n_ellipses > 1:
raise ValueError("Only one Ellipsis is allowed")
self.slices = slices
def __init__(self, slices):
if isinstance(slices, list):
if all([isinstance(s, int) for s in slices]):
slices = slices,
slices = tuple(slices)
elif not isinstance(slices, tuple):
slices = slices,
if chainer.is_debug():
n_ellipses = 0
for s in slices:
if s is Ellipsis:
n_ellipses += 1
if n_ellipses > 1:
raise ValueError('Only one Ellipsis is allowed')
self.slices = slices
def forward(self, inputs):
x, W = inputs
xp = cuda.get_array_module(*inputs)
if chainer.is_debug():
valid_x = xp.logical_and(0 <= x, x < len(W))
if self.ignore_label is not None:
valid_x = xp.logical_or(valid_x, x == self.ignore_label)
if not valid_x.all():
raise ValueError('Each not ignored `x` value need to satisfy'
'`0 <= x < len(W)`')
if self.ignore_label is not None:
mask = (x == self.ignore_label)
return xp.where(
mask[..., None], 0, W.take(xp.where(mask, 0, x), axis=0)),
return W.take(x, axis=0),
def bias(x, y, axis=1):
"""Elementwise summation with broadcasting.
Computes a elementwise summation of two input variables, with the shape of
the latter variable broadcasted to match the shape of the former. ``axis``
is the first axis of the first variable along which the second variable is
applied.
The term "broadcasting" here comes from Caffe's bias layer so the
"broadcasting" with the following arguments::
x : 100 x 3 x 40 x 5 x 6
y : 3 x 40
axis : 1
is equivalent to the following numpy broadcasting::
x : 100 x 3 x 40 x 5 x 6
y : (1 x) 3 x 40 x 1 x 1
Note that the axis of ``x`` to which we apply ``y`` is specified by the
argument ``axis``, whose meaning is different from numpy's ``axis``.
Args:
x (:class:`~chainer.Variable` or :ref:`ndarray`):
Input variable to be summed.
y (:class:`~chainer.Variable` or :ref:`ndarray`):
Input variable to sum, broadcasted.
axis (int): The first axis of ``x`` along which ``y`` is applied.
Returns:
~chainer.Variable: Output variable.
"""
x_shape = x.shape
y_shape = y.shape
if chainer.is_debug():
assert x_shape[axis:axis + len(y_shape)] == y_shape
y1_shape = tuple([1] * axis + list(y_shape) +
[1] * (len(x_shape) - axis - len(y_shape)))
y1 = reshape.reshape(y, y1_shape)
y2 = broadcast.broadcast_to(y1, x_shape)
return x + y2
def forward_cpu(self, inputs):
x, t = inputs
if chainer.is_debug():
self._check_input_values(x, t)
log_y = numpy.log(x)
if self.cache_score:
self.y = x
log_yd = numpy.rollaxis(log_y, 1)
log_yd = log_yd.reshape(len(log_yd), -1)
log_p = log_yd[numpy.maximum(t.ravel(), 0), six.moves.range(t.size)]
if getattr(self, 'normalize', True):
count = (t != self.ignore_label).sum()
else:
count = len(x)
self._coeff = 1.0 / max(count, 1)
y = (log_p * (t.ravel() != self.ignore_label)).sum(keepdims=True) \
* (-self._coeff)
return y.reshape(()),
def _get_indices_or_sections(indices_or_sections):
"""Checks and convert ``indices_or_sections`` argument
Converted value is one of: 1-D numpy.ndarray, list, int, and
NumPy int scalar.
Returns:
A binary tuple in which the 1st element is indices (sequence) and
the 2nd element is sections (scalar).
Only one of the two is not ``None`` and the other is ``None``.
"""
ios = indices_or_sections
is_seq = False
if isinstance(ios, numpy.ndarray):
# numpy.ndarray
if ios.dtype.kind != 'i' and ios.size > 0:
# Note: numpy.array([]) (dtype is float64) should be accepted.
raise TypeError('indices_or_sections must be integers')
if ios.ndim >= 2:
raise TypeError('indices_or_sections must be 1-D sequence')
is_seq = ios.ndim != 0
elif isinstance(ios, collections_abc.Sequence):
# Any sequence except numpy.ndarray
ios = list(ios)
is_seq = True
elif isinstance(indices_or_sections, six.integer_types):
# int
pass
else:
raise TypeError(
'indices_or_sections must be integer or 1-D array.\n'
'Actual: {}'.format(type(indices_or_sections)))
if is_seq and chainer.is_debug():
for p, n in six.moves.zip(ios, ios[1:]):
if p > n:
raise ValueError('indices_or_sections must be sorted')
if is_seq:
return ios, None
else:
return None, ios
def forward(self, inputs):
self.retain_inputs((0,))
x, W = inputs
self._w_shape = W.shape
xp = backend.get_array_module(*inputs)
if chainer.is_debug():
valid_x = xp.logical_and(0 <= x, x < len(W))
if self.ignore_label is not None:
valid_x = xp.logical_or(valid_x, x == self.ignore_label)
if not valid_x.all():
raise ValueError('Each not ignored `x` value need to satisfy'
'`0 <= x < len(W)`')
if self.ignore_label is not None:
mask = (x == self.ignore_label)
return xp.where(mask[..., None], 0, W[xp.where(mask, 0, x)]),
return W[x],
def _double_backward_softmax_cross_entropy(x, t, normalize, class_weight,
ignore_label, reduce):
if isinstance(t, variable.Variable):
t = t.data
_check_class_weight_option(class_weight)
_check_reduce_option(reduce)
if chainer.is_debug():
_check_input_values(x, t, ignore_label)
loss = -chainer.functions.log_softmax(x)
if class_weight is not None:
shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
class_weight = chainer.functions.broadcast_to(
class_weight.reshape(shape), x.shape)
loss = loss * class_weight
in_use = (t != ignore_label).astype(x.dtype)
loss = chainer.functions.rollaxis(loss, 1, loss.ndim)
loss = chainer.functions.reshape(loss, (-1, loss.shape[-1]))
# Replace ignore_label value with one valid for F.select_item below.
t = t.clip(0, loss.shape[1] - 1)
loss = chainer.functions.select_item(loss, t.ravel())
loss = chainer.functions.reshape(loss, t.shape)
loss = loss * in_use
if reduce == 'mean':
if normalize:
count = in_use.sum()
else:
count = len(x)
count = max(count, 1.)
loss = loss / count
return chainer.functions.sum(loss)
else:
return loss
def forward_gpu(self, inputs):
cupy = cuda.cupy
x, t = inputs
if chainer.is_debug():
self._check_input_values(x, t)
log_y = softmax_log(x, self.use_cudnn)
self.y = cupy.exp(log_y)
if getattr(self, 'normalize', True):
coeff = cupy.maximum(1, (t != self.ignore_label).sum())
else:
coeff = max(1, len(t))
self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)
log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
ret = cuda.reduce(
'S t, raw T log_y, int32 n_channel, raw T coeff', 'T out',
't == -1 ? 0 : log_y[_j * n_channel + t]',
'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'
)(t, log_y.reduced_view(), log_y.shape[-1], self._coeff)
return ret,
def forward(self, inputs):
x, t = inputs
if chainer.is_debug():
if not ((0 <= t).all() and
(t < x.shape[1]).all()):
msg = 'Each label `t` need to satisfty `0 <= t < x.shape[1]`'
raise ValueError(msg)
xp = cuda.get_array_module(x)
if xp is numpy:
# This code is equivalent to `t.choose(x.T)`, but `numpy.choose`
# does not work when `x.shape[1] > 32`.
return x[six.moves.range(t.size), t],
else:
y = cuda.elementwise(
'S t, raw T x',
'T y',
'int ind[] = {i, t}; y = x[ind];',
'getitem_fwd'
)(t, x)
return y,
def scale(x, y, axis=1):
"""Elementwise product with broadcasting.
Computes a elementwise product of two input variables, with the shape of
the latter variable broadcasted to match the shape of the former. ``axis``
is the first axis of the first variable along which the second variable is
applied.
The term "broadcasting" here comes from Caffe's scale layer so the
"broadcasting" with the following arguments::
x : 100 x 3 x 40 x 60
y : 3 x 40
axis : 1
is equivalent to the following numpy broadcasting::
x : 100 x 3 x 40 x 60
y : 1 x 3 x 40 x 1
Note that how the ``axis`` indicates to which axis of ``x`` we apply ``y``.
Args:
x (~chainer.Variable): Input variable to be scaled.
y (~chainer.Variable): Input variable to scale, broadcasted.
axis (int): The first axis of ``x`` along which ``y`` is applied.
Returns:
~chainer.Variable: Output variable.
"""
x_shape = x.shape
y_shape = y.shape
if chainer.is_debug():
assert x_shape[axis:axis + len(y_shape)] == y_shape
y1_shape = tuple([1] * axis + list(y_shape) +
[1] * (len(x_shape) - axis - len(y_shape)))
y1 = reshape.reshape(y, y1_shape)
y2 = broadcast.broadcast_to(y1, x_shape)
return x * y2
def forward_cpu(self, inputs):
x, t = inputs
if chainer.is_debug():
self._check_input_values(x, t)
log_y = softmax_log(x, False)
if self.cache_score:
self.y = numpy.exp(log_y)
log_yd = numpy.rollaxis(log_y, 1)
log_yd = log_yd.reshape(len(log_yd), -1)
log_p = log_yd[numpy.maximum(t.ravel(), 0), six.moves.range(t.size)]
# deal with the case where the SoftmaxCrossEntropy is
# unpickled from the old version
if getattr(self, "normalize", True):
count = (t != self.ignore_label).sum()
else:
count = len(x)
self._coeff = 1.0 / max(count, 1)
y = (log_p * (t.ravel() != self.ignore_label)).sum(keepdims=True) * (-self._coeff)
return (y.reshape(()),)
def apply(self, inputs):
"""Computes output variables and grows the computational graph.
Basic behavior is expressed in the documentation of
:class:`FunctionNode`.
.. note::
If the :data:`~Variable.data` attribute of input variables exist on
a GPU device, that device is made current before calling
:meth:`forward`, so implementors do not need to take care of device
selection in most cases.
Args:
inputs: Tuple of input variables. Each element can be either
:class:`~chainer.Variable` or :ref:`ndarray`. If the element
is an ndarray, it is automatically wrapped with
:class:`~chainer.Variable`.
Returns:
A tuple of output :class:`~chainer.Variable` objects.
"""
chainerx_in_data = None
chainerx_device = None
is_chainerx, in_data = _extract_apply_in_data(inputs)
if is_chainerx:
# Try ChainerX C++ implementation.
# If it's supported, the output arrays are wrapped with Variables
# and returned.
# If not supported, FunctionNode.forward_chainerx should return
# Fallback.
# In that case the input arrays are converted to numpy.ndarray
# or cupy.ndarray (depending on the ChainerX backend) and
# forward computation falls back to the conventional
# FunctionNode.forward() implementaion.
outputs = self.forward_chainerx(in_data)
if outputs is not chainer.Fallback:
# Supported. Wrap with variables and return
assert isinstance(outputs, tuple)
return tuple([
variable.Variable._init_unchecked(
y, requires_grad=y.is_backprop_required(),
is_chainerx_array=True)
for y in outputs])
# Fall back to FunctionNode.forward()
chainerx_in_data, in_data, chainerx_device = (
self._chainerx_apply_fallback_preprocess(in_data, inputs))
self._is_chainerx_fallback_mode = True
self.chainerx_device = chainerx_device
utils._check_arrays_forward_compatible(in_data, self.label)
is_debug = chainer.is_debug()
if is_debug:
# Keep stack trace for debug
self.stack = traceback.extract_stack()
if configuration.config.type_check:
self._check_data_type_forward(in_data)
hooks = chainer.get_function_hooks()
if self._n_local_function_hooks > 0:
hooks = collections.OrderedDict(hooks)
hooks.update(self.local_function_hooks)
hooks = hooks.values() # avoid six for performance
for hook in hooks:
hook.forward_preprocess(self, in_data)
# Forward propagation
with cuda.get_device_from_array(*in_data):
self._input_indexes_to_retain = None
self._output_indexes_to_retain = None
if chainer.config.schedule_func is not None:
outputs = static_forward_optimizations(self, in_data)
elif self._is_chainerx_fallback_mode:
# In ChainerX fallback, __class__ is temporarily replaced with
# the fabricated one with automatic attirbute fallback.
with _chainerx_attribute_fallback(self, chainerx_device):
outputs = self.forward(in_data)
else:
# In normal case, simply run the forward method.
outputs = self.forward(in_data)
# Check for output array types
if not isinstance(outputs, tuple):
raise TypeError(
'forward output must be a tuple ({})\n'
'Actual: {}'.format(self.label, type(outputs)))
if not chainer.is_arrays_compatible(outputs):
raise TypeError(
'incompatible array types are mixed in the forward output '
'({}).\n'
'Actual: {}'.format(
self.label,
', '.join(str(type(x)) for x in outputs)))
for hook in hooks:
hook.forward_postprocess(self, in_data)
# NaN check of output values
if is_debug:
if any(chainer.backend._contains_nan(out)
for out in outputs):
msg = ('NaN is detected on forward computation of '
'{}'.format(self.label))
raise RuntimeError(msg)
self._output_count = len(outputs)
if self._is_chainerx_fallback_mode:
ret = self._chainerx_apply_fallback_postprocess(
chainerx_in_data, inputs, outputs)
else:
input_vars = [chainer.as_variable(x) for x in inputs]
requires_grad = any([x.requires_grad for x in input_vars])
ret = tuple(
[variable.Variable(y, requires_grad=requires_grad)
for y in outputs])
if configuration.config.enable_backprop:
# Topological ordering
self.rank = max(
[x.rank for x in input_vars]) if input_vars else 0
# Add backward edges
for y in ret:
y.creator_node = self
self.inputs = tuple([x.node for x in input_vars])
# Add forward edges (must be weak references)
self.outputs = tuple([weakref.ref(y.node) for y in ret])
if self._input_indexes_to_retain is not None:
for index in self._input_indexes_to_retain:
input_vars[index].retain_data()
if self._output_indexes_to_retain is not None:
retained_data = []
for index in self._output_indexes_to_retain:
ret[index].retain_data()
retained_data.append(outputs[index])
self._retained_output_data = tuple(retained_data)
self.lazy_grad_sum = configuration.config.lazy_grad_sum
return ret
def backward(self, axis, gamma, gy, x, xp, expander, mean, inv_std, eps,
var):
return cudnn.batch_normalization_backward(x, gamma, gy, mean, inv_std,
eps, self.is_for_conv2d,
self.cudnn_mode,
chainer.is_debug())
def setUp(self):
self.link = links.EmbedID(2, 2, ignore_label=self.ignore_label)
self.t = numpy.array([self.t_value], dtype=numpy.int32)
self.original_debug = chainer.is_debug()
chainer.set_debug(True)
def backward(self, retain_grad=False):
"""Runs error backpropagation (a.k.a. backprop) from this variable.
On backprop, :meth:`Function.backward` is called on each
:class:`Function` object appearing in the backward graph starting from
this variable. The backward graph is represented by backward references
from variables to their creators, and from functions to their inputs.
The backprop stops at all root variables. Some functions set ``None``
as gradients of some inputs, where further backprop does not take place
at such input variables.
This method uses :data:`grad` as the initial error array. User can
manually set a gradient array before calling this method. If
:data:`data` contains only one element (i.e., it is scalar) and
:data:`grad` is ``None``, then this method automatically complements
1.0 as the initial error. This is useful on starting backprop from
some scalar loss value.
Args:
retain_grad (bool): If ``True``, the gradient arrays of all
intermediate variables are kept. Otherwise, :data:`grad` of the
intermediate variables are set to ``None`` on appropriate
timing, which may reduce the maximum memory consumption.
In most cases of training some models, the purpose of backprop
is to compute gradients of parameters, not of variables, so it
is recommended to set this flag ``False``.
"""
if self.creator is None:
return
initial_device = None
if cuda.available and isinstance(self.data, cuda.cupy.ndarray):
try:
initial_device = cuda.Device()
except cuda.cupy.cuda.runtime.CUDARuntimeError as e:
if e.status != 38: # cudaErrorNoDevice
raise
is_debug = chainer.is_debug()
cand_funcs = []
seen_set = set()
seen_vars = set()
need_copy = set()
# Initialize error by 1, if this is a loss variable
if self.data.size == 1 and self.grad is None:
with cuda.get_device(self.data) as device:
if device is cuda.DummyDevice:
self.grad = numpy.ones_like(self.data)
else:
self.grad = cuda.cupy.ones_like(self.data)
def add_cand(cand):
if cand not in seen_set:
# Negate since heapq is min-heap
heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand))
seen_set.add(cand)
add_cand(self.creator)
while cand_funcs:
_, _, func = heapq.heappop(cand_funcs)
outputs = [y() for y in func.outputs] # access via weak ref
in_data = tuple([x.data for x in func.inputs])
out_grad = tuple([None if y is None else y.grad for y in outputs])
hooks = chainer.get_function_hooks()
if func._n_local_function_hooks != 0:
hooks = collections.OrderedDict(hooks)
hooks.update(func.local_function_hooks)
cuda.get_device(*(in_data + out_grad)).use()
for hook in six.itervalues(hooks):
hook.backward_preprocess(func, in_data, out_grad)
gxs = func.backward(in_data, out_grad)
assert len(gxs) == len(in_data)
for hook in six.itervalues(hooks):
hook.backward_postprocess(func, in_data, out_grad)
if is_debug:
for gx in gxs:
if gx is None:
continue
cuda.get_device(gx).use()
if cuda.get_array_module(gx).isnan(gx).any():
msg = 'NaN is detected on backward computation'
raise RuntimeError(msg)
if not retain_grad:
for y in outputs:
if y is not None and y is not self:
y.grad = None
for x, gx in zip(func.inputs, gxs):
if gx is None:
continue
_check_grad_type(func, x, gx)
# Accumulate the gradient to x. It is a bit tricky to handle
# branches and parameter gradient accumulation correctly.
id_x = id(x)
if x.creator is None: # leaf
if x._grad is None:
x.grad = gx
need_copy.add(id_x)
else:
cuda.get_device(gx).use()
if id_x in need_copy:
x.grad = utils.force_array(x.grad + gx) # copy
need_copy.remove(id_x)
else:
x._grad += gx
else: # not a leaf
add_cand(x.creator)
if id_x not in seen_vars: # 1st visit
x.grad = gx
seen_vars.add(id_x)
need_copy.add(id_x)
else:
cuda.get_device(gx).use()
if id_x in need_copy: # 2nd visit
x._grad = utils.force_array(gx + x._grad) # copied
need_copy.remove(id_x)
else: # 3rd or later visit
x._grad += gx
del gxs # to reduce memory usage
if initial_device is not None:
initial_device.use()
def apply(self, inputs):
"""Computes output variables and grows the computational graph.
Basic behavior is expressed in the documentation of
:class:`FunctionNode`.
.. note::
If the :data:`~Variable.data` attribute of input variables exist on
a GPU device, that device is made current before calling
:meth:`forward`, so implementors do not need to take care of device
selection in most cases.
Args:
inputs: Tuple of input variables. Each element can be either
:class:`~chainer.Variable`, :class:`numpy.ndarray`,
or :class:`cupy.ndarray`. If the element is an ndarray, it is
automatically wrapped with :class:`~chainer.Variable`.
Returns:
A tuple of output :class:`~chainer.Variable` objects.
"""
input_vars = [chainer.as_variable(x) for x in inputs]
in_data = tuple([x.data for x in input_vars])
requires_grad = any([x.requires_grad for x in input_vars])
# Check for input array types
if not chainer.is_arrays_compatible(in_data):
raise TypeError(
'incompatible array types are mixed in the forward input '
'({}).\n'
'Actual: {}'.format(self.label,
', '.join(str(type(x)) for x in in_data)))
is_debug = chainer.is_debug()
if is_debug:
# Keep stack trace for debug
self.stack = traceback.extract_stack()
if configuration.config.type_check:
self._check_data_type_forward(in_data)
hooks = chainer.get_function_hooks()
if self._n_local_function_hooks > 0:
hooks = collections.OrderedDict(hooks)
hooks.update(self.local_function_hooks)
hooks = hooks.values() # avoid six for performance
for hook in hooks:
hook.forward_preprocess(self, in_data)
# Forward propagation
with cuda.get_device_from_array(*in_data):
self._input_indexes_to_retain = None
self._output_indexes_to_retain = None
if chainer.config.schedule_func is not None:
outputs = static_forward_optimizations(self, in_data)
else:
outputs = self.forward(in_data)
# Check for output array types
if not isinstance(outputs, tuple):
raise TypeError('forward output must be a tuple ({})\n'
'Actual: {}'.format(self.label, type(outputs)))
if not chainer.is_arrays_compatible(outputs):
raise TypeError(
'incompatible array types are mixed in the forward output '
'({}).\n'
'Actual: {}'.format(self.label,
', '.join(str(type(x)) for x in outputs)))
for hook in hooks:
hook.forward_postprocess(self, in_data)
# NaN check of output values
if is_debug:
if any(chainer.backend._contains_nan(out) for out in outputs):
msg = ('NaN is detected on forward computation of '
'{}'.format(self.label))
raise RuntimeError(msg)
ret = tuple([
variable.Variable(y, requires_grad=requires_grad) for y in outputs
])
if configuration.config.enable_backprop:
# Topological ordering
self.rank = max([x.rank for x in input_vars]) if input_vars else 0
# Add backward edges
for y in ret:
y.creator_node = self
self.inputs = tuple([x.node for x in input_vars])
# Add forward edges (must be weak references)
self.outputs = tuple([weakref.ref(y.node) for y in ret])
if self._input_indexes_to_retain is not None:
for index in self._input_indexes_to_retain:
input_vars[index].retain_data()
if self._output_indexes_to_retain is not None:
retained_data = []
for index in self._output_indexes_to_retain:
ret[index].retain_data()
retained_data.append(outputs[index])
self._retained_output_data = tuple(retained_data)
self.lazy_grad_sum = configuration.config.lazy_grad_sum
return ret
def setUp(self):
self.x = numpy.random.uniform(-1, 1, (2, 2)).astype(numpy.float32)
# `0` is required to avoid NaN
self.t = numpy.array([self.t_value, 0], dtype=numpy.int32)
self.original_debug = chainer.is_debug()
chainer.set_debug(True)
def setUp(self):
self.x = numpy.random.uniform(-1, 1, (1, 2)).astype(numpy.float32)
self.t = numpy.array([self.t_value], dtype=numpy.int32)
self.original_debug = chainer.is_debug()
chainer.set_debug(True)
def forward(self, inputs):
x, t = inputs[:2]
rest = len(inputs) - 2
head_W, Ws = inputs[2], inputs[3:2 + (rest - 1) // 2 + 1]
Rs = inputs[2 + (rest - 1) // 2 + 1:]
n_tails = len(Rs)
# minus_inf = -1024.
minus_inf = -numpy.inf
xp = cuda.get_array_module(x)
if chainer.is_debug():
_check_input_values(x, t, self.ignore_label)
self.retain_inputs(tuple(six.moves.range(len(inputs))))
cluster_hots = []
for i in six.moves.range(1, n_tails + 1):
lower, upper = self.cutoff[i], self.cutoff[i + 1]
in_cluster = xp.logical_and(lower <= t, t < upper)
if self.output_all:
in_cluster = xp.ones(
in_cluster.shape, dtype=in_cluster.dtype)
cluster_hots.append(in_cluster)
self.cluster_hots = cluster_hots
self.head = self.linear(x, head_W)
self.ls_head = log_softmax._log_softmax(self.head)
self.reduced_xs = []
self.tails = []
self.ls_tails = []
for i, in_cluster in enumerate(cluster_hots, start=1):
tail_idx = i - 1
if xp.any(in_cluster):
reduced_x = self.linear(x[in_cluster], Rs[tail_idx])
self.reduced_xs.append(reduced_x)
out = self.linear(reduced_x, Ws[tail_idx])
self.tails.append(out)
ls_out = log_softmax._log_softmax(out)
self.ls_tails.append(ls_out)
else:
self.reduced_xs.append(None)
self.tails.append(None)
self.ls_tails.append(None)
n_head_out = head_W.shape[0] - n_tails
n_out = n_head_out + sum(W.shape[0] for W in Ws)
shape = (x.shape[0], n_out)
log_y = xp.full(shape, minus_inf, dtype=x.dtype)
log_y[:, :n_head_out] = self.ls_head[:, :n_head_out]
for i, (in_cluster, tail) in enumerate(
zip(cluster_hots, self.ls_tails), start=1):
if tail is None:
continue
lower, upper = self.cutoff[i], self.cutoff[i + 1]
tail_main = self.ls_head[:, n_head_out + i - 1]
tail_main_in = xp.broadcast_to(
tail_main[in_cluster][:, None], tail.shape)
log_y[xp.nonzero(in_cluster)[0], lower:upper] = tail_main_in + tail
not_in_cluster = xp.logical_not(in_cluster)
log_y[xp.nonzero(not_in_cluster)[0],
lower] = tail_main[not_in_cluster]
return log_y,
def forward_grad(self, rho=1e-3, decay=0.50, loss_scale=None):
"""test
"""
self._node._check_old_style_gradient()
if self.creator_node is None:
return
initial_device = None
if cuda.available and isinstance(self.data, cuda.ndarray):
try:
initial_device = cuda.Device()
except cuda.cupy.cuda.runtime.CUDARuntimeError as e:
if e.status != 38: # cudaErrorNoDevice
raise
is_debug = chainer.is_debug()
cand_funcs = []
seen_set = set()
def add_cand(cand):
if cand not in seen_set:
# Negate since heapq is min-heap
heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand))
seen_set.add(cand)
add_cand(self.creator_node)
cur_decay = 1.0
while cand_funcs:
_, _, func = heapq.heappop(cand_funcs)
inputs = func.inputs
target_input_indexes = [
i for i, x in enumerate(inputs) if x.requires_grad
]
if not target_input_indexes:
continue
in_data = tuple([x.data for x in inputs])
cuda.get_device_from_array(*in_data).use()
if hasattr(func, 'with_frad') and func.with_frad:
gW, gb = func.forward_grad(in_data, rho)
gxs = [None, Variable(gW * cur_decay), Variable(gb * cur_decay)]
cur_decay *= decay
else:
gxs = [None] * len(inputs)
if is_debug:
for gx in gxs:
if gx is None:
continue
gx_data = gx.data
if gx_data.dtype.kind == 'f':
cuda.get_device_from_array(gx_data).use()
if cuda.get_array_module(gx_data).isnan(gx_data).any():
raise RuntimeError(
'NaN is detected on forward-grad computation of '
'{}'.format(func.label))
for i, gx in enumerate(gxs):
x = inputs[i]
if x.creator_node is not None:
add_cand(x.creator_node)
if gx is None:
continue
if not x.requires_grad:
continue
_check_grad_type(func, x, gx.data)
x_var = x.get_variable_or_none()
if x_var is not None:
x_var._grad_var = gx
x_var._loss_scale = loss_scale
del gxs # to reduce memory usage
if initial_device is not None:
initial_device.use()
def _backprop(outputs, inputs, grad_required, retain_grad, grads, loss_scale):
candidate_funcs, push_candidate, pop_candidate = _get_ordered_func_heap()
for y in outputs:
creator = y.creator_node
if creator is not None:
push_candidate(creator)
input_nodes = set(x.node for x in inputs)
ret_dict = {}
is_debug = chainer.is_debug()
base_hooks = chainer.get_function_hooks().values()
while candidate_funcs:
func = pop_candidate()
# Collect the gradients w.r.t. the outputs
ys = [y() for y in func.outputs] # access via weak ref
gys = tuple([grads.pop(y) for y in ys])
for node, gy in six.moves.zip(ys, gys):
if node is not None:
if node in input_nodes:
ret_dict[node] = gy
if retain_grad:
y = node.get_variable_or_none()
if y is not None:
y.grad_var = gy
y._loss_scale = loss_scale
# Collect the gradients w.r.t. the inputs
input_indexes = []
x_grads = collections.OrderedDict()
for i, x in enumerate(func.inputs):
if x not in grad_required:
continue
input_indexes.append(i)
if x not in x_grads:
x_grads[x] = grads.get_as_list(x)
if not input_indexes:
continue
input_indexes = tuple(input_indexes)
# Do backward
# Call pre-backward hooks
if func._n_local_function_hooks != 0:
local_hooks = collections.OrderedDict(chainer.get_function_hooks())
local_hooks.update(func.local_function_hooks)
hooks = local_hooks.values() # avoid six for performance
else:
hooks = base_hooks
in_data = [x.data for x in func.inputs]
out_grad_data = [None if g is None else g.data for g in gys]
with cuda.get_device_from_array(*in_data):
for hook in hooks:
hook.backward_preprocess(
func, tuple(in_data), tuple(out_grad_data))
_backprop_utils.backprop_step(func, input_indexes, gys, x_grads,
is_debug)
# Call post-backward hooks
for hook in hooks:
hook.backward_postprocess(
func, tuple(in_data), tuple(out_grad_data))
# Update grads
for node, g in x_grads.items():
if not g: # gradient == None
continue
creator = node.creator_node
if creator is not None:
push_candidate(creator)
for x in input_nodes:
if x not in ret_dict:
ret_dict[x] = grads.pop(x)
return ret_dict
def setUp(self):
self.x = numpy.arange(10).reshape((2, 5)).astype('f')
self.ind = numpy.array(self.indices, 'i')
self.debug = chainer.is_debug()
chainer.set_debug(True)
def forward(self, inputs):
self.retain_inputs((0, 1))
x, gamma, beta = inputs
xp = backend.get_array_module(x)
if self.running_mean is None:
self.running_mean = xp.zeros_like(gamma, dtype=x.dtype)
self.running_var = xp.zeros_like(gamma, dtype=x.dtype)
self.axis = _compute_axis(x.ndim, gamma.ndim, self.axis)
self.key_axis = _compute_key_axis(x.ndim, gamma.ndim, self.axis)
if all(x.shape[i] == 1 for i in self.axis):
if 0 in self.axis:
warnings.warn(
'A batch with no more than one sample has been given'
' to F.batch_normalization. F.batch_normalization'
' will always output a zero tensor for such batches.'
' This could be caused by incorrect configuration in'
' your code (such as running evaluation while'
' chainer.config.train=True),'
' but could also happen in the last batch of training'
' if non-repeating iterator is used.', UserWarning)
else:
warnings.warn(
'F.batch_normalization received a batch with single'
' dimensions along all axes that are used for aggregating'
' statistics. F.batch_normalization'
' will always output a zero tensor for such batches.',
UserWarning)
# TODO(niboshi): Refactor calculation of expander and axis into a
# function and call it just before they are used.
# expander inserts singleton dimensions to gamma and beta so that they
# can be broadcasted with x.
expander = [None for _ in range(x.ndim)]
for i in self.key_axis:
expander[i] = slice(None)
expander = tuple(expander)
self.expander = expander
self.mode = _BNMode(x, gamma, self.key_axis)
self.use_cudnn = self.mode.can_use_cudnn(xp)
self.use_ideep = self.mode.can_use_ideep()
if self.use_ideep:
# TODO(niboshi): Refactor iDeep part into a separate method
expand_dim = False
if x.ndim == 2:
expand_dim = True
x = x[:, :, None, None]
y, self.mean, self.var, self.inv_std = (
intel64.ideep.batchNormalization.Forward(
intel64.ideep.array(x.astype(gamma.dtype, copy=False)),
intel64.ideep.array(gamma), intel64.ideep.array(beta),
None, None, self.eps))
y = y.astype(x.dtype, copy=False)
m = x.size // gamma.size
adjust = m / max(m - 1., 1.)
# Update running_mean
if isinstance(self.running_mean, intel64.ideep.mdarray):
self.running_mean.inplace_axpby(self.decay, (1 - self.decay),
self.mean)
else:
self.running_mean *= self.decay
self.running_mean += self.mean * (1 - self.decay)
# Update running_var
if isinstance(self.running_var, intel64.ideep.mdarray):
self.running_var.inplace_axpby(self.decay, (1 - self.decay),
self.var * adjust)
else:
self.running_var *= self.decay
self.running_var += self.var * adjust * (1 - self.decay)
if expand_dim:
y = numpy.squeeze(y, axis=(2, 3))
elif self.use_cudnn:
# self.mean and self.inv_std are used as buffers to save
# intermediate results computed during forward pass. These buffers
# are used to speed-up backward pass.
y, self.mean, self.inv_std = (
cudnn.batch_normalization_forward_training(
x, gamma, beta, self.running_mean, self.running_var, None,
None, self.eps, self.decay, self.mode.is_for_conv2d,
self.mode.get_cudnn_mode(), chainer.is_debug()))
else:
# Generic CPU and GPU implementation
gamma = gamma[expander]
beta = beta[expander]
self.mean = x.mean(axis=self.axis, dtype=gamma.dtype)
var = x.var(axis=self.axis, dtype=gamma.dtype)
if xp is numpy:
self.inv_std = numpy.reciprocal(
numpy.sqrt(var + self.eps, dtype=gamma.dtype))
else:
self.inv_std = cuda.cupyx.rsqrt(var + self.eps,
dtype=gamma.dtype)
y = _apply_bn_fwd(xp, x, self.mean[expander],
self.inv_std[expander], gamma, beta)
# Update running statistics
m = x.size // gamma.size
adjust = m / max(m - 1., 1.) # unbiased estimation
xp = backend.get_array_module(self.running_mean, self.running_var)
if xp is chainerx:
self.running_mean, self.running_var = backend.from_chx(
(self.running_mean, self.running_var))
self.running_mean *= self.decay
self.running_mean += (1 - self.decay) * self.mean
self.running_var *= self.decay
self.running_var += (1 - self.decay) * adjust * var
if xp is chainerx:
self.running_mean = backend.to_chx(self.running_mean)
self.running_var = backend.to_chx(self.running_var)
return y,
def _backward_main(self, retain_grad, loss_scale):
self._node._check_old_style_gradient()
if self.creator_node is None:
return
is_debug = chainer.is_debug()
cand_funcs = []
seen_set = set()
grads = _backprop_utils.GradTable(load_if_new=True)
# Initialize error by 1, if this is a loss variable
if self.data.size == 1 and self._grad_var is None:
if self.data.ndim != 0:
warnings.warn(
'Treating a scalar as a variable with only one element'
' in Variable.backward is deprecated. A scalar variable'
' must be a 0-dimensional array. Apply'
' chainer.functions.squeeze to obtain a scalar variable.'
' If the size of this variable accidentally becomes one,'
' set zero to grad.', DeprecationWarning)
with cuda.get_device_from_array(self.data) as device:
if device is cuda.DummyDevice:
self.grad = numpy.ones_like(self.data)
else:
self.grad = cuda.cupy.ones_like(self.data)
if loss_scale is not None:
self.grad *= loss_scale
grads[self._node] = self._grad_var
def add_cand(cand):
if cand not in seen_set:
# Negate since heapq is min-heap
heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand))
seen_set.add(cand)
add_cand(self.creator_node)
leaf_nodes = set()
while cand_funcs:
_, _, func = heapq.heappop(cand_funcs)
inputs = func.inputs
target_input_indexes = tuple(
[i for i, x in enumerate(inputs) if x.requires_grad])
outputs = [y() for y in func.outputs] # access via weak ref
out_grad = tuple([grads.pop(y) for y in outputs])
if not target_input_indexes:
continue
in_data = tuple([x.data for x in inputs])
out_grad_data = tuple(
[None if g is None else g.data for g in out_grad])
hooks = chainer.get_function_hooks()
if func._n_local_function_hooks != 0:
hooks = collections.OrderedDict(hooks)
hooks.update(func.local_function_hooks)
hooks = hooks.values() # avoid six for performance
with cuda.get_device_from_array(*(in_data + out_grad_data)):
for hook in hooks:
hook.backward_preprocess(func, in_data, out_grad_data)
# Collect the current input gradients.
target_inputs = [inputs[i] for i in target_input_indexes]
# Keep the order for the portability, rather than
# in_grad = {x: grads.get_as_list(x)
# for x in set(target_inputs)}
in_grad = collections.OrderedDict()
for x in target_inputs:
if x not in in_grad:
in_grad[x] = grads.get_as_list(x)
_backprop_utils.backprop_step(func, target_input_indexes,
out_grad, in_grad)
for hook in hooks:
hook.backward_postprocess(func, in_data, out_grad_data)
if is_debug:
# each grad is a list of variables
# iter_gxs expands it as a sequence of variables.
def iter_gxs(gxs):
for gx in gxs:
for gx_elem in gx:
yield gx_elem
for gx in iter_gxs(in_grad.values()):
gx_data = gx.data
if gx_data.dtype.kind == 'f':
with cuda.get_device_from_array(gx_data):
xp = cuda.get_array_module(gx_data)
if xp.isnan(gx_data).any():
raise RuntimeError(
'NaN is detected on backward computation '
'of {}'.format(func.label))
for y, gy in six.moves.zip(outputs, out_grad):
if y is not None and y is not self.node:
y_var = y.get_variable_or_none()
if y_var is not None:
y_var._grad_var = gy if retain_grad else None
for x, gx in in_grad.items():
if not gx: # gradient == None
continue
for gx_elem in gx:
_check_grad_type(func, x, gx_elem.data)
if x.creator_node is None: # leaf
leaf_nodes.add(x)
else:
add_cand(x.creator_node)
del in_grad # to reduce memory usage
for x in leaf_nodes:
x_var = x.get_variable_or_none()
gx = grads.pop(x)
if x_var is not None:
x_var._grad_var = gx
x_var._loss_scale = loss_scale
grads.assert_no_grads()
def setUp(self):
self.default_debug = chainer.is_debug()
chainer.set_debug(True)
def setUp(self):
self.default_debug = chainer.is_debug()
chainer.set_debug(True)
self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))
def _backprop_to_all(outputs, retain_grad, loss_scale):
"""Backprop to all input variables
Args:
outputs (list of tuple): each tuple is (y_node, y_grad_var).
y_grad_var should not be None.
retain_grad (bool): see docstring of Variable.backward
loss_scale (float): see docstring of Variable.backward
"""
OrderedDict = chainer.utils._collections.OrderedDict # fix py2 memory leak
cand_funcs = []
seen_set = set()
def add_cand(cand):
if cand not in seen_set:
# Negate since heapq is min-heap
heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand))
seen_set.add(cand)
grads = _backprop_utils.GradTable(accumulate_grad_inputs=True)
leaf_nodes = set()
for y, gy in outputs:
grads.accumulate(y, gy)
func = y.creator_node
if func is None: # leaf
leaf_nodes.add(y)
else:
add_cand(func)
# Fix F812 (Python 2)
y = None
del y
is_debug = chainer.is_debug()
base_hooks = chainer.get_function_hooks().values()
while cand_funcs:
_, _, func = heapq.heappop(cand_funcs)
inputs = func.inputs
target_input_indexes = tuple([
i for i, x in enumerate(inputs) if x.requires_grad
])
outputs = [y() for y in func.outputs] # access via weak ref
out_grad = tuple([grads.pop(y)
if y is not None and y.creator_node is not None
else None
for y in outputs])
if not target_input_indexes:
continue
in_data = [x.data for x in inputs]
out_grad_array = [None if g is None else g.raw_array for g in out_grad]
if func._n_local_function_hooks != 0:
local_hooks = collections.OrderedDict(chainer.get_function_hooks())
local_hooks.update(func.local_function_hooks)
hooks = local_hooks.values() # avoid six for performance
else:
hooks = base_hooks
with chainer.using_device(
backend.get_device_from_array(*(in_data + out_grad_array))):
for hook in hooks:
hook.backward_preprocess(
func, tuple(in_data), tuple(out_grad_array))
# Collect the current input gradients.
target_inputs = [inputs[i] for i in target_input_indexes]
# Keep the order for the portability, rather than
# in_grad = {x: grads.get_as_list(x)
# for x in set(target_inputs)}
in_grad = OrderedDict()
for x in target_inputs:
if x not in in_grad:
in_grad[x] = grads.get_as_list(x)
_backprop_utils.backprop_step(
func, target_input_indexes, out_grad, in_grad, is_debug)
for hook in hooks:
hook.backward_postprocess(
func, tuple(in_data), tuple(out_grad_array))
if retain_grad:
# The gradients of the outputs of `func` are final. Store them if
# retain_grad=True.
for y, gy in six.moves.zip(outputs, out_grad):
if y is not None:
y._set_grad_var_if_available(gy)
del gy # to reduce memory usage
del out_grad # to reduce memory usage
for x, gx in in_grad.items():
if not gx: # gradient == None
continue
for gx_elem in gx:
if gx_elem is not None:
chainer.variable._check_grad_type(
func, x, True, gx_elem.raw_array)
del gx_elem # to reduce memory usage
if x.creator_node is None: # leaf
leaf_nodes.add(x)
else:
add_cand(x.creator_node)
del gx, in_grad # to reduce memory usage
for x in leaf_nodes:
x_var = x.get_variable_or_none()
gx = grads.pop(x)
if x_var is not None:
x_var._set_grad_var_without_check(gx)
x_var._loss_scale = loss_scale
grads.assert_no_grads()
def backward(self, retain_grad=False):
"""Runs error backpropagation (a.k.a. backprop) from this variable.
On backprop, :meth:`FunctionNode.backward` is called on each
:class:`FunctionNode` object appearing in the backward graph starting
from this variable. The backward graph is represented by backward
references from variable nodes to their creators, and from function
nodes to their input variable nodes. The backprop stops at all root
nodes. Some function nodes set ``None`` as gradients of some inputs,
where further backprop does not take place at such inputs.
This method uses :data:`grad` as the initial error array. User can
manually set a gradient array before calling this method. If
:data:`data` contains only one element (i.e., it is scalar) and
:data:`grad` is ``None``, then this method automatically complements
1.0 as the initial error. This is useful on starting backprop from
some scalar loss value.
Note that this method does not support *differentiable backprop*. Use
:func:`grad` to compute the gradient of gradients.
Args:
retain_grad (bool): If ``True``, the gradient arrays of all
intermediate variables are kept. Otherwise, :data:`grad` of the
intermediate variables are set to ``None`` on appropriate
timing, which may reduce the maximum memory consumption.
In most cases of training some models, the purpose of backprop
is to compute gradients of parameters, not of all variables,
and therefore it is recommended to set this flag ``False``.
"""
self._node._check_old_style_gradient()
if self.creator_node is None:
return
initial_device = None
if cuda.available and isinstance(self.data, cuda.cupy.ndarray):
try:
initial_device = cuda.Device()
except cuda.cupy.cuda.runtime.CUDARuntimeError as e:
if e.status != 38: # cudaErrorNoDevice
raise
is_debug = chainer.is_debug()
cand_funcs = []
seen_set = set()
grads = {}
# Initialize error by 1, if this is a loss variable
if self.data.size == 1 and self._grad_var is None:
with cuda.get_device_from_array(self.data) as device:
if device is cuda.DummyDevice:
self.grad = numpy.ones_like(self.data)
else:
self.grad = cuda.cupy.ones_like(self.data)
grads[self._node] = self._grad_var
def add_cand(cand):
if cand not in seen_set:
# Negate since heapq is min-heap
heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand))
seen_set.add(cand)
add_cand(self.creator_node)
def get_grad(node):
if node is None:
return None
if node in grads:
return grads[node]
return node.grad_var
while cand_funcs:
_, _, func = heapq.heappop(cand_funcs)
inputs = func.inputs
outputs = [y() for y in func.outputs] # access via weak ref
in_data = tuple([x.data for x in inputs])
out_grad = tuple([get_grad(y) for y in outputs])
out_grad_data = tuple(
[None if g is None else g.data for g in out_grad])
hooks = chainer.get_function_hooks()
if func._n_local_function_hooks != 0:
hooks = collections.OrderedDict(hooks)
hooks.update(func.local_function_hooks)
hooks = hooks.values() # avoid six for performance
cuda.get_device_from_array(*in_data).use()
for hook in hooks:
hook.backward_preprocess(func, in_data, out_grad_data)
# Collect the current input gradients.
#
# Note (Tokui): When the same variable is passed to multiple input
# slots (e.g. an expression like ``f(x, x)``), it makes the
# gradient accumulation complicated since the back-propagated
# gradients w.r.t. the first and second argument should be
# accumulated to the current gradient w.r.t. the same variable.
# In this case, the current implementation passes the current
# gradient only to the first occurrence of the variable in the
# input tuple and passes ``None`` to the rest of the occurrences.
# For example, when the input variables are ``(x, x)``, the
# input gradient passed to the ``backward_accumulate`` method is
# ``(gx, None)`` where ``gx`` is the current gradient of ``x``.
# See also the docstring of ``FunctionNode.backward_accumulate``.
target_input_indexes = [
i for i, x in enumerate(inputs) if x.requires_grad
]
target_inputs = [inputs[i] for i in target_input_indexes]
in_grad = []
for i, index_i in enumerate(target_input_indexes):
x = inputs[index_i]
if x in target_inputs[:i]:
# Pass ``None`` for duplicated input variables except for
# the first occurrence (see the comment above).
gx = None
elif x in grads:
gx = grads[x]
elif x.creator_node is None:
x._check_old_style_gradient()
# accumulate the gradient only if the node is a leaf
gx = x.grad_var
else:
gx = None
in_grad.append(gx)
gxs = func.backward_accumulate(target_input_indexes, out_grad,
in_grad)
assert len(gxs) == len(in_grad)
for hook in hooks:
hook.backward_postprocess(func, in_data, out_grad_data)
if is_debug:
for gx in gxs:
if gx is None:
continue
gx_data = gx.data
cuda.get_device_from_array(gx_data).use()
if cuda.get_array_module(gx_data).isnan(gx_data).any():
msg = 'NaN is detected on backward computation'
raise RuntimeError(msg)
if not retain_grad:
for y in outputs:
if y is not None and y is not self.node:
grads[y] = None
y_var = y.get_variable()
if y_var is not None:
y_var._grad_var = None
for i, gx in enumerate(gxs):
if gx is None:
continue
x = target_inputs[i]
if not x.requires_grad:
continue
_check_grad_type(func, x, gx.data)
if x in target_inputs[:i]:
# Accumulate the duplicated gradients here. See the comment
# above the code that builds ``in_grad``.
cur_gx = grads[x]
grads[x] = gx if cur_gx is None else gx + cur_gx
else:
grads[x] = gx
x_var = x.get_variable()
if x_var is not None:
x_var._grad_var = grads[x]
if x.creator_node is not None:
add_cand(x.creator_node)
del gxs # to reduce memory usage
if initial_device is not None:
initial_device.use()
def apply(self, inputs):
"""Computes output variables and grows the computational graph.
Basic behavior is expressed in the documentation of
:class:`FunctionNode`.
.. note::
If the :data:`~Variable.data` attribute of input variables exist on
a GPU device, that device is made current before calling
:meth:`forward`, so implementors do not need to take care of device
selection in most cases.
Args:
inputs: Tuple of input variables. Each element can be either
:class:`Variable`, :class:`numpy.ndarray`,
or :class:`cupy.ndarray`. If the element is an ndarray, it is
automatically wrapped with :class:`Variable`.
Returns:
A tuple of output :class:`Variable` objects.
"""
input_vars = [
x if isinstance(x, variable.Variable) else variable.Variable(
x, requires_grad=False) for x in inputs
]
in_data = tuple([x.data for x in input_vars])
requires_grad = any([x.requires_grad for x in input_vars])
if chainer.is_debug():
self.stack = traceback.extract_stack()
if configuration.config.type_check:
self._check_data_type_forward(in_data)
hooks = chainer.get_function_hooks()
if self._n_local_function_hooks > 0:
hooks = collections.OrderedDict(hooks)
hooks.update(self.local_function_hooks)
hooks = hooks.values() # avoid six for performance
for hook in hooks:
hook.forward_preprocess(self, in_data)
# Forward propagation
with cuda.get_device_from_array(*in_data):
self._input_indexes_to_retain = None
self._output_indexes_to_retain = None
outputs = self.forward(in_data)
assert type(outputs) is tuple
for hook in hooks:
hook.forward_postprocess(self, in_data)
# NaN check of output values
if chainer.is_debug():
if any(out.dtype.kind == 'f'
and cuda.get_array_module(out).isnan(out).any()
for out in outputs):
msg = ('NaN is detected on forward computation of '
'{}'.format(self.label))
raise RuntimeError(msg)
ret = tuple([
variable.Variable(y, requires_grad=requires_grad) for y in outputs
])
if configuration.config.enable_backprop:
# Topological ordering
self.rank = max([x.rank for x in input_vars]) if input_vars else 0
# Add backward edges
for i, y in enumerate(ret):
y.creator_node = self
self.inputs = tuple([x.node for x in input_vars])
# Add forward edges (must be weak references)
self.outputs = tuple([weakref.ref(y.node) for y in ret])
if self._input_indexes_to_retain is not None:
for index in self._input_indexes_to_retain:
input_vars[index].retain_data()
if self._output_indexes_to_retain is not None:
retained_data = []
for index in self._output_indexes_to_retain:
ret[index].retain_data()
retained_data.append(outputs[index])
self._retained_output_data = tuple(retained_data)
return ret
def __call__(self, *inputs):
"""Applies forward propagation with chaining backward references.
Basic behavior is expressed in documentation of :class:`Function`
class.
.. note::
If the :data:`~Variable.data` attribute of input variables exist on
GPU device, then, before it calls :meth:`forward` method, the
appropriate device is selected, so in most cases implementers do
not need to take care of device selection.
Args:
inputs: Tuple of input :class:`Variable`, :class:`numpy.ndarray` or
:class:`cupy.ndarray` objects.
If the input is an :class:`numpy.ndarray` or a
:class:`cupy.ndarray`, it is automatically wrapped with
:class:`Variable`.
Returns:
One :class:`Variable` object or a tuple of multiple
:class:`Variable` objects.
"""
inputs = [
x if isinstance(x, variable.Variable) else variable.Variable(
x, requires_grad=False) for x in inputs
]
in_data = tuple([x.data for x in inputs])
requires_grad = any([x.requires_grad for x in inputs])
if chainer.is_debug():
self._stack = traceback.extract_stack()
if configuration.config.type_check:
self._check_data_type_forward(in_data)
hooks = chainer.get_function_hooks()
if self._n_local_function_hooks != 0:
hooks = collections.OrderedDict(hooks)
hooks.update(self.local_function_hooks)
for hook in six.itervalues(hooks):
hook.forward_preprocess(self, in_data)
# Forward prop
with cuda.get_device_from_array(*in_data):
self._input_indexes_to_retain = None
self._output_indexes_to_retain = None
outputs = self.forward(in_data)
assert type(outputs) == tuple
for hook in six.itervalues(hooks):
hook.forward_postprocess(self, in_data)
if chainer.is_debug():
if any(out.dtype.kind == 'f'
and cuda.get_array_module(out).isnan(out).any()
for out in outputs):
msg = 'NaN is detected on forward computation'
raise RuntimeError(msg)
ret = tuple([
variable.Variable(y, requires_grad=requires_grad) for y in outputs
])
if configuration.config.enable_backprop:
# Topological ordering
self.rank = max([x.rank for x in inputs]) if inputs else 0
# Backward edges
for y in ret:
y.set_creator(self)
self.inputs = tuple([x.node for x in inputs])
# Forward edges (must be weak references)
self.outputs = tuple([weakref.ref(y.node) for y in ret])
input_indexes_to_retain = self._input_indexes_to_retain
if input_indexes_to_retain is None:
# input arrays are retained by default
input_indexes_to_retain = six.moves.range(len(inputs))
for index in input_indexes_to_retain:
inputs[index].retain_data()
del self._input_indexes_to_retain
output_indexes_to_retain = self._output_indexes_to_retain
if output_indexes_to_retain is not None:
for index in output_indexes_to_retain:
ret[index].retain_data()
del self._output_indexes_to_retain
if len(ret) == 1:
return ret[0]
else:
return ret
def _backward_main(self, retain_grad):
self._node._check_old_style_gradient()
if self.creator_node is None:
return
initial_device = None
if cuda.available and isinstance(self.data, cuda.cupy.ndarray):
try:
initial_device = cuda.Device()
except cuda.cupy.cuda.runtime.CUDARuntimeError as e:
if e.status != 38: # cudaErrorNoDevice
raise
is_debug = chainer.is_debug()
cand_funcs = []
seen_set = set()
grads = {}
# Initialize error by 1, if this is a loss variable
if self.data.size == 1 and self._grad_var is None:
with cuda.get_device_from_array(self.data) as device:
if device is cuda.DummyDevice:
self.grad = numpy.ones_like(self.data)
else:
self.grad = cuda.cupy.ones_like(self.data)
grads[self._node] = self._grad_var
def add_cand(cand):
if cand not in seen_set:
# Negate since heapq is min-heap
heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand))
seen_set.add(cand)
add_cand(self.creator_node)
def get_grad(node):
if node is None:
return None
if node in grads:
return grads[node]
return node.grad_var
while cand_funcs:
_, _, func = heapq.heappop(cand_funcs)
inputs = func.inputs
target_input_indexes = [
i for i, x in enumerate(inputs) if x.requires_grad
]
if not target_input_indexes:
continue
outputs = [y() for y in func.outputs] # access via weak ref
in_data = tuple([x.data for x in inputs])
out_grad = tuple([get_grad(y) for y in outputs])
out_grad_data = tuple(
[None if g is None else g.data for g in out_grad])
hooks = chainer.get_function_hooks()
if func._n_local_function_hooks != 0:
hooks = collections.OrderedDict(hooks)
hooks.update(func.local_function_hooks)
hooks = hooks.values() # avoid six for performance
cuda.get_device_from_array(*in_data).use()
for hook in hooks:
hook.backward_preprocess(func, in_data, out_grad_data)
# Collect the current input gradients.
#
# Note (Tokui): When the same variable is passed to multiple input
# slots (e.g. an expression like ``f(x, x)``), it makes the
# gradient accumulation complicated since the back-propagated
# gradients w.r.t. the first and second argument should be
# accumulated to the current gradient w.r.t. the same variable.
# In this case, the current implementation passes the current
# gradient only to the first occurrence of the variable in the
# input tuple and passes ``None`` to the rest of the occurrences.
# For example, when the input variables are ``(x, x)``, the
# input gradient passed to the ``backward_accumulate`` method is
# ``(gx, None)`` where ``gx`` is the current gradient of ``x``.
# See also the docstring of ``FunctionNode.backward_accumulate``.
target_inputs = [inputs[i] for i in target_input_indexes]
in_grad = []
for i, index_i in enumerate(target_input_indexes):
x = inputs[index_i]
if x in target_inputs[:i]:
# Pass ``None`` for duplicated input variables except for
# the first occurrence (see the comment above).
gx = None
elif x in grads:
gx = grads[x]
elif x.creator_node is None:
x._check_old_style_gradient()
# accumulate the gradient only if the node is a leaf
gx = x.grad_var
else:
gx = None
in_grad.append(gx)
gxs = func.backward_accumulate(target_input_indexes, out_grad,
in_grad)
assert len(gxs) == len(in_grad)
for hook in hooks:
hook.backward_postprocess(func, in_data, out_grad_data)
if is_debug:
for gx in gxs:
if gx is None:
continue
gx_data = gx.data
if gx_data.dtype.kind == 'f':
cuda.get_device_from_array(gx_data).use()
if cuda.get_array_module(gx_data).isnan(gx_data).any():
raise RuntimeError(
'NaN is detected on backward computation of '
'{}'.format(func.label))
if not retain_grad:
for y in outputs:
if y is not None and y is not self.node:
grads[y] = None
y_var = y.get_variable()
if y_var is not None:
y_var._grad_var = None
for i, gx in enumerate(gxs):
if gx is None:
continue
x = target_inputs[i]
if not x.requires_grad:
continue
_check_grad_type(func, x, gx.data)
if x in target_inputs[:i]:
# Accumulate the duplicated gradients here. See the comment
# above the code that builds ``in_grad``.
cur_gx = grads[x]
grads[x] = gx if cur_gx is None else gx + cur_gx
else:
grads[x] = gx
x_var = x.get_variable()
if x_var is not None:
x_var._grad_var = grads[x]
if x.creator_node is not None:
add_cand(x.creator_node)
del gxs # to reduce memory usage
if initial_device is not None:
initial_device.use()
def setUp(self):
self.original_debug = chainer.is_debug()
chainer.set_debug(True)
self.one = numpy.array([1], numpy.float32)
self.f = chainer.FunctionNode()
def forward_gpu(self, inputs):
class_weight = backend.from_chx(self.class_weight)
self.retain_inputs((0, 1))
x, t = inputs
if x.ndim == t.ndim and x.shape == t.shape:
self.soft_target = True
cupy = cuda.cupy
if chainer.is_debug() and not self.soft_target:
_check_input_values(x, t, self.ignore_label)
if x.size == 0:
y = cupy.zeros(t.shape, dtype=x.dtype)
if self.cache_score:
self.y = y
if self.reduce == 'mean':
return y.sum(),
else:
return y,
log_y = log_softmax._log_softmax(x)
if self.cache_score:
self.y = cupy.exp(log_y)
if self.soft_target:
return self._soft_target_loss(cupy, x, t, log_y)
if class_weight is not None:
shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
log_y *= cupy.broadcast_to(class_weight.reshape(shape), x.shape)
log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
if self.reduce == 'mean':
# Reduction is performed in a promoted dtype
reduc_dtype = _reduction_dtype(x.dtype)
if self.normalize:
count = (t != self.ignore_label).sum(dtype=reduc_dtype)
count = cupy.maximum(1, count)
coeff = 1. / count
else:
coeff = cupy.array(1. / max(1, len(t)), dtype=reduc_dtype)
self._coeff = coeff
ret = cuda.reduce(
'S t, raw T log_y, int32 n_channel, raw U coeff, '
'S ignore_label', 'U out',
't == ignore_label ? T(0) : log_y[_j * n_channel + t]',
'a + b', 'out = static_cast<U>(a * -coeff[0])', '0',
'crossent_fwd')(t, log_y.reduced_view(), log_y.shape[-1],
self._coeff, self.ignore_label)
ret = ret.astype(log_y.dtype, copy=False)
else:
ret = cuda.elementwise(
'S t, raw T log_y, int32 n_channel, T ignore', 'T out', '''
if (t == ignore) {
out = 0;
} else {
out = -log_y[i * n_channel + t];
}
''', 'softmax_crossent_no_reduce_fwd')(t, log_y.reduced_view(),
log_y.shape[-1],
self.ignore_label)
ret = ret.reshape(t.shape)
return ret,
def __call__(self, *inputs):
"""Applies forward propagation with chaining backward references.
Basic behavior is expressed in documentation of :class:`Function`
class.
.. note::
If the :data:`~Variable.data` attribute of input variables exist on
GPU device, then, before it calls :meth:`forward` method, the
appropriate device is selected, so in most cases implementers do
not need to take care of device selection.
Args:
inputs: Tuple of input :class:`Variable`, :class:`numpy.ndarray` or
:class:`cupy.ndarray` objects. The volatile flags of all input
variables must agree. If the input is an :class:`numpy.ndarray`
or a :class:`cupy.ndarray`, it is automatically wrapped with
:class:`Variable`.
Returns:
One :class:`Variable` object or a tuple of multiple
:class:`Variable` objects.
"""
inputs = [
x if isinstance(x, chainer.Variable) else chainer.Variable(
x, volatile=flag.AUTO) for x in inputs
]
in_data = tuple([x.data for x in inputs])
if chainer.is_debug():
self._stack = traceback.extract_stack()
if self.type_check_enable:
self._check_data_type_forward(in_data)
hooks = chainer.get_function_hooks()
if self._n_local_function_hooks != 0:
hooks = collections.OrderedDict(hooks)
hooks.update(self.local_function_hooks)
for hook in six.itervalues(hooks):
hook.forward_preprocess(self, in_data)
# Forward prop
with cuda.get_device(*in_data):
outputs = self.forward(in_data)
assert type(outputs) == tuple
for hook in six.itervalues(hooks):
hook.forward_postprocess(self, in_data)
if chainer.is_debug():
if any(out.dtype.kind == 'f'
and cuda.get_array_module(out).isnan(out).any()
for out in outputs):
msg = 'NaN is detected on forward computation'
raise RuntimeError(msg)
out_v = flag.aggregate_flags([x.volatile for x in inputs])
ret = tuple([variable.Variable(y, volatile=out_v) for y in outputs])
if out_v == 'on':
build_graph = False
elif out_v == 'off':
build_graph = True
else:
build_graph = getattr(_thread_local, 'default_backprop', True)
if build_graph:
# Topological ordering
self.rank = max([x.rank for x in inputs]) if inputs else 0
# Backward edges
for y in ret:
y.set_creator(self)
self.inputs = inputs
# Forward edges (must be weak references)
self.outputs = tuple([weakref.ref(y) for y in ret])
if len(ret) == 1:
return ret[0]
else:
return ret
def backprop_step(func, target_input_indexes, grad_outputs, grad_inputs):
"""Accumulates gradients of a FunctionNode
This routine is used by :meth:`chainer.Variable.backward` and
:func:`chainer.grad`.
Args:
func (~chainer.FunctionNode): The function for which gradients are
accumulated.
target_input_indexes (tuple of int): Sorted indices of the inputs
that require gradients. It is guaranteed that this tuple contains
at least one element.
grad_outputs (tuple of Variable): Gradients w.r.t. the output
variables. If the gradient w.r.t. an output variable is not
given, the corresponding element is ``None``.
grad_inputs (dict): References of the gradients w.r.t. the input
variables.
"""
is_debug = chainer.is_debug()
if is_debug:
assert isinstance(target_input_indexes, tuple)
assert target_input_indexes == tuple(sorted(target_input_indexes))
assert isinstance(grad_outputs, tuple)
if func.backward_accumulate.__code__ \
is not chainer.FunctionNode.backward_accumulate.__code__:
# backward_accumulate is overridden
grad_inputs_tuple = tuple([
_pop_or_none(grad_inputs[func.inputs[i]])
for i in target_input_indexes
])
# Call backward_accumulate()
try:
gxs = func.backward_accumulate(target_input_indexes, grad_outputs,
grad_inputs_tuple)
except Exception as e:
_reraise_with_stack(func, e)
else: # otherwise, backward should be overridden
# Call backward()
try:
gxs = func.backward(target_input_indexes, grad_outputs)
except Exception as e:
_reraise_with_stack(func, e)
if is_debug:
for gx in gxs:
if not (gx is None or isinstance(gx, chainer.Variable)):
raise ValueError(
func._get_error_message(
'type of gradients returned from backward is '
'incorrect: '
'{} != expected {}'.format(type(gx),
chainer.Variable)))
len_gxs = len(gxs)
if len_gxs == len(func.inputs):
gxs = tuple([gxs[i] for i in target_input_indexes])
elif len_gxs != len(target_input_indexes):
msg = 'number of gradients returned from backward is incorrect: '
if len(func.inputs) == len(target_input_indexes):
msg += ('%s != expected %s' % (len_gxs, len(func.inputs)))
else:
msg += ('%s != expected %s or %s' %
(len_gxs, len(func.inputs), len(target_input_indexes)))
raise ValueError(func._get_error_message(msg))
for i, gx in six.moves.zip(target_input_indexes, gxs):
if gx is not None:
grad_inputs[func.inputs[i]].append(gx)
if is_debug:
node_x = func.inputs[i]
g_input_list = grad_inputs[node_x]
if gx.shape != node_x.shape:
raise ValueError(
func._get_error_message(
'shape of gradients returned from backward is '
'incorrect: '
'input-index={}, actual {} != expected {}'.format(
i, gx.shape, node_x.shape)))
if gx is not None and g_input_list:
g_input = g_input_list[0]
if gx.shape != g_input.shape:
raise ValueError(
func._get_error_message(
'shape of gradients returned from backward is '
'incorrect: '
'input-index={}, actual {} != expected {}'.
format(i, gx.shape, g_input.shape)))
if gx.dtype != g_input.dtype:
raise ValueError(
func._get_error_message(
'dtype of gradients returned from backward is '
'incorrect: '
'input-index={}, actual {} != expected {}'.
format(i, gx.dtype, g_input.dtype)))
del gxs
if is_debug:
# each grad is a list of variables
# iter_gxs expands it as a sequence of variables.
def iter_gxs(gxs):
for gx in gxs:
for gx_elem in gx:
yield gx_elem
for gx in iter_gxs(grad_inputs.values()):
if chainer.backend._contains_nan(gx.data):
raise RuntimeError(
'NaN is detected on backward computation of {}'.format(
func.label))
if not func.lazy_grad_sum:
for gx in grad_inputs.values():
_reduce(gx)
