def forward_cpu(self, inputs): self.retain_inputs((0, 1, 2)) x1, x2, gy = inputs sqnorm = x1 ** 2 + x2 ** 2 gx1 = utils.force_array(x2 / sqnorm * gy) gx2 = utils.force_array(-x1 / sqnorm * gy) return gx1, gx2
def check_anchor_target_creator( self, anchor_target_layer, bbox, anchor, img_size): xp = cuda.get_array_module(bbox) loc, label = self.anchor_target_layer( bbox, anchor, img_size) # Test types self.assertIsInstance(loc, xp.ndarray) self.assertIsInstance(label, xp.ndarray) # Test shapes self.assertEqual(loc.shape, (self.n_anchor, 4)) self.assertEqual(label.shape, (self.n_anchor,)) # Test dtype self.assertEqual(loc.dtype, np.float32) self.assertEqual(label.dtype, np.int32) # Test ratio of foreground and background labels np.testing.assert_equal( cuda.to_cpu(utils.force_array(xp.sum(label >= 0))), self.n_sample) n_pos = cuda.to_cpu(utils.force_array(xp.sum(label == 1))) n_neg = cuda.to_cpu(utils.force_array(xp.sum(label == 0))) self.assertLessEqual( n_pos, self.n_sample * self.pos_ratio) self.assertLessEqual(n_neg, self.n_sample - n_pos)
def forward_cpu(self, inputs): gy, = inputs gx1 = utils.force_array(numpy.where(self.cond, gy, gy.dtype.type(0))) gx2 = utils.force_array(numpy.where(self.cond, gy.dtype.type(0), gy)) return ( utils.sum_to(gx1, self.x1_shape), utils.sum_to(gx2, self.x2_shape))
def forward(self, inputs): self.retain_inputs((0, 1)) xp = cuda.get_array_module(*inputs) x, t = inputs self.ignore_mask = (t != self.ignore_label) # stable computation of the cross entropy. loss = -( self.ignore_mask * (x * (t - (x >= 0)) - xp.log1p(xp.exp(-xp.abs(x))))) if not self.reduce == 'mean': return utils.force_array(loss.astype(x.dtype)), if self.normalize: count = xp.maximum(1, self.ignore_mask.sum()) else: count = max(1, len(x)) self.count = count # TODO(takagi): Fix to perform division in a specific dtype. See # cupy/cupy#1534. return utils.force_array( xp.divide(xp.sum(loss), self.count), dtype=x.dtype),
def forward_cpu(self, inputs): self.retain_inputs((0, 1, 2, 3)) p, x, y, gz = inputs pg = p * gz return (utils.force_array((x - y) * gz), utils.force_array(pg), utils.force_array(gz - pg))
def log_prob(self, x): if isinstance(x, chainer.Variable): x = x.data x = x.astype(self.lam.dtype) xp1 = (x + 1).astype(self.lam.dtype) x, xp1 = utils.force_array(x), utils.force_array(xp1) return x * exponential.log(self.lam) - lgamma.lgamma(xp1) - self.lam
def backward_cpu(self, inputs, grads): p, x, y = inputs g = grads[0] pg = p * g return (utils.force_array((x - y) * g), utils.force_array(pg), utils.force_array(g - pg))
def backward(self, inputs, grads): x1, x2 = inputs gy, = grads gx = gy * 2 * self.difference gx = utils.force_array(gx, dtype=x1.dtype) gx_minus = utils.force_array(-gx, dtype=x1.dtype) return gx, gx_minus
def forward_cpu(self, inputs): self.retain_inputs((0, 1, 2)) x0, x1, gy = inputs one = x1.dtype.type(1) gx0 = utils.force_array(x1 * (x0 ** (x1 - one)) * gy) gx1 = utils.force_array(numpy.log(x0) * self.y * gy) return gx0, gx1
def forward(self, inputs): x1, x2 = inputs if self.check_on == 'forward_input': self._check_contiguousness(x1) self._check_contiguousness(x2) self.retain_inputs((0, 1)) y1, y2 = _forward_correct(x1, x2) return utils.force_array(y1), utils.force_array(y2)
def forward(self, inputs): device = self.device x1, x2 = inputs if device.xp is chainerx: fallback_device = device.fallback_device assert isinstance(x1, fallback_device.supported_array_types) assert isinstance(x2, fallback_device.supported_array_types) self.retain_inputs((0, 1)) y1, y2 = _forward_correct(x1, x2) return utils.force_array(y1), utils.force_array(y2)
def forward(self, inputs_and_grad_outputs): device = self.device x1, x2, gy1, gy2 = inputs_and_grad_outputs if device.xp is chainerx: fallback_device = device.fallback_device assert isinstance(gy1, fallback_device.supported_array_types) assert isinstance(gy2, fallback_device.supported_array_types) self.retain_inputs((0, 1, 2, 3)) ggx1, ggx2 = _backward_correct(x1, x2, gy1, gy2) return utils.force_array(ggx1), utils.force_array(ggx2)
def assert_allclose(x, y, atol=1e-5, rtol=1e-4, verbose=True): """Asserts if some corresponding element of x and y differs too much. This function can handle both CPU and GPU arrays simultaneously. Args: x: Left-hand-side array. y: Right-hand-side array. atol (float): Absolute tolerance. rtol (float): Relative tolerance. verbose (bool): If ``True``, it outputs verbose messages on error. """ x = backend.CpuDevice().send(utils.force_array(x)) y = backend.CpuDevice().send(utils.force_array(y)) try: numpy.testing.assert_allclose( x, y, atol=atol, rtol=rtol, verbose=verbose) except AssertionError as e: f = six.StringIO() f.write(str(e) + '\n\n') f.write( 'assert_allclose failed: \n' + ' shape: {} {}\n'.format(x.shape, y.shape) + ' dtype: {} {}\n'.format(x.dtype, y.dtype)) if x.shape == y.shape: xx = numpy.atleast_1d(x) yy = numpy.atleast_1d(y) err = numpy.abs(xx - yy) tol_err = atol + rtol * numpy.abs(yy).astype(numpy.float64) i = numpy.unravel_index( numpy.argmax(err.astype(numpy.float64) - tol_err), err.shape) if yy[i] == 0: rel_err = 'inf' else: rel_err = err[i] / numpy.abs(yy[i]) f.write( ' i: {}\n'.format(i) + ' x[i]: {}\n'.format(xx[i]) + ' y[i]: {}\n'.format(yy[i]) + ' relative error[i]: {}\n'.format(rel_err) + ' absolute error[i]: {}\n'.format(err[i])) opts = numpy.get_printoptions() try: numpy.set_printoptions(threshold=10000) f.write('x: ' + numpy.array2string(x, prefix='x: ') + '\n') f.write('y: ' + numpy.array2string(y, prefix='y: ') + '\n') finally: numpy.set_printoptions(**opts) raise AssertionError(f.getvalue())
def setUp_configure(self): from scipy import stats self.dist = distributions.Cauchy self.scipy_dist = stats.cauchy self.test_targets = set(["batch_shape", "cdf", "entropy", "event_shape", "icdf", "log_prob", "support"]) loc = utils.force_array( numpy.random.uniform(-1, 1, self.shape).astype(numpy.float32)) scale = utils.force_array(numpy.exp( numpy.random.uniform(-1, 1, self.shape)).astype(numpy.float32)) self.params = {"loc": loc, "scale": scale} self.scipy_params = {"loc": loc, "scale": scale}
def setUp_configure(self): from scipy import stats self.dist = distributions.Cauchy self.scipy_dist = stats.cauchy self.test_targets = set(['batch_shape', 'cdf', 'entropy', 'event_shape', 'icdf', 'log_prob', 'support']) loc = utils.force_array( numpy.random.uniform(-1, 1, self.shape).astype(numpy.float32)) scale = utils.force_array(numpy.exp( numpy.random.uniform(-1, 1, self.shape)).astype(numpy.float32)) self.params = {'loc': loc, 'scale': scale} self.scipy_params = {'loc': loc, 'scale': scale}
def forward_cpu(self, inputs): self.retain_inputs((0, 1)) x, gy = inputs gx = utils.force_array(numpy.sin(x)) numpy.negative(gx, out=gx) gx *= gy return gx,
def forward(self, inputs): xp = cuda.get_array_module(inputs[0]) self.input_length = inputs[0] label_length = inputs[1] t = inputs[2] xs = inputs[3:] if chainer.is_debug(): # Batch size check. assert len(xs[0]) == len(t) assert len(xs[0]) == len(self.input_length) assert len(xs[0]) == len(label_length) # Length check. assert len(xs) >= xp.max(self.input_length) assert len(t[0]) >= xp.max(label_length) self.path_length = 2 * label_length + 1 yseq_shape = (len(xs),) + xs[0].shape self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp) log_yseq = self.log_matrix(self.yseq, xp) self.path = _label_to_path(t, self.blank_symbol, xp) self.prob_trans = self.calc_trans( log_yseq, self.input_length, t, label_length, self.path, self.path_length, xp) loss = -_logsumexp(self.prob_trans[0], xp, axis=1) if self.reduce == 'mean': loss = utils.force_array(xp.mean(loss)) return loss,
def forward(self, inputs): self.retain_inputs((0,)) x, = inputs xp = cuda.get_array_module(x) norm = (xp.sqrt(xp.sum(xp.square(x), axis=self.axis, keepdims=True)) + x.dtype.type(self.eps)) return utils.force_array(x / norm),
def sign(x): """Elementwise sign function. For a given input :math:`x`, this function returns :math:`sgn(x)` defined as .. math:: sgn(x) = \\left \\{ \\begin{array}{cc} -1 & {\\rm if~x < 0} \\\\ 0 & {\\rm if~x = 0} \\\\ 1 & {\\rm if~x > 0} \\\\ \\end{array} \\right. .. note:: The gradient of this function is ``None`` everywhere and therefore unchains the computational graph. Args: x (~chainer.Variable): Input variable for which the sign is computed. Returns: ~chainer.Variable: Output variable. """ if isinstance(x, chainer.variable.Variable): x = x.array xp = backend.get_array_module(x) return chainer.as_variable(utils.force_array(xp.sign(x)))
def forward(self, inputs): self.retain_inputs((0, 1)) sp, dn = inputs c = _coo_matmul(sp, self.sp_row, self.sp_col, self.sp_shape, self.sp_order, dn, self.transa, self.transb, self.transc, self.dtype) return utils.force_array(c, self.dtype),
def forward(self, inputs): self.retain_inputs((0, 1)) xp = backend.get_array_module(*inputs) x1, x2 = inputs difference = x1 - x2 y = xp.square(difference) return utils.force_array(y, dtype=x1.dtype),
def forward_cpu(self, inputs): x = inputs[0] half = x.dtype.type(0.5) y = utils.force_array(numpy.tanh(x * half) * half + half) self.retain_outputs((0,)) self._use_cudnn = False return y,
def forward_cpu(self, x): self.retain_outputs((0,)) try: invx = utils.force_array(numpy.linalg.inv(x[0])) except numpy.linalg.LinAlgError: raise ValueError('Input has singular matrices.') return invx,
def forward(self, inputs): xp = backend.get_array_module(inputs[0]) self.input_length, label_length, t, xs = inputs if self.zero_padding is None: if xs.dtype == numpy.float16: self.zero_padding = -10000.0 else: self.zero_padding = -10000000000.0 if chainer.is_debug(): assert len(xs) >= xp.max(self.input_length) assert t.shape[1] >= xp.max(label_length) self.path_length = 2 * label_length + 1 self.yseq = _softmax(xs, xp) log_yseq = self.log_matrix(self.yseq, xp) self.path = _label_to_path(t, self.blank_symbol, xp) self.prob_trans = self.calc_trans( log_yseq, self.input_length, t, label_length, self.path, self.path_length, xp) loss = -_logsumexp(self.prob_trans[0], xp, axis=1) if self.reduce == 'mean': loss = utils.force_array(xp.mean(loss)) return loss,
def forward_cpu(self, inputs): x, = inputs # y = log(1 + exp(beta * x)) / beta bx = self.beta * x y = (numpy.fmax(bx, 0) + numpy.log1p(numpy.exp(-numpy.fabs(bx)))) * self.beta_inv return utils.force_array(y, x.dtype),
def test_0dim_array(self): x = utils.force_array(numpy.array(1, numpy.float32), dtype=self.dtype) self.assertIsInstance(x, numpy.ndarray) if self.dtype is None: self.assertEqual(x.dtype, numpy.float32) else: self.assertEqual(x.dtype, self.dtype)
def forward(self, inputs): self.retain_inputs((0, 1)) a, b = inputs if self.a_axes is None or self.b_axes is None: a_axes = [[], []] # 0:row axes, 1:col axes b_axes = [[], []] # 0:row axes, 1:col axes axes = self.axes if isinstance(axes, collections.Sequence): a_axes[1], b_axes[0] = axes if numpy.isscalar(a_axes[1]): a_axes[1] = a_axes[1], if numpy.isscalar(b_axes[0]): b_axes[0] = b_axes[0], else: a_axes[1] = six.moves.range(a.ndim - axes, a.ndim) b_axes[0] = six.moves.range(axes) a_range = six.moves.range(a.ndim) a_axes[0] = [i for i in a_range if i not in a_axes[1]] b_range = six.moves.range(b.ndim) b_axes[1] = [i for i in b_range if i not in b_axes[0]] self.a_axes = a_axes self.b_axes = b_axes c = _tensordot(a, b, self.a_axes, self.b_axes, self.c_axes) if self.c_axes is None: c_axes = [[], []] # 0:row axes, 1:col axes c_row_ndim = len(self.a_axes[0]) c_col_ndim = len(self.b_axes[1]) c_axes[0] = six.moves.range(c_row_ndim) c_axes[1] = six.moves.range(c_row_ndim, c_row_ndim + c_col_ndim) self.c_axes = c_axes return utils.force_array(c, self.dtype),
def forward_cpu(self, x): if not available_cpu: raise ImportError('SciPy is not available. Forward computation' ' of erfinv in CPU can not be done.' + str(_import_error)) self.retain_outputs((0,)) return utils.force_array(special.erfinv(x[0]), dtype=x[0].dtype),
def forward(self, inputs): self.retain_inputs((0, 1)) a, b = inputs c = _coo_matmul_gradsp(a, b, self.sp_row, self.sp_col, self.sp_shape, self.transa, self.transb, self.transc, self.dtype) return utils.force_array(c),
def backward(self, x, gy): xp = cuda.get_array_module(*x) gx = utils.force_array(xp.cos(x[0])) xp.square(gx, out=gx) xp.reciprocal(gx, out=gx) gx *= gy[0] return gx,
def forward_cpu(self, x): self.retain_inputs((0,)) return utils.force_array(numpy.log(x[0])),
def backward(self, indexes, gy): x = self.get_retained_inputs()[0] return utils.force_array(gy[0] / x),
def forward_cpu(self, inputs): self.retain_inputs((0, 1)) y, gy = inputs one = y.dtype.type(1) return utils.force_array(gy * (one - y * y)),
def backward_cpu(self, x, gy): return utils.force_array(gy[0] * (x[0] > 0)),
def forward(self, inputs): self.retain_inputs((0,)) x = inputs[0] xp = cuda.get_array_module(x) return utils.force_array(xp.log2(x)),
def forward(self, inputs): x, t = inputs xp = cuda.get_array_module(x) loss = -xp.mean(t * xp.log2(x + 1e-6) + (1 - t) * xp.log2((1 - x) + 1e-6)) return force_array(loss),
def forward_cpu(self, x): self.retain_outputs((0, )) return utils.force_array(numpy.expm1(x[0])),
def backward_cpu(self, x, gy): gx = utils.force_array(numpy.cos(x[0])) gx *= gy[0] return gx,
def backward(self, retain_grad=False): """Runs error backpropagation (a.k.a. backprop) from this variable. On backprop, :meth:`Function.backward` is called on each :class:`Function` object appearing in the backward graph starting from this variable. The backward graph is represented by backward references from variables to their creators, and from functions to their inputs. The backprop stops at all root variables. Some functions set ``None`` as gradients of some inputs, where further backprop does not take place at such input variables. This method uses :data:`grad` as the initial error array. User can manually set a gradient array before calling this method. If :data:`data` contains only one element (i.e., it is scalar) and :data:`grad` is ``None``, then this method automatically complements 1.0 as the initial error. This is useful on starting backprop from some scalar loss value. Args: retain_grad (bool): If ``True``, the gradient arrays of all intermediate variables are kept. Otherwise, :data:`grad` of the intermediate variables are set to ``None`` on appropriate timing, which may reduce the maximum memory consumption. In most cases of training some models, the purpose of backprop is to compute gradients of parameters, not of variables, so it is recommended to set this flag ``False``. """ if self.creator is None: return initial_device = None if cuda.available and isinstance(self.data, cuda.cupy.ndarray): try: initial_device = cuda.Device() except cuda.cupy.cuda.runtime.CUDARuntimeError as e: if e.status != 38: # cudaErrorNoDevice raise is_debug = chainer.is_debug() cand_funcs = [] seen_set = set() seen_vars = set() need_copy = set() # Initialize error by 1, if this is a loss variable if self.data.size == 1 and self.grad is None: with cuda.get_device(self.data) as device: if device is cuda.DummyDevice: self.grad = numpy.ones_like(self.data) else: self.grad = cuda.cupy.ones_like(self.data) def add_cand(cand): if cand not in seen_set: # Negate since heapq is min-heap heapq.heappush(cand_funcs, (-cand.rank, len(seen_set), cand)) seen_set.add(cand) add_cand(self.creator) while cand_funcs: _, _, func = heapq.heappop(cand_funcs) outputs = [y() for y in func.outputs] # access via weak ref in_data = tuple([x.data for x in func.inputs]) out_grad = tuple([None if y is None else y.grad for y in outputs]) hooks = chainer.get_function_hooks() if func._n_local_function_hooks != 0: hooks = collections.OrderedDict(hooks) hooks.update(func.local_function_hooks) cuda.get_device(*(in_data + out_grad)).use() for hook in six.itervalues(hooks): hook.backward_preprocess(func, in_data, out_grad) gxs = func.backward(in_data, out_grad) assert len(gxs) == len(in_data) for hook in six.itervalues(hooks): hook.backward_postprocess(func, in_data, out_grad) if is_debug: for gx in gxs: if gx is None: continue cuda.get_device(gx).use() if cuda.get_array_module(gx).isnan(gx).any(): msg = 'NaN is detected on backward computation' raise RuntimeError(msg) if not retain_grad: for y in outputs: if y is not None and y is not self: y.grad = None for x, gx in zip(func.inputs, gxs): if gx is None: continue _check_grad_type(func, x, gx) # Accumulate the gradient to x. It is a bit tricky to handle # branches and parameter gradient accumulation correctly. id_x = id(x) if x.creator is None: # leaf if x._grad is None: x.grad = gx need_copy.add(id_x) else: cuda.get_device(gx).use() if id_x in need_copy: x.grad = utils.force_array(x.grad + gx) # copy need_copy.remove(id_x) else: x._grad += gx else: # not a leaf add_cand(x.creator) if id_x not in seen_vars: # 1st visit x.grad = gx seen_vars.add(id_x) need_copy.add(id_x) else: cuda.get_device(gx).use() if id_x in need_copy: # 2nd visit x._grad = utils.force_array(gx + x._grad) # copied need_copy.remove(id_x) else: # 3rd or later visit x._grad += gx del gxs # to reduce memory usage if initial_device is not None: initial_device.use()
def forward_cpu(self, x): return utils.force_array(numpy.maximum(x[0], 0, dtype=x[0].dtype)),
def forward(self, x): xp = cuda.get_array_module(*x) return utils.force_array(xp.floor(x[0]), x[0].dtype),
def backward_cpu(self, x, gy): gx = utils.force_array(numpy.sin(x[0])) numpy.negative(gx, out=gx) gx *= gy[0] return gx,
def forward_cpu(self, x): self.invx = utils.force_array(numpy.linalg.inv(x[0])) return self.invx,
def forward(self, x): xp = cuda.get_array_module(*x) return utils.force_array(xp.log10(x[0])),
def forward_cpu(self, x): zero = utils.force_type(x[0].dtype, 0) return utils.force_array(numpy.maximum(zero, x[0])),
def forward(self, x): self.retain_outputs((0, )) xp = backend.get_array_module(*x) return utils.force_array(xp.sqrt(x[0], dtype=x[0].dtype)),
def forward_cpu(self, x): y = utils.force_array(numpy.tanh(x[0])) self.retain_outputs((0, )) self._use_cudnn = False return y,
def forward_gpu(self, inputs): self.retain_outputs((0, )) x, = inputs out = cuda.cupyx.rsqrt(x, dtype=x.dtype) return utils.force_array(out),