def copyto(dst, src): """Copies the elements of an ndarray to those of another one. This function can copy the CPU/GPU arrays to the destination arrays on another device. Args: dst (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`): Destination array. src (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`): Source array. """ if isinstance(dst, numpy.ndarray): numpy.copyto(dst, _cpu._to_cpu(src)) elif isinstance(dst, intel64.mdarray): intel64.ideep.basic_copyto(dst, _cpu._to_cpu(src)) elif isinstance(dst, cuda.ndarray): if isinstance(src, chainer.get_cpu_array_types()): src = numpy.asarray(src) if dst.flags.c_contiguous or dst.flags.f_contiguous: dst.set(src) else: cuda.cupy.copyto(dst, cuda.to_gpu(src, device=dst.device)) elif isinstance(src, cuda.ndarray): cuda.cupy.copyto(dst, src) else: raise TypeError( 'cannot copy from non-array object of type {}'.format( type(src))) else: raise TypeError('cannot copy to non-array object of type {}'.format( type(dst)))
def test_evaluate(self, backend_config): data = backend_config.get_array(self.data) batches = [backend_config.get_array(b) for b in self.batches] device = backend_config.device iterator, converter, target, evaluator = (self.prepare( data, batches, device)) reporter = chainer.Reporter() reporter.add_observer('target', target) with reporter: mean = evaluator.evaluate() # The converter gets results of the iterator and the device number. self.assertEqual(len(converter.args), len(data)) if backend_config.use_cuda: expected_device_arg = backend_config.cuda_device else: expected_device_arg = -1 for i in range(len(data)): numpy.testing.assert_array_equal( _cpu._to_cpu(converter.args[i]['batch']), self.data[i]) self.assertEqual(converter.args[i]['device'], expected_device_arg) # The model gets results of converter. self.assertEqual(len(target.args), len(batches)) for i in range(len(batches)): numpy.testing.assert_array_equal(_cpu._to_cpu(target.args[i]), self.batches[i]) expect_mean = numpy.mean([numpy.sum(x) for x in self.batches]) self.assertAlmostEqual(_cpu._to_cpu(mean['target/loss']), expect_mean, places=4)
def test_evaluate(self, backend_config): data = backend_config.get_array(self.data) batches = [backend_config.get_array(b) for b in self.batches] device = backend_config.device iterator, converter, target, evaluator = ( self.prepare(data, batches, device)) reporter = chainer.Reporter() reporter.add_observer('target', target) with reporter: mean = evaluator.evaluate() # The converter gets results of the iterator and the device number. self.assertEqual(len(converter.args), len(data)) if backend_config.use_cuda: expected_device_arg = backend_config.cuda_device else: expected_device_arg = -1 for i in range(len(data)): numpy.testing.assert_array_equal( _cpu._to_cpu(converter.args[i]['batch']), self.data[i]) self.assertEqual(converter.args[i]['device'], expected_device_arg) # The model gets results of converter. self.assertEqual(len(target.args), len(batches)) for i in range(len(batches)): numpy.testing.assert_array_equal( _cpu._to_cpu(target.args[i]), self.batches[i]) expect_mean = numpy.mean([numpy.sum(x) for x in self.batches]) self.assertAlmostEqual( _cpu._to_cpu(mean['target/loss']), expect_mean, places=4)
def copyto(dst, src): """Copies the elements of an ndarray to those of another one. This function can copy the CPU/GPU arrays to the destination arrays on another device. Args: dst (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`): Destination array. src (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`): Source array. """ if isinstance(dst, numpy.ndarray): numpy.copyto(dst, _cpu._to_cpu(src)) elif isinstance(dst, intel64.mdarray): intel64.ideep.basic_copyto( dst, _cpu._to_cpu(src)) elif isinstance(dst, cuda.ndarray): if isinstance(src, chainer.get_cpu_array_types()): src = numpy.asarray(src) if dst.flags.c_contiguous or dst.flags.f_contiguous: dst.set(src) else: cuda.cupy.copyto(dst, cuda.to_gpu(src, device=dst.device)) elif isinstance(src, cuda.ndarray): cuda.cupy.copyto(dst, src) else: raise TypeError('cannot copy from non-array object of type {}' .format(type(src))) else: raise TypeError('cannot copy to non-array object of type {}'.format( type(dst)))
def check_u_updated_in_train(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) y1 = layer(x).array u1 = getattr(layer, hook.vector_name).copy() y2 = layer(x).array u2 = getattr(layer, hook.vector_name) y1, y2 = _cpu._to_cpu(y1), _cpu._to_cpu(y2) u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2) assert not numpy.array_equal(u1, u2) assert not numpy.array_equal(y1, y2)
def check_u_updated_in_train(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) y1 = layer(x).array u1 = getattr(layer, hook.vector_name).copy() y2 = layer(x).array u2 = getattr(layer, hook.vector_name) y1, y2 = _cpu._to_cpu(y1), _cpu._to_cpu(y2) u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2) assert not numpy.array_equal(u1, u2) assert not numpy.array_equal(y1, y2)
def check_deleted(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) with chainer.using_device(backend_config.device): y1 = layer(x).array with chainer.using_config('train', False): y2 = layer(x).array layer.delete_hook(hook.name) assert not hasattr(layer, hook.vector_name) y3 = layer(x).array y1, y2, y3 = _cpu._to_cpu(y1), _cpu._to_cpu(y2), _cpu._to_cpu(y3) assert not numpy.array_equal(y1, y3) assert not numpy.array_equal(y2, y3)
def check_deleted(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) with chainer.using_device(backend_config.device): y1 = layer(x).array with chainer.using_config('train', False): y2 = layer(x).array layer.delete_hook(hook.name) assert not hasattr(layer, hook.vector_name) y3 = layer(x).array y1, y2, y3 = _cpu._to_cpu(y1), _cpu._to_cpu(y2), _cpu._to_cpu(y3) assert not numpy.array_equal(y1, y3) assert not numpy.array_equal(y2, y3)
def check_in_recomputing(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) y1 = layer(x).array u1 = getattr(layer, hook.vector_name).copy() v1 = hook.v.copy() with chainer.using_config('in_recomputing', True): y2 = layer(x).array u2 = getattr(layer, hook.vector_name) v2 = hook.v u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2) v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2) numpy.testing.assert_array_equal(u1, u2) numpy.testing.assert_array_equal(v1, v2) testing.assert_allclose(y1, y2)
def check_in_recomputing(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) y1 = layer(x).array u1 = getattr(layer, hook.vector_name).copy() v1 = hook.v.copy() with chainer.using_config('in_recomputing', True): y2 = layer(x).array u2 = getattr(layer, hook.vector_name) v2 = hook.v u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2) v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2) numpy.testing.assert_array_equal(u1, u2) numpy.testing.assert_array_equal(v1, v2) testing.assert_allclose(y1, y2)
def check_u_not_updated_in_test(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) with chainer.using_config('train', False): y1 = layer(x).array u1 = getattr(layer, hook.vector_name).copy() v1 = hook.v.copy() y2 = layer(x).array u2 = getattr(layer, hook.vector_name) v2 = hook.v.copy() u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2) v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2) numpy.testing.assert_array_equal(u1, u2) numpy.testing.assert_array_equal(v1, v2) testing.assert_allclose(y1, y2)
def check_u_not_updated_in_test(self, backend_config): layer, hook = self.layer, self.hook layer.add_hook(hook) layer.to_device(backend_config.device) x = backend_config.get_array(self.x) with chainer.using_config('train', False): y1 = layer(x).array u1 = getattr(layer, hook.vector_name).copy() v1 = hook.v.copy() y2 = layer(x).array u2 = getattr(layer, hook.vector_name) v2 = hook.v.copy() u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2) v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2) numpy.testing.assert_array_equal(u1, u2) numpy.testing.assert_array_equal(v1, v2) testing.assert_allclose(y1, y2)
def check_serialization(self, backend_config): with utils.tempdir() as root: filename = os.path.join(root, 'tmp.npz') layer1 = self.layer.copy('copy') hook1 = copy.deepcopy(self.hook) layer1.add_hook(hook1) layer1.to_device(backend_config.device) x = backend_config.get_array(self.x) with backend_config: layer1(x) with chainer.using_config('train', False): y1 = layer1(x) serializers.save_npz(filename, layer1) layer2 = self.layer.copy('copy') hook2 = copy.deepcopy(self.hook) layer2.add_hook(hook2) # Test loading is nice. msg = None try: serializers.load_npz(filename, layer2) except Exception as e: msg = e assert msg is None with chainer.using_config('train', False): y2 = layer2(self.x.copy()) # Test attributes are the same. orig_weight = _cpu._to_cpu( getattr(layer1, hook1.weight_name).array) orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name)) numpy.testing.assert_array_equal( orig_weight, getattr(layer2, hook2.weight_name).array) numpy.testing.assert_array_equal( orig_vector, getattr(layer2, hook2.vector_name)) testing.assert_allclose(y1.array, y2.array)
def check_serialization(self, backend_config): with utils.tempdir() as root: filename = os.path.join(root, 'tmp.npz') layer1 = self.layer.copy('copy') hook1 = copy.deepcopy(self.hook) layer1.add_hook(hook1) layer1.to_device(backend_config.device) x = backend_config.get_array(self.x) with backend_config: layer1(x) with chainer.using_config('train', False): y1 = layer1(x) serializers.save_npz(filename, layer1) layer2 = self.layer.copy('copy') hook2 = copy.deepcopy(self.hook) layer2.add_hook(hook2) # Test loading is nice. msg = None try: serializers.load_npz(filename, layer2) except Exception as e: msg = e assert msg is None with chainer.using_config('train', False): y2 = layer2(self.x.copy()) # Test attributes are the same. orig_weight = _cpu._to_cpu( getattr(layer1, hook1.weight_name).array) orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name)) numpy.testing.assert_array_equal( orig_weight, getattr(layer2, hook2.weight_name).array) numpy.testing.assert_array_equal( orig_vector, getattr(layer2, hook2.vector_name)) testing.assert_allclose(y1.array, y2.array)
def __call__(self, key, value): if value is None: # use Empty to represent None if h5py.version.version_tuple < (2, 7, 0): raise RuntimeError( 'h5py>=2.7.0 is required to serialize None.') arr = h5py.Empty('f') compression = None else: arr = _cpu._to_cpu(value) compression = None if arr.size <= 1 else self.compression self.group.create_dataset(key, data=arr, compression=compression) return value
def __call__(self, key, value): if value is None: # use Empty to represent None if h5py.version.version_tuple < (2, 7, 0): raise RuntimeError( 'h5py>=2.7.0 is required to serialize None.') arr = h5py.Empty('f') compression = None else: arr = _cpu._to_cpu(value) compression = None if arr.size <= 1 else self.compression self.group.create_dataset(key, data=arr, compression=compression) return value
def _array_to_chainerx(array, device=None): # If device is None, appropriate device is chosen according to the input # arrays. assert device is None or isinstance(device, chainerx.Device) if array is None: return None if array.dtype not in chainerx.all_dtypes: raise TypeError( 'Dtype {} is not supported in ChainerX.'.format(array.dtype.name)) if isinstance(array, chainerx.ndarray): if device is None: return array if device is array.device: return array return array.to_device(device) if isinstance(array, numpy.ndarray): if device is None: device = chainerx.get_device('native', 0) return chainerx.array(array, device=device, copy=False) if isinstance(array, cuda.ndarray): if device is None: device = chainerx.get_device('cuda', array.device.id) elif device.backend.name != 'cuda': # cupy to non-cuda backend # TODO(niboshi): Remove conversion to numpy when both CuPy and # ChainerX support the array interface. array = _cpu._to_cpu(array) return chainerx.array(array, device=device, copy=False) elif device.index != array.device.id: # cupy to cuda backend but different device array = cuda.to_gpu(array, device=device.index) # cupy to cuda backend with the same device return chainerx._core._fromrawpointer( array.data.mem.ptr, array.shape, array.dtype, array.strides, device, array.data.ptr - array.data.mem.ptr, array) if isinstance(array, intel64.mdarray): return _array_to_chainerx(numpy.array(array), device) if numpy.isscalar(array): return chainerx.asarray(array) raise TypeError( 'Array cannot be converted into chainerx.ndarray' '\nActual type: {0}.'.format(type(array)))
def send_array(self, array): if isinstance(array, ideep.mdarray): return array if not isinstance(array, numpy.ndarray): array = _cpu._to_cpu(array) # to numpy.ndarray if (isinstance(array, numpy.ndarray) and array.ndim in (1, 2, 4) and 0 not in array.shape): # TODO(kmaehashi): Remove ndim validation once iDeep has fixed. # Currently iDeep only supports (1, 2, 4)-dim arrays. # Note that array returned from `ideep.array` may not be an # iDeep mdarray, e.g., when the dtype is not float32. array = ideep.array(array, itype=ideep.wgt_array) return array
def send_array(self, array): if isinstance(array, ideep.mdarray): return array if not isinstance(array, numpy.ndarray): array = _cpu._to_cpu(array) # to numpy.ndarray if (isinstance(array, numpy.ndarray) and array.ndim in (1, 2, 4) and 0 not in array.shape): # TODO(kmaehashi): Remove ndim validation once iDeep has fixed. # Currently iDeep only supports (1, 2, 4)-dim arrays. # Note that array returned from `ideep.array` may not be an # iDeep mdarray, e.g., when the dtype is not float32. array = ideep.array(array, itype=ideep.wgt_array) return array
def _array_from_chainerx(array): if array is None: return None if not isinstance(array, chainerx.ndarray): if isinstance(array, chainer.get_array_types()): return array raise TypeError( 'Tried to convert to a non-ChainerX array from an invalid type: ' '{}'.format(type(array))) backend_name = array.device.backend.name if backend_name == 'native': return _cpu._to_cpu(array) if backend_name == 'cuda': return cuda.to_gpu(array, array.device.index) raise ValueError( 'Only ChainerX arrays with native or cuda backends can be converted ' 'to non-ChainerX arrays.\nActual: {0}.'.format(backend_name))
def _array_from_chainerx(array): if array is None: return None if not isinstance(array, chainerx.ndarray): if isinstance(array, chainer.get_array_types()): return array raise TypeError( 'Tried to convert to a non-ChainerX array from an invalid type: ' '{}'.format(type(array))) backend_name = array.device.backend.name if backend_name == 'native': return _cpu._to_cpu(array) if backend_name == 'cuda': return cuda.to_gpu(array, array.device.index) raise ValueError( 'Only ChainerX arrays with native or cuda backends can be converted ' 'to non-ChainerX arrays.\nActual: {0}.'.format(backend_name))
def iterate_single_input(i_in, x, orig_x, x_ind): orig = orig_x[x_ind] # `yss` holds a list of output arrays for each of 2 or 5 sampling # points. if detect_nondifferentiable: yss = [ eval_func(x, x_ind, -eps * 1., orig), eval_func(x, x_ind, -eps * .5, orig), ys0, eval_func(x, x_ind, +eps * .5, orig), eval_func(x, x_ind, +eps * 1., orig), ] else: yss = [ eval_func(x, x_ind, -eps * 1, orig), eval_func(x, x_ind, +eps * 1, orig), ] assert all([ y is None or (y.shape == yss[0][i].shape and y.dtype == yss[0][i].dtype) for ys in yss for i, y in enumerate(ys) ]) # If all the outputs are 0-size, skip non-differentiable check. if all([y is None or y.size == 0 for y in yss[0]]): detect_nondifferentiable_ = False else: detect_nondifferentiable_ = detect_nondifferentiable if detect_nondifferentiable_: # Detect non-differentiable point by quadratic fitting # Check for non-finite output. # If any single element in the output arrays has different # finiteness among sampled points, that means this is a # non-differentiable point. # If the function consistently generates non-finite values # around the point, we do not treat the point as # non-differentiable. # (Example: x<0 region for the logarithm function) any_nonfinite = False for i_out in range(nout): isfinites = [xp.isfinite(ys[i_out]) for ys in yss] if any((isfinites[0] != isfinites[i]).any() for i in range(1, len(yss))): s = six.StringIO() s.write('Tried to compute the numeric gradient on a ' 'non-differentiable point.\n\n') s.write('i_in: {}\n'.format(i_in)) s.write('i_out: {}\n'.format(i_out)) s.write('x: {}\n'.format(inputs[i_in])) s.write('index on x: {}\n'.format(x_ind)) s.write('eps: {}\n'.format(eps)) s.write('y[x-eps ]: {}\n'.format(yss[0][i_out])) s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out])) s.write('y[x ]: {}\n'.format(yss[2][i_out])) s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out])) s.write('y[x+eps ]: {}\n'.format(yss[4][i_out])) raise NondifferentiableError(s.getvalue()) any_nonfinite |= not all((_).all() for _ in isfinites) if not any_nonfinite: # Stack flattened outputs to make (5, *)-shaped 2D array ystack = xp.vstack( [xp.hstack([y.ravel() for y in ys]) for ys in yss]) assert ystack.ndim == 2 and ystack.shape[0] == len(yss) # Fit to quadratic if xp is not numpy: ystack = _cpu._to_cpu(ystack) polyfit = numpy.polynomial.polynomial.polyfit _, (residuals, _, _, _) = polyfit(range(len(yss)), ystack, deg=2, full=True) if xp is not numpy: residuals = device.send(residuals) residuals = xp.sqrt(residuals / len(yss)) # Check for error for each output array for i_out in range(nout): size = sizes[i_out] cumsize = cumsizes[i_out] shape = shapes[i_out] # TODO(niboshi): The following two lines could be # rewritten using xp.stack, which is supported in # NumPy>=1.10 ymax = xp.concatenate([ys[i_out][None] for ys in yss]).max(axis=0) ymin = xp.concatenate([ys[i_out][None] for ys in yss]).min(axis=0) # Restore the shape of flattened residual res = residuals[cumsize - size:cumsize] res = res.reshape(shape) det = utils.force_array(diff_atol + diff_rtol * (ymax - ymin) < res) # Constant output = not nondifferentiable det[ymax == ymin] = False if det.any(): s = six.StringIO() s.write('Tried to compute the numeric gradient on a ' 'non-differentiable point.\n\n') s.write('i_in: {}\n'.format(i_in)) s.write('i_out: {}\n'.format(i_out)) s.write('x: {}\n'.format(inputs[i_in])) s.write('index on x: {}\n'.format(x_ind)) s.write('eps: {}\n'.format(eps)) s.write('diff_rtol: {}\n'.format(diff_rtol)) s.write('diff_atol: {}\n'.format(diff_atol)) s.write('ymax: {}\n'.format(ymax)) s.write('ymin: {}\n'.format(ymin)) s.write( 'diff_atol + diff_rtol * (ymax-ymin): {}\n'.format( diff_atol + diff_rtol * (ymax - ymin))) s.write('fitting errors: {}\n'.format(res)) s.write('y[x-eps ]: {}\n'.format(yss[0][i_out])) s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out])) s.write('y[x ]: {}\n'.format(yss[2][i_out])) s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out])) s.write('y[x+eps ]: {}\n'.format(yss[4][i_out])) raise NondifferentiableError(s.getvalue()) # Calculate numerical gradient for i_out, gy in enumerate(grad_outputs): if gy is None: continue if not numpy.isscalar(gy): gy = gy.astype(numpy.float64, copy=False) gpu_ = (xp is cuda.cupy and all(isinstance(ys[i_out], cuda.ndarray) for ys in yss)) # If any output sample is None, all others must be. assert all([(yss[0][i_out] is None) == (yss[j][i_out] is None) for j in range(len(yss))]) # If outputs samples are None, the part of numeric gradient for # this output is considered as zero: skip the accumulation. if yss[0][i_out] is None: continue if len(yss) == 2: # 1st order y0 = yss[0][i_out] y1 = yss[1][i_out] if gpu_: numerical_grad_kernel_1(y1, y0, xp.asarray(gy), eps, gx[x_ind]) else: dot = ((y1 - y0) * gy).sum() gx[x_ind] = gx[x_ind] + dot / (2 * eps) elif len(yss) == 5: # 3rd order y0 = yss[0][i_out] y1 = yss[1][i_out] y2 = yss[3][i_out] y3 = yss[4][i_out] if gpu_: numerical_grad_kernel_3(y3, y2, y1, y0, gy, eps, gx[x_ind]) else: num = -y3 + 8 * y2 - 8 * y1 + y0 dot = (num * gy).sum() gx[x_ind] = gx[x_ind] + dot / (6 * eps) else: assert False
def __call__(self, key, value): key = key.lstrip('/') self.target[self.path + key] = (_cpu._to_cpu(value) if value is not None else numpy.asarray(None)) return value
def generate_array(initializer, shape, xp, dtype=None, device=None): # type: (types.AbstractInitializer, types.ShapeSpec, types.Xp, types.DTypeSpec, types.DeviceSpec) -> types.NdArray # NOQA """Return initialized array. The algorithms used to make the new values depend on the concrete derived classes. If the initializer has the ``dtype`` attribute, it is used to construct the array. Otherwise, ``chainer.config.dtype`` is used instead. See :ref:`configuration` for the dtype config. Args: initializer: A callable object that takes :ref:`ndarray` and edits its value. shape (tuple): Shape of a return array. xp (module): :mod:`cupy`, :mod:`numpy`, or :mod:`chainerx`. dtype: Dtype specifier. If omitted, ``initializer.dtype`` is used. device: Target device specifier. If omitted, the current device is used for :mod:`cupy`, and the default device is used for :mod:`chainerx`. Returns: :ref:`ndarray`: An initialized array. """ dtype_attr = getattr(initializer, 'dtype', None) if dtype is not None and dtype_attr is not None \ and numpy.dtype(dtype) != numpy.dtype(dtype_attr): raise ValueError( 'dtype mismatch: {} != {}'.format(dtype, dtype_attr)) if dtype is None: dtype = dtype_attr dtype = chainer.get_dtype(dtype) if device is None: backend_device = backend._guess_device_from_array_module(xp) else: backend_device = chainer.get_device(device) if xp != backend_device.xp: raise ValueError('xp and device arguments are inconsistent.') if xp is chainerx: # Initialize with NumPy/CuPy array that shares memory with the # ChainerX array. # TODO(sonots): Directly use initializer after ChainerX # supports random. chx_device = backend_device.device # type: ignore # TODO(okapies): remove 'type: ignore' when chainerx implements sequence support for empty() # NOQA array = chainerx.empty(shape, dtype=dtype, device=chx_device) # type: ignore # NOQA if chx_device.backend.name == 'native': temp_array = _cpu._to_cpu(array) temp_device = cuda.DummyDevice # type: cuda.Device elif chx_device.backend.name == 'cuda': temp_array = cuda.to_gpu(array, chx_device.index) temp_device = cuda.Device(chx_device.index) else: raise RuntimeError('ChainerX backend: {} is not supported.'.format( chx_device.backend.name)) with temp_device: initializer(temp_array) return array with chainer.using_device(backend_device): array = xp.empty(shape, dtype=dtype) initializer(array) return array
def iterate_single_input(i_in, x, orig_x, i): orig = orig_x[i] # `yss` holds a list of output arrays for each of 2 or 5 sampling # points. if detect_nondifferentiable: yss = [ eval_func(x, i, -eps * 1., orig), eval_func(x, i, -eps * .5, orig), ys0, eval_func(x, i, +eps * .5, orig), eval_func(x, i, +eps * 1., orig), ] else: yss = [ eval_func(x, i, -eps * 1, orig), eval_func(x, i, +eps * 1, orig), ] if detect_nondifferentiable: # Detect non-differentiable point by quadratic fitting # Check for non-finite output. # If any single element in the output arrays has different # finiteness among sampled points, that means this is a # non-differentiable point. # If the function consistently generates non-finite values # around the point, we do not treat the point as # non-differentiable. # (Example: x<0 region for the logarithm function) any_nonfinite = False for i_out in range(nout): isfinites = [xp.isfinite(ys[i_out]) for ys in yss] if any((isfinites[0] != isfinites[i]).any() for i in range(1, len(yss))): s = six.StringIO() s.write( 'Tried to compute the numeric gradient on a ' 'non-differentiable point.\n\n') s.write('i_in: {}\n'.format(i_in)) s.write('i_out: {}\n'.format(i_out)) s.write('x: {}\n'.format(inputs[i_in])) s.write('index on x: {}\n'.format(i)) s.write('eps: {}\n'.format(eps)) s.write('y[x-eps ]: {}\n'.format(yss[0][i_out])) s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out])) s.write('y[x ]: {}\n'.format(yss[2][i_out])) s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out])) s.write('y[x+eps ]: {}\n'.format(yss[4][i_out])) raise NondifferentiableError(s.getvalue()) any_nonfinite |= not all((_).all() for _ in isfinites) if not any_nonfinite: # Stack flattened outputs to make (5, *)-shaped 2D array ystack = xp.vstack( [xp.hstack([y.ravel() for y in ys]) for ys in yss]) assert ystack.ndim == 2 and ystack.shape[0] == len(yss) # Fit to quadratic if xp is not numpy: ystack = _cpu._to_cpu(ystack) polyfit = numpy.polynomial.polynomial.polyfit _, (residuals, _, _, _) = polyfit( range(len(yss)), ystack, deg=2, full=True) if xp is not numpy: residuals = device.send(residuals) residuals = xp.sqrt(residuals / len(yss)) # Check for error for each output array for i_out in range(nout): size = sizes[i_out] cumsize = cumsizes[i_out] shape = shapes[i_out] # TODO(niboshi): The following two lines could be # rewritten using xp.stack, which is supported in # NumPy>=1.10 ymax = xp.concatenate( [ys[i_out][None] for ys in yss]).max(axis=0) ymin = xp.concatenate( [ys[i_out][None] for ys in yss]).min(axis=0) # Restore the shape of flattened residual res = residuals[cumsize - size:cumsize] res = res.reshape(shape) det = utils.force_array( diff_atol + diff_rtol * (ymax - ymin) < res) # Constant output = not nondifferentiable det[ymax == ymin] = False if det.any(): s = six.StringIO() s.write( 'Tried to compute the numeric gradient on a ' 'non-differentiable point.\n\n') s.write('i_in: {}\n'.format(i_in)) s.write('i_out: {}\n'.format(i_out)) s.write('x: {}\n'.format(inputs[i_in])) s.write('index on x: {}\n'.format(i)) s.write('eps: {}\n'.format(eps)) s.write('diff_rtol: {}\n'.format(diff_rtol)) s.write('diff_atol: {}\n'.format(diff_atol)) s.write('ymax: {}\n'.format(ymax)) s.write('ymin: {}\n'.format(ymin)) s.write( 'diff_atol + diff_rtol * (ymax-ymin): {}\n'.format( diff_atol + diff_rtol * (ymax - ymin))) s.write('fitting errors: {}\n'.format(res)) s.write('y[x-eps ]: {}\n'.format(yss[0][i_out])) s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out])) s.write('y[x ]: {}\n'.format(yss[2][i_out])) s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out])) s.write('y[x+eps ]: {}\n'.format(yss[4][i_out])) raise NondifferentiableError(s.getvalue()) # Calculate numerical gradient for i_out, gy in enumerate(grad_outputs): if gy is None: continue if not numpy.isscalar(gy): gy = gy.astype(numpy.float64, copy=False) gpu_ = (xp is cuda.cupy and all(isinstance(ys[i_out], cuda.ndarray) for ys in yss)) # If any output sample is None, all others must be. assert all([ (yss[0][i_out] is None) == (yss[j][i_out] is None) for j in range(len(yss))]) # If outputs samples are None, the part of numeric gradient for # this output is considered as zero: skip the accumulation. if yss[0][i_out] is None: continue if len(yss) == 2: # 1st order y0 = yss[0][i_out] y1 = yss[1][i_out] if gpu_: numerical_grad_kernel_1( y1, y0, xp.asarray(gy), eps, gx[i]) else: dot = ((y1 - y0) * gy).sum() gx[i] = gx[i] + dot / (2 * eps) elif len(yss) == 5: # 3rd order y0 = yss[0][i_out] y1 = yss[1][i_out] y2 = yss[3][i_out] y3 = yss[4][i_out] if gpu_: numerical_grad_kernel_3( y3, y2, y1, y0, gy, eps, gx[i]) else: num = -y3 + 8 * y2 - 8 * y1 + y0 dot = (num * gy).sum() gx[i] = gx[i] + dot / (6 * eps) else: assert False
def generate_array(initializer, shape, xp, dtype=None, device=None): # type: (types.AbstractInitializer, types.ShapeSpec, types.Xp, types.DTypeSpec, types.DeviceSpec) -> types.NdArray # NOQA """Return initialized array. The algorithms used to make the new values depend on the concrete derived classes. If the initializer has the ``dtype`` attribute, it is used to construct the array. Otherwise, ``chainer.config.dtype`` is used instead. See :ref:`configuration` for the dtype config. Args: initializer: A callable object that takes :class:`numpy.ndarray` or :class:`cupy.ndarray` and edits its value. shape (tuple): Shape of a return array. xp (module): :mod:`cupy`, :mod:`numpy`, or :mod:`chainerx`. dtype: Dtype specifier. If omitted, ``initializer.dtype`` is used. device: Target device specifier. If omitted, the current device is used for :mod:`cupy`, and the default device is used for :mod:`chainerx`. Returns: numpy.ndarray, cupy.ndarray, or chainerx.ndarray: An initialized array. """ dtype_attr = getattr(initializer, 'dtype', None) if dtype is not None and dtype_attr is not None \ and numpy.dtype(dtype) != numpy.dtype(dtype_attr): raise ValueError( 'dtype mismatch: {} != {}'.format(dtype, dtype_attr)) if dtype is None: dtype = dtype_attr dtype = chainer.get_dtype(dtype) if device is None: if xp is cuda.cupy: backend_device = chainer.get_device(cuda.Device()) elif xp is chainerx: backend_device = chainer.get_device(chainerx.get_default_device()) else: backend_device = chainer.get_device(numpy) else: backend_device = chainer.get_device(device) if xp != backend_device.xp: raise ValueError('xp and device arguments are inconsistent.') if xp is chainerx: # Initialize with NumPy/CuPy array that shares memory with the # ChainerX array. # TODO(sonots): Directly use initializer after ChainerX # supports random. chx_device = backend_device.device # type: ignore # TODO(okapies): remove 'type: ignore' when chainerx implements sequence support for empty() # NOQA array = chainerx.empty(shape, dtype=dtype, device=chx_device) # type: ignore # NOQA if chx_device.backend.name == 'native': temp_array = _cpu._to_cpu(array) temp_device = cuda.DummyDevice # type: cuda.Device elif chx_device.backend.name == 'cuda': temp_array = cuda.to_gpu(array, chx_device.index) temp_device = cuda.Device(chx_device.index) else: raise RuntimeError('ChainerX backend: {} is not supported.'.format( chx_device.backend.name)) with temp_device: initializer(temp_array) return array with chainer.using_device(backend_device): array = xp.empty(shape, dtype=dtype) initializer(array) return array
def __call__(self, key, value): key = key.lstrip('/') self.target[self.path + key] = ( _cpu._to_cpu(value) if value is not None else numpy.asarray(None)) return value