def copyto(dst, src):
"""Copies the elements of an ndarray to those of another one.
This function can copy the CPU/GPU arrays to the destination arrays on
another device.
Args:
dst (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`):
Destination array.
src (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`):
Source array.
"""
if isinstance(dst, numpy.ndarray):
numpy.copyto(dst, _cpu._to_cpu(src))
elif isinstance(dst, intel64.mdarray):
intel64.ideep.basic_copyto(dst, _cpu._to_cpu(src))
elif isinstance(dst, cuda.ndarray):
if isinstance(src, chainer.get_cpu_array_types()):
src = numpy.asarray(src)
if dst.flags.c_contiguous or dst.flags.f_contiguous:
dst.set(src)
else:
cuda.cupy.copyto(dst, cuda.to_gpu(src, device=dst.device))
elif isinstance(src, cuda.ndarray):
cuda.cupy.copyto(dst, src)
else:
raise TypeError(
'cannot copy from non-array object of type {}'.format(
type(src)))
else:
raise TypeError('cannot copy to non-array object of type {}'.format(
type(dst)))
def test_evaluate(self, backend_config):
data = backend_config.get_array(self.data)
batches = [backend_config.get_array(b) for b in self.batches]
device = backend_config.device
iterator, converter, target, evaluator = (self.prepare(
data, batches, device))
reporter = chainer.Reporter()
reporter.add_observer('target', target)
with reporter:
mean = evaluator.evaluate()
# The converter gets results of the iterator and the device number.
self.assertEqual(len(converter.args), len(data))
if backend_config.use_cuda:
expected_device_arg = backend_config.cuda_device
else:
expected_device_arg = -1
for i in range(len(data)):
numpy.testing.assert_array_equal(
_cpu._to_cpu(converter.args[i]['batch']), self.data[i])
self.assertEqual(converter.args[i]['device'], expected_device_arg)
# The model gets results of converter.
self.assertEqual(len(target.args), len(batches))
for i in range(len(batches)):
numpy.testing.assert_array_equal(_cpu._to_cpu(target.args[i]),
self.batches[i])
expect_mean = numpy.mean([numpy.sum(x) for x in self.batches])
self.assertAlmostEqual(_cpu._to_cpu(mean['target/loss']),
expect_mean,
places=4)
def test_evaluate(self, backend_config):
data = backend_config.get_array(self.data)
batches = [backend_config.get_array(b) for b in self.batches]
device = backend_config.device
iterator, converter, target, evaluator = (
self.prepare(data, batches, device))
reporter = chainer.Reporter()
reporter.add_observer('target', target)
with reporter:
mean = evaluator.evaluate()
# The converter gets results of the iterator and the device number.
self.assertEqual(len(converter.args), len(data))
if backend_config.use_cuda:
expected_device_arg = backend_config.cuda_device
else:
expected_device_arg = -1
for i in range(len(data)):
numpy.testing.assert_array_equal(
_cpu._to_cpu(converter.args[i]['batch']), self.data[i])
self.assertEqual(converter.args[i]['device'], expected_device_arg)
# The model gets results of converter.
self.assertEqual(len(target.args), len(batches))
for i in range(len(batches)):
numpy.testing.assert_array_equal(
_cpu._to_cpu(target.args[i]), self.batches[i])
expect_mean = numpy.mean([numpy.sum(x) for x in self.batches])
self.assertAlmostEqual(
_cpu._to_cpu(mean['target/loss']), expect_mean, places=4)
def copyto(dst, src):
"""Copies the elements of an ndarray to those of another one.
This function can copy the CPU/GPU arrays to the destination arrays on
another device.
Args:
dst (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`):
Destination array.
src (`numpy.ndarray`, `cupy.ndarray` or `ideep4py.mdarray`):
Source array.
"""
if isinstance(dst, numpy.ndarray):
numpy.copyto(dst, _cpu._to_cpu(src))
elif isinstance(dst, intel64.mdarray):
intel64.ideep.basic_copyto(
dst, _cpu._to_cpu(src))
elif isinstance(dst, cuda.ndarray):
if isinstance(src, chainer.get_cpu_array_types()):
src = numpy.asarray(src)
if dst.flags.c_contiguous or dst.flags.f_contiguous:
dst.set(src)
else:
cuda.cupy.copyto(dst, cuda.to_gpu(src, device=dst.device))
elif isinstance(src, cuda.ndarray):
cuda.cupy.copyto(dst, src)
else:
raise TypeError('cannot copy from non-array object of type {}'
.format(type(src)))
else:
raise TypeError('cannot copy to non-array object of type {}'.format(
type(dst)))
def check_u_updated_in_train(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
y1 = layer(x).array
u1 = getattr(layer, hook.vector_name).copy()
y2 = layer(x).array
u2 = getattr(layer, hook.vector_name)
y1, y2 = _cpu._to_cpu(y1), _cpu._to_cpu(y2)
u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2)
assert not numpy.array_equal(u1, u2)
assert not numpy.array_equal(y1, y2)
def check_u_updated_in_train(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
y1 = layer(x).array
u1 = getattr(layer, hook.vector_name).copy()
y2 = layer(x).array
u2 = getattr(layer, hook.vector_name)
y1, y2 = _cpu._to_cpu(y1), _cpu._to_cpu(y2)
u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2)
assert not numpy.array_equal(u1, u2)
assert not numpy.array_equal(y1, y2)
def check_deleted(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with chainer.using_device(backend_config.device):
y1 = layer(x).array
with chainer.using_config('train', False):
y2 = layer(x).array
layer.delete_hook(hook.name)
assert not hasattr(layer, hook.vector_name)
y3 = layer(x).array
y1, y2, y3 = _cpu._to_cpu(y1), _cpu._to_cpu(y2), _cpu._to_cpu(y3)
assert not numpy.array_equal(y1, y3)
assert not numpy.array_equal(y2, y3)
def check_deleted(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with chainer.using_device(backend_config.device):
y1 = layer(x).array
with chainer.using_config('train', False):
y2 = layer(x).array
layer.delete_hook(hook.name)
assert not hasattr(layer, hook.vector_name)
y3 = layer(x).array
y1, y2, y3 = _cpu._to_cpu(y1), _cpu._to_cpu(y2), _cpu._to_cpu(y3)
assert not numpy.array_equal(y1, y3)
assert not numpy.array_equal(y2, y3)
def check_in_recomputing(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
y1 = layer(x).array
u1 = getattr(layer, hook.vector_name).copy()
v1 = hook.v.copy()
with chainer.using_config('in_recomputing', True):
y2 = layer(x).array
u2 = getattr(layer, hook.vector_name)
v2 = hook.v
u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2)
v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2)
numpy.testing.assert_array_equal(u1, u2)
numpy.testing.assert_array_equal(v1, v2)
testing.assert_allclose(y1, y2)
def check_in_recomputing(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
y1 = layer(x).array
u1 = getattr(layer, hook.vector_name).copy()
v1 = hook.v.copy()
with chainer.using_config('in_recomputing', True):
y2 = layer(x).array
u2 = getattr(layer, hook.vector_name)
v2 = hook.v
u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2)
v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2)
numpy.testing.assert_array_equal(u1, u2)
numpy.testing.assert_array_equal(v1, v2)
testing.assert_allclose(y1, y2)
def check_u_not_updated_in_test(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with chainer.using_config('train', False):
y1 = layer(x).array
u1 = getattr(layer, hook.vector_name).copy()
v1 = hook.v.copy()
y2 = layer(x).array
u2 = getattr(layer, hook.vector_name)
v2 = hook.v.copy()
u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2)
v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2)
numpy.testing.assert_array_equal(u1, u2)
numpy.testing.assert_array_equal(v1, v2)
testing.assert_allclose(y1, y2)
def check_u_not_updated_in_test(self, backend_config):
layer, hook = self.layer, self.hook
layer.add_hook(hook)
layer.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with chainer.using_config('train', False):
y1 = layer(x).array
u1 = getattr(layer, hook.vector_name).copy()
v1 = hook.v.copy()
y2 = layer(x).array
u2 = getattr(layer, hook.vector_name)
v2 = hook.v.copy()
u1, u2 = _cpu._to_cpu(u1), _cpu._to_cpu(u2)
v1, v2 = _cpu._to_cpu(v1), _cpu._to_cpu(v2)
numpy.testing.assert_array_equal(u1, u2)
numpy.testing.assert_array_equal(v1, v2)
testing.assert_allclose(y1, y2)
def check_serialization(self, backend_config):
with utils.tempdir() as root:
filename = os.path.join(root, 'tmp.npz')
layer1 = self.layer.copy('copy')
hook1 = copy.deepcopy(self.hook)
layer1.add_hook(hook1)
layer1.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with backend_config:
layer1(x)
with chainer.using_config('train', False):
y1 = layer1(x)
serializers.save_npz(filename, layer1)
layer2 = self.layer.copy('copy')
hook2 = copy.deepcopy(self.hook)
layer2.add_hook(hook2)
# Test loading is nice.
msg = None
try:
serializers.load_npz(filename, layer2)
except Exception as e:
msg = e
assert msg is None
with chainer.using_config('train', False):
y2 = layer2(self.x.copy())
# Test attributes are the same.
orig_weight = _cpu._to_cpu(
getattr(layer1, hook1.weight_name).array)
orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name))
numpy.testing.assert_array_equal(
orig_weight,
getattr(layer2, hook2.weight_name).array)
numpy.testing.assert_array_equal(
orig_vector, getattr(layer2, hook2.vector_name))
testing.assert_allclose(y1.array, y2.array)
def check_serialization(self, backend_config):
with utils.tempdir() as root:
filename = os.path.join(root, 'tmp.npz')
layer1 = self.layer.copy('copy')
hook1 = copy.deepcopy(self.hook)
layer1.add_hook(hook1)
layer1.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with backend_config:
layer1(x)
with chainer.using_config('train', False):
y1 = layer1(x)
serializers.save_npz(filename, layer1)
layer2 = self.layer.copy('copy')
hook2 = copy.deepcopy(self.hook)
layer2.add_hook(hook2)
# Test loading is nice.
msg = None
try:
serializers.load_npz(filename, layer2)
except Exception as e:
msg = e
assert msg is None
with chainer.using_config('train', False):
y2 = layer2(self.x.copy())
# Test attributes are the same.
orig_weight = _cpu._to_cpu(
getattr(layer1, hook1.weight_name).array)
orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name))
numpy.testing.assert_array_equal(
orig_weight, getattr(layer2, hook2.weight_name).array)
numpy.testing.assert_array_equal(
orig_vector, getattr(layer2, hook2.vector_name))
testing.assert_allclose(y1.array, y2.array)
def __call__(self, key, value):
if value is None:
# use Empty to represent None
if h5py.version.version_tuple < (2, 7, 0):
raise RuntimeError(
'h5py>=2.7.0 is required to serialize None.')
arr = h5py.Empty('f')
compression = None
else:
arr = _cpu._to_cpu(value)
compression = None if arr.size <= 1 else self.compression
self.group.create_dataset(key, data=arr, compression=compression)
return value
def __call__(self, key, value):
if value is None:
# use Empty to represent None
if h5py.version.version_tuple < (2, 7, 0):
raise RuntimeError(
'h5py>=2.7.0 is required to serialize None.')
arr = h5py.Empty('f')
compression = None
else:
arr = _cpu._to_cpu(value)
compression = None if arr.size <= 1 else self.compression
self.group.create_dataset(key, data=arr, compression=compression)
return value
def _array_to_chainerx(array, device=None):
# If device is None, appropriate device is chosen according to the input
# arrays.
assert device is None or isinstance(device, chainerx.Device)
if array is None:
return None
if array.dtype not in chainerx.all_dtypes:
raise TypeError(
'Dtype {} is not supported in ChainerX.'.format(array.dtype.name))
if isinstance(array, chainerx.ndarray):
if device is None:
return array
if device is array.device:
return array
return array.to_device(device)
if isinstance(array, numpy.ndarray):
if device is None:
device = chainerx.get_device('native', 0)
return chainerx.array(array, device=device, copy=False)
if isinstance(array, cuda.ndarray):
if device is None:
device = chainerx.get_device('cuda', array.device.id)
elif device.backend.name != 'cuda':
# cupy to non-cuda backend
# TODO(niboshi): Remove conversion to numpy when both CuPy and
# ChainerX support the array interface.
array = _cpu._to_cpu(array)
return chainerx.array(array, device=device, copy=False)
elif device.index != array.device.id:
# cupy to cuda backend but different device
array = cuda.to_gpu(array, device=device.index)
# cupy to cuda backend with the same device
return chainerx._core._fromrawpointer(
array.data.mem.ptr,
array.shape,
array.dtype,
array.strides,
device,
array.data.ptr - array.data.mem.ptr,
array)
if isinstance(array, intel64.mdarray):
return _array_to_chainerx(numpy.array(array), device)
if numpy.isscalar(array):
return chainerx.asarray(array)
raise TypeError(
'Array cannot be converted into chainerx.ndarray'
'\nActual type: {0}.'.format(type(array)))
def send_array(self, array):
if isinstance(array, ideep.mdarray):
return array
if not isinstance(array, numpy.ndarray):
array = _cpu._to_cpu(array) # to numpy.ndarray
if (isinstance(array, numpy.ndarray) and array.ndim in (1, 2, 4)
and 0 not in array.shape):
# TODO(kmaehashi): Remove ndim validation once iDeep has fixed.
# Currently iDeep only supports (1, 2, 4)-dim arrays.
# Note that array returned from `ideep.array` may not be an
# iDeep mdarray, e.g., when the dtype is not float32.
array = ideep.array(array, itype=ideep.wgt_array)
return array
def send_array(self, array):
if isinstance(array, ideep.mdarray):
return array
if not isinstance(array, numpy.ndarray):
array = _cpu._to_cpu(array) # to numpy.ndarray
if (isinstance(array, numpy.ndarray) and
array.ndim in (1, 2, 4) and
0 not in array.shape):
# TODO(kmaehashi): Remove ndim validation once iDeep has fixed.
# Currently iDeep only supports (1, 2, 4)-dim arrays.
# Note that array returned from `ideep.array` may not be an
# iDeep mdarray, e.g., when the dtype is not float32.
array = ideep.array(array, itype=ideep.wgt_array)
return array
def _array_from_chainerx(array):
if array is None:
return None
if not isinstance(array, chainerx.ndarray):
if isinstance(array, chainer.get_array_types()):
return array
raise TypeError(
'Tried to convert to a non-ChainerX array from an invalid type: '
'{}'.format(type(array)))
backend_name = array.device.backend.name
if backend_name == 'native':
return _cpu._to_cpu(array)
if backend_name == 'cuda':
return cuda.to_gpu(array, array.device.index)
raise ValueError(
'Only ChainerX arrays with native or cuda backends can be converted '
'to non-ChainerX arrays.\nActual: {0}.'.format(backend_name))
def _array_from_chainerx(array):
if array is None:
return None
if not isinstance(array, chainerx.ndarray):
if isinstance(array, chainer.get_array_types()):
return array
raise TypeError(
'Tried to convert to a non-ChainerX array from an invalid type: '
'{}'.format(type(array)))
backend_name = array.device.backend.name
if backend_name == 'native':
return _cpu._to_cpu(array)
if backend_name == 'cuda':
return cuda.to_gpu(array, array.device.index)
raise ValueError(
'Only ChainerX arrays with native or cuda backends can be converted '
'to non-ChainerX arrays.\nActual: {0}.'.format(backend_name))
def iterate_single_input(i_in, x, orig_x, x_ind):
orig = orig_x[x_ind]
# `yss` holds a list of output arrays for each of 2 or 5 sampling
# points.
if detect_nondifferentiable:
yss = [
eval_func(x, x_ind, -eps * 1., orig),
eval_func(x, x_ind, -eps * .5, orig),
ys0,
eval_func(x, x_ind, +eps * .5, orig),
eval_func(x, x_ind, +eps * 1., orig),
]
else:
yss = [
eval_func(x, x_ind, -eps * 1, orig),
eval_func(x, x_ind, +eps * 1, orig),
]
assert all([
y is None
or (y.shape == yss[0][i].shape and y.dtype == yss[0][i].dtype)
for ys in yss for i, y in enumerate(ys)
])
# If all the outputs are 0-size, skip non-differentiable check.
if all([y is None or y.size == 0 for y in yss[0]]):
detect_nondifferentiable_ = False
else:
detect_nondifferentiable_ = detect_nondifferentiable
if detect_nondifferentiable_:
# Detect non-differentiable point by quadratic fitting
# Check for non-finite output.
# If any single element in the output arrays has different
# finiteness among sampled points, that means this is a
# non-differentiable point.
# If the function consistently generates non-finite values
# around the point, we do not treat the point as
# non-differentiable.
# (Example: x<0 region for the logarithm function)
any_nonfinite = False
for i_out in range(nout):
isfinites = [xp.isfinite(ys[i_out]) for ys in yss]
if any((isfinites[0] != isfinites[i]).any()
for i in range(1, len(yss))):
s = six.StringIO()
s.write('Tried to compute the numeric gradient on a '
'non-differentiable point.\n\n')
s.write('i_in: {}\n'.format(i_in))
s.write('i_out: {}\n'.format(i_out))
s.write('x: {}\n'.format(inputs[i_in]))
s.write('index on x: {}\n'.format(x_ind))
s.write('eps: {}\n'.format(eps))
s.write('y[x-eps ]: {}\n'.format(yss[0][i_out]))
s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out]))
s.write('y[x ]: {}\n'.format(yss[2][i_out]))
s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out]))
s.write('y[x+eps ]: {}\n'.format(yss[4][i_out]))
raise NondifferentiableError(s.getvalue())
any_nonfinite |= not all((_).all() for _ in isfinites)
if not any_nonfinite:
# Stack flattened outputs to make (5, *)-shaped 2D array
ystack = xp.vstack(
[xp.hstack([y.ravel() for y in ys]) for ys in yss])
assert ystack.ndim == 2 and ystack.shape[0] == len(yss)
# Fit to quadratic
if xp is not numpy:
ystack = _cpu._to_cpu(ystack)
polyfit = numpy.polynomial.polynomial.polyfit
_, (residuals, _, _, _) = polyfit(range(len(yss)),
ystack,
deg=2,
full=True)
if xp is not numpy:
residuals = device.send(residuals)
residuals = xp.sqrt(residuals / len(yss))
# Check for error for each output array
for i_out in range(nout):
size = sizes[i_out]
cumsize = cumsizes[i_out]
shape = shapes[i_out]
# TODO(niboshi): The following two lines could be
# rewritten using xp.stack, which is supported in
# NumPy>=1.10
ymax = xp.concatenate([ys[i_out][None]
for ys in yss]).max(axis=0)
ymin = xp.concatenate([ys[i_out][None]
for ys in yss]).min(axis=0)
# Restore the shape of flattened residual
res = residuals[cumsize - size:cumsize]
res = res.reshape(shape)
det = utils.force_array(diff_atol + diff_rtol *
(ymax - ymin) < res)
# Constant output = not nondifferentiable
det[ymax == ymin] = False
if det.any():
s = six.StringIO()
s.write('Tried to compute the numeric gradient on a '
'non-differentiable point.\n\n')
s.write('i_in: {}\n'.format(i_in))
s.write('i_out: {}\n'.format(i_out))
s.write('x: {}\n'.format(inputs[i_in]))
s.write('index on x: {}\n'.format(x_ind))
s.write('eps: {}\n'.format(eps))
s.write('diff_rtol: {}\n'.format(diff_rtol))
s.write('diff_atol: {}\n'.format(diff_atol))
s.write('ymax: {}\n'.format(ymax))
s.write('ymin: {}\n'.format(ymin))
s.write(
'diff_atol + diff_rtol * (ymax-ymin): {}\n'.format(
diff_atol + diff_rtol * (ymax - ymin)))
s.write('fitting errors: {}\n'.format(res))
s.write('y[x-eps ]: {}\n'.format(yss[0][i_out]))
s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out]))
s.write('y[x ]: {}\n'.format(yss[2][i_out]))
s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out]))
s.write('y[x+eps ]: {}\n'.format(yss[4][i_out]))
raise NondifferentiableError(s.getvalue())
# Calculate numerical gradient
for i_out, gy in enumerate(grad_outputs):
if gy is None:
continue
if not numpy.isscalar(gy):
gy = gy.astype(numpy.float64, copy=False)
gpu_ = (xp is cuda.cupy
and all(isinstance(ys[i_out], cuda.ndarray) for ys in yss))
# If any output sample is None, all others must be.
assert all([(yss[0][i_out] is None) == (yss[j][i_out] is None)
for j in range(len(yss))])
# If outputs samples are None, the part of numeric gradient for
# this output is considered as zero: skip the accumulation.
if yss[0][i_out] is None:
continue
if len(yss) == 2: # 1st order
y0 = yss[0][i_out]
y1 = yss[1][i_out]
if gpu_:
numerical_grad_kernel_1(y1, y0, xp.asarray(gy), eps,
gx[x_ind])
else:
dot = ((y1 - y0) * gy).sum()
gx[x_ind] = gx[x_ind] + dot / (2 * eps)
elif len(yss) == 5: # 3rd order
y0 = yss[0][i_out]
y1 = yss[1][i_out]
y2 = yss[3][i_out]
y3 = yss[4][i_out]
if gpu_:
numerical_grad_kernel_3(y3, y2, y1, y0, gy, eps, gx[x_ind])
else:
num = -y3 + 8 * y2 - 8 * y1 + y0
dot = (num * gy).sum()
gx[x_ind] = gx[x_ind] + dot / (6 * eps)
else:
assert False
def __call__(self, key, value):
key = key.lstrip('/')
self.target[self.path + key] = (_cpu._to_cpu(value) if value
is not None else numpy.asarray(None))
return value
def generate_array(initializer, shape, xp, dtype=None, device=None):
# type: (types.AbstractInitializer, types.ShapeSpec, types.Xp, types.DTypeSpec, types.DeviceSpec) -> types.NdArray # NOQA
"""Return initialized array.
The algorithms used to make the new values depend on the
concrete derived classes. If the initializer has the ``dtype`` attribute,
it is used to construct the array. Otherwise, ``chainer.config.dtype`` is
used instead. See :ref:`configuration` for the dtype config.
Args:
initializer: A callable object that takes :ref:`ndarray` and edits its
value.
shape (tuple): Shape of a return array.
xp (module): :mod:`cupy`, :mod:`numpy`, or :mod:`chainerx`.
dtype: Dtype specifier. If omitted, ``initializer.dtype`` is used.
device: Target device specifier. If omitted, the current device is
used for :mod:`cupy`, and the default device is used for
:mod:`chainerx`.
Returns:
:ref:`ndarray`: An initialized array.
"""
dtype_attr = getattr(initializer, 'dtype', None)
if dtype is not None and dtype_attr is not None \
and numpy.dtype(dtype) != numpy.dtype(dtype_attr):
raise ValueError(
'dtype mismatch: {} != {}'.format(dtype, dtype_attr))
if dtype is None:
dtype = dtype_attr
dtype = chainer.get_dtype(dtype)
if device is None:
backend_device = backend._guess_device_from_array_module(xp)
else:
backend_device = chainer.get_device(device)
if xp != backend_device.xp:
raise ValueError('xp and device arguments are inconsistent.')
if xp is chainerx:
# Initialize with NumPy/CuPy array that shares memory with the
# ChainerX array.
# TODO(sonots): Directly use initializer after ChainerX
# supports random.
chx_device = backend_device.device # type: ignore
# TODO(okapies): remove 'type: ignore' when chainerx implements sequence support for empty() # NOQA
array = chainerx.empty(shape, dtype=dtype, device=chx_device) # type: ignore # NOQA
if chx_device.backend.name == 'native':
temp_array = _cpu._to_cpu(array)
temp_device = cuda.DummyDevice # type: cuda.Device
elif chx_device.backend.name == 'cuda':
temp_array = cuda.to_gpu(array, chx_device.index)
temp_device = cuda.Device(chx_device.index)
else:
raise RuntimeError('ChainerX backend: {} is not supported.'.format(
chx_device.backend.name))
with temp_device:
initializer(temp_array)
return array
with chainer.using_device(backend_device):
array = xp.empty(shape, dtype=dtype)
initializer(array)
return array
def iterate_single_input(i_in, x, orig_x, i):
orig = orig_x[i]
# `yss` holds a list of output arrays for each of 2 or 5 sampling
# points.
if detect_nondifferentiable:
yss = [
eval_func(x, i, -eps * 1., orig),
eval_func(x, i, -eps * .5, orig),
ys0,
eval_func(x, i, +eps * .5, orig),
eval_func(x, i, +eps * 1., orig),
]
else:
yss = [
eval_func(x, i, -eps * 1, orig),
eval_func(x, i, +eps * 1, orig),
]
if detect_nondifferentiable:
# Detect non-differentiable point by quadratic fitting
# Check for non-finite output.
# If any single element in the output arrays has different
# finiteness among sampled points, that means this is a
# non-differentiable point.
# If the function consistently generates non-finite values
# around the point, we do not treat the point as
# non-differentiable.
# (Example: x<0 region for the logarithm function)
any_nonfinite = False
for i_out in range(nout):
isfinites = [xp.isfinite(ys[i_out]) for ys in yss]
if any((isfinites[0] != isfinites[i]).any()
for i in range(1, len(yss))):
s = six.StringIO()
s.write(
'Tried to compute the numeric gradient on a '
'non-differentiable point.\n\n')
s.write('i_in: {}\n'.format(i_in))
s.write('i_out: {}\n'.format(i_out))
s.write('x: {}\n'.format(inputs[i_in]))
s.write('index on x: {}\n'.format(i))
s.write('eps: {}\n'.format(eps))
s.write('y[x-eps ]: {}\n'.format(yss[0][i_out]))
s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out]))
s.write('y[x ]: {}\n'.format(yss[2][i_out]))
s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out]))
s.write('y[x+eps ]: {}\n'.format(yss[4][i_out]))
raise NondifferentiableError(s.getvalue())
any_nonfinite |= not all((_).all() for _ in isfinites)
if not any_nonfinite:
# Stack flattened outputs to make (5, *)-shaped 2D array
ystack = xp.vstack(
[xp.hstack([y.ravel() for y in ys]) for ys in yss])
assert ystack.ndim == 2 and ystack.shape[0] == len(yss)
# Fit to quadratic
if xp is not numpy:
ystack = _cpu._to_cpu(ystack)
polyfit = numpy.polynomial.polynomial.polyfit
_, (residuals, _, _, _) = polyfit(
range(len(yss)), ystack, deg=2, full=True)
if xp is not numpy:
residuals = device.send(residuals)
residuals = xp.sqrt(residuals / len(yss))
# Check for error for each output array
for i_out in range(nout):
size = sizes[i_out]
cumsize = cumsizes[i_out]
shape = shapes[i_out]
# TODO(niboshi): The following two lines could be
# rewritten using xp.stack, which is supported in
# NumPy>=1.10
ymax = xp.concatenate(
[ys[i_out][None] for ys in yss]).max(axis=0)
ymin = xp.concatenate(
[ys[i_out][None] for ys in yss]).min(axis=0)
# Restore the shape of flattened residual
res = residuals[cumsize - size:cumsize]
res = res.reshape(shape)
det = utils.force_array(
diff_atol + diff_rtol * (ymax - ymin) < res)
# Constant output = not nondifferentiable
det[ymax == ymin] = False
if det.any():
s = six.StringIO()
s.write(
'Tried to compute the numeric gradient on a '
'non-differentiable point.\n\n')
s.write('i_in: {}\n'.format(i_in))
s.write('i_out: {}\n'.format(i_out))
s.write('x: {}\n'.format(inputs[i_in]))
s.write('index on x: {}\n'.format(i))
s.write('eps: {}\n'.format(eps))
s.write('diff_rtol: {}\n'.format(diff_rtol))
s.write('diff_atol: {}\n'.format(diff_atol))
s.write('ymax: {}\n'.format(ymax))
s.write('ymin: {}\n'.format(ymin))
s.write(
'diff_atol + diff_rtol * (ymax-ymin): {}\n'.format(
diff_atol + diff_rtol * (ymax - ymin)))
s.write('fitting errors: {}\n'.format(res))
s.write('y[x-eps ]: {}\n'.format(yss[0][i_out]))
s.write('y[x-eps/2]: {}\n'.format(yss[1][i_out]))
s.write('y[x ]: {}\n'.format(yss[2][i_out]))
s.write('y[x+eps/2]: {}\n'.format(yss[3][i_out]))
s.write('y[x+eps ]: {}\n'.format(yss[4][i_out]))
raise NondifferentiableError(s.getvalue())
# Calculate numerical gradient
for i_out, gy in enumerate(grad_outputs):
if gy is None:
continue
if not numpy.isscalar(gy):
gy = gy.astype(numpy.float64, copy=False)
gpu_ = (xp is cuda.cupy and
all(isinstance(ys[i_out], cuda.ndarray)
for ys in yss))
# If any output sample is None, all others must be.
assert all([
(yss[0][i_out] is None) == (yss[j][i_out] is None)
for j in range(len(yss))])
# If outputs samples are None, the part of numeric gradient for
# this output is considered as zero: skip the accumulation.
if yss[0][i_out] is None:
continue
if len(yss) == 2: # 1st order
y0 = yss[0][i_out]
y1 = yss[1][i_out]
if gpu_:
numerical_grad_kernel_1(
y1, y0, xp.asarray(gy), eps, gx[i])
else:
dot = ((y1 - y0) * gy).sum()
gx[i] = gx[i] + dot / (2 * eps)
elif len(yss) == 5: # 3rd order
y0 = yss[0][i_out]
y1 = yss[1][i_out]
y2 = yss[3][i_out]
y3 = yss[4][i_out]
if gpu_:
numerical_grad_kernel_3(
y3, y2, y1, y0, gy, eps, gx[i])
else:
num = -y3 + 8 * y2 - 8 * y1 + y0
dot = (num * gy).sum()
gx[i] = gx[i] + dot / (6 * eps)
else:
assert False
def generate_array(initializer, shape, xp, dtype=None, device=None):
# type: (types.AbstractInitializer, types.ShapeSpec, types.Xp, types.DTypeSpec, types.DeviceSpec) -> types.NdArray # NOQA
"""Return initialized array.
The algorithms used to make the new values depend on the
concrete derived classes. If the initializer has the ``dtype`` attribute,
it is used to construct the array. Otherwise, ``chainer.config.dtype`` is
used instead. See :ref:`configuration` for the dtype config.
Args:
initializer: A callable object that takes :class:`numpy.ndarray`
or :class:`cupy.ndarray` and edits its value.
shape (tuple): Shape of a return array.
xp (module): :mod:`cupy`, :mod:`numpy`, or :mod:`chainerx`.
dtype: Dtype specifier. If omitted, ``initializer.dtype`` is used.
device: Target device specifier. If omitted, the current device is
used for :mod:`cupy`, and the default device is used for
:mod:`chainerx`.
Returns:
numpy.ndarray, cupy.ndarray, or chainerx.ndarray: An initialized array.
"""
dtype_attr = getattr(initializer, 'dtype', None)
if dtype is not None and dtype_attr is not None \
and numpy.dtype(dtype) != numpy.dtype(dtype_attr):
raise ValueError(
'dtype mismatch: {} != {}'.format(dtype, dtype_attr))
if dtype is None:
dtype = dtype_attr
dtype = chainer.get_dtype(dtype)
if device is None:
if xp is cuda.cupy:
backend_device = chainer.get_device(cuda.Device())
elif xp is chainerx:
backend_device = chainer.get_device(chainerx.get_default_device())
else:
backend_device = chainer.get_device(numpy)
else:
backend_device = chainer.get_device(device)
if xp != backend_device.xp:
raise ValueError('xp and device arguments are inconsistent.')
if xp is chainerx:
# Initialize with NumPy/CuPy array that shares memory with the
# ChainerX array.
# TODO(sonots): Directly use initializer after ChainerX
# supports random.
chx_device = backend_device.device # type: ignore
# TODO(okapies): remove 'type: ignore' when chainerx implements sequence support for empty() # NOQA
array = chainerx.empty(shape, dtype=dtype, device=chx_device) # type: ignore # NOQA
if chx_device.backend.name == 'native':
temp_array = _cpu._to_cpu(array)
temp_device = cuda.DummyDevice # type: cuda.Device
elif chx_device.backend.name == 'cuda':
temp_array = cuda.to_gpu(array, chx_device.index)
temp_device = cuda.Device(chx_device.index)
else:
raise RuntimeError('ChainerX backend: {} is not supported.'.format(
chx_device.backend.name))
with temp_device:
initializer(temp_array)
return array
with chainer.using_device(backend_device):
array = xp.empty(shape, dtype=dtype)
initializer(array)
return array
def __call__(self, key, value):
key = key.lstrip('/')
self.target[self.path + key] = (
_cpu._to_cpu(value) if value is not None
else numpy.asarray(None))
return value
