def check_forward_outputs(self, outputs, expected_outputs):
assert isinstance(outputs, tuple)
assert isinstance(expected_outputs, tuple)
assert all(isinstance(a, chainer.get_array_types()) for a in outputs)
assert all(
isinstance(a, chainer.get_array_types()) for a in expected_outputs)
_check_arrays_equal(outputs, expected_outputs, LinkTestError,
**self.check_forward_options)
def check_forward_outputs(self, outputs, expected_outputs):
assert isinstance(outputs, tuple)
assert isinstance(expected_outputs, tuple)
assert all(isinstance(a, chainer.get_array_types()) for a in outputs)
assert all(
isinstance(a, chainer.get_array_types()) for a in expected_outputs)
_check_arrays_equal(
outputs, expected_outputs, LinkTestError,
**self.check_forward_options)
def copyparams(self, link, copy_persistent=True):
"""Copies all parameters from given link.
This method copies data arrays of all parameters in the hierarchy. The
copy is even done across the host and devices. Note that this method
does not copy the gradient arrays.
*From v5.0.0:* this method also copies the persistent values (e.g. the
moving statistics of :class:`~chainer.links.BatchNormalization`). If
the persistent value is an ndarray, the elements are copied. Otherwise,
it is copied using :func:`copy.deepcopy`. The old behavior (not copying
persistent values) can be reproduced with ``copy_persistent=False``.
Args:
link (Link): Source link object.
copy_persistent (bool): If ``True``, persistent values are also
copied. ``True`` by default.
"""
src = link.__dict__
dst = self.__dict__
for name in self._params:
dst[name].copydata(src[name])
if copy_persistent:
array_types = chainer.get_array_types()
for name in self._persistent:
d = dst[name]
s = src[name]
if isinstance(d, array_types) and isinstance(s, array_types):
cuda.copyto(d, s)
else:
dst[name] = copy.deepcopy(s)
def __init__(self, func, x_data, y_grad, params, eps, atol, rtol, no_grads,
dtype, detect_nondifferentiable, is_immutable_params):
# If `is_immutable_params` is `False`, `params` are expected to be of
# type `chainer.Parameter` and are updated in-place.
# To run `_CheckBackward` with ChainerX ndarrays however which cannot
# be updated in-place when wrapped in `chainer.Parameter`s, this flag
# should be `True` and parameters should be given as ndarrays.
# `func` in the former case must take inputs as arguments only. In the
# latter, it must take the parameters in addition.
if dtype is not None and numpy.dtype(dtype).kind != 'f':
raise ValueError('`dtype` is allowed only float type')
if is_immutable_params:
if not all(
isinstance(p, chainer.get_array_types()) for p in params):
raise ValueError(
'All parameters in `params` must be ndarrays if '
'`is_immutable_params` is `True`. Actual: {}.'.format(
', '.join(str(type(p)) for p in params)))
x_data = _as_tuple(x_data)
if y_grad is not None:
y_grad = _as_tuple(y_grad)
params = _as_tuple(params)
if no_grads is None:
no_grads = [x.dtype.kind != 'f' for x in x_data]
else:
if len(no_grads) != len(x_data):
raise ValueError(
'Length of no_grads param and xs should be same.\n'
'Actual: {0} != {1}'.format(len(no_grads), len(x_data)))
device = backend.get_device_from_array(*x_data)
if device.xp is chainerx:
if len(params) > 0 and not is_immutable_params:
raise NotImplementedError(
'gradient_check must be called with '
'is_immutable_params=True to test parameters with '
'ChainerX.')
if any(no_grads):
raise NotImplementedError(
'gradient_check does not support no_grads argument for '
'ChainerX arrays')
self.device = device
self.func = func
self.x_data = x_data
self.y_grad = y_grad
self.params = params
self.no_grads = no_grads
self.atol = atol
self.rtol = rtol
self.is_immutable_params = is_immutable_params
# options for numeric gradients
self.eps = eps
self.dtype = dtype
self.detect_nondifferentiable = detect_nondifferentiable
def visit_array(self, arr):
assert isinstance(arr, chainer.get_array_types())
device = backend.get_device_from_array(arr)
if self._skip_visiting(device):
self._warn_to_gpu(device, self._device)
return arr
return self._device.send(arr)
def visit_array(self, arr):
assert isinstance(arr, chainer.get_array_types())
if not (self._skip_between_cupy_devices
and self._device.xp is cuda.cupy
and isinstance(arr, cuda.ndarray)):
return self._device.send(arr)
return arr
def copyparams(self, link, copy_persistent=True):
"""Copies all parameters from given link.
This method copies data arrays of all parameters in the hierarchy. The
copy is even done across the host and devices. Note that this method
does not copy the gradient arrays.
*From v5.0.0:* this method also copies the persistent values (e.g. the
moving statistics of :class:`~chainer.links.BatchNormalization`). If
the persistent value is an ndarray, the elements are copied. Otherwise,
it is copied using :func:`copy.deepcopy`. The old behavior (not copying
persistent values) can be reproduced with ``copy_persistent=False``.
Args:
link (Link): Source link object.
copy_persistent (bool): If ``True``, persistent values are also
copied. ``True`` by default.
"""
src = link.__dict__
dst = self.__dict__
for name in self._params:
dst[name].copydata(src[name])
if copy_persistent:
array_types = chainer.get_array_types()
for name in self._persistent:
d = dst[name]
s = src[name]
if isinstance(d, array_types) and isinstance(s, array_types):
backend.copyto(d, s)
else:
dst[name] = copy.deepcopy(s)
def __init__(self, dtype=None):
if not (isinstance(self.fill_value, chainer.get_array_types())
or numpy.isscalar(self.fill_value)):
raise ValueError(
'fill_value must be either scalar, numpy.ndarray, '
'cupy.ndarray or chainerx.ndarray.')
super(_Constant, self).__init__(dtype)
def _check_backward_with_params(
# This function was introduced along with the `is_immutable_params`
# argument to `_CheckBackward`.
# It allows passing `params` as ndarrays instead of `Parameter`s and thus
# depends less on the state of the parameter held by the caller.
# It is required by the `LinkTestCase` to check ChainerX parameter
# gradients, since those parameters cannot perturbed in-place for the
# numerical gradients if passed as `Parameter`s as those requiring
# gradients cannot be updated in-place.
func,
x_data,
y_grad,
params=(),
eps=1e-3,
atol=1e-5,
rtol=1e-4,
no_grads=None,
dtype=None,
detect_nondifferentiable=False):
assert all(isinstance(p, chainer.get_array_types()) for p in params)
_CheckBackward(func,
x_data,
y_grad,
params,
eps,
atol,
rtol,
no_grads,
dtype,
detect_nondifferentiable,
is_immutable_params=True).run()
def _as_array(data):
if isinstance(data, chainer.get_array_types()):
return data
else:
device = chainer.backend.get_device_from_array(data[0])
with chainer.using_device(device):
return device.xp.asarray(data)
def _validate_inout(xs):
# print(xs)
# We use a scalar false as a None.
# TODO(hamaji): Revisit to check if this decision is OK.
if xs is None:
xs = False
if isinstance(xs, chainer.Variable):
xs = xs.array
elif isinstance(xs, chainer.get_array_types()):
xs = chainer.cuda.to_cpu(xs)
elif isinstance(xs, bool):
xs = np.array(xs, dtype=np.bool)
elif isinstance(xs, int):
xs = np.array(xs, dtype=np.int64)
elif isinstance(xs, collections.Iterable):
xs = [_validate_inout(x) for x in xs]
elif (isinstance(xs, np.float32) or isinstance(xs, np.float64)
or isinstance(xs, np.int32) or isinstance(xs, np.int64)):
pass
else:
raise ValueError('Unknown type: {}'.format(type(xs)))
return xs
def visit_array(self, arr):
assert isinstance(arr, chainer.get_array_types())
if not (self._skip_between_cupy_devices
and self._device.xp is cuda.cupy
and isinstance(arr, cuda.ndarray)):
return self._device.send(arr)
return arr
def _naive_batch_renormalization(
x,
gamma,
beta, # variables
mean,
var, # variables
running_mean,
running_var, # arrays
rmax,
dmax,
eps,
decay):
# If decay is not None, the running stats are updated.
F = chainer.functions
assert isinstance(x, chainer.Variable)
assert isinstance(gamma, chainer.Variable)
assert isinstance(beta, chainer.Variable)
assert isinstance(mean, chainer.Variable)
assert isinstance(var, chainer.Variable)
assert isinstance(running_mean, chainer.get_array_types())
assert isinstance(running_var, chainer.get_array_types())
assert mean.shape == var.shape
assert mean.shape == running_mean.shape
assert mean.shape == running_var.shape
assert mean.shape == gamma.shape
dt = x.dtype.type
std = F.sqrt(var + dt(eps))
# r and d are gradient-stopped
running_std = numpy.sqrt(running_var + dt(eps))
r = (std.array / running_std).clip(1. / rmax, rmax)
d = ((mean.array - running_mean) / running_std).clip(-dmax, dmax)
xhat = (x - mean) / std * r + d
y = gamma * xhat + beta
# Update running stats
if decay is not None:
running_mean *= decay
running_mean += mean.array * dt(1. - decay)
# unbiased estimation
m = x.size // gamma.size
adjust = m / max(m - 1., 1.)
running_var *= decay
running_var += var.array * dt((1. - decay) * adjust)
return y
def __init__(self, in_size, out_size, pool_size,
initialW=None, initial_bias=0):
super(Maxout, self).__init__()
linear_out_size = out_size * pool_size
if initialW is None or \
numpy.isscalar(initialW) or \
isinstance(initialW, initializer.Initializer):
pass
elif isinstance(initialW, chainer.get_array_types()):
if initialW.ndim != 3:
raise ValueError('initialW.ndim should be 3')
initialW = initialW.reshape(linear_out_size, in_size)
elif callable(initialW):
initialW_orig = initialW
def initialW(array):
array.shape = (out_size, pool_size, in_size)
initialW_orig(array)
array.shape = (linear_out_size, in_size)
if initial_bias is None or \
numpy.isscalar(initial_bias) or \
isinstance(initial_bias, initializer.Initializer):
pass
elif isinstance(initial_bias, chainer.get_array_types()):
if initial_bias.ndim != 2:
raise ValueError('initial_bias.ndim should be 2')
initial_bias = initial_bias.reshape(linear_out_size)
elif callable(initial_bias):
initial_bias_orig = initial_bias
def initial_bias(array):
array.shape = (out_size, pool_size)
initial_bias_orig(array)
array.shape = linear_out_size,
with self.init_scope():
self.linear = linear.Linear(
in_size, linear_out_size,
nobias=initial_bias is None, initialW=initialW,
initial_bias=initial_bias)
self.out_size = out_size
self.pool_size = pool_size
def _check_is_initializer_like(initializer):
if not (initializer is None or isinstance(initializer, chainer.Initializer)
or callable(initializer)
or isinstance(initializer, chainer.get_array_types())
or numpy.isscalar(initializer)):
raise TypeError(
'Initializer is of wrong type: {}. Allowed types are Initializer, '
'ndarray and scalar.'.format(type(initializer)))
def _get_type(name, index, array, accept_none):
var = '{0}[{1}]'.format(name, index)
if accept_none and array is None:
# case that gradient is not given
return Variable(TypeInfo((), None), var)
assert isinstance(array, chainer.get_array_types())
return Variable(TypeInfo(array.shape, array.dtype), var)
def _get_type(name, index, array, accept_none):
var = '{0}[{1}]'.format(name, index)
if accept_none and array is None:
# case that gradient is not given
return Variable(TypeInfo((), None), var)
assert isinstance(array, chainer.get_array_types())
return Variable(TypeInfo(array.shape, array.dtype), var)
def _check_is_initializer_like(initializer):
if not (initializer is None
or isinstance(initializer, chainer.Initializer)
or callable(initializer)
or isinstance(initializer, chainer.get_array_types())
or numpy.isscalar(initializer)):
raise TypeError(
'Initializer is of wrong type: {}. Allowed types are Initializer, '
'ndarray and scalar.'.format(type(initializer)))
def __init__(
self, func, xs, gys, params, eps, atol, rtol, no_gxs,
dtype, detect_nondifferentiable, is_immutable_params):
# If `is_immutable_params` is `False`, `params` are expected to be of
# type `chainer.Parameter` and are updated in-place.
# To run `_CheckBackward` with ChainerX ndarrays however which cannot
# be updated in-place when wrapped in `chainer.Parameter`s, this flag
# should be `True` and parameters should be given as ndarrays.
# `func` in the former case must take inputs as arguments only. In the
# latter, it must take the parameters in addition.
if dtype is not None and numpy.dtype(dtype).kind != 'f':
raise ValueError('`dtype` is allowed only float type')
if is_immutable_params:
if not all(
isinstance(p, chainer.get_array_types()) for p in params):
raise ValueError(
'All parameters in `params` must be ndarrays if '
'`is_immutable_params` is `True`. Actual: {}.'.format(
', '.join(str(type(p)) for p in params)))
xs = _as_tuple(xs)
if gys is not None:
gys = _as_tuple(gys)
params = _as_tuple(params)
if no_gxs is None:
no_gxs = [x.dtype.kind != 'f' for x in xs]
else:
if len(no_gxs) != len(xs):
raise ValueError(
'Length of no_grads param and xs should be same.\n'
'Actual: {0} != {1}'.format(len(no_gxs), len(xs)))
device = backend.get_device_from_array(*xs)
if device.xp is chainerx:
if params and not is_immutable_params:
raise NotImplementedError(
'gradient_check does not support params argument for '
'ChainerX arrays')
self.device = device
self.func = func
self.xs = xs
self.gys = gys
self.params = params
self.no_gxs = no_gxs
self.atol = atol
self.rtol = rtol
self.is_immutable_params = is_immutable_params
# options for numeric gradients
self.eps = eps
self.dtype = dtype
self.detect_nondifferentiable = detect_nondifferentiable
def _make_variable_from_array(array, name):
if not isinstance(array, chainer.get_array_types()):
raise InvalidType(
'isinstance({}, ndarray)'.format(name),
'type({}) == {}'.format(name, type(array)),
)
if in_light_mode():
return array
else:
return Variable(TypeInfo(array.shape, array.dtype), name)
def device_resident_accept(self, visitor):
super(Link, self).device_resident_accept(visitor)
d = self.__dict__
for name in self._params:
x = d[name]
visitor.visit_variable(x)
for name in self._persistent:
x = d[name]
if isinstance(x, chainer.get_array_types()):
d[name] = visitor.visit_array(x)
def device_resident_accept(self, visitor):
super(Link, self).device_resident_accept(visitor)
d = self.__dict__
for name in self._params:
x = d[name]
visitor.visit_variable(x)
for name in self._persistent:
x = d[name]
if isinstance(x, chainer.get_array_types()):
d[name] = visitor.visit_array(x)
def _preprocess_rhs(x, value):
if isinstance(value, chainer.Variable):
return value
if not (numpy.isscalar(value)
or isinstance(value, chainer.get_array_types())):
raise TypeError(
'Value must be a scalar, `numpy.ndarray`, `cupy.ndarray` '
'or a `Variable`.\nActual: {}'.format(type(value)))
return value.astype(x.dtype, copy=False)
def __call__(self, array):
if self.dtype is not None:
assert array.dtype == self.dtype
# Calling copy to ensures that the fill_value array
# is moved to the device where array resides
if isinstance(self.fill_value, chainer.get_array_types()):
backend.copyto(array, self.fill_value)
else:
device = backend.get_device_from_array(array)
array[...] = device.xp.asarray(self.fill_value)
def _prepare(self, param):
device = param.device
with chainer.using_device(device):
state = self.state
if state is None:
state = self._state = {}
self.init_state(param)
for name, value in six.iteritems(state):
if not isinstance(value, chainer.get_array_types()):
continue
state[name] = device.send(value)
def _prepare(self, param):
device = param.device
with chainer.using_device(device):
state = self.state
if state is None:
state = self._state = {}
self.init_state(param)
for name, value in six.iteritems(state):
if not isinstance(value, chainer.get_array_types()):
continue
state[name] = device.send(value)
def _get_initializer(initializer):
# type: (tp.Optional[types.InitializerSpec]) -> types.AbstractInitializer # NOQA
if initializer is None:
return LeCunNormal()
if (isinstance(initializer, chainer.get_array_types())
or numpy.isscalar(initializer)):
return Constant(initializer)
if not callable(initializer):
raise TypeError('invalid type of initializer: %s' % type(initializer))
return initializer
def __call__(self, trainer):
"""Execute the statistics extension.
Collect statistics for the current state of parameters.
Note that this method will merely update its statistic summary, unless
the internal trigger is fired. If the trigger is fired, the summary
will also be reported and then reset for the next accumulation.
Args:
trainer (~chainer.training.Trainer): Associated trainer that
invoked this extension.
"""
statistics = {}
for link in self._links:
link_name = getattr(link, 'name', 'None')
for param_name, param in link.namedparams():
for attr_name in self._attrs:
for function_name, function in \
six.iteritems(self._statistics):
# Get parameters as a flattened one-dimensional array
# since the statistics function should make no
# assumption about the axes
params = getattr(param, attr_name).ravel()
if (self._skip_nan_params
and (
backend.get_array_module(params).isnan(params)
.any())):
value = numpy.nan
else:
value = function(params)
key = self.report_key_template.format(
prefix=self._prefix + '/' if self._prefix else '',
link_name=link_name,
param_name=param_name,
attr_name=attr_name,
function_name=function_name
)
if (isinstance(value, chainer.get_array_types())
and value.size > 1):
# Append integer indices to the keys if the
# statistic function return multiple values
statistics.update({'{}/{}'.format(key, i): v for
i, v in enumerate(value)})
else:
statistics[key] = value
self._summary.add(statistics)
if self._trigger(trainer):
reporter.report(self._summary.compute_mean())
self._summary = reporter.DictSummary() # Clear summary
def _naive_batch_renormalization(
x, gamma, beta, # variables
mean, var, # variables
running_mean, running_var, # arrays
rmax, dmax, eps, decay):
# If decay is not None, the running stats are updated.
F = chainer.functions
assert isinstance(x, chainer.Variable)
assert isinstance(gamma, chainer.Variable)
assert isinstance(beta, chainer.Variable)
assert isinstance(mean, chainer.Variable)
assert isinstance(var, chainer.Variable)
assert isinstance(running_mean, chainer.get_array_types())
assert isinstance(running_var, chainer.get_array_types())
assert mean.shape == var.shape
assert mean.shape == running_mean.shape
assert mean.shape == running_var.shape
assert mean.shape == gamma.shape
dt = x.dtype.type
std = F.sqrt(var + dt(eps))
# r and d are gradient-stopped
running_std = numpy.sqrt(running_var + dt(eps))
r = (std.array / running_std).clip(1. / rmax, rmax)
d = ((mean.array - running_mean) / running_std).clip(-dmax, dmax)
xhat = (x - mean) / std * r + d
y = gamma * xhat + beta
# Update running stats
if decay is not None:
running_mean *= decay
running_mean += mean.array * dt(1. - decay)
# unbiased estimation
m = x.size // gamma.size
adjust = m / max(m - 1., 1.)
running_var *= decay
running_var += (var.array + dt(eps)) * dt((1. - decay) * adjust)
return y
def convert_parameter(parameter):
if isinstance(parameter, chainer.Parameter):
array = parameter.array
elif isinstance(parameter, chainer.Variable):
array = parameter.array
elif isinstance(parameter, chainer.get_array_types()):
array = parameter
else:
raise ValueError(
'The type of parameter is unknown. It should be either Parameter '
'or Variable or ndarray, but the type was {}.'.format(
type(parameter)))
array = chainer.cuda.to_cpu(array)
return numpy_helper.from_array(array, str(id(parameter)))
def _concat_arrays(arrays, padding):
# Convert `arrays` to numpy.ndarray if `arrays` consists of the built-in
# types such as int, float or list.
if not isinstance(arrays[0], chainer.get_array_types()):
arrays = numpy.asarray(arrays)
if padding is not None:
arr_concat = _concat_arrays_with_padding(arrays, padding)
else:
device = backend.get_device_from_array(arrays[0])
with chainer.using_device(device):
arr_concat = device.xp.concatenate(
[array[None] for array in arrays])
return arr_concat
def _make_dataset(key, data, size):
if isinstance(data, chainer.get_array_types()):
if key is None:
key = '_{}'.format(id(data))
return _Array(key, data)
elif isinstance(data, list):
if key is None:
key = '_{}'.format(id(data))
return _List(key, data)
elif callable(data):
if key is None:
raise ValueError('key(s) must be specified for callable')
if size is None:
raise ValueError('size must be specified for callable')
return _Index(size).transform(key, data)
def _check_backward_with_params(
# This function was introduced along with the `is_immutable_params`
# argument to `_CheckBackward`.
# It allows passing `params` as ndarrays instead of `Parameter`s and thus
# depends less on the state of the parameter held by the caller.
# It is required by the `LinkTestCase` to check ChainerX parameter
# gradients, since those parameters cannot perturbed in-place for the
# numerical gradients if passed as `Parameter`s as those requiring
# gradients cannot be updated in-place.
func, x_data, y_grad, params=(),
eps=1e-3, atol=1e-5, rtol=1e-4, no_grads=None, dtype=None,
detect_nondifferentiable=False):
assert all(isinstance(p, chainer.get_array_types()) for p in params)
_CheckBackward(
func, x_data, y_grad, params, eps, atol, rtol, no_grads, dtype,
detect_nondifferentiable, is_immutable_params=True
).run()
def __init__(self, data=None, **kwargs):
name, grad, requires_grad = argument.parse_kwargs(
kwargs, ('name', None), ('grad', None), ('requires_grad', True),
volatile='volatile argument is not supported anymore. '
'Use chainer.using_config')
if (data is not None and
not isinstance(data, chainer.get_array_types())):
msg = '''numpy.ndarray or cuda.ndarray are expected.
Actual: {0}'''.format(type(data))
raise TypeError(msg)
# Use a list as a data structure to hold the data array indirectly to
# abstract its initialized/uninitialized state.
self._data = [data]
self._requires_grad = requires_grad
self._node = VariableNode(self, name)
self._grad_var = None if grad is None else Variable(grad)
self._loss_scale = None
def _convert_value_to_string(value):
if isinstance(value, variable.Variable):
value = value.data
if numpy.isscalar(value):
if value < 0:
return '({})'.format(value)
else:
return str(value)
array_types = chainer.get_array_types()
if isinstance(value, array_types):
return 'constant array'
else:
raise ValueError(
'Value must be a Variable, scalar, {} or {}. Actual: {}'.format(
', '.join([str(at) for at in array_types[:-1]]),
array_types[-1], type(value)))
def __init__(self, data=None, **kwargs):
name, grad, requires_grad = argument.parse_kwargs(
kwargs, ('name', None), ('grad', None), ('requires_grad', True),
volatile='volatile argument is not supported anymore. '
'Use chainer.using_config')
if (data is not None
and not isinstance(data, chainer.get_array_types())):
msg = '''numpy.ndarray or cuda.ndarray are expected.
Actual: {0}'''.format(type(data))
raise TypeError(msg)
# Use a list as a data structure to hold the data array indirectly to
# abstract its initialized/uninitialized state.
self._data = [data]
self._requires_grad = requires_grad
self._node = VariableNode(self, name)
self._grad_var = None if grad is None else Variable(grad)
self._loss_scale = None
def _array_from_chainerx(array):
if array is None:
return None
if not isinstance(array, chainerx.ndarray):
if isinstance(array, chainer.get_array_types()):
return array
raise TypeError(
'Tried to convert to a non-ChainerX array from an invalid type: '
'{}'.format(type(array)))
backend_name = array.device.backend.name
if backend_name == 'native':
return _cpu._to_cpu(array)
if backend_name == 'cuda':
return cuda.to_gpu(array, array.device.index)
raise ValueError(
'Only ChainerX arrays with native or cuda backends can be converted '
'to non-ChainerX arrays.\nActual: {0}.'.format(backend_name))
def _array_from_chainerx(array):
if array is None:
return None
if not isinstance(array, chainerx.ndarray):
if isinstance(array, chainer.get_array_types()):
return array
raise TypeError(
'Tried to convert to a non-ChainerX array from an invalid type: '
'{}'.format(type(array)))
backend_name = array.device.backend.name
if backend_name == 'native':
return _cpu._to_cpu(array)
if backend_name == 'cuda':
return cuda.to_gpu(array, array.device.index)
raise ValueError(
'Only ChainerX arrays with native or cuda backends can be converted '
'to non-ChainerX arrays.\nActual: {0}.'.format(backend_name))
def format_customized_shapes(args, shapes):
if isinstance(args, (list, tuple)):
if not isinstance(shapes, list) or len(args) != len(shapes):
raise ValueError('Customized shapes cannot fit for input list')
for i, (arg, shape) in enumerate(zip(args, shapes)):
if len(arg.shape) != len(shape):
raise ValueError(
'Index-{} shape length must be same as input'.format(i))
return shapes
elif isinstance(args, dict):
if not isinstance(shapes, (list, dict)) or\
len(args) != len(shapes):
raise ValueError('Customized shapes cannot fit for input dict')
if isinstance(shapes, list):
shapes = {k: v for k, v in zip(args.keys(), shapes)}
formatted_shapes = []
for k, arg in args.items():
if k not in shapes:
raise ValueError(
'Key "{}" is not found in customized shapes'.format(k))
if len(arg.shape) != len(shapes[k]):
raise ValueError(
'Key "{}" shape length must be same as input'.format(k))
formatted_shapes.append(shapes[k])
return formatted_shapes
else:
assert isinstance(args, (chainer.Variable, chainer.get_array_types()))
if isinstance(shapes, list):
if len(shapes) != 1:
raise ValueError('Customized shape must be single')
elif not isinstance(shapes, tuple):
raise ValueError(
'Type {} is not supported for single input'.format(
type(shapes)))
else:
shapes = [shapes]
if len(args.shape) != len(shapes[0]):
raise ValueError('Shape length must be same as input')
return shapes
def convert_GetItem(func, opset_version, input_names, output_names, context):
x = func.inputs[0]
axes, starts, ends = [], [], []
squeeze_idxs, unsqueeze_idxs = [], []
skipped = 0 # when set ellipsis, need to skip index rolling
prev_gathered_axis = -1
gather_axis = -1
gather_idx = None # when GatherND, set first array for broadcasting
gather_nd_idx = None
is_used_slice_whole = False # GatherND does not support axis, need to care
for i, idx in enumerate(func.slices):
# axis means the index of input x, adjust None and Ellipsis counts
axis = i - len(unsqueeze_idxs) + skipped
if isinstance(idx, slice):
if idx.step is not None and idx.step != 1:
raise ValueError(
'GetItem with {}step slicing is not supported in ONNX '
'Slice operator'.format(idx.step))
if idx.start is None and idx.stop is None:
is_used_slice_whole = True
continue
axes.append(axis)
starts.append(0 if idx.start is None else idx.start)
ends.append(x.shape[axis] if idx.stop is None else idx.stop)
elif isinstance(idx, int):
axes.append(axis)
starts.append(idx)
ends.append(idx + 1)
squeeze_idxs.append(axis)
elif isinstance(idx, np.ndarray) and idx.ndim == 0:
scalar_idx = idx.item()
axes.append(axis)
starts.append(scalar_idx)
ends.append(scalar_idx + 1)
squeeze_idxs.append(axis)
elif idx is None:
unsqueeze_idxs.append(i - len(squeeze_idxs) + skipped)
elif idx is Ellipsis:
# calculate rest slice number except None, GetItem does not allow
# multiple Ellipsis, so ignore latter Ellipsis count
rest_slice_len = len(
[idx_ for idx_ in func.slices[i + 1:] if idx_ is not None])
assert skipped == 0
skipped = len(x.shape) - axis - rest_slice_len - 1
elif isinstance(idx, (list, ) + chainer.get_array_types()):
if prev_gathered_axis >= 0:
if (i - 1) != prev_gathered_axis:
raise ValueError(
'ONNX-Chainer does not support non-consecutive'
'multiple advanced indexing')
if is_used_slice_whole:
raise ValueError(
'ONNX-Chainer does not support whole indexing(`[:]`)'
'in front of multiple advanced indexing')
if unsqueeze_idxs:
raise ValueError(
'ONNX-Chainer does not support new axis in front of '
'multiple advanced indexing')
# multiple advanced index, convert to GatherND
idx_array = _to_ndarray(idx)
base_idx = gather_idx if gather_nd_idx is None else\
gather_nd_idx
gather_nd_idx = np.vstack((base_idx, idx_array))
prev_gathered_axis = i
else:
# convert to Gather, if next index is also list, change to
# GatherND
gather_axis = axis - len(squeeze_idxs) + len(unsqueeze_idxs)
gather_idx = _to_ndarray(idx)
prev_gathered_axis = i
else:
raise ValueError(
'GetItem with type {} cannot handle in ONNX Slice, so that '
'ONNX-Chainer does not accept the type'.format(type(idx)))
gb = onnx_helper.GraphBuilder()
slice_output = input_names
if axes:
output = get_slice_node(gb, opset_version, context, slice_output, axes,
starts, ends)
slice_output = [output]
if squeeze_idxs:
output = gb.op('Squeeze', slice_output, axes=squeeze_idxs)
slice_output = [output]
if unsqueeze_idxs:
output = gb.op('Unsqueeze', slice_output, axes=unsqueeze_idxs)
slice_output = [output]
if gather_nd_idx is not None:
gather_nd_idx_name = context.add_const(gather_nd_idx.T, 'indices')
slice_output.append(gather_nd_idx_name)
gb.op('GatherND', slice_output)
elif gather_idx is not None:
gather_idx_name = context.add_const(gather_idx, 'indices')
slice_output.append(gather_idx_name)
gb.op('Gather', slice_output, axis=gather_axis)
return gb.nodes(output_names=output_names)
def _check_contiguousness(self, arr):
assert isinstance(arr, chainer.get_array_types())
_check_contiguousness(arr, self.contiguous)
def _export(model, args, filename, export_params, graph_name, save_text,
opset_version, input_names, output_names, return_named_inout,
external_converters, external_opset_imports, input_shapes):
if opset_version is None:
opset_version = min(int(onnx.defs.onnx_opset_version()),
MAXIMUM_OPSET_VERSION)
elif opset_version < MINIMUM_OPSET_VERSION or \
opset_version > MAXIMUM_OPSET_VERSION:
warnings.warn(
'ONNX-Chainer has been tested only with opset_version {} ~ {}'
'The ONNX file exported with your requested opset_version ({}) '
'may cause some problems because the converters used for the '
'opset_version have not been tested.'.format(
MINIMUM_OPSET_VERSION, MAXIMUM_OPSET_VERSION, opset_version))
if input_shapes is not None:
# if input shapes are invalid, raise exception before forwarding.
input_shapes = format_customized_shapes(args, input_shapes)
with RetainInputHook():
# Forward computation
context = Context(model)
network_inputs = OrderedDict()
if isinstance(args, tuple):
args = list(args)
if isinstance(args, list):
for i, arg in enumerate(args):
if isinstance(arg, chainer.get_array_types()):
args[i] = chainer.Variable(arg)
network_inputs[context.get_name(args[i])] = args[i]
outputs = model(*args)
elif isinstance(args, dict):
for key, arg in args.items():
if isinstance(arg, chainer.get_array_types()):
args[key] = chainer.Variable(arg)
network_inputs[context.get_name(args[key])] = args[key]
outputs = model(**args)
elif isinstance(args, chainer.get_array_types()):
args = chainer.Variable(args)
network_inputs[context.get_name(args)] = args
outputs = model(args)
elif isinstance(args, chainer.Variable):
network_inputs[context.get_name(args)] = args
outputs = model(args)
else:
raise ValueError(
'The \'args\' argument should be a list, tuple, dict, '
'numpy array, or Chainer Variable. But a {} object was '
'given.'.format(type(args)))
rename_variable_name(context, args, network_inputs, input_names)
initializers = []
input_tensors = []
param_names = set()
for org_name, param in model.namedparams():
# `model.namedparams()` has `include_uninit` flag but not use, to
# output user warning
if param.array is None:
warnings.warn(
'The parameter \'{}\' is not initialized, skip setting to '
'ONNX graph'.format(org_name))
continue
name = context.get_name(param)
param_names.add(name)
tensor = convert_parameter(param, context)
initializers.append(tensor)
input_tensors.append(
helper.make_tensor_value_info(name, tensor.data_type,
tensor.dims))
for i, (name, var) in enumerate(network_inputs.items()):
shape = var.shape if input_shapes is None else input_shapes[i]
input_tensors.append(
helper.make_tensor_value_info(
name, NP_TYPE_TO_TENSOR_TYPE[var.dtype], shape))
if external_converters:
chainer.utils.experimental('external_converters')
converters = dict(mapping.converters, **external_converters)
else:
converters = mapping.converters
if isinstance(outputs, (list, tuple)):
flat_outputs = outputs
elif isinstance(outputs, dict):
flat_outputs = list(outputs.values())
elif isinstance(outputs, chainer.Variable):
flat_outputs = [outputs]
else:
raise RuntimeError(
'Unexpected output type from the model: {}'.format(
type(outputs)))
if not all([isinstance(o, chainer.Variable) for o in flat_outputs]):
raise ValueError('The all \'outputs\' must be Chainer Variable')
network_outputs = OrderedDict([(context.get_name(var), var)
for var in flat_outputs])
if output_names:
rename_variable_name(context, outputs, network_outputs,
output_names)
o = Graph(context, converters, opset_version,
param_names | set(network_inputs.keys()), network_outputs)
o.to_onnx_graph()
implicit_input_names = set(context.implicit_inputs.keys())
for name in implicit_input_names:
tensor = convert_parameter(context.implicit_inputs[name], context)
initializers.append(tensor)
input_tensors.append(
helper.make_tensor_value_info(name, tensor.data_type, tensor.dims))
# If additional parameters are created during conversion
for param in context.parameters:
tensor = convert_parameter(param, context)
initializers.append(tensor)
input_tensors.append(
helper.make_tensor_value_info(context.get_name(param),
tensor.data_type, tensor.dims))
# Convert output tensors
output_tensors = []
for name, var in network_outputs.items():
output_tensors.append(
helper.make_tensor_value_info(name,
NP_TYPE_TO_TENSOR_TYPE[var.dtype],
var.shape))
if not export_params:
initializers = []
onnx_graph = helper.make_graph(o.graph,
graph_name,
input_tensors,
output_tensors,
initializer=initializers)
opset_imports = [helper.make_operatorsetid('', opset_version)]
if external_opset_imports:
chainer.utils.experimental('external_opset_imports')
for domain, version in external_opset_imports.items():
opset_imports.append(helper.make_operatorsetid(domain, version))
model = helper.make_model(onnx_graph,
producer_name='Chainer',
producer_version=chainer.__version__,
opset_imports=opset_imports)
model.ir_version = onnx.IR_VERSION
check_onnx_model(model, external_converters, external_opset_imports)
if input_shapes is not None:
for output in model.graph.output:
for d in output.type.tensor_type.shape.dim:
d.Clear()
model = shape_inference.infer_shapes(model)
check_onnx_model(model, external_converters, external_opset_imports)
if filename is not None and isinstance(filename, str):
with open(filename, 'wb') as fp:
fp.write(model.SerializeToString())
if save_text:
with open(filename + '.txt', 'w') as fp:
print(model, file=fp)
elif hasattr(filename, 'write'):
filename.write(model.SerializeToString())
if return_named_inout:
chainer.utils.experimental('return_named_inout')
return model, network_inputs, network_outputs
return model
def visit_array(self, arr):
assert isinstance(arr, chainer.get_array_types())
return backend.from_chx(arr)
def _check_contiguousness(self, arr):
assert isinstance(arr, chainer.get_array_types())
_check_contiguousness(arr, self.contiguous)
