def _check_chainerx_numpy_result_array(check_result_func, chainerx_result, numpy_result, indices): # Compares `chainerx_result` and `numpy_result` as arrays. is_chainerx_valid_type = isinstance(chainerx_result, chainerx.ndarray) is_numpy_valid_type = _is_numpy_type(numpy_result) if not (is_chainerx_valid_type and is_numpy_valid_type): raise _ResultsCheckFailure( 'Using decorator without returning ndarrays. ' 'If you want to explicitly ignore certain tests, ' 'return chainerx.testing.ignore() to avoid this error', indices) if chainerx_result.shape != numpy_result.shape: raise _ResultsCheckFailure( 'Shape mismatch', indices, lambda np_r, chx_r: ('chainerx: {}, numpy: {}'.format(chx_r.shape, np_r.shape))) if chainerx_result.device is not chainerx.get_default_device(): raise _ResultsCheckFailure( 'ChainerX bad device', indices, lambda np_r, chx_r: ('default: {}, chainerx: {}'.format( chainerx.get_default_device(), chx_r.device))) try: check_result_func(chainerx_result, numpy_result) except AssertionError as e: # Convert AssertionError to _ResultsCheckFailure raise _ResultsCheckFailure(str(e), indices)
def _check_chainerx_numpy_result_array( check_result_func, chainerx_result, numpy_result, indices): # Compares `chainerx_result` and `numpy_result` as arrays. is_chainerx_valid_type = isinstance(chainerx_result, chainerx.ndarray) is_numpy_valid_type = _is_numpy_type(numpy_result) if not (is_chainerx_valid_type and is_numpy_valid_type): raise _ResultsCheckFailure( 'Using decorator without returning ndarrays. ' 'If you want to explicitly ignore certain tests, ' 'return chainerx.testing.ignore() to avoid this error', indices) if chainerx_result.shape != numpy_result.shape: raise _ResultsCheckFailure( 'Shape mismatch', indices, lambda np_r, chx_r: ( 'chainerx: {}, numpy: {}'.format(chx_r.shape, np_r.shape))) if chainerx_result.device is not chainerx.get_default_device(): raise _ResultsCheckFailure( 'ChainerX bad device', indices, lambda np_r, chx_r: ( 'default: {}, chainerx: {}'.format( chainerx.get_default_device(), chx_r.device))) try: check_result_func(chainerx_result, numpy_result) except AssertionError as e: # Convert AssertionError to _ResultsCheckFailure raise _ResultsCheckFailure(str(e), indices)
def test_using_device_with_name(device_instance1, device_instance2): if device_instance1 == device_instance2: return device1 = device_instance1 device2 = device_instance2 chainerx.set_default_device(device1) with chainerx.using_device(device2.name) as scope: assert chainerx.get_default_device() == device2 assert scope.device is device2 with chainerx.using_device(device2.backend.name, device2.index) as scope: assert chainerx.get_default_device() == device2 assert scope.device is device2
def setup(self, float_dtype): dtype = float_dtype ksize = self.ksize device = chainerx.get_default_device() if (device.backend.name == 'cuda' and len(ksize) != 2 and len(ksize) != 3): raise unittest.SkipTest( 'cuDNN supports only 2 and 3 spatial dimensions.') # TODO(niboshi): average_pool can return nan if pad_mode is 'ignore', # and numeric gradients cannot be calculated. # If chainerx.where is implemented, we can replace nans and remove # this skip. if self.pad_mode in ('ignore', None): self.skip_backward_test = True self.skip_double_backward_test = True self.check_double_backward_options.update({'rtol': 5e-3, 'atol': 5e-3}) if dtype == 'float16': self.check_forward_options.update({'rtol': 5e-3, 'atol': 5e-4}) self.check_backward_options.update({'rtol': 5e-2, 'atol': 5e-3}) else: self.check_backward_options.update({'rtol': 5e-3, 'atol': 5e-3, }) self.dtype = dtype
def setup(self): device = chainerx.get_default_device() a_dtype, b_dtype = self.in_dtypes a_kind = numpy.dtype(a_dtype).kind b_kind = numpy.dtype(b_dtype).kind # TODO(beam2d): Remove the skip after supporting non-float dot on CUDA if device.name == 'cuda:0' and (a_kind != 'f' and b_kind != 'f'): pytest.skip('non-float dot is not supported on CUDA') # Skip backward/double-backward tests for int dtypes if a_kind != 'f' or b_kind != 'f': self.skip_backward_test = True self.skip_double_backward_test = True # Skip backward/double-backward tests if the output will be # disconnected. # TODO(niboshi): Remove this skip condition after enabling backward() # for such cases. if self.a_shape and self.a_shape[-1] == 0: self.skip_backward_test = True self.skip_double_backward_test = True if a_dtype == 'float16' or b_dtype == 'float16': self.check_forward_options.update({ 'rtol': 1e-2, 'atol': 1e-2}) self.check_backward_options.update({ 'rtol': 1e-2, 'atol': 1e-2}) self.check_double_backward_options.update({ 'rtol': 1e-2, 'atol': 1e-2})
def setup(self): device = chainerx.get_default_device() a_dtype, b_dtype = self.in_dtypes a_kind = numpy.dtype(a_dtype).kind b_kind = numpy.dtype(b_dtype).kind # TODO(beam2d): Remove the skip after supporting non-float dot on CUDA if device.name == 'cuda:0' and (a_kind != 'f' and b_kind != 'f'): pytest.skip('non-float dot is not supported on CUDA') # Skip backward/double-backward tests for int dtypes if a_kind != 'f' or b_kind != 'f': self.skip_backward_test = True self.skip_double_backward_test = True # Skip backward/double-backward tests if the output will be # disconnected. # TODO(niboshi): Remove this skip condition after enabling backward() # for such cases. if self.a_shape and self.a_shape[-1] == 0: self.skip_backward_test = True self.skip_double_backward_test = True if a_dtype == 'float16' or b_dtype == 'float16': self.check_forward_options.update({'rtol': 1e-2, 'atol': 1e-2}) self.check_backward_options.update({'rtol': 1e-2, 'atol': 1e-2}) self.check_double_backward_options.update({ 'rtol': 1e-2, 'atol': 1e-2 })
def setup(self, float_dtype): dtype = float_dtype ksize = self.ksize device = chainerx.get_default_device() if (device.backend.name == 'cuda' and len(ksize) != 2 and len(ksize) != 3): raise unittest.SkipTest( 'cuDNN supports only 2 and 3 spatial dimensions.') # TODO(niboshi): average_pool can return nan if pad_mode is 'ignore', # and numeric gradients cannot be calculated. # If chainerx.where is implemented, we can replace nans and remove # this skip. if self.pad_mode in ('ignore', None): self.skip_backward_test = True self.skip_double_backward_test = True self.check_double_backward_options.update({'rtol': 5e-3, 'atol': 5e-3}) if dtype == 'float16': self.check_forward_options.update({'rtol': 5e-3, 'atol': 5e-4}) self.check_backward_options.update({'rtol': 5e-2, 'atol': 5e-3}) else: self.check_backward_options.update({ 'rtol': 5e-3, 'atol': 5e-3, }) self.dtype = dtype
def setup(self): device = chainerx.get_default_device() if (device.backend.name == 'native' and not chainerx.linalg._is_lapack_available()): pytest.skip('LAPACK is not linked to ChainerX') self.check_backward_options.update({'rtol': 5e-3}) self.check_double_backward_options.update({'rtol': 5e-3})
def setup(self, float_dtype): device = chainerx.get_default_device() if device.backend.name == 'cuda' and len(self.x_shape) <= 3: # TODO(hvy): Support 1 dimensional convolution with CUDA. pytest.skip('cudnn does not support 1-dim convolution') if device.backend.name == 'cuda' and self.cover_all: pytest.skip('cudnn does not support cover_all') if device.backend.name == 'native' and float_dtype == 'float16': # TODO(niboshi): Fix accuracy pytest.skip('Native float16 operation has insufficient accuracy') self.dtype = float_dtype if float_dtype == 'float16': self.check_forward_options.update({'rtol': 5e-2, 'atol': 5e-3}) self.check_backward_options.update({ 'eps': 2**-3, 'rtol': 1e-1, 'atol': 1e-2 }) else: self.check_forward_options.update({'rtol': 1e-3}) self.check_backward_options.update({ 'eps': 1e-2, 'rtol': 1e-3, 'atol': 1e-4 }) self.check_double_backward_options.update({'rtol': 5e-2, 'atol': 5e-3})
def test_chainerx_get_device(): context = chainerx.Context() with chainerx.context_scope(context): device = chainerx.get_device('native:0') assert device.context is context assert device.name == 'native:0' assert device is chainerx.get_device('native', 0) assert device is chainerx.get_device(device) assert chainerx.get_default_device() is chainerx.get_device()
def setup(self): super().setup() device = chainerx.get_default_device() _skip_if_native_and_lapack_unavailable(device) self.check_forward_options.update({'rtol': 1e-4, 'atol': 1e-4}) self.check_backward_options.update({'rtol': 5e-3}) self.check_double_backward_options.update({'rtol': 5e-3})
def test_ascontiguousarray_from_chainerx_array_device(): with chainerx.using_device(chainerx.get_device('native:0')): dev = chainerx.get_device('native:1') # Non default one assert chainerx.get_default_device() is not dev a = chainerx.arange(10, device=dev) b = chainerx.ascontiguousarray(a) assert b.is_contiguous is True assert b.device is dev
def prepare_chainerx_inputs(num_input, config, grad=False): device = chainerx.get_default_device() dtype = config['dtype'] inputs = prepare_numpy_inputs(num_input, config) inputs = [chainerx.array(i, dtype=dtype) for i in inputs] if grad: for i in inputs: i.require_grad() device.synchronize() return inputs
def cache_restore_context(request): device = chainerx.get_default_device() context = chainerx.get_default_context() global_context = chainerx.get_global_default_context() def restore_context(): chainerx.set_global_default_context(global_context) chainerx.set_default_context(context) chainerx.set_default_device(device) request.addfinalizer(restore_context)
def check_device_spec_chainerx(self, device_spec, expected_device_name): device = backend.get_device(device_spec) assert isinstance(device, backend.ChainerxDevice) assert device.xp is chainerx assert isinstance(device.device, chainerx.Device) assert device.device.name == expected_device_name with backend.using_device(device_spec): # TODO(niboshi): Test the Chainer default device assert (chainerx.get_default_device() == chainerx.get_device( expected_device_name))
def _check_array_from_chainerx_array_with_dtype( shape, src_dtype, dst_dtype_spec, copy, device=None): t = array_utils.create_dummy_ndarray( chainerx, shape, src_dtype, device=device) a = chainerx.array(t, dtype=dst_dtype_spec, copy=copy) src_dtype = chainerx.dtype(src_dtype) dst_dtype = src_dtype if dst_dtype_spec is None else chainerx.dtype( dst_dtype_spec) device = chainerx.get_device(device) if (not copy and src_dtype == dst_dtype and device is chainerx.get_default_device()): assert t is a else: assert t is not a chainerx.testing.assert_array_equal_ex(a, t.astype(dst_dtype)) assert a.dtype == dst_dtype assert a.device is chainerx.get_default_device()
def test_view(shape, dtype): array = array_utils.create_dummy_ndarray(chainerx, shape, dtype) view = array.view() chainerx.testing.assert_array_equal_ex(view, array) assert view.device is chainerx.get_default_device() # inplace modification if array.size > 0: array *= array assert array._debug_flat_data == view._debug_flat_data
def test_using_device(device_instance1, device_instance2): if device_instance1 == device_instance2: return device1 = device_instance1 device2 = device_instance2 chainerx.set_default_device(device1) with chainerx.using_device(device2) as scope: assert chainerx.get_default_device() is device2 assert scope.device is device2 scope = chainerx.using_device(device2) assert chainerx.get_default_device() == device1 assert scope.device is device2 with scope: assert chainerx.get_default_device() == device2 assert scope.device is device2 assert chainerx.get_default_device() == device1 assert scope.device is device2
def test_view(shape, dtype): array = array_utils.create_dummy_ndarray(chainerx, shape, dtype) view = array.view() chainerx.testing.assert_array_equal_ex(view, array) assert view.device is chainerx.get_default_device() # inplace modification if array.size > 0: array += array assert array._debug_flat_data == view._debug_flat_data
def check_device_spec_chainerx(self, device_spec, expected_device_name): device = backend.get_device(device_spec) assert isinstance(device, backend.ChainerxDevice) assert device.xp is chainerx assert isinstance(device.device, chainerx.Device) assert device.device.name == expected_device_name with backend.using_device(device_spec): # TODO(niboshi): Test the Chainer default device assert ( chainerx.get_default_device() == chainerx.get_device(expected_device_name))
def setup(self, float_dtype): self.dtype = float_dtype device = chainerx.get_default_device() if device.backend.name == 'cuda' and len(self.x_shape) <= 3: # TODO(sonots): Support 1 dimensional convolution with CUDA. pytest.skip( 'cuDNN does not support 1 dimensional convolution and throws ' 'DimensionError') if device.backend.name == 'cuda' and self.cover_all is True: pytest.skip( 'outsize (for cover_all=True) is not supported by CUDA') if float_dtype == 'float16': self.check_forward_options.update({'rtol': 5e-2, 'atol': 5e-3}) self.check_backward_options.update({ 'eps': 2**-3, 'rtol': 1e-1, 'atol': 1e-2 }) else: self.check_forward_options.update({'rtol': 1e-3}) self.check_backward_options.update({ 'eps': 1e-2, 'rtol': 1e-3, 'atol': 1e-4 }) self.check_double_backward_options.update({'rtol': 5e-2, 'atol': 5e-3}) # Determine outsize cover_all = self.cover_all if cover_all is None: outsize = None else: x_shape = self.x_shape w_shape = self.w_shape stride = self.stride pad = self.pad in_dims = x_shape[2:] kernel_size = w_shape[2:] ndim = len(in_dims) stride_tup = ((stride, ) * ndim if isinstance(stride, int) else stride) pad_tup = (pad, ) * ndim if isinstance(pad, int) else pad outsize = tuple( chainer.utils.conv.get_deconv_outsize(d, k, s, p, cover_all) for (d, k, s, p) in zip(in_dims, kernel_size, stride_tup, pad_tup)) self.outsize = outsize
def setup(self): device = chainerx.get_default_device() _skip_if_native_and_lapack_unavailable(device) self.check_backward_options.update({ 'eps': 1e-5, 'rtol': 1e-3, 'atol': 1e-3 }) self.check_double_backward_options.update({ 'eps': 1e-5, 'rtol': 1e-3, 'atol': 1e-3 })
def _guess_device_from_array_module(xp): """Returns a plausible device from array module .. warning:: There can be multiple devices for a module """ if xp is cuda.cupy: return cuda.GpuDevice(cuda.Device()) elif xp is chainerx: return _chainerx.ChainerxDevice(chainerx.get_default_device()) else: # Cannot detect intel64, because xp of intel64 is numpy. return _cpu.CpuDevice()
def setup(self, float_dtype): dtype = float_dtype ksize = self.ksize device = chainerx.get_default_device() if (device.backend.name == 'cuda' and len(ksize) != 2 and len(ksize) != 3): raise unittest.SkipTest( 'cuDNN supports only 2 and 3 spatial dimensions') if dtype == 'float16': self.check_backward_options.update({'rtol': 5e-2, 'atol': 1e-3}) self.check_double_backward_options.update({ 'rtol': 5e-2, 'atol': 1e-3}) self.dtype = dtype
def setup(self): self.check_forward_options.update({'rtol': 1e-2, 'atol': 1e-2}) self.check_backward_options.update({'rtol': 1e-2, 'atol': 1e-2}) self.check_double_backward_options.update({'rtol': 5e-3, 'atol': 5e-2}) if self.in_dtypes[0] == 'float16': self.check_forward_options.update({'rtol': 1e-1, 'atol': 1e-1}) self.check_backward_options.update({'rtol': 1e-1, 'atol': 1e-1}) self.check_double_backward_options.update({ 'rtol': 1e-1, 'atol': 1e-1 }) device = chainerx.get_default_device() if device.backend.name == 'cuda': if self.in_dtypes[0] != 'float32': self.skip_backward_test = True self.skip_double_backward_test = True
def setup(self, float_dtype): dtype = float_dtype ksize = self.ksize device = chainerx.get_default_device() if (device.backend.name == 'cuda' and len(ksize) != 2 and len(ksize) != 3): raise unittest.SkipTest( 'cuDNN supports only 2 and 3 spatial dimensions') if dtype == 'float16': self.check_backward_options.update({'rtol': 5e-2, 'atol': 1e-3}) self.check_double_backward_options.update({ 'rtol': 5e-2, 'atol': 1e-3 }) self.dtype = dtype
def setup(self, float_dtype): self.dtype = float_dtype device = chainerx.get_default_device() if device.backend.name == 'cuda' and len(self.x_shape) <= 3: # TODO(sonots): Support 1 dimensional convolution with CUDA. pytest.skip( 'cuDNN does not support 1 dimensional convolution and throws ' 'DimensionError') if device.backend.name == 'cuda' and self.cover_all is True: pytest.skip( 'outsize (for cover_all=True) is not supported by CUDA') if float_dtype == 'float16': self.check_forward_options.update({'rtol': 5e-2, 'atol': 5e-3}) self.check_backward_options.update({ 'eps': 2 ** -3, 'rtol': 1e-1, 'atol': 1e-2}) else: self.check_forward_options.update({'rtol': 1e-3}) self.check_backward_options.update({ 'eps': 1e-2, 'rtol': 1e-3, 'atol': 1e-4}) self.check_double_backward_options.update({ 'rtol': 5e-2, 'atol': 5e-3}) # Determine outsize cover_all = self.cover_all if cover_all is None: outsize = None else: x_shape = self.x_shape w_shape = self.w_shape stride = self.stride pad = self.pad in_dims = x_shape[2:] kernel_size = w_shape[2:] ndim = len(in_dims) stride_tup = ( (stride,) * ndim if isinstance(stride, int) else stride) pad_tup = (pad,) * ndim if isinstance(pad, int) else pad outsize = tuple( chainer.utils.conv.get_deconv_outsize(d, k, s, p, cover_all) for (d, k, s, p) in zip(in_dims, kernel_size, stride_tup, pad_tup)) self.outsize = outsize
def setup(self, dtype): device = chainerx.get_default_device() # TODO(imanishi): Remove the skip after supporting non-float dot on # CUDA if device.name == 'cuda:0' and numpy.dtype(dtype).kind != 'f': pytest.skip('non-float dot is not supported') # Skip backward/double-backward tests for int dtypes if numpy.dtype(dtype).kind != 'f': self.skip_backward_test = True self.skip_double_backward_test = True if dtype == 'float16': self.check_forward_options.update({ 'rtol': 1e-2, 'atol': 1e-2}) self.check_backward_options.update({ 'rtol': 1e-2, 'atol': 1e-2}) self.check_double_backward_options.update({ 'rtol': 1e-2, 'atol': 1e-2}) self.dtype = dtype
def setup(self): if len(self.in_dtypes) == 3: x_dtype, w_dtype, b_dtype = self.in_dtypes else: (x_dtype, w_dtype), b_dtype = self.in_dtypes, None x_kind = numpy.dtype(x_dtype).kind w_kind = numpy.dtype(w_dtype).kind b_kind = None if b_dtype is None else numpy.dtype(b_dtype).kind device = chainerx.get_default_device() if device.backend.name == 'cuda' and len(self.x_shape) <= 3: # TODO(hvy): Support 1 dimensional convolution with CUDA. pytest.skip('cudnn does not support 1-dim convolution') if device.backend.name == 'cuda' and self.cover_all: pytest.skip('cudnn does not support cover_all') # Skip backward/double-backward tests for int dtypes if (x_kind != 'f' and w_kind != 'f' and (b_kind is None or b_kind != 'f')): self.skip_backward_test = True self.skip_double_backward_test = True if (x_dtype == 'float16' or w_dtype == 'float16' or b_dtype == 'float16'): self.check_forward_options.update({'rtol': 5e-2, 'atol': 5e-3}) self.check_backward_options.update({ 'eps': 2**-3, 'rtol': 1e-1, 'atol': 1e-2 }) else: self.check_forward_options.update({'rtol': 1e-3}) self.check_backward_options.update({ 'eps': 1e-2, 'rtol': 1e-3, 'atol': 1e-4 }) self.check_double_backward_options.update({'rtol': 5e-2, 'atol': 5e-3})
def setup(self): if len(self.in_dtypes) == 3: x_dtype, w_dtype, b_dtype = self.in_dtypes else: (x_dtype, w_dtype), b_dtype = self.in_dtypes, None x_kind = numpy.dtype(x_dtype).kind w_kind = numpy.dtype(w_dtype).kind b_kind = None if b_dtype is None else numpy.dtype(b_dtype).kind device = chainerx.get_default_device() if device.backend.name == 'cuda' and (x_kind != 'f' or w_kind != 'f' or b_kind != 'f'): raise unittest.SkipTest('CUDA dot does not support integers.') # Skip backward/double-backward tests for int dtypes if (x_kind != 'f' and w_kind != 'f' and (b_kind is None or b_kind != 'f')): self.skip_backward_test = True self.skip_double_backward_test = True # Skip backward/double-backward tests if the output will be # disconnected. # TODO(niboshi): Remove this skip condition after enabling backward() # for such cases. if 0 in self.x_shape or 0 in self.w_shape: self.skip_backward_test = True self.skip_double_backward_test = True if (x_dtype == 'float16' or w_dtype == 'float16' or b_dtype == 'float16'): self.check_forward_options.update({'rtol': 1e-2, 'atol': 1e-2}) self.check_backward_options.update({'rtol': 1e-2, 'atol': 1e-2}) self.check_double_backward_options.update({ 'rtol': 1e-2, 'atol': 1e-2 })
def setup(self, float_dtype): device = chainerx.get_default_device() if device.backend.name == 'cuda' and len(self.x_shape) <= 3: # TODO(hvy): Support 1 dimensional convolution with CUDA. pytest.skip('cudnn does not support 1-dim convolution') if device.backend.name == 'cuda' and self.cover_all: pytest.skip('cudnn does not support cover_all') if device.backend.name == 'native' and float_dtype == 'float16': # TODO(niboshi): Fix accuracy pytest.skip('Native float16 operation has insufficient accuracy') self.dtype = float_dtype if float_dtype == 'float16': self.check_forward_options.update({'rtol': 5e-2, 'atol': 5e-3}) self.check_backward_options.update({ 'eps': 2 ** -3, 'rtol': 1e-1, 'atol': 1e-2}) else: self.check_forward_options.update({'rtol': 1e-3}) self.check_backward_options.update({ 'eps': 1e-2, 'rtol': 1e-3, 'atol': 1e-4}) self.check_double_backward_options.update({ 'rtol': 5e-2, 'atol': 5e-3})
def setup(self): reduced_shape = self.reduced_shape x_dtype = self.x_dtype param_dtype = self.param_dtype eps = self.eps decay = self.decay axis = self.axis contiguous = self.contiguous # - Non-contiguous running values which are updated in-place are not # supported by CUDA. # - Non-contiguous gamma and beta is not supported by CUDA. # TODO(hvy): Support non-contiguous gamma and beta with CUDA. Create a # contiguous copy in the cuDNN wrapper. if (chainerx.get_default_device().backend.name == 'cuda' and contiguous is None): raise unittest.SkipTest( 'batch_norm with CUDA currently has limited support for ' 'non-contiguous inputs.') # BatchNorm is unstable for fp16 for both native and CUDA. # TODO(hvy): Fix backward and double backward for fp16. if x_dtype == 'float16' and param_dtype == 'float16': self.skip_backward_test = True self.skip_double_backward_test = True self.running_mean = numpy.random.uniform( -1, 1, reduced_shape).astype(param_dtype) self.running_var = numpy.random.uniform( 0.1, 1, reduced_shape).astype(param_dtype) optional_args = {} if eps is not None: optional_args['eps'] = eps if decay is not None: optional_args['decay'] = decay if axis is not None: optional_args['axis'] = axis self.optional_args = optional_args # TODO(hvy): Fix forward, backward and double backward for fp16. if x_dtype == 'float16' or param_dtype == 'float16': self.check_forward_options.update({ 'rtol': 1e-1, 'atol': 1e-1}) self.check_backward_options.update({ 'rtol': 1e-1, 'atol': 1e-1}) self.check_double_backward_options.update({ 'rtol': 1e-1, 'atol': 1e-1}) else: self.check_forward_options.update({ 'rtol': 1e-6, 'atol': 1e-5}) self.check_backward_options.update({ 'rtol': 5e-3, 'atol': 5e-4}) self.check_double_backward_options.update({ 'rtol': 5e-2, 'atol': 5e-3}) # Running values that are recorded in forward for similarity checks. self.running_mean_chx = None self.running_var_chx = None self.running_mean_ch = None self.running_var_ch = None
def generate_array(initializer, shape, xp, dtype=None, device=None): """Return initialized array. The algorithms used to make the new values depend on the concrete derived classes. If the initializer has the ``dtype`` attribute, it is used to construct the array. Otherwise, ``chainer.config.dtype`` is used instead. See :ref:`configuration` for the dtype config. Args: initializer: A callable object that takes :class:`numpy.ndarray` or :class:`cupy.ndarray` and edits its value. shape (tuple): Shape of a return array. xp (module): :mod:`cupy`, :mod:`numpy`, or :mod:`chainerx`. dtype: Dtype specifier. If omitted, ``initializer.dtype`` is used. device: Target device specifier. If omitted, the current device is used for :mod:`cupy`, and the default device is used for :mod:`chainerx`. Returns: numpy.ndarray, cupy.ndarray, or chainerx.ndarray: An initialized array. """ dtype_attr = getattr(initializer, 'dtype', None) if dtype is not None and dtype_attr is not None \ and numpy.dtype(dtype) != numpy.dtype(dtype_attr): raise ValueError( 'dtype mismatch: {} != {}'.format(dtype, dtype_attr)) if dtype is None: dtype = dtype_attr dtype = chainer.get_dtype(dtype) if device is None: if xp is cuda.cupy: device = chainer.get_device(cuda.Device()) elif xp is chainerx: device = chainer.get_device(chainerx.get_default_device()) else: device = chainer.get_device(numpy) else: device = chainer.get_device(device) if xp != device.xp: raise ValueError('xp and device arguments are inconsistent.') if xp is chainerx: # Initialize with NumPy/CuPy array that shares memory with the # ChainerX array. # TODO(sonots): Directly use initializer after ChainerX # supports random. chx_device = device.device array = chainerx.empty(shape, dtype=dtype, device=chx_device) if chx_device.backend.name == 'native': temp_array = _cpu._to_cpu(array) temp_device = cuda.DummyDevice elif chx_device.backend.name == 'cuda': temp_array = cuda.to_gpu(array, chx_device.index) temp_device = cuda.Device(chx_device.index) else: raise RuntimeError('ChainerX backend: {} is not supported.'.format( chx_device.backend.name)) with temp_device: initializer(temp_array) return array with chainer.using_device(device): array = xp.empty(shape, dtype=dtype) initializer(array) return array
def check_device(a, device=None): if device is None: device = chainerx.get_default_device() elif isinstance(device, str): device = chainerx.get_device(device) assert a.device is device
def setup(self, float_dtype): self.dtype = float_dtype eps = self.eps decay = self.decay axis = self.axis optional_args = {} if eps is not None: optional_args['eps'] = eps if decay is not None: optional_args['decay'] = decay if axis is not None: optional_args['axis'] = axis self.optional_args = optional_args # - Non-contiguous running values which are updated in-place are not # supported by CUDA. # - Non-contiguous gamma and beta is not supported by CUDA. # TODO(hvy): Support non-contiguous gamma and beta with CUDA. Create a # contiguous copy in the cuDNN wrapper. self.is_cuda = chainerx.get_default_device().backend.name == 'cuda' if self.is_cuda and self.contiguous is None: raise unittest.SkipTest( 'batch_norm with CUDA currently has limited support for ' 'non-contiguous inputs.') # Float16 backward is unstable. if float_dtype == 'float16': self.skip_backward_test = True self.skip_double_backward_test = True if float_dtype == 'float16': self.check_forward_options.update({'rtol': 1e-1, 'atol': 1e-1}) self.check_backward_options.update({ 'eps': 1e-3, 'rtol': 1e-1, 'atol': 1e-2 }) self.check_double_backward_options.update({ 'rtol': 1e-1, 'atol': 1e-2 }) else: self.check_forward_options.update({'rtol': 1e-6, 'atol': 1e-5}) self.check_backward_options.update({ 'eps': 1e-3, 'rtol': 5e-3, 'atol': 5e-4 }) self.check_double_backward_options.update({ 'rtol': 5e-2, 'atol': 5e-3 }) reduced_shape = self.reduced_shape running_mean = numpy.random.uniform(-1, 1, reduced_shape).astype(float_dtype) running_var = numpy.random.uniform(.1, 1, reduced_shape).astype(float_dtype) self.running_mean = running_mean self.running_var = running_var # Used to verify running mean and variance similarity with Chainer. self.running_mean_chx = None self.running_var_chx = None self.running_mean_ch = None self.running_var_ch = None
def test_use(self, backend_config): device = chainer.get_device(backend_config.chainerx_device) with chainerx.using_device('native:1'): device.use() assert device.device is chainerx.get_default_device()
def cache_restore_device(request): device = chainerx.get_default_device() def restore_device(): chainerx.set_default_device(device) request.addfinalizer(restore_device)
def test_default_device_with_name(device_instance1): device = device_instance1 chainerx.set_default_device(device.name) assert chainerx.get_default_device() is device
def generate_array(initializer, shape, xp, dtype=None, device=None): # type: (types.AbstractInitializer, types.ShapeSpec, types.Xp, types.DTypeSpec, types.DeviceSpec) -> types.NdArray # NOQA """Return initialized array. The algorithms used to make the new values depend on the concrete derived classes. If the initializer has the ``dtype`` attribute, it is used to construct the array. Otherwise, ``chainer.config.dtype`` is used instead. See :ref:`configuration` for the dtype config. Args: initializer: A callable object that takes :class:`numpy.ndarray` or :class:`cupy.ndarray` and edits its value. shape (tuple): Shape of a return array. xp (module): :mod:`cupy`, :mod:`numpy`, or :mod:`chainerx`. dtype: Dtype specifier. If omitted, ``initializer.dtype`` is used. device: Target device specifier. If omitted, the current device is used for :mod:`cupy`, and the default device is used for :mod:`chainerx`. Returns: numpy.ndarray, cupy.ndarray, or chainerx.ndarray: An initialized array. """ dtype_attr = getattr(initializer, 'dtype', None) if dtype is not None and dtype_attr is not None \ and numpy.dtype(dtype) != numpy.dtype(dtype_attr): raise ValueError( 'dtype mismatch: {} != {}'.format(dtype, dtype_attr)) if dtype is None: dtype = dtype_attr dtype = chainer.get_dtype(dtype) if device is None: if xp is cuda.cupy: backend_device = chainer.get_device(cuda.Device()) elif xp is chainerx: backend_device = chainer.get_device(chainerx.get_default_device()) else: backend_device = chainer.get_device(numpy) else: backend_device = chainer.get_device(device) if xp != backend_device.xp: raise ValueError('xp and device arguments are inconsistent.') if xp is chainerx: # Initialize with NumPy/CuPy array that shares memory with the # ChainerX array. # TODO(sonots): Directly use initializer after ChainerX # supports random. chx_device = backend_device.device # type: ignore # TODO(okapies): remove 'type: ignore' when chainerx implements sequence support for empty() # NOQA array = chainerx.empty(shape, dtype=dtype, device=chx_device) # type: ignore # NOQA if chx_device.backend.name == 'native': temp_array = _cpu._to_cpu(array) temp_device = cuda.DummyDevice # type: cuda.Device elif chx_device.backend.name == 'cuda': temp_array = cuda.to_gpu(array, chx_device.index) temp_device = cuda.Device(chx_device.index) else: raise RuntimeError('ChainerX backend: {} is not supported.'.format( chx_device.backend.name)) with temp_device: initializer(temp_array) return array with chainer.using_device(backend_device): array = xp.empty(shape, dtype=dtype) initializer(array) return array
def run_binary_broadcast_op_benchmark(op, config, mode='forward', warmup=10, runs=25): if not op.get_forward_func(): return (None, None), config backend = backend_switcher[op.get_backend()] func = op.get_forward_func() config1 = {"shape": config["shape1"], "dtype": config["dtype"]} config2 = {"shape": config["shape2"], "dtype": config["dtype"]} if backend == 'numpy': if mode == 'forward': def benchmark_func(inputs): result = func(*inputs) return result def input_func(): return prepare_numpy_inputs(1, config1) + prepare_numpy_inputs( 1, config2) forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: return (None, None), config elif backend == 'mxnet.numpy': if mode == 'forward': def benchmark_func(inputs): result = func(*inputs) return result def input_func(): return prepare_mxnet_inputs(1, config1, False) + prepare_mxnet_inputs( 1, config2, False) forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: def input_func(): inputs = prepare_mxnet_inputs( 1, config1, True) + prepare_mxnet_inputs(1, config2, True) with mxnet.autograd.record(): result = func(*inputs) return result def benchmark_func(result): result.backward() return result backward_time, backward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (backward_time, backward_std), config elif backend == 'jax.numpy': def input_func(): return prepare_jax_inputs(1, config1) + prepare_jax_inputs( 1, config2) if mode == 'forward': jit_func = jax.jit(func) def benchmark_func(inputs): result = jit_func(*inputs) try: result.block_until_ready() except Exception: pass return result forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: def grad_func(*args): return jax.numpy.sum(func(*args)) jit_func = jax.jit(jax.grad(grad_func, [0, 1])) def benchmark_func(inputs): result = jit_func(*inputs) try: result.block_until_ready() except Exception: pass return result backward_time, backward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (backward_time, backward_std), config elif backend == 'chainerx': device = chainerx.get_default_device() if mode == 'forward': def input_func(): return prepare_chainerx_inputs( 1, config1, False) + prepare_chainerx_inputs( 1, config2, False) def benchmark_func(inputs): res = func(*inputs) device.synchronize() return res forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: def input_func(): inputs = prepare_chainerx_inputs( 1, config1, True) + prepare_chainerx_inputs( 1, config2, True) result = func(*inputs) result.grad = chainerx.ones_like(result) return result def benchmark_func(result): result.backward() device.synchronize() backward_time, backward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (backward_time, backward_std), config
def _run_simple_op_benchmark(num_input, op, config, mode='forward', warmup=10, runs=25): if not op.get_forward_func(): return (None, None), config backend = backend_switcher[op.get_backend()] func = op.get_forward_func() if num_input: config_ = deepcopy(config) tensor_config = { 'shape': config_.pop('shape'), 'dtype': config_.pop('dtype') } if num_input else None func = functools.partial(func, **config_) if backend == 'numpy': if mode == 'forward': def benchmark_func(inputs): result = func(*inputs) return result input_func = functools.partial(prepare_numpy_inputs, num_input, tensor_config) forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: return (None, None), config elif backend == 'mxnet.numpy': if mode == 'forward': def benchmark_func(inputs): result = func(*inputs) return result input_func = functools.partial(prepare_mxnet_inputs, num_input, tensor_config, False) forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: def input_func(): inputs = prepare_mxnet_inputs(num_input, tensor_config, True) with mxnet.autograd.record(): result = func(*inputs) return result def benchmark_func(result): result.backward() backward_time, backward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (backward_time, backward_std), config elif backend == 'jax.numpy': input_func = functools.partial(prepare_jax_inputs, num_input, tensor_config) if mode == 'forward': jit_func = jax.jit(func) def benchmark_func(inputs): result = jit_func(*inputs) try: result.block_until_ready() except Exception: pass return result forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: def grad_func(*args): return jax.numpy.sum(func(*args)) jit_func = jax.jit(jax.grad(grad_func, list(range(num_input)))) def benchmark_func(inputs): result = jit_func(*inputs) try: result.block_until_ready() except Exception: pass return result backward_time, backward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (backward_time, backward_std), config elif backend == 'chainerx': device = chainerx.get_default_device() if mode == 'forward': input_func = functools.partial(prepare_chainerx_inputs, num_input, tensor_config, False) def benchmark_func(inputs): res = func(*inputs) device.synchronize() return res forward_time, forward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (forward_time, forward_std), config else: def input_func(): inputs = prepare_chainerx_inputs(num_input, tensor_config, True) result = func(*inputs) result.grad = chainerx.ones_like(result) return result def benchmark_func(result): result.backward() device.synchronize() backward_time, backward_std = get_time_metric( benchmark_func, input_func, warmup, runs) return (backward_time, backward_std), config else: if mode == 'forward': func = functools.partial(func, **config) forward_time, forward_std = get_time_metric( func, None, warmup, runs) return (forward_time, forward_std), config else: return (None, None), config
def setup(self): if len(self.in_dtypes) == 3: x_dtype, w_dtype, b_dtype = self.in_dtypes else: (x_dtype, w_dtype), b_dtype = self.in_dtypes, None x_kind = numpy.dtype(x_dtype).kind w_kind = numpy.dtype(w_dtype).kind b_kind = None if b_dtype is None else numpy.dtype(b_dtype).kind device = chainerx.get_default_device() if device.backend.name == 'cuda' and len(self.x_shape) <= 3: # TODO(sonots): Support 1 dimensional convolution with CUDA. pytest.skip( 'cuDNN does not support 1 dimensional convolution and throws ' 'DimensionError') if device.backend.name == 'cuda' and self.cover_all is True: pytest.skip( 'outsize (for cover_all=True) is not supported by CUDA') # Skip backward/double-backward tests for int dtypes if (x_kind != 'f' and w_kind != 'f' and (b_kind is None or b_kind != 'f')): self.skip_backward_test = True self.skip_double_backward_test = True if (x_dtype == 'float16' or w_dtype == 'float16' or b_dtype == 'float16'): self.check_forward_options.update({'rtol': 5e-2, 'atol': 5e-3}) self.check_backward_options.update({ 'eps': 2**-3, 'rtol': 1e-1, 'atol': 1e-2 }) else: self.check_forward_options.update({'rtol': 1e-3}) self.check_backward_options.update({ 'eps': 1e-2, 'rtol': 1e-3, 'atol': 1e-4 }) self.check_double_backward_options.update({'rtol': 5e-2, 'atol': 5e-3}) # Determine outsize cover_all = self.cover_all if cover_all is None: outsize = None else: x_shape = self.x_shape w_shape = self.w_shape stride = self.stride pad = self.pad in_dims = x_shape[2:] kernel_size = w_shape[2:] ndim = len(in_dims) stride_tup = ((stride, ) * ndim if isinstance(stride, int) else stride) pad_tup = (pad, ) * ndim if isinstance(pad, int) else pad outsize = tuple( chainer.utils.conv.get_deconv_outsize(d, k, s, p, cover_all) for (d, k, s, p) in zip(in_dims, kernel_size, stride_tup, pad_tup)) self.outsize = outsize