def set_deterministic_debug_mode(debug_mode: Union[builtins.int, str]) -> None: r"""Sets the debug mode for deterministic operations. .. note:: This is an alternative interface for :func:`torch.use_deterministic_algorithms`. Refer to that function's documentation for details about affected operations. Args: debug_mode(str or int): If "default" or 0, don't error or warn on nondeterministic operations. If "warn" or 1, warn on nondeterministic operations. If "error" or 2, error on nondeterministic operations. """ # NOTE: builtins.int is used here because int in this scope resolves # to torch.int if not isinstance(debug_mode, (builtins.int, str)): raise TypeError(f'debug_mode must be str or int, but got {type(debug_mode)}') if isinstance(debug_mode, str): if debug_mode == 'default': debug_mode = 0 elif debug_mode == 'warn': debug_mode = 1 elif debug_mode == 'error': debug_mode = 2 else: raise RuntimeError( 'invalid value of debug_mode, expected one of `default`, ' f'`warn`, `error`, but got {debug_mode}') if debug_mode == 0: _C._set_deterministic_algorithms(False) elif debug_mode == 1: _C._set_deterministic_algorithms(True, warn_only=True) elif debug_mode == 2: _C._set_deterministic_algorithms(True) else: raise RuntimeError( 'invalid value of debug_mode, expected 0, 1, or 2, ' f'but got {debug_mode}')
def use_deterministic_algorithms(d): r""" Sets whether PyTorch operations must use "deterministic" algorithms. That is, algorithms which, given the same input, and when run on the same software and hardware, always produce the same output. When True, operations will use deterministic algorithms when available, and if only nondeterministic algorithms are available they will throw a :class:RuntimeError when called. .. warning:: This feature is in beta, and its design and implementation may change in the future. The following normally-nondeterministic operations will act deterministically when `d=True`: * :class:`torch.nn.Conv1d` when called on CUDA tensor * :class:`torch.nn.Conv2d` when called on CUDA tensor * :class:`torch.nn.Conv3d` when called on CUDA tensor * :class:`torch.nn.ConvTranspose1d` when called on CUDA tensor * :class:`torch.nn.ConvTranspose2d` when called on CUDA tensor * :class:`torch.nn.ConvTranspose3d` when called on CUDA tensor * :func:`torch.bmm` when called on sparse-dense CUDA tensors The following normally-nondeterministic operations will throw a :class:`RuntimeError` when `d=True`: * :class:`torch.nn.AvgPool3d` when called on a CUDA tensor that requires grad * :class:`torch.nn.AdaptiveAvgPool2d` when called on a CUDA tensor that requires grad * :class:`torch.nn.AdaptiveAvgPool3d` when called on a CUDA tensor that requires grad * :class:`torch.nn.MaxPool3d` when called on a CUDA tensor that requires grad * :class:`torch.nn.AdaptiveMaxPool2d` when called on a CUDA tensor that requires grad * :class:`torch.nn.FractionalMaxPool2d` when called on a CUDA tensor that requires grad * :class:`torch.nn.FractionalMaxPool3d` when called on a CUDA tensor that requires grad * :func:`torch.nn.functional.interpolate` when called on a CUDA tensor that requires grad and one of the following modes is used: - `linear` - `bilinear` - `bicubic` - `trilinear` * :class:`torch.nn.ReflectionPad1d` when called on a CUDA tensor that requires grad * :class:`torch.nn.ReflectionPad2d` when called on a CUDA tensor that requires grad * :class:`torch.nn.ReplicationPad1d` when called on a CUDA tensor that requires grad * :class:`torch.nn.ReplicationPad2d` when called on a CUDA tensor that requires grad * :class:`torch.nn.ReplicationPad3d` when called on a CUDA tensor that requires grad * :class:`torch.nn.NLLLoss` when called on a CUDA tensor that requires grad * :class:`torch.nn.CTCLoss` when called on a CUDA tensor that requires grad * :class:`torch.nn.EmbeddingBag` when called on a CUDA tensor that requires grad * :func:`torch.scatter_add_` when called on a CUDA tensor * :func:`torch.index_add_` when called on a CUDA tensor * :func:`torch.index_copy` * :func:`torch.index_select` when called on a CUDA tensor that requires grad * :func:`torch.repeat_interleave` when called on a CUDA tensor that requires grad * :func:`torch.histc` when called on a CUDA tensor * :func:`torch.bincount` when called on a CUDA tensor * :func:`torch.kthvalue` with called on a CUDA tensor * :func:`torch.median` with indices output when called on a CUDA tensor A handful of CUDA operations are nondeterministic if the CUDA version is 10.2 or greater, unless the environment variable `CUBLAS_WORKSPACE_CONFIG=:4096:8` or `CUBLAS_WORKSPACE_CONFIG=:16:8` is set. See the CUDA documentation for more details: `<https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility>`_ If one of these environment variable configurations is not set, a :class:`RuntimeError` will be raised from these operations when called with CUDA tensors: * :func:`torch.mm` * :func:`torch.mv` * :func:`torch.bmm` Note that deterministic operations tend to have worse performance than non-deterministic operations. Args: d (:class:`bool`): If True, force operations to be deterministic. If False, allow non-deterministic operations. """ _C._set_deterministic_algorithms(d)
def use_deterministic_algorithms(mode): r""" Sets whether PyTorch operations must use "deterministic" algorithms. That is, algorithms which, given the same input, and when run on the same software and hardware, always produce the same output. When enabled, operations will use deterministic algorithms when available, and if only nondeterministic algorithms are available they will throw a :class:`RuntimeError` when called. The following normally-nondeterministic operations will act deterministically when ``mode=True``: * :class:`torch.nn.Conv1d` when called on CUDA tensor * :class:`torch.nn.Conv2d` when called on CUDA tensor * :class:`torch.nn.Conv3d` when called on CUDA tensor * :class:`torch.nn.ConvTranspose1d` when called on CUDA tensor * :class:`torch.nn.ConvTranspose2d` when called on CUDA tensor * :class:`torch.nn.ConvTranspose3d` when called on CUDA tensor * :func:`torch.bmm` when called on sparse-dense CUDA tensors * :func:`torch.Tensor.__getitem__` when attempting to differentiate a CPU tensor and the index is a list of tensors * :func:`torch.Tensor.index_put` with ``accumulate=False`` * :func:`torch.Tensor.index_put` with ``accumulate=True`` when called on a CPU tensor * :func:`torch.Tensor.put_` with ``accumulate=True`` when called on a CPU tensor * :func:`torch.Tensor.scatter_add_` when ``input`` dimension is one and called on a CUDA tensor * :func:`torch.gather` when ``input`` dimension is one and called on a CUDA tensor that requires grad * :func:`torch.index_add` when called on CUDA tensor * :func:`torch.index_select` when attempting to differentiate a CUDA tensor * :func:`torch.repeat_interleave` when attempting to differentiate a CUDA tensor * :func:`torch.Tensor.index_copy` when called on a CPU or CUDA tensor The following normally-nondeterministic operations will throw a :class:`RuntimeError` when ``mode=True``: * :class:`torch.nn.AvgPool3d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.AdaptiveAvgPool2d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.AdaptiveAvgPool3d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.MaxPool3d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.AdaptiveMaxPool2d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.FractionalMaxPool2d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.FractionalMaxPool3d` when attempting to differentiate a CUDA tensor * :func:`torch.nn.functional.interpolate` when attempting to differentiate a CUDA tensor and one of the following modes is used: - ``linear`` - ``bilinear`` - ``bicubic`` - ``trilinear`` * :class:`torch.nn.ReflectionPad1d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.ReflectionPad2d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.ReplicationPad1d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.ReplicationPad2d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.ReplicationPad3d` when attempting to differentiate a CUDA tensor * :class:`torch.nn.NLLLoss` when called on a CUDA tensor * :class:`torch.nn.CTCLoss` when attempting to differentiate a CUDA tensor * :class:`torch.nn.EmbeddingBag` when attempting to differentiate a CUDA tensor when ``mode='max'`` * :func:`torch.Tensor.scatter_add_` when ``input`` dimension is larger than one and called on a CUDA tensor * :func:`torch.gather` when ``input`` dimension is larger than one and called on a CUDA tensor that requires grad * :func:`torch.Tensor.put_` when ``accumulate=False`` * :func:`torch.Tensor.put_` when ``accumulate=True`` and called on a CUDA tensor * :func:`torch.histc` when called on a CUDA tensor * :func:`torch.bincount` when called on a CUDA tensor * :func:`torch.kthvalue` with called on a CUDA tensor * :func:`torch.median` with indices output when called on a CUDA tensor * :func:`torch.nn.functional.grid_sample` when attempting to differentiate a CUDA tensor A handful of CUDA operations are nondeterministic if the CUDA version is 10.2 or greater, unless the environment variable ``CUBLAS_WORKSPACE_CONFIG=:4096:8`` or ``CUBLAS_WORKSPACE_CONFIG=:16:8`` is set. See the CUDA documentation for more details: `<https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility>`_ If one of these environment variable configurations is not set, a :class:`RuntimeError` will be raised from these operations when called with CUDA tensors: * :func:`torch.mm` * :func:`torch.mv` * :func:`torch.bmm` Note that deterministic operations tend to have worse performance than nondeterministic operations. .. note:: This flag does not detect or prevent nondeterministic behavior caused by calling an inplace operation on a tensor with an internal memory overlap or by giving such a tensor as the :attr:`out` argument for an operation. In these cases, multiple writes of different data may target a single memory location, and the order of writes is not guaranteed. Args: mode (:class:`bool`): If True, makes potentially nondeterministic operations switch to a deterministic algorithm or throw a runtime error. If False, allows nondeterministic operations. Example:: >>> torch.use_deterministic_algorithms(True) # Forward mode nondeterministic error >>> torch.randn(10).index_copy(0, torch.tensor([0]), torch.randn(1)) ... RuntimeError: index_copy does not have a deterministic implementation... # Backward mode nondeterministic error >>> torch.randn(10, requires_grad=True, device='cuda').index_select(0, torch.tensor([0], device='cuda')).backward() ... RuntimeError: index_add_cuda_ does not have a deterministic implementation... """ _C._set_deterministic_algorithms(mode)