def set_deterministic_debug_mode(debug_mode: Union[builtins.int, str]) -> None:
r"""Sets the debug mode for deterministic operations.
.. note:: This is an alternative interface for
:func:`torch.use_deterministic_algorithms`. Refer to that function's
documentation for details about affected operations.
Args:
debug_mode(str or int): If "default" or 0, don't error or warn on
nondeterministic operations. If "warn" or 1, warn on
nondeterministic operations. If "error" or 2, error on
nondeterministic operations.
"""
# NOTE: builtins.int is used here because int in this scope resolves
# to torch.int
if not isinstance(debug_mode, (builtins.int, str)):
raise TypeError(f'debug_mode must be str or int, but got {type(debug_mode)}')
if isinstance(debug_mode, str):
if debug_mode == 'default':
debug_mode = 0
elif debug_mode == 'warn':
debug_mode = 1
elif debug_mode == 'error':
debug_mode = 2
else:
raise RuntimeError(
'invalid value of debug_mode, expected one of `default`, '
f'`warn`, `error`, but got {debug_mode}')
if debug_mode == 0:
_C._set_deterministic_algorithms(False)
elif debug_mode == 1:
_C._set_deterministic_algorithms(True, warn_only=True)
elif debug_mode == 2:
_C._set_deterministic_algorithms(True)
else:
raise RuntimeError(
'invalid value of debug_mode, expected 0, 1, or 2, '
f'but got {debug_mode}')
def use_deterministic_algorithms(d):
r""" Sets whether PyTorch operations must use "deterministic"
algorithms. That is, algorithms which, given the same input, and when
run on the same software and hardware, always produce the same output.
When True, operations will use deterministic algorithms when available,
and if only nondeterministic algorithms are available they will throw a
:class:RuntimeError when called.
.. warning::
This feature is in beta, and its design and implementation may change
in the future.
The following normally-nondeterministic operations will act
deterministically when `d=True`:
* :class:`torch.nn.Conv1d` when called on CUDA tensor
* :class:`torch.nn.Conv2d` when called on CUDA tensor
* :class:`torch.nn.Conv3d` when called on CUDA tensor
* :class:`torch.nn.ConvTranspose1d` when called on CUDA tensor
* :class:`torch.nn.ConvTranspose2d` when called on CUDA tensor
* :class:`torch.nn.ConvTranspose3d` when called on CUDA tensor
* :func:`torch.bmm` when called on sparse-dense CUDA tensors
The following normally-nondeterministic operations will throw a
:class:`RuntimeError` when `d=True`:
* :class:`torch.nn.AvgPool3d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.AdaptiveAvgPool2d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.AdaptiveAvgPool3d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.MaxPool3d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.AdaptiveMaxPool2d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.FractionalMaxPool2d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.FractionalMaxPool3d` when called on a CUDA tensor that requires grad
* :func:`torch.nn.functional.interpolate` when called on a CUDA tensor that requires grad
and one of the following modes is used:
- `linear`
- `bilinear`
- `bicubic`
- `trilinear`
* :class:`torch.nn.ReflectionPad1d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.ReflectionPad2d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.ReplicationPad1d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.ReplicationPad2d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.ReplicationPad3d` when called on a CUDA tensor that requires grad
* :class:`torch.nn.NLLLoss` when called on a CUDA tensor that requires grad
* :class:`torch.nn.CTCLoss` when called on a CUDA tensor that requires grad
* :class:`torch.nn.EmbeddingBag` when called on a CUDA tensor that requires grad
* :func:`torch.scatter_add_` when called on a CUDA tensor
* :func:`torch.index_add_` when called on a CUDA tensor
* :func:`torch.index_copy`
* :func:`torch.index_select` when called on a CUDA tensor that requires grad
* :func:`torch.repeat_interleave` when called on a CUDA tensor that requires grad
* :func:`torch.histc` when called on a CUDA tensor
* :func:`torch.bincount` when called on a CUDA tensor
* :func:`torch.kthvalue` with called on a CUDA tensor
* :func:`torch.median` with indices output when called on a CUDA tensor
A handful of CUDA operations are nondeterministic if the CUDA version is
10.2 or greater, unless the environment variable `CUBLAS_WORKSPACE_CONFIG=:4096:8`
or `CUBLAS_WORKSPACE_CONFIG=:16:8` is set. See the CUDA documentation for more
details: `<https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility>`_
If one of these environment variable configurations is not set, a :class:`RuntimeError`
will be raised from these operations when called with CUDA tensors:
* :func:`torch.mm`
* :func:`torch.mv`
* :func:`torch.bmm`
Note that deterministic operations tend to have worse performance than
non-deterministic operations.
Args:
d (:class:`bool`): If True, force operations to be deterministic.
If False, allow non-deterministic operations.
"""
_C._set_deterministic_algorithms(d)
def use_deterministic_algorithms(mode):
r""" Sets whether PyTorch operations must use "deterministic"
algorithms. That is, algorithms which, given the same input, and when
run on the same software and hardware, always produce the same output.
When enabled, operations will use deterministic algorithms when available,
and if only nondeterministic algorithms are available they will throw a
:class:`RuntimeError` when called.
The following normally-nondeterministic operations will act
deterministically when ``mode=True``:
* :class:`torch.nn.Conv1d` when called on CUDA tensor
* :class:`torch.nn.Conv2d` when called on CUDA tensor
* :class:`torch.nn.Conv3d` when called on CUDA tensor
* :class:`torch.nn.ConvTranspose1d` when called on CUDA tensor
* :class:`torch.nn.ConvTranspose2d` when called on CUDA tensor
* :class:`torch.nn.ConvTranspose3d` when called on CUDA tensor
* :func:`torch.bmm` when called on sparse-dense CUDA tensors
* :func:`torch.Tensor.__getitem__` when attempting to differentiate a CPU tensor
and the index is a list of tensors
* :func:`torch.Tensor.index_put` with ``accumulate=False``
* :func:`torch.Tensor.index_put` with ``accumulate=True`` when called on a CPU
tensor
* :func:`torch.Tensor.put_` with ``accumulate=True`` when called on a CPU
tensor
* :func:`torch.Tensor.scatter_add_` when ``input`` dimension is one and called
on a CUDA tensor
* :func:`torch.gather` when ``input`` dimension is one and called
on a CUDA tensor that requires grad
* :func:`torch.index_add` when called on CUDA tensor
* :func:`torch.index_select` when attempting to differentiate a CUDA tensor
* :func:`torch.repeat_interleave` when attempting to differentiate a CUDA tensor
* :func:`torch.Tensor.index_copy` when called on a CPU or CUDA tensor
The following normally-nondeterministic operations will throw a
:class:`RuntimeError` when ``mode=True``:
* :class:`torch.nn.AvgPool3d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.AdaptiveAvgPool2d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.AdaptiveAvgPool3d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.MaxPool3d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.AdaptiveMaxPool2d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.FractionalMaxPool2d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.FractionalMaxPool3d` when attempting to differentiate a CUDA tensor
* :func:`torch.nn.functional.interpolate` when attempting to differentiate a CUDA tensor
and one of the following modes is used:
- ``linear``
- ``bilinear``
- ``bicubic``
- ``trilinear``
* :class:`torch.nn.ReflectionPad1d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.ReflectionPad2d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.ReplicationPad1d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.ReplicationPad2d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.ReplicationPad3d` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.NLLLoss` when called on a CUDA tensor
* :class:`torch.nn.CTCLoss` when attempting to differentiate a CUDA tensor
* :class:`torch.nn.EmbeddingBag` when attempting to differentiate a CUDA tensor when
``mode='max'``
* :func:`torch.Tensor.scatter_add_` when ``input`` dimension is larger than one
and called on a CUDA tensor
* :func:`torch.gather` when ``input`` dimension is larger than one
and called on a CUDA tensor that requires grad
* :func:`torch.Tensor.put_` when ``accumulate=False``
* :func:`torch.Tensor.put_` when ``accumulate=True`` and called on a CUDA tensor
* :func:`torch.histc` when called on a CUDA tensor
* :func:`torch.bincount` when called on a CUDA tensor
* :func:`torch.kthvalue` with called on a CUDA tensor
* :func:`torch.median` with indices output when called on a CUDA tensor
* :func:`torch.nn.functional.grid_sample` when attempting to differentiate a CUDA tensor
A handful of CUDA operations are nondeterministic if the CUDA version is
10.2 or greater, unless the environment variable ``CUBLAS_WORKSPACE_CONFIG=:4096:8``
or ``CUBLAS_WORKSPACE_CONFIG=:16:8`` is set. See the CUDA documentation for more
details: `<https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility>`_
If one of these environment variable configurations is not set, a :class:`RuntimeError`
will be raised from these operations when called with CUDA tensors:
* :func:`torch.mm`
* :func:`torch.mv`
* :func:`torch.bmm`
Note that deterministic operations tend to have worse performance than
nondeterministic operations.
.. note::
This flag does not detect or prevent nondeterministic behavior caused
by calling an inplace operation on a tensor with an internal memory
overlap or by giving such a tensor as the :attr:`out` argument for an
operation. In these cases, multiple writes of different data may target
a single memory location, and the order of writes is not guaranteed.
Args:
mode (:class:`bool`): If True, makes potentially nondeterministic
operations switch to a deterministic algorithm or throw a runtime
error. If False, allows nondeterministic operations.
Example::
>>> torch.use_deterministic_algorithms(True)
# Forward mode nondeterministic error
>>> torch.randn(10).index_copy(0, torch.tensor([0]), torch.randn(1))
...
RuntimeError: index_copy does not have a deterministic implementation...
# Backward mode nondeterministic error
>>> torch.randn(10, requires_grad=True, device='cuda').index_select(0, torch.tensor([0], device='cuda')).backward()
...
RuntimeError: index_add_cuda_ does not have a deterministic implementation...
"""
_C._set_deterministic_algorithms(mode)
