def allow_non_deterministic():
prev_state = torch.are_deterministic_algorithms_enabled()
try:
torch.use_deterministic_algorithms(False)
yield
finally:
torch.use_deterministic_algorithms(prev_state)
def main():
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
torch.use_deterministic_algorithms(True)
pl.seed_everything(seed=123, workers=True)
logging.basicConfig(level=logging.INFO)
datamodule = clevr.ClevrWInstructionsDataModule(
"data", batch_size=16, nhops=[0]
)
# most params obtained via inspection of dataset
model = NSMLightningModule(
input_size=45,
n_node_properties=4,
computation_steps=0 + 1, # lul
encoded_question_size=100,
output_size=28,
learn_rate=0.001,
use_instruction_loss=False,
)
metric_to_track = "train_loss"
trainer = pl.Trainer(
gpus=-1 if torch.cuda.is_available() else 0,
max_epochs=1000,
callbacks=[
pl.callbacks.EarlyStopping(monitor=metric_to_track, patience=300),
pl.callbacks.ModelCheckpoint(save_top_k=-1,every_n_epochs=50),
],
)
trainer.fit(model, datamodule)
def set_seed(seed=0, rank=None):
"""
seed: basic seed
rank: rank of the current process, using which to mutate basic seed to have a unique seed per process
"""
os.environ['DEEPLIIF_SEED'] = str(seed)
if seed is not None:
if rank is not None:
seed_final = seed + int(rank)
else:
seed_final = seed
os.environ['PYTHONHASHSEED'] = str(seed_final)
random.seed(seed_final)
np.random.seed(seed_final)
torch.manual_seed(seed_final)
torch.cuda.manual_seed(seed_final)
torch.cuda.manual_seed_all(seed_final)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.use_deterministic_algorithms(True)
print(f'deterministic training, seed set to {seed_final}')
else:
print(f'not using deterministic training')
def random_seed(seed, backends=False, deterministic_torch=True):
"""Set random seed.
Set random seed to ``random``, ``np.random``, ``torch.backends.cudnn`` and ``torch.manual_seed``.
Also advise torch to use deterministic algorithms.
References:
https://pytorch.org/docs/stable/notes/randomness.html
Args:
seed: Random seed.
backends: Whether to also adapt backends. If set True cuDNN's benchmark feature is disabled. This
causes cuDNN to deterministically select an algorithm, possibly at the cost of reduced performance.
Also the selected algorithm is set to run deterministically.
deterministic_torch: Whether to set PyTorch operations to behave deterministically.
"""
from torch import manual_seed
from torch.backends import cudnn
import random
random.seed(seed)
manual_seed(seed)
np.random.seed(seed)
if backends:
cudnn.deterministic = True
cudnn.benchmark = False
if deterministic_torch and 'use_deterministic_algorithms' in dir(torch):
torch.use_deterministic_algorithms(True)
def set_determinism(
seed: Optional[int] = NP_MAX,
use_deterministic_algorithms: Optional[bool] = None,
additional_settings: Optional[Union[Sequence[Callable[[int], Any]], Callable[[int], Any]]] = None,
) -> None:
"""
Set random seed for modules to enable or disable deterministic training.
Args:
seed: the random seed to use, default is np.iinfo(np.int32).max.
It is recommended to set a large seed, i.e. a number that has a good balance
of 0 and 1 bits. Avoid having many 0 bits in the seed.
if set to None, will disable deterministic training.
use_deterministic_algorithms: Set whether PyTorch operations must use "deterministic" algorithms.
additional_settings: additional settings that need to set random seed.
Note:
This function will not affect the randomizable objects in :py:class:`monai.transforms.Randomizable`, which
have independent random states. For those objects, the ``set_random_state()`` method should be used to
ensure the deterministic behavior (alternatively, :py:class:`monai.data.DataLoader` by default sets the seeds
according to the global random state, please see also: :py:class:`monai.data.utils.worker_init_fn` and
:py:class:`monai.data.utils.set_rnd`).
"""
if seed is None:
# cast to 32 bit seed for CUDA
seed_ = torch.default_generator.seed() % MAX_SEED
torch.manual_seed(seed_)
else:
seed = int(seed) % MAX_SEED
torch.manual_seed(seed)
global _seed
_seed = seed
random.seed(seed)
np.random.seed(seed)
if additional_settings is not None:
additional_settings = ensure_tuple(additional_settings)
for func in additional_settings:
func(seed)
if torch.backends.flags_frozen():
warnings.warn("PyTorch global flag support of backends is disabled, enable it to set global `cudnn` flags.")
torch.backends.__allow_nonbracketed_mutation_flag = True
if seed is not None:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
else: # restore the original flags
torch.backends.cudnn.deterministic = _flag_deterministic
torch.backends.cudnn.benchmark = _flag_cudnn_benchmark
if use_deterministic_algorithms is not None:
if hasattr(torch, "use_deterministic_algorithms"): # `use_deterministic_algorithms` is new in torch 1.8.0
torch.use_deterministic_algorithms(use_deterministic_algorithms)
elif hasattr(torch, "set_deterministic"): # `set_deterministic` is new in torch 1.7.0
torch.set_deterministic(use_deterministic_algorithms) # type: ignore
else:
warnings.warn("use_deterministic_algorithms=True, but PyTorch version is too old to set the mode.")
def init_seed(torch_seed=100, random_seed=100, np_random_seed=100):
'''
Initialize all random seeds
'''
torch.manual_seed(torch_seed)
random.seed(random_seed)
np.random.seed(np_random_seed)
torch.use_deterministic_algorithms(True)
def setup_method(self):
super().setup_method()
self.set_up_model(
FIXTURES_ROOT / "rc" / "bidaf" / "experiment.json",
FIXTURES_ROOT / "rc" / "squad.json",
seed=27,
)
torch.use_deterministic_algorithms(True)
def set_deterministic(deterministic: bool = True):
# if hasattr(torch, 'backends') and hasattr(torch.backends, 'cudnn'):
# torch.backends.cudnn.enabled = not deterministic
# torch.backends.cudnn.benchmark = not deterministic
# torch.backends.cudnn.deterministic = deterministic
if hasattr(torch, 'set_deterministic'):
torch.set_deterministic(deterministic)
else:
torch.use_deterministic_algorithms(deterministic)
def set_deterministic():
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
if torch.__version__ <= Version("1.7"):
torch.set_deterministic(True)
else:
torch.use_deterministic_algorithms(True)
def _init_deterministic(self, deterministic: bool) -> None:
self.deterministic = deterministic
torch.use_deterministic_algorithms(deterministic)
if deterministic:
# fixing non-deterministic part of horovod
# https://github.com/PyTorchLightning/pytorch-lightning/pull/1572/files#r420279383
os.environ["HOROVOD_FUSION_THRESHOLD"] = "0"
# https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
def set_seed(seed):
"""Set seeds for numpy and pytorch."""
if seed is not None:
if seed >= 0:
seed_everything(seed=seed)
torch.use_deterministic_algorithms(True)
torch.backends.cudnn.benchmark = False
else:
logging.warning(
f'the random seed should be a non-negative integer')
def seed_prng(seed, use_cuda=False, deterministic=False):
if deterministic:
torch.use_deterministic_algorithms(True)
if use_cuda:
torch.backends.cudnn.benchmark = True
random.seed(seed)
numpy.random.seed(random.randint(1, 100000))
torch.random.manual_seed(random.randint(1, 100000))
if use_cuda is True:
torch.cuda.manual_seed_all(random.randint(1, 100000))
def setup_method(self):
super().setup_method()
self.set_up_model(
FIXTURES_ROOT / "rc" / "dialog_qa" / "experiment.json",
FIXTURES_ROOT / "rc" / "dialog_qa" / "quac_sample.json",
seed=42,
)
self.batch = Batch(self.instances)
self.batch.index_instances(self.vocab)
torch.use_deterministic_algorithms(True)
def set_random_seed(deterministic: bool, seed: int):
if deterministic:
torch.use_deterministic_algorithms(True)
# set the seed for reproducibility
np.random.seed(seed)
torch.manual_seed(seed)
random.seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
def main():
torch.use_deterministic_algorithms(True)
torch.distributed.init_process_group("nccl")
rank = torch.distributed.get_rank()
world_size = torch.distributed.get_world_size()
torch.cuda.set_device(rank)
explicit_nhwc = True
dtype = torch.float16
N, C, H, W = 1, 64, 200, 336
Hs = ((H + 8 * world_size - 1) // (8 * world_size)) * 8
H = Hs * world_size
gt_bottleneck = ground_truth_bottleneck(C, dtype, explicit_nhwc)
gt = ground_truth(N, C, H, W, dtype, 1, gt_bottleneck)
# verify that spatial bottleneck with group_size 1 produces same results as ground truth bottleneck
spatial_bottleneck = spatial_parallel_bottleneck(C, dtype, explicit_nhwc,
gt_bottleneck, None)
bt = apply_to_different_bottleneck(gt, spatial_bottleneck)
compare(gt, bt)
#print_bottleneck_p_and_b(gt_bottleneck)
#print_bottleneck_p_and_b(spatial_bottleneck)
spatial_group_size = world_size
spatial_communicator = None
peer_pool = PeerMemoryPool(rank, world_size, spatial_group_size,
64 * 1024 * 1024, 2 * 1024 * 1024)
#halex = HaloExchangerAllGather(world_size, spatial_group_size, rank, spatial_communicator)
#halex = HaloExchangerSendRecv(world_size, spatial_group_size, rank, spatial_communicator)
halex = HaloExchangerPeer(world_size,
spatial_group_size,
rank,
spatial_communicator,
peer_pool,
explicit_nhwc,
numSM=1)
#print("halex.signals = %s" % (str(halex.signals)))
# Make sure peer memory halo exchanger has finished initializing flags on all ranks before proceeding
#torch.cuda.synchronize()
#torch.distributed.barrier()
bt2 = n_way_spatial(halex,
gt_bottleneck,
gt,
explicit_nhwc,
world_size,
rank,
fp32_reduce=True)
compare(gt, bt2)
def test_equalize(device):
torch.use_deterministic_algorithms(False)
check_functional_vs_PIL_vs_scripted(
F.equalize,
F_pil.equalize,
F_t.equalize,
{},
device,
dtype=None,
tol=1.0,
agg_method="max",
)
def _init_deterministic(self, deterministic: Union[bool, _LITERAL_WARN]) -> None:
self.deterministic = deterministic
if _TORCH_GREATER_EQUAL_1_11 and deterministic == "warn":
torch.use_deterministic_algorithms(True, warn_only=True)
else:
torch.use_deterministic_algorithms(deterministic)
if deterministic:
# fixing non-deterministic part of horovod
# https://github.com/PyTorchLightning/pytorch-lightning/pull/1572/files#r420279383
os.environ["HOROVOD_FUSION_THRESHOLD"] = "0"
# https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
def set_random_seed(seed, set_cuda=False): _rseed = torch.initial_seed() if seed is None else seed rpyseed(_rseed) torch.manual_seed(_rseed) if set_cuda: torch.cuda.manual_seed_all(_rseed) # Make cudnn methods deterministic according to: https://pytorch.org/docs/stable/notes/randomness.html#cudnn try: torch.use_deterministic_algorithms(use_deterministic) except: torch.backends.cudnn.deterministic = use_deterministic torch.backends.cudnn.benchmark = False
def init_seeds(seed=0, deterministic=False):
# Initialize random number generator (RNG) seeds https://pytorch.org/docs/stable/notes/randomness.html
# cudnn seed 0 settings are slower and more reproducible, else faster and less reproducible
import torch.backends.cudnn as cudnn
if deterministic and check_version(torch.__version__, '1.12.0'): # https://github.com/ultralytics/yolov5/pull/8213
torch.use_deterministic_algorithms(True)
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'
# os.environ['PYTHONHASHSEED'] = str(seed)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
cudnn.benchmark, cudnn.deterministic = (False, True) if seed == 0 else (True, False)
def set_random_seeds(seed: int = 0) -> None:
"""
Set torch.backends.cudnn.benchmark = True to speed up training,
but may cause some non-determinism.
Disabling the benchmarking feature with torch.backends.cudnn.benchmark = False
causes cuDNN to deterministically select an algorithm, possibly at the cost of
reduced performance.
"""
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.use_deterministic_algorithms(True)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = torch.cuda.is_available()
def enable_reproducibility(self, seed: int = 0) -> None:
"""Limits sources of nondeterministic behavior."""
self._seed = seed
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
torch.use_deterministic_algorithms(True)
torch.backends.cudnn.benchmark = False
# If you want to use deterministic algorithms with CUDA, then you need to set
# the CUBLAS_WORKSPACE_CONFIG environment variable; otherwise, Torch errors.
# See https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility.
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
def test_choice_function_fixed(trivial_choice_problem, name):
np.random.seed(123)
# Pytorch does not guarantee full reproducibility in different settings
# [1]. This may become a problem in the test suite, in which case we should
# increase the tolerance. These are only "sanity checks" on small data sets
# anyway and the exact values do not mean much here.
# [1] https://pytorch.org/docs/stable/notes/randomness.html
torch.manual_seed(123)
# Trade off performance for better reproducibility.
torch.use_deterministic_algorithms(True)
x, y = trivial_choice_problem
choice_function = choice_functions[name][0]
params, accuracies = choice_functions[name][1], choice_functions[name][2]
learner = choice_function(**params)
if name == GLM_CHOICE:
learner.fit(
x,
y,
vi_params={
"n": 100,
"method": "advi",
"callbacks": [CheckParametersConvergence()],
},
)
else:
learner.fit(x, y)
s_pred = learner.predict_scores(x)
y_pred = learner.predict_for_scores(s_pred)
y_pred_2 = learner.predict(x)
rtol = 1e-2
atol = 5e-2
assert np.isclose(0.0,
subset_01_loss(y_pred, y_pred_2),
rtol=rtol,
atol=atol,
equal_nan=False)
for key, value in accuracies.items():
metric = choice_metrics[key]
if metric in metrics_on_predictions:
pred_loss = metric(y, y_pred)
else:
pred_loss = metric(y, s_pred)
assert np.isclose(value,
pred_loss,
rtol=rtol,
atol=atol,
equal_nan=False)
def set_deterministic(_seed_: int = 2020, disable_uda=False):
random.seed(_seed_)
np.random.seed(_seed_)
torch.manual_seed(
_seed_
) # use torch.manual_seed() to seed the RNG for all devices (both CPU and CUDA)
torch.cuda.manual_seed_all(_seed_)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if disable_uda:
pass
else:
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'
# set a debug environment variable CUBLAS_WORKSPACE_CONFIG to ":16:8" (may limit overall performance) or ":4096:8" (will increase library footprint in GPU memory by approximately 24MiB).
torch.use_deterministic_algorithms(True)
def test_discrete_choice_function_fixed(trivial_discrete_choice_problem, name):
np.random.seed(123)
# There are some caveats with pytorch reproducibility. See the comment on
# the corresponding line of `test_choice_functions.py` for details.
torch.manual_seed(123)
torch.use_deterministic_algorithms(True)
x, y = trivial_discrete_choice_problem
choice_function = discrete_choice_functions[name][0]
params, accuracies = (
discrete_choice_functions[name][1],
discrete_choice_functions[name][2],
)
learner = choice_function(**params)
if name in [MNL, NLM, GEV, PCL, MLM]:
learner.fit(
x,
y,
vi_params={
"n": 100,
"method": "advi",
"callbacks": [CheckParametersConvergence()],
},
)
else:
learner.fit(x, y)
s_pred = learner.predict_scores(x)
y_pred = learner.predict_for_scores(s_pred)
y_pred_2 = learner.predict(x)
rtol = 1e-2
atol = 5e-2
assert np.isclose(0.0,
subset_01_loss(y_pred, y_pred_2),
rtol=rtol,
atol=atol,
equal_nan=False)
for key, value in accuracies.items():
metric = metrics[key]
if metric in metrics_on_predictions:
pred_loss = metric(y, y_pred)
else:
pred_loss = metric(y, s_pred)
assert np.isclose(value,
pred_loss,
rtol=rtol,
atol=atol,
equal_nan=False)
def set_sibyl_seed(seed=41):
import torch
import random
import numpy as np
SIBYL_SEED = seed
# torch
torch.manual_seed(SIBYL_SEED)
if torch.cuda.is_available():
torch.cuda.manual_seed(SIBYL_SEED)
torch.use_deterministic_algorithms(True)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
# python
random.seed(SIBYL_SEED)
#numpy
np.random.seed(SIBYL_SEED)
def seed_everything(seed, deterministic=False, compensation="memory") -> None:
"""
Function that sets seed for pseudo-random number generators in:
pytorch, numpy, python.random
Adapted from pytorch-lightning
https://pytorch-lightning.readthedocs.io/en/latest/_modules/pytorch_lightning/utilities/seed.html#seed_everything
Args:
seed (int): Value of the seed for all pseudo-random number generators
deterministic (bool): If set to True will raise an error if non-deterministic behaviour is encountered
compensation (str): Chooses which computational aspect is affected when deterministic is set to True.
Must be chosen between time and memory.
Raises:
ClinicaDLConfigurationError: if compensation is not in {"time", "memory"}.
RuntimeError: if a non-deterministic behaviour was encountered.
"""
from clinicadl.utils.exceptions import ClinicaDLConfigurationError
max_seed_value = np.iinfo(np.uint32).max
min_seed_value = np.iinfo(np.uint32).min
if not (min_seed_value <= seed <= max_seed_value):
seed = random.randint(min_seed_value, max_seed_value)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
if deterministic:
torch.backends.cudnn.benchmark = False
if compensation == "memory":
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
elif compensation == "time":
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
else:
raise ClinicaDLConfigurationError(
f"The compensation for a deterministic CUDA setting "
f"must be chosen between 'time' and 'memory'.")
torch.use_deterministic_algorithms(True)
def test_bincount():
"""test that bincount works in deterministic setting on GPU."""
torch.use_deterministic_algorithms(True)
x = torch.randint(100, size=(100, ))
# uses custom implementation
res1 = _bincount(x, minlength=10)
torch.use_deterministic_algorithms(False)
# uses torch.bincount
res2 = _bincount(x, minlength=10)
# explicit call to make sure, that res2 is not by accident using our manual implementation
res3 = torch.bincount(x, minlength=10)
# check for correctness
assert torch.allclose(res1, res2)
assert torch.allclose(res1, res3)
def test_object_ranker_fixed(trivial_ranking_problem, ranker_name):
np.random.seed(123)
# There are some caveats with pytorch reproducibility. See the comment on
# the corresponding line of `test_choice_functions.py` for details.
torch.manual_seed(123)
torch.use_deterministic_algorithms(True)
x, y = trivial_ranking_problem
ranker, params, (loss, acc) = object_rankers[ranker_name]
ranker = ranker(**params)
ranker.fit(x, y)
pred_scores = ranker.predict_scores(x)
pred_loss = zero_one_rank_loss_for_scores_ties_np(y, pred_scores)
rtol = 1e-2
atol = 1e-4
assert np.isclose(loss, pred_loss, rtol=rtol, atol=atol, equal_nan=False)
pred = ranker.predict_for_scores(pred_scores)
pred_2 = ranker.predict(x)
pred_acc = zero_one_accuracy_np(pred, pred_2)
assert np.isclose(1.0, pred_acc, rtol=rtol, atol=atol, equal_nan=False)
pred_acc = zero_one_accuracy_np(pred, y)
assert np.isclose(acc, pred_acc, rtol=rtol, atol=atol, equal_nan=False)
def compute_opacus_grad_sample(
self,
x: Union[torch.Tensor, PackedSequence],
module: nn.Module,
batch_first=True,
loss_reduction="mean",
) -> Dict[str, torch.tensor]:
"""
Runs Opacus to compute per-sample gradients and return them for testing purposes.
Args:
x: The tensor in input to the ``module``
module: The ``ModelWithLoss`` that wraps the nn.Module you want to test.
batch_first: Whether batch size is the first dimension (as opposed to the second).
Defaults to True.
loss_reduction: What reduction to apply to the loss. Defaults to "mean".
Returns:
Dictionary mapping parameter_name -> per-sample-gradient for that parameter
"""
torch.use_deterministic_algorithms(True)
torch.manual_seed(0)
np.random.seed(0)
gs_module = GradSampleModule(clone_module(module),
batch_first=batch_first,
loss_reduction=loss_reduction)
grad_sample_module = ModelWithLoss(gs_module, loss_reduction)
grad_sample_module.zero_grad()
loss = grad_sample_module(x)
loss.backward()
opacus_grad_samples = {
name: p.grad_sample
for name, p in
grad_sample_module.wrapped_module._module.named_parameters()
}
return opacus_grad_samples
def enable_full_determinism(seed: int):
"""
Helper function for reproducible behavior during distributed training. See
- https://pytorch.org/docs/stable/notes/randomness.html for pytorch
- https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_op_determinism for tensorflow
"""
# set seed first
set_seed(seed)
if is_torch_available():
# Enable PyTorch deterministic mode. This potentially requires either the environment
# variable 'CUDA_LAUNCH_BLOCKING' or 'CUBLAS_WORKSPACE_CONFIG' to be set,
# depending on the CUDA version, so we set them both here
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
torch.use_deterministic_algorithms(True)
# Enable CUDNN deterministic mode
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if is_tf_available():
tf.config.experimental.enable_op_determinism()
