def make_reproducible(seed=42):
""" Make the run reproducible by setting the seed """
torch.set_deterministic(True)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
def prepare_device(n_gpu: int, keep_reproducibility=False):
"""
Choose to use CPU or GPU depend on the value of "n_gpu".
Args:
n_gpu(int): the number of GPUs used in the experiment. if n_gpu == 0, use CPU; if n_gpu >= 1, use GPU.
keep_reproducibility (bool): if we need to consider the repeatability of experiment, set keep_reproducibility to True.
See Also
Reproducibility: https://pytorch.org/docs/stable/notes/randomness.html
"""
if n_gpu == 0:
print("Using CPU in the experiment.")
device = torch.device("cpu")
else:
# possibly at the cost of reduced performance
if keep_reproducibility:
print("Using CuDNN deterministic mode in the experiment.")
torch.backends.cudnn.benchmark = False # ensures that CUDA selects the same convolution algorithm each time
torch.set_deterministic(
True
) # configures PyTorch only to use deterministic implementation
else:
# causes cuDNN to benchmark multiple convolution algorithms and select the fastest
torch.backends.cudnn.benchmark = True
device = torch.device("cuda:0")
return device
def main(params):
# reproducitiblity
torch.manual_seed(0)
np.random.seed(0)
torch.backends.cudnn.deterministic = False # cuDNN deterministically select an algorithm, possibly at the cost of reduced performance
torch.set_deterministic(
True
) # optional, some operation is without a deterministic alternative and would throw an error
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print('using device', device)
# prepare dataset
transform = get_transform()
trainset = XXXDataset(root='./data/train', transform=transform)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=4,
shuffle=True,
num_workers=2)
# prepare network
net = XXXNet().to(device)
# define loss function and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),
lr=params['lr'],
momentum=params['momentum'])
# train
train(trainloader, net, criterion, optimizer, params, device)
# save model
save_model_path = os.path.join('models', params['model_name'])
torch.save(net.state_dict(), save_model_path)
def process_params(params, dest_dir):
# Fill params
params = update_default_dict(params)
# Restart model
if params['model_restart']:
model_file = params['model_restart']
with open(
op.join(dest_dir, "models", params['model_restart'] + '.json'),
"r") as f:
params = json.load(f)
params['model_restart'] = model_file
# Seed
if params['seed'] is None:
params['seed'] = np.random.choice(2147483648) # 2^31
# Hidden layers
if type(params['model']['hidden_layers']) == str:
params['model']['hidden_layers'] = [
int(h) for h in params['model']['hidden_layers'].strip(']').strip(
'[').split(',')
]
# Set random seed
torch.manual_seed(params['seed'])
torch.set_deterministic(True)
np.random.seed(params['seed'])
# Data normalization is not possible with Bjorck layers because they guarnatee a
# Lipshitz constant of exactly one
if params['normalize_input'] or params['normalize_output']:
assert params['model'][
'linear'] != 'bjorck', "Bjorck layers incompatible with data normalization"
return params
def set_seeds():
random.seed(42)
np.random.seed(12345)
torch.manual_seed(1234)
torch.set_deterministic(True)
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = False
def __init__(
self,
learning_rate: float = 1e-3,
buffer_capacity: int = 200,
max_epochs_per_task: int = 10,
weight_decay: float = 1e-6,
seed: int = None,
):
self.learning_rate = learning_rate
self.weight_decay = weight_decay
self.buffer_capacity = buffer_capacity
self.net: ResNet
self.buffer: Optional[Buffer] = None
self.optim: torch.optim.Optimizer
self.task: int = 0
self.rng = np.random.RandomState(seed)
self.seed = seed
if seed:
torch.manual_seed(seed)
torch.set_deterministic(True)
self.epochs_per_task: int = max_epochs_per_task
self.early_stop_patience: int = 2
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def set_seed(seed, torch_deterministic=False):
if seed == -1 and torch_deterministic:
seed = 42
elif seed == -1:
seed = np.random.randint(0, 10000)
print("Setting seed: {}".format(seed))
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
if torch_deterministic:
# refer to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.set_deterministic(True)
else:
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.deterministic = False
return seed
def init_env(config):
# set a debug environment variable CUBLAS_WORKSPACE_CONFIG to ":16:8" (may limit overall performance) or ":4096:8" (will increase library footprint in GPU memory by approximately 24MiB).
# https://docs.nvidia.com/cuda/cublas/index.html
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8"
if config.seed is not None:
if config.seed >= 0:
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.set_deterministic(True)
torch.backends.cudnn.benchmark = False
else:
logging.warning(
f'the random seed should be a non-negative integer')
config.device = None
if not config.cpu and torch.cuda.is_available():
config.device = torch.device('cuda')
else:
config.device = torch.device('cpu')
# https://github.com/pytorch/pytorch/issues/11201
torch.multiprocessing.set_sharing_strategy('file_system')
logging.info(f'Using device: {config.device}')
config.run_name = '{}_{}_{}'.format(
config.data_name,
Path(config.config).stem if config.config else config.model_name,
datetime.now().strftime('%Y%m%d%H%M%S'),
)
logging.info(f'Run name: {config.run_name}')
return config
def fit(self, model, dataset):
# init loggers
self.tensorboard = SummaryWriter()
self.save_dir = self.tensorboard.log_dir
self.logger = self.init_logger()
self.logger.info(f'seed torch and numpy: {self.seed}')
torch.manual_seed(self.seed)
np.random.seed(self.seed)
torch.set_deterministic(True)
self.logger.info(f'setup dataset ...')
dataset.setup()
self.logger.info('done.')
self.logger.info(f'start training ...')
self.train_tqdm = tqdm(range(self.max_epochs),
desc='epoch',
dynamic_ncols=True,
postfix=0)
for i in self.train_tqdm:
self.num_steps_this_epoch = 0
self.train_epoch(model, dataset)
if self.should_stop:
break
self.train_tqdm.close()
self.tensorboard.close()
def __init__(self, args: Namespace, wildcards: dict, descriptions: dict = None):
super().__init__()
# args, seed
self.args = args
self.save_dir = self._parsed_argument('save_dir', args)
self.is_test_run = self._parsed_argument('is_test_run', args)
self.seed = self._parsed_argument('seed', args)
self.is_deterministic = self._parsed_argument('is_deterministic', args)
random.seed(self.seed)
np.random.seed(self.seed)
torch.manual_seed(self.seed)
if self.is_deterministic:
# see https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
torch.set_deterministic(self.is_deterministic)
# maybe delete old dir, note arguments, save run_config
if self._parsed_argument('save_del_old', args):
shutil.rmtree(self.save_dir, ignore_errors=True)
os.makedirs(self.save_dir, exist_ok=True)
save_as_json(args, get_task_config_path(self.save_dir), wildcards)
dump_system_info(self.save_dir + 'sysinfo.txt')
# logging
self.log_file = '%slog_task.txt' % self.save_dir
LoggerManager().set_logging(default_save_file=self.log_file)
self.logger = self.new_logger(index=None)
log_args(self.logger, None, self.args, add_git_hash=True, descriptions=descriptions)
Register.log_all(self.logger)
# reset weight strategies so that consecutive tasks do not conflict with each other
StrategyManager().reset()
self.methods = []
def set_random_seeds(random_seed=0):
torch.manual_seed(random_seed)
#torch.backends.cudnn.deterministic = True
torch.set_deterministic(True)
#torch.backends.cudnn.benchmark = False
np.random.seed(random_seed)
random.seed(random_seed)
def test_linear_transformation(self):
c, h, w = 3, 24, 32
tensor, _ = self._create_data(h, w, channels=c, device=self.device)
matrix = torch.rand(c * h * w, c * h * w, device=self.device)
mean_vector = torch.rand(c * h * w, device=self.device)
fn = T.LinearTransformation(matrix, mean_vector)
scripted_fn = torch.jit.script(fn)
self._test_transform_vs_scripted(fn, scripted_fn, tensor)
batch_tensors = torch.rand(4, c, h, w, device=self.device)
# We skip some tests from _test_transform_vs_scripted_on_batch as
# results for scripted and non-scripted transformations are not exactly the same
torch.manual_seed(12)
torch.set_deterministic(True)
transformed_batch = fn(batch_tensors)
torch.manual_seed(12)
s_transformed_batch = scripted_fn(batch_tensors)
self.assertTrue(transformed_batch.equal(s_transformed_batch))
with get_tmp_dir() as tmp_dir:
scripted_fn.save(os.path.join(tmp_dir, "t_norm.pt"))
def force_determinism(is_deterministic=False):
if not is_deterministic:
return
torch.manual_seed(0)
np.random.seed(0)
torch.backends.cudnn.benchmark = False
torch.set_deterministic(True)
def set_determinism(
seed: Optional[int] = NP_MAX,
use_deterministic_algorithms: Optional[bool] = None,
additional_settings: Optional[Union[Sequence[Callable[[int], Any]], Callable[[int], Any]]] = None,
) -> None:
"""
Set random seed for modules to enable or disable deterministic training.
Args:
seed: the random seed to use, default is np.iinfo(np.int32).max.
It is recommended to set a large seed, i.e. a number that has a good balance
of 0 and 1 bits. Avoid having many 0 bits in the seed.
if set to None, will disable deterministic training.
use_deterministic_algorithms: Set whether PyTorch operations must use "deterministic" algorithms.
additional_settings: additional settings that need to set random seed.
Note:
This function will not affect the randomizable objects in :py:class:`monai.transforms.Randomizable`, which
have independent random states. For those objects, the ``set_random_state()`` method should be used to
ensure the deterministic behavior (alternatively, :py:class:`monai.data.DataLoader` by default sets the seeds
according to the global random state, please see also: :py:class:`monai.data.utils.worker_init_fn` and
:py:class:`monai.data.utils.set_rnd`).
"""
if seed is None:
# cast to 32 bit seed for CUDA
seed_ = torch.default_generator.seed() % MAX_SEED
torch.manual_seed(seed_)
else:
seed = int(seed) % MAX_SEED
torch.manual_seed(seed)
global _seed
_seed = seed
random.seed(seed)
np.random.seed(seed)
if additional_settings is not None:
additional_settings = ensure_tuple(additional_settings)
for func in additional_settings:
func(seed)
if torch.backends.flags_frozen():
warnings.warn("PyTorch global flag support of backends is disabled, enable it to set global `cudnn` flags.")
torch.backends.__allow_nonbracketed_mutation_flag = True
if seed is not None:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
else: # restore the original flags
torch.backends.cudnn.deterministic = _flag_deterministic
torch.backends.cudnn.benchmark = _flag_cudnn_benchmark
if use_deterministic_algorithms is not None:
if hasattr(torch, "use_deterministic_algorithms"): # `use_deterministic_algorithms` is new in torch 1.8.0
torch.use_deterministic_algorithms(use_deterministic_algorithms)
elif hasattr(torch, "set_deterministic"): # `set_deterministic` is new in torch 1.7.0
torch.set_deterministic(use_deterministic_algorithms) # type: ignore
else:
warnings.warn("use_deterministic_algorithms=True, but PyTorch version is too old to set the mode.")
def set_seed(cfg):
if cfg.use_seed:
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.manual_seed(cfg.seed)
torch.cuda.manual_seed(cfg.seed)
torch.cuda.manual_seed_all(cfg.seed)
torch.set_deterministic(True)
def set_seed(seed):
print(f"setting seed to {seed}")
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.set_deterministic(True)
torch.backends.cudnn.benchmark = False
def __init__(self,
text,
*,
lr=.07,
image_size=512,
gradient_accumulate_every=1,
save_every=50,
epochs=20,
iterations=1050,
save_progress=False,
bilinear=False,
open_folder=True,
seed=None,
torch_deterministic=False,
max_classes=None,
class_temperature=2.,
save_date_time=False,
save_best=False):
super().__init__()
if exists(seed):
print(f'setting seed of {seed}')
if seed == 0:
print(
'you can override this with --seed argument in the command line, or --random for a randomly chosen one'
)
torch.manual_seed(seed)
if torch_deterministic:
assert not bilinear, 'the deterministic (seeded) operation does not work with interpolation (PyTorch 1.7.1)'
torch.set_deterministic(True)
self.epochs = epochs
self.iterations = iterations
model = BigSleep(image_size=image_size,
bilinear=bilinear,
max_classes=max_classes,
class_temperature=class_temperature).cuda()
self.model = model
self.lr = lr
self.optimizer = Adam(model.model.latents.parameters(), lr)
self.gradient_accumulate_every = gradient_accumulate_every
self.save_every = save_every
self.save_progress = save_progress
self.save_date_time = save_date_time
self.save_best = save_best
self.current_best_score = 0
self.open_folder = open_folder
self.total_image_updates = (self.epochs *
self.iterations) / self.save_every
self.set_text(text)
def set_deterministic(deterministic: bool = True):
# if hasattr(torch, 'backends') and hasattr(torch.backends, 'cudnn'):
# torch.backends.cudnn.enabled = not deterministic
# torch.backends.cudnn.benchmark = not deterministic
# torch.backends.cudnn.deterministic = deterministic
if hasattr(torch, 'set_deterministic'):
torch.set_deterministic(deterministic)
else:
torch.use_deterministic_algorithms(deterministic)
def set_deterministic():
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
if torch.__version__ <= Version("1.7"):
torch.set_deterministic(True)
else:
torch.use_deterministic_algorithms(True)
def fix_all_seed(seed):
random.seed(seed)
np.random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
torch.set_deterministic(True)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
def torch_seed(s: int = 72163) -> torch.Generator:
"""
seeding for reproducibility"""
generator = torch.manual_seed(s)
if False: # Disabled for now
torch.set_deterministic(True)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(s)
torch.backends.cudnn.deterministic = True
return generator
def test_equalize(self):
torch.set_deterministic(False)
self._test_adjust_fn(
F.equalize,
F_pil.equalize,
F_t.equalize,
[{}],
tol=1.0,
agg_method="max",
dts=(None,)
)
def test_equalize(device):
torch.set_deterministic(False)
check_functional_vs_PIL_vs_scripted(
F.equalize,
F_pil.equalize,
F_t.equalize,
{},
device,
dtype=None,
tol=1.0,
agg_method="max",
)
def make_deterministic(seed):
np.random.seed(seed)
torch.manual_seed(seed)
# see https://github.com/pytorch/pytorch/issues/47672
cuda_version = torch.version.cuda
if cuda_version is not None and float(torch.version.cuda) >= 10.2:
os.environ['CUBLAS_WORKSPACE_CONFIG'] = '4096:8'
else:
torch.set_deterministic(True) # Not all Operations support this.
# This is only for Convolution no problem
torch.backends.cudnn.deterministic = True
def _init_deterministic(self, deterministic: bool) -> None:
self.deterministic = deterministic
if _TORCH_GREATER_EQUAL_1_8:
torch.use_deterministic_algorithms(deterministic)
else:
torch.set_deterministic(deterministic)
if deterministic:
# fixing non-deterministic part of horovod
# https://github.com/PyTorchLightning/pytorch-lightning/pull/1572/files#r420279383
os.environ["HOROVOD_FUSION_THRESHOLD"] = str(0)
# https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
def __init__(self, model, seed, **kwargs):
super().__init__(**kwargs, seed=seed, model=model)
# set deterministic behavior
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
seed_everything(seed)
if re.match('^vgg.*', model.lower()) is not None:
log.warning("VGG requires non-deterministic implementation, "
"disabling pytorch determinism...")
return
torch.set_deterministic(True)
torch.backends.cudnn.deterministic = True
def __init__(self, value: int, cuda: bool = False):
self.value = value
self.cuda = cuda
self.no_side_effect = False
if self.no_side_effect:
self._last_seed = torch.initial_seed()
numpy.random.seed(self.value)
torch.manual_seed(self.value)
if False: # Disabled for now
torch.set_deterministic(True)
if self.cuda:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
def setup_cudnn(deterministic: bool = False, use_gpu: bool = torch.cuda.is_available(), seed: Union[int, Tuple[int]] = None):
""" Setup CUDNN backend and Random Number Generators to given seed(s).
Args:
- deterministic: Set to `True` for deterministic/reproducible results: Makes training procedure reproducible by calling `torch.set_deterministic` (may have small performance impact).
- seed: Seed(s) argument which can either be a single seed passed to `deepcv.utils.set_seeds` or a Tuple of seeds which will be passed to `deepcv.utils.set_each_seeds` as positional seed arguments
.. See `deepcv.utils.set_seeds` and [`torch.set_deterministic` torch documentation](https://pytorch.org/docs/stable/generated/torch.set_deterministic.html) for more details.
"""
torch.set_deterministic(deterministic) # (Pytorch 1.7+)
if use_gpu:
torch.backends.cudnn.benchmark = use_gpu and not torch.backends.cudnn.deterministic
torch.backends.cudnn.fastest = use_gpu # Disable this if memory issues
if isinstance(seed, int):
set_seeds(seed)
else:
# Assume seed contains a tuple of seeds to be given as `deepcv.utils.set_each_seeds` positional arguments
set_each_seeds(seed)
def set_deterministic(seed: int = None, by_rank: bool = False):
""" Set seed of `torch`, `random` and `numpy` to `seed` for making it deterministic. Because of CUDA's limitation,
this may not make everything deterministic, however.
"""
with set_seed(seed, by_rank):
if seed is not None:
if hasattr(torch, "set_deterministic"):
torch.set_deterministic(True)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
logger.info(
"Set deterministic. But some GPU computations might be still non-deterministic. "
"Also, this may affect the performance.")
yield
if hasattr(torch, "set_deterministic"):
torch.set_deterministic(False)
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
logger.info("Back to non-deterministic.")
def set_all_seeds(seed=42, deterministic_cudnn=True):
"""
Setting multiple seeds to make runs reproducible.
Enabling "deterministic_cudnn" gives full reproducibility with CUDA, but might slow down training
Args:
param seed: number to use as seed
type deterministic_cudnn: bool
return: None
"""
os.environ['PYTHONHASHSEED'] = str(seed)
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8" # or ":16:8"
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
if deterministic_cudnn:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.set_deterministic(True)
