def on_validation_end(self, **kwargs):
# Consolidate the state dict of sharded optimizers
consolidate_optim_state_dict(self.trainer.optimizer)
stop = self.early_stopping(self.trainer.num_updates,
self.trainer.current_iteration,
kwargs["meter"])
stop = bool(broadcast_scalar(stop, src=0, device=self.trainer.device))
return stop
def change_dataloader(self):
choice = 0
if self.num_datasets <= 1:
self.current_index = choice
return
if self._is_main:
choice = self.iteration_strategy()
# self._finished_iterators will always be empty in case of
# non-proportional (equal) sampling
while self.dataset_list[choice] in self._finished_iterators:
choice = self.iteration_strategy()
choice = broadcast_scalar(choice, 0, device=get_current_device())
self.current_index = choice
def change_dataloader(self):
choice = 0
if self.num_datasets <= 1:
self.current_index = choice
return
if self._is_master:
choice = np.random.choice(
self.num_datasets, 1, p=self._dataset_probabilities
)[0]
# self._finished_iterators will always be empty in case of
# non-proportional (equal) sampling
while self.dataset_list[choice] in self._finished_iterators:
choice = np.random.choice(
self.num_datasets, 1, p=self._dataset_probabilities
)[0]
choice = broadcast_scalar(choice, 0, device=get_current_device())
self.current_index = choice
def change_dataloader(self):
if self.num_datasets <= 1:
return
choice = 0
if self._is_master:
choice = np.random.choice(self.num_datasets,
1,
p=self._dataset_probabilities)[0]
while choice in self._finished_iterators:
choice = np.random.choice(self.num_datasets,
1,
p=self._dataset_probabilities)[0]
choice = broadcast_scalar(choice,
0,
device=registry.get("current_device"))
self.current_index = choice
self.current_dataset = self.datasets[self.current_index]
self.current_loader = self.loaders[self.current_index]
self._chosen_iterator = self.iterators[self.current_index]
def _try_full_validation(self, force=False):
should_break = False
if self.num_updates % self.evaluation_interval == 0 or force:
self.snapshot_timer.reset()
self.writer.write(
"Evaluation time. Running on full validation set...")
# Validation and Early stopping
# Create a new meter for this case
report, meter = self.evaluate(self.val_loader)
extra = {
"num_updates": self.num_updates,
"epoch": self.current_epoch,
"iterations": self.current_iteration,
"max_updates": self.max_updates,
"val_time": self.snapshot_timer.get_time_since_start(),
}
stop = self.early_stopping(self.num_updates,
self.current_iteration, meter)
stop = bool(broadcast_scalar(stop, src=0, device=self.device))
extra.update(self.early_stopping.get_info())
self._summarize_report(meter, extra=extra)
gc.collect()
if "cuda" in str(self.device):
torch.cuda.empty_cache()
if stop is True:
self.writer.write("Early stopping activated")
should_break = True
self.train_timer.reset()
return should_break
def change_dataloader(self):
if self.num_datasets <= 1:
return
choice = 0
if self._is_master:
choice = np.random.choice(self.num_datasets,
1,
p=self._dataset_probabilities)[0]
# self._finished_iterators will always be empty in case of
# non-proportional (equal) sampling
while choice in self._finished_iterators:
choice = np.random.choice(self.num_datasets,
1,
p=self._dataset_probabilities)[0]
choice = broadcast_scalar(choice,
0,
device=registry.get("current_device"))
self.current_index = choice
self.current_dataset = self.datasets[self.current_index]
self.current_loader = self.loaders[self.current_index]
self._chosen_iterator = self.iterators[self.current_index]
def on_validation_end(self, **kwargs):
stop = self.early_stopping(self.trainer.num_updates,
self.trainer.current_iteration,
kwargs["meter"])
stop = bool(broadcast_scalar(stop, src=0, device=self.trainer.device))
return stop
