def epoch_callback(self, model, epoch): """ Challenges the current baseline with the model and replaces the baseline model if it is improved. :param model: The model to challenge the baseline by :param epoch: The current epoch """ rank = torch.distributed.get_rank() if rank == 0: print("Evaluating candidate model on evaluation dataset") candidate_vals = rollout(model, self.dataset, self.opts).cpu().numpy() candidate_mean = candidate_vals.mean() if rank == 0: print( "Epoch {} candidate mean {}, baseline epoch {} mean {}, difference {}" .format(epoch, candidate_mean, self.epoch, self.mean, candidate_mean - self.mean)) if candidate_mean - self.mean < 0: # Calc p value t, p = ttest_rel(candidate_vals, self.bl_vals) p_val = p / 2 # one-sided assert t < 0, "T-statistic should be negative" if rank == 0: print("p-value: {}".format(p_val)) if p_val < self.opts.bl_alpha: if rank == 0: print('Update baseline') self._update_model(model, epoch)
def _update_model(self, model, epoch, dataset=None): self.model = copy.deepcopy(model) # Always generate baseline dataset when updating model to prevent overfitting to the baseline dataset if dataset is not None: if len(dataset) != self.opts.val_size: print( "Warning: not using saved baseline dataset since val_size does not match" ) dataset = None elif (dataset[0] if self.problem.NAME == 'tsp' else dataset[0]['loc']).size(0) != self.opts.graph_size: print( "Warning: not using saved baseline dataset since graph_size does not match" ) dataset = None if dataset is None: self.dataset = self.problem.make_dataset( size=self.opts.graph_size, num_samples=self.opts.val_size, distribution=self.opts.data_distribution) else: self.dataset = dataset if torch.distributed.get_rank() == 0: print("Evaluating baseline model on evaluation dataset") self.bl_vals = rollout(self.model, self.dataset, self.opts).cpu().numpy() self.mean = self.bl_vals.mean() self.epoch = epoch
def wrap_dataset(self, dataset): print("Evaluating baseline on dataset...") # Need to convert baseline to 2D to prevent converting to double, see # https://discuss.pytorch.org/t/dataloader-gives-double-instead-of-float/717/3 return BaselineDataset( dataset, rollout(self.model, dataset, self.opts).view(-1, 1))
def _update_model(self, model, epoch, dataset=None): self.model = copy.deepcopy(model) # Always generate baseline dataset when updating model to prevent overfitting to the baseline dataset if dataset is None: self.dataset = self.problem.make_dataset( size=self.opts.graph_size, num_samples=self.opts.val_size) else: self.dataset = dataset print("Evaluating baseline model on evaluation dataset") self.bl_vals = rollout(self.model, self.dataset, self.opts).cpu().numpy() self.mean = self.bl_vals.mean() self.epoch = epoch
def _update_model(self, model, epoch, dataset=None): self.model = copy.deepcopy(model) # Always generate baseline dataset when updating model to prevent overfitting to the baseline dataset self.dataset = self.problem.make_dataset( min_size=self.opts.min_size, max_size=self.opts.max_size, batch_size=self.opts.batch_size, num_samples=self.opts.rollout_size, distribution=self.opts.data_distribution, neighbors=self.opts.neighbors, knn_strat=self.opts.knn_strat ) print("\nEvaluating Rollout baseline model on evaluation dataset") self.bl_vals = rollout(self.model, self.dataset, self.opts).cpu().numpy() self.mean = self.bl_vals.mean() self.epoch = epoch
def _update_model(self, model, epoch, dataset=None): self.model = copy.deepcopy(model) if dataset is not None: if len(dataset) != self.opts.stat_size: print( "Warning: not using saved baseline dataset since val_size does not match" ) elif (dataset[0] if self.problem.NAME == 'tsp' else dataset[0]['loc']).size(0) != self.opts.graph_size: print( "Warning: not using saved baseline dataset since graph_size does not match" ) else: self.dataset = dataset print("Evaluating baseline model on evaluation dataset") self.bl_vals = rollout(self.model, self.dataset, self.opts).cpu().numpy() self.mean = self.bl_vals.mean() self.epoch = epoch