def epoch_callback(self, model, epoch):
"""
Challenges the current baseline with the model and replaces the baseline model if it is improved.
:param model: The model to challenge the baseline by
:param epoch: The current epoch
"""
rank = torch.distributed.get_rank()
if rank == 0:
print("Evaluating candidate model on evaluation dataset")
candidate_vals = rollout(model, self.dataset, self.opts).cpu().numpy()
candidate_mean = candidate_vals.mean()
if rank == 0:
print(
"Epoch {} candidate mean {}, baseline epoch {} mean {}, difference {}"
.format(epoch, candidate_mean, self.epoch, self.mean,
candidate_mean - self.mean))
if candidate_mean - self.mean < 0:
# Calc p value
t, p = ttest_rel(candidate_vals, self.bl_vals)
p_val = p / 2 # one-sided
assert t < 0, "T-statistic should be negative"
if rank == 0:
print("p-value: {}".format(p_val))
if p_val < self.opts.bl_alpha:
if rank == 0:
print('Update baseline')
self._update_model(model, epoch)
def _update_model(self, model, epoch, dataset=None):
self.model = copy.deepcopy(model)
# Always generate baseline dataset when updating model to prevent overfitting to the baseline dataset
if dataset is not None:
if len(dataset) != self.opts.val_size:
print(
"Warning: not using saved baseline dataset since val_size does not match"
)
dataset = None
elif (dataset[0] if self.problem.NAME == 'tsp' else
dataset[0]['loc']).size(0) != self.opts.graph_size:
print(
"Warning: not using saved baseline dataset since graph_size does not match"
)
dataset = None
if dataset is None:
self.dataset = self.problem.make_dataset(
size=self.opts.graph_size,
num_samples=self.opts.val_size,
distribution=self.opts.data_distribution)
else:
self.dataset = dataset
if torch.distributed.get_rank() == 0:
print("Evaluating baseline model on evaluation dataset")
self.bl_vals = rollout(self.model, self.dataset,
self.opts).cpu().numpy()
self.mean = self.bl_vals.mean()
self.epoch = epoch
def wrap_dataset(self, dataset):
print("Evaluating baseline on dataset...")
# Need to convert baseline to 2D to prevent converting to double, see
# https://discuss.pytorch.org/t/dataloader-gives-double-instead-of-float/717/3
return BaselineDataset(
dataset,
rollout(self.model, dataset, self.opts).view(-1, 1))
def _update_model(self, model, epoch, dataset=None):
self.model = copy.deepcopy(model)
# Always generate baseline dataset when updating model to prevent overfitting to the baseline dataset
if dataset is None:
self.dataset = self.problem.make_dataset(
size=self.opts.graph_size, num_samples=self.opts.val_size)
else:
self.dataset = dataset
print("Evaluating baseline model on evaluation dataset")
self.bl_vals = rollout(self.model, self.dataset,
self.opts).cpu().numpy()
self.mean = self.bl_vals.mean()
self.epoch = epoch
def _update_model(self, model, epoch, dataset=None):
self.model = copy.deepcopy(model)
# Always generate baseline dataset when updating model to prevent overfitting to the baseline dataset
self.dataset = self.problem.make_dataset(
min_size=self.opts.min_size, max_size=self.opts.max_size, batch_size=self.opts.batch_size,
num_samples=self.opts.rollout_size, distribution=self.opts.data_distribution,
neighbors=self.opts.neighbors, knn_strat=self.opts.knn_strat
)
print("\nEvaluating Rollout baseline model on evaluation dataset")
self.bl_vals = rollout(self.model, self.dataset, self.opts).cpu().numpy()
self.mean = self.bl_vals.mean()
self.epoch = epoch
def _update_model(self, model, epoch, dataset=None):
self.model = copy.deepcopy(model)
if dataset is not None:
if len(dataset) != self.opts.stat_size:
print(
"Warning: not using saved baseline dataset since val_size does not match"
)
elif (dataset[0] if self.problem.NAME == 'tsp' else
dataset[0]['loc']).size(0) != self.opts.graph_size:
print(
"Warning: not using saved baseline dataset since graph_size does not match"
)
else:
self.dataset = dataset
print("Evaluating baseline model on evaluation dataset")
self.bl_vals = rollout(self.model, self.dataset,
self.opts).cpu().numpy()
self.mean = self.bl_vals.mean()
self.epoch = epoch
