def on_train_begin(self, *args, **kwargs):
if self.__log_iteration <= 0:
return
for log_name in self.__class_names:
self.__averagers['train_' + log_name] = Averager()
self.__averagers['valid_' + log_name] = Averager()
def __init__(self, logger: Optional[DefaultLogger] = None, best_in_iter: int = 5000,
output_dir: str = './'):
super().__init__(logger)
self._averager = Averager()
self.__best_in_iter = best_in_iter
self.__output_dir = output_dir
self.__best = np.inf
self.__stage = -1
self.__predicitons = []
class ModelCheckpoint(Callback):
def __init__(self,
logger: Optional[DefaultLogger] = None,
best_in_iter: int = 5000,
output_dir: str = './'):
super().__init__(logger)
self._averager = Averager()
self.__best_in_iter = best_in_iter
self.__output_dir = output_dir
self.__best = np.inf
self.__stage = -1
if self.__best_in_iter == 0:
self.__best_in_iter = 5000
def on_validation_begin(self, epoch, iteration, *args, **kwargs):
if iteration % self.__best_in_iter == 0:
self._averager.reset()
self.__best = np.inf
self.__stage = self.__stage + 1
def on_validation_end(self, epoch, iteration, *args, **kwargs):
super().on_validation_end(epoch, iteration, *args, **kwargs)
network = kwargs[
'network'] if 'network' in kwargs else None # type: Optional[AbstractNetwork]
if network is None:
return
lloss = self._averager.value
if lloss < self.__best:
self.__best = lloss
self.__save_best_validation_result(epoch, iteration, network)
def __save_best_validation_result(self, epoch, iteration,
network: AbstractNetwork):
dl = network.get_dataloader('validation') # type: DataLoader
# noinspection PyTypeChecker
ds = dl.dataset # type: AbstractDataset
dataset = ds.data_set
datafold = ds.data_fold
file = "{}/ds_{}__fold_{}__best_in_stage_{}.pt".format(
self.__output_dir, dataset, datafold, self.__stage)
self._info("Saving best iteration result to {}...".format(file))
model_mode = network.model.mode
network.model.mode = 'eval'
if torch.cuda.is_available():
network.model.cpu()
model_state_dict = network.model.state_dict()
optimizer_state_dict = network.optimizer.state_dict()
network.model.cuda()
else:
model_state_dict = network.model.state_dict()
optimizer_state_dict = network.optimizer.state_dict()
persist_torch_model(file, epoch, iteration, model_state_dict,
optimizer_state_dict)
network.model.mode = model_mode
def on_validation_begin(self, epoch, iteration, *args, **kwargs):
if 'valid_loss' not in self.__loss_averagers:
self.__loss_averagers['valid_loss'] = Averager()
self.__loss_averagers['valid_loss'].reset()
def on_train_begin(self, *args, **kwargs):
if 'loss' not in self.__loss_averagers:
self.__loss_averagers['loss'] = Averager()
class OutputSaver(Callback):
def __init__(self, logger: Optional[DefaultLogger] = None, best_in_iter: int = 5000,
output_dir: str = './'):
super().__init__(logger)
self._averager = Averager()
self.__best_in_iter = best_in_iter
self.__output_dir = output_dir
self.__best = np.inf
self.__stage = -1
self.__predicitons = []
def on_validation_begin(self, epoch, iteration, *args, **kwargs):
if iteration % self.__best_in_iter == 0:
self._averager.reset()
self.__best = np.inf
self.__stage = self.__stage + 1
self.__predicitons = []
def on_validation_iteration_end(self, epoch: int, iteration: int, batch_idx: int, batch_data: any, outputs: Tensor,
targets: Tensor, loss: Tensor, *args, **kwargs):
super().on_validation_iteration_end(epoch, iteration, batch_idx, batch_data, outputs, targets, loss, *args,
**kwargs)
indices, images, targets, extras = batch_data
self._averager.send(loss.item())
outputs = torch.sigmoid(outputs)
np_targets = targets.view(-1).data.cpu().numpy()
np_outputs = outputs.view(-1).data.cpu().numpy()
for i, index in enumerate(indices):
self.__predicitons.append({
'id': extras[i]['id'],
'label': np_targets[i],
'prediction': np_outputs[i]
})
def on_validation_end(self, epoch, iteration, *args, **kwargs):
super().on_validation_end(epoch, iteration, *args, **kwargs)
network = kwargs['network'] if 'network' in kwargs else None # type: Optional[AbstractNetwork]
if network is None:
return
lloss = self._averager.value
if lloss < self.__best:
self.__best = lloss
self.__save_best_validation_result(epoch, iteration, network)
def __save_best_validation_result(self, epoch, iteration, network: AbstractNetwork):
dl = network.get_dataloader('validation') # type: DataLoader
# noinspection PyTypeChecker
ds = dl.dataset # type: AbstractDataset
dataset = ds.data_set
datafold = ds.data_fold
file = "{}/outputs__ds_{}__fold_{}__best_in_stage_{}.csv".format(
self.__output_dir, dataset, datafold, self.__stage)
self._info("Saving best prediction (based on best valid loss) result to {}...".format(file))
df = pd.DataFrame(self.__predicitons)
df.to_csv(file, index=False)
self.__predicitons = []