def _validation(self): sly.logger.info("Before validation", extra={'epoch': self.epoch_flt}) if self.config['validate_with_model_eval']: self.model.eval() metrics_values = defaultdict(int) samples_cnt = 0 for val_it, (inputs, targets) in enumerate(self.data_loaders['val']): inputs, targets = cuda_variable(inputs, volatile=True), cuda_variable(targets) outputs = self.model(inputs) full_batch_size = inputs.size(0) for name, metric in self.val_metrics.items(): metric_value = metric(outputs, targets) if isinstance(metric_value, torch.autograd.Variable): # for val loss metric_value = metric_value.data[0] metrics_values[name] += metric_value * full_batch_size samples_cnt += full_batch_size sly.logger.info("Validation in progress", extra={'epoch': self.epoch_flt, 'val_iter': val_it, 'val_iters': self.val_iters}) for name in metrics_values: metrics_values[name] /= float(samples_cnt) sly.report_metrics_validation(self.epoch_flt, metrics_values) self.model.train() sly.logger.info("Validation has been finished", extra={'epoch': self.epoch_flt}) return metrics_values
def infer_per_pixel_scores_single_image(model, raw_input, out_shape, apply_softmax=True): """ Performs inference with PyTorch model and resize predictions to a given size. Args: model: PyTorch model inherited from torch.Module class. raw_input: PyTorch Tensor out_shape: Output size (height, width). apply_softmax: Whether to apply softmax function after inference or not. Returns: Inference resulting numpy array resized to a given size. """ model_input = torch.stack([raw_input], 0) # add dim #0 (batch size 1) model_input = cuda_variable(model_input, volatile=True) output = model(model_input) if apply_softmax: output = torch_functional.softmax(output, dim=1) output = output.data.cpu().numpy()[0] # from batch to 3d pred = np.transpose(output, (1, 2, 0)) return sly_image.resize(pred, out_shape)
def dice_loss(preds, trues, weight=None, is_average=True, ignore_index=None): num = preds.size(0) preds = preds.view(num, -1) trues = trues.view(num, -1) if ignore_index is not None: ignore_mask = trues.data == ignore_index preds = preds.clone() preds.data[ignore_mask] = 0 trues = trues.clone() trues.data[ignore_mask] = 0 if weight is not None: w = cuda_variable(weight).view(num, -1) preds = preds * w trues = trues * w intersection = (preds * trues).sum(1) scores = 2. * (intersection + 1) / (preds.sum(1) + trues.sum(1) + 1) if is_average: score = scores.sum() / num return torch.clamp(score, 0., 1.) else: return scores
def train(self): progress = sly.Progress('Model training: ', self.epochs * self.train_iters) self.model.train() lr_decr = self.config['lr_decreasing'] policy = LRPolicyWithPatience(optim_cls=Adam, init_lr=self.config['lr'], patience=lr_decr['patience'], lr_divisor=lr_decr['lr_divisor'], model=self.model) best_val_loss = float('inf') debug_saver = None debug_save_prob = float(os.getenv('DEBUG_PATCHES_PROB', 0.0)) if debug_save_prob > 0: target_multi = int(255.0 / len(self.out_classes)) debug_saver = DebugSaver(odir=os.path.join(sly.TaskPaths.DEBUG_DIR, 'debug_patches'), prob=debug_save_prob, target_multi=target_multi) for epoch in range(self.epochs): sly.logger.info("Before new epoch", extra={'epoch': self.epoch_flt}) for train_it, (inputs_cpu, targets_cpu) in enumerate( self.data_loaders['train']): inputs, targets = cuda_variable(inputs_cpu), cuda_variable( targets_cpu) outputs = self.model(inputs) loss = self.criterion(outputs, targets) if debug_saver is not None: out_cls = functional.softmax(outputs, dim=1) debug_saver.process(inputs_cpu, targets_cpu, out_cls.data.cpu()) policy.optimizer.zero_grad() loss.backward() policy.optimizer.step() metric_values_train = {'loss': loss.data[0]} for name, metric in self.metrics.items(): metric_values_train[name] = metric(outputs, targets) progress.iter_done_report() self.epoch_flt = epoch_float(epoch, train_it + 1, self.train_iters) sly.report_metrics_training(self.epoch_flt, metric_values_train) if self.eval_planner.need_validation(self.epoch_flt): metrics_values_val = self._validation() self.eval_planner.validation_performed() val_loss = metrics_values_val['loss'] model_is_best = val_loss < best_val_loss if model_is_best: best_val_loss = val_loss sly.logger.info( 'It\'s been determined that current model is the best one for a while.' ) self._save_model_snapshot(model_is_best, opt_data={ 'epoch': self.epoch_flt, 'val_metrics': metrics_values_val, }) policy.reset_if_needed(val_loss, self.model) sly.logger.info("Epoch was finished", extra={'epoch': self.epoch_flt})