def evaluate_ensemble(config: ConfigClass, models_list):
device = torch.device(
f'cuda:{config.gpu_node}' if torch.cuda.is_available() else 'cpu')
# Create dataloader class
loader_class = get_dataloaders(config.data)
segment_metrics = SegmentationMetrics(num_classes=loader_class.num_classes,
threshold=config.binarize_threshold)
# Evaluate models
for model in models_list:
model.eval()
with torch.no_grad():
for batch in loader_class.evaluation_loader:
x, y = batch
x = x.to(device=device, non_blocking=True)
y = y.to(device=device, non_blocking=True)
averaged_y = torch.zeros_like(y)
for model in models_list:
y_pred = model(x)
averaged_y += torch.sigmoid(y_pred)
averaged_y = averaged_y / len(models_list)
segment_metrics.update((averaged_y, y), process=False)
metrics = segment_metrics.compute()
return metrics
def evaluate_monte_carlo(config: ConfigClass, model):
device = torch.device(
f'cuda:{config.gpu_node}' if torch.cuda.is_available() else 'cpu')
mc_passes = config.prediction.mc_passes
# Create dataloader class
loader_class = get_dataloaders(config.data)
segment_metrics = SegmentationMetrics(num_classes=loader_class.num_classes,
threshold=config.binarize_threshold)
# Evaluate model
model.eval()
model.apply(apply_dropout)
with torch.no_grad():
for batch in loader_class.evaluation_loader:
x, y = batch
x = x.to(device=device, non_blocking=True)
y = y.to(device=device, non_blocking=True)
summed_y = torch.zeros_like(y)
for idx in range(mc_passes):
y_pred = model(x)
summed_y += torch.sigmoid(y_pred)
averaged_y = summed_y / mc_passes
segment_metrics.update((averaged_y, y), process=False)
metrics = segment_metrics.compute()
return metrics
def _init_engines(self) -> Tuple[Engine, Engine]:
self.train_metrics = {
'total_loss':
metrics.RunningAverage(output_transform=lambda x: x['loss']),
'segment_loss':
metrics.RunningAverage(
output_transform=lambda x: x['segment_loss']),
'kl_div':
metrics.RunningAverage(output_transform=lambda x: x['kl_div'])
}
self.val_metrics = {
'vae_metrics':
VAEMetrics(loss_fn=self.criterion,
mse_factor=0,
kld_factor=self.starting_kld_factor),
'segment_metrics':
SegmentationMetrics(num_classes=self.data_loaders.num_classes,
threshold=self.config.binarize_threshold),
}
trainer = self._init_trainer_engine()
evaluator = self._init_evaluator_engine()
return trainer, evaluator
def _init_train_components_ensemble(self, reinitialise=False):
self.val_metrics = {
'loss': metrics.Loss(BCEAndJaccardLoss(eval_ensemble=True, gpu_node=self.config.gpu_node)),
'segment_metrics': SegmentationMetrics(num_classes=self.data_loaders.num_classes,
threshold=self.config.binarize_threshold,
eval_ensemble=True)
}
self.model_cfg.network_params.input_channels = self.data_loaders.input_channels
self.model_cfg.network_params.num_classes = self.data_loaders.num_classes
self.model_cfg.network_params.image_size = self.data_loaders.image_size
self.criterion = self._init_criterion(not reinitialise)
self.ens_models = list()
self.ens_optimizers = list()
self.ens_lr_schedulers = list()
optimizer_cls = get_optimizer(self.optim_cfg)
init_param_names = retrieve_class_init_parameters(optimizer_cls)
optimizer_params = {k: v for k, v in self.optim_cfg.items() if k in init_param_names}
for _ in range(self.len_models):
self.ens_models.append(get_model(self.model_cfg).to(device=self.device))
self.ens_optimizers.append(optimizer_cls(self.ens_models[-1].parameters(), **optimizer_params))
lr_scheduler = self._init_lr_scheduler(self.ens_optimizers[-1])
self.ens_lr_schedulers.append(lr_scheduler)
if not reinitialise:
self.main_logger.info(f'Using ensemble of {self.len_models} {self.ens_models[0]}')
self.main_logger.info(f'Using optimizers {self.ens_optimizers[0].__class__.__name__}')
self.trainer, self.evaluator = self._init_engines()
self._init_handlers()
def mds_evaluate_monte_carlo(config: ConfigClass, model, name):
device = torch.device(
f'cuda:{config.gpu_node}' if torch.cuda.is_available() else 'cpu')
# Create dataloader wrapper
loader_wrapper = MDSDataLoaders(config.data)
segment_metrics = SegmentationMetrics(
num_classes=loader_wrapper.num_classes,
threshold=config.binarize_threshold)
# Evaluate
for dir_id in loader_wrapper.dir_list:
data_loader = loader_wrapper.get_evaluation_loader(dir_id)
model.eval()
model.apply(apply_dropout)
segmentation = torch.zeros(data_loader.dataset.shape)
ground_truth = torch.zeros_like(segmentation)
idx = 0
with torch.no_grad():
for batch in data_loader:
x, y = batch
x = x.to(device=device, non_blocking=True)
for _ in range(config.prediction.mc_passes):
y_pred = model(x)
segmentation[idx:idx +
config.data.batch_size_val] += torch.sigmoid(
y_pred).cpu()
ground_truth[idx:idx + config.data.batch_size_val] = y
idx += config.data.batch_size_val
segmentation = segmentation / config.prediction.mc_passes
segment_metrics.update((segmentation, ground_truth), process=False)
segmentation = segmentation.numpy()
save_segmentation_to_file(segmentation, config.binarize_threshold,
loader_wrapper.predict_path, dir_id, name)
metrics = segment_metrics.compute()
return metrics
def mds_evaluate_one_pass(config: ConfigClass, model, name):
device = torch.device(
f'cuda:{config.gpu_node}' if torch.cuda.is_available() else 'cpu')
# Create dataloader wrapper
loader_wrapper = MDSDataLoaders(config.data)
segment_metrics = SegmentationMetrics(
num_classes=loader_wrapper.num_classes,
threshold=config.binarize_threshold)
# Evaluate
for dir_id in loader_wrapper.dir_list:
data_loader = loader_wrapper.get_evaluation_loader(dir_id)
model.eval()
segmentation = np.zeros(data_loader.dataset.shape)
idx = 0
with torch.no_grad():
for batch in data_loader:
x, y = batch
x = x.to(device=device, non_blocking=True)
y = y.to(device=device, non_blocking=True)
y_pred = model(x)
segment_metrics.update((y_pred, y))
segmentation[idx:idx +
config.data.batch_size_val] = torch.sigmoid(
y_pred.cpu()).numpy()
idx += config.data.batch_size_val
save_segmentation_to_file(segmentation, config.binarize_threshold,
loader_wrapper.predict_path, dir_id, name)
metrics = segment_metrics.compute()
return metrics
def evaluate_one_pass(config: ConfigClass, model):
device = torch.device(
f'cuda:{config.gpu_node}' if torch.cuda.is_available() else 'cpu')
# Create dataloader class
loader_class = get_dataloaders(config.data)
segment_metrics = SegmentationMetrics(num_classes=loader_class.num_classes,
threshold=config.binarize_threshold)
# Evaluate model
model.eval()
with torch.no_grad():
for batch in loader_class.evaluation_loader:
x, y = batch
x = x.to(device=device, non_blocking=True)
y = y.to(device=device, non_blocking=True)
y_pred = model(x)
segment_metrics.update((y_pred, y))
metrics = segment_metrics.compute()
return metrics
def _init_train_components(self, reinitialise=False):
self.val_metrics = {
'loss': metrics.Loss(get_loss_function(self.loss_cfg)),
'segment_metrics': SegmentationMetrics(num_classes=self.data_loaders.num_classes,
threshold=self.config.binarize_threshold)
}
self.model_cfg.network_params.input_channels = self.data_loaders.input_channels
self.model_cfg.network_params.num_classes = self.data_loaders.num_classes
self.model_cfg.network_params.image_size = self.data_loaders.image_size
self.model = self._init_model(not reinitialise)
self.criterion = self._init_criterion(not reinitialise)
self.optimizer = self._init_optimizer(not reinitialise)
self.lr_scheduler = self._init_lr_scheduler(self.optimizer)
self.trainer, self.evaluator = self._init_engines()
self._init_handlers()
def _init_train_components(self):
self.metrics = {
'loss':
metrics.Loss(
GaussianVariationalInference(
get_loss_function(self.train_cfg.loss_fn))),
'segment_metrics':
SegmentationMetrics(num_classes=self.data_loaders.num_classes,
threshold=self.config.binarize_threshold)
}
self.model_cfg.network_params.input_channels = self.data_loaders.input_channels
self.model_cfg.network_params.num_classes = self.data_loaders.num_classes
self.model = self._init_model()
self.optimizer = self._init_optimizer()
self.vi = GaussianVariationalInference(self._init_criterion())
self.lr_scheduler = self._init_lr_scheduler(self.optimizer)
self.trainer, self.evaluator = self._init_engines()
self._init_handlers()