def evaluate(self):
model = self.model
# TODO: compute predictions in this function (similar to train, test...)
# this allows use "evaluations" in addition to "total_evaluations"
# keep track inside a metrics_meter (so tp, fp, ... does not need to be computed in the eval function)
for dataset_name in self.config.datasets:
dataset_config = AttrDefault(lambda: None,
self.config.datasets[dataset_name])
if dataset_config.evaluating:
print("evaluate on ", dataset_name)
# TODO: do not allow "evaluations" because this is not called after every batch
data_loader = self.data_loaders[dataset_name]
eval_metrics = {}
for ef in dataset_config._mapping.get("total_evaluations", []):
ev_func = get_total_evaluation(ef["name"])
eval_metrics = {
**eval_metrics,
**ev_func(None,
model=model,
data_loader=data_loader,
config=self.config,
current_dataset_config=dataset_config,
eval_args=ef.get("eval_args", {}))
}
# logger.info('total metrics: {}'.format(str(eval_metrics)))
shared_globals.console.info("evaluation " + dataset_name +
":\n" + str(eval_metrics))
def test(self,
epoch,
dataset_name,
dataset_config,
model=None,
extra_name=""):
logger.info('Testing on ({}) epoch {}:'.format(
dataset_name + extra_name, epoch))
if model is None:
model = self.model
scheduler = self.scheduler
optimizer = self.optimizer
optim_config = self.config.optim_config
model_config = self.config.model_config
if self.config.tensorboard:
writer = self.writer
# training mode
model.eval()
loss_meter = AverageMeter()
correct_meter = AverageMeter()
accuracy_meter = AverageMeter()
metrics_meter = DictAverageMeter()
start = time.time()
test_loader = self.data_loaders[dataset_name]
dataset_name = dataset_name + extra_name
for step, (data, _, targets) in enumerate(test_loader):
if self.config.tensorboard_test_images:
if epoch == 0 and step == 0:
image = torchvision.utils.make_grid(data,
normalize=True,
scale_each=True)
writer.add_image(dataset_name + '/Image', image, epoch)
if self.config.use_gpu:
data = data.cuda()
targets = targets.cuda()
with torch.no_grad():
outputs = model(data)
if model_config.binary_classifier:
targets = targets.float(
) # https://discuss.pytorch.org/t/data-type-mismatch-in-loss-function/34860
loss = self.criterion(outputs, targets)
# if data_config['use_mixup']:
# _, targets = targets.max(dim=1)
if model_config['multi_label']:
preds = (outputs >
model_config['prediction_threshold']).float()
elif model_config.binary_classifier:
if model_config.sigmoid_output:
preds = outputs > 0.5
else:
preds = outputs > 0.
elif model_config.regression:
preds = outputs
else:
_, preds = torch.max(outputs, dim=1)
loss_ = loss.item()
if model_config.binary_classifier:
targets_binary = targets > 0.5 # accounting for smoothed labels
correct_ = preds.eq(targets_binary).sum().item()
elif model_config.regression:
# in regression accuracy is L1 loss
correct_ = torch.abs(preds - targets).sum().item()
else:
correct_ = preds.eq(targets).sum().item()
if model_config['multi_label']:
num = data.size(0) * model_config['n_classes']
else:
num = data.size(0)
if model_config['multi_label']:
total_num = len(
test_loader.dataset) * model_config['n_classes']
else:
total_num = len(test_loader.dataset)
eval_metrics = {}
for ef in dataset_config._mapping.get("evaluations", []):
ev_func = get_evaluation(ef["name"])
if epoch % ef.get("frequency", 1) == 0:
eval_metrics = {
**eval_metrics,
**ev_func(outputs,
targets,
eval_args=ef.get("eval_args", {}))
}
metrics_meter.update(eval_metrics, num)
loss_meter.update(loss_, num)
correct_meter.update(correct_, 1)
accuracy = correct_meter.sum / total_num
accuracy_meter.update(accuracy, num)
if step % ((len(test_loader) + 10) // 10) == 0:
print('\x1b[2K',
'Test[{}]{}: Step {}/{} '
'Loss {:.4f} ({:.4f}) '
'Accuracy {:.4f} ({:.4f})'.format(
epoch, dataset_name, step + 1, len(test_loader),
loss_meter.val, loss_meter.avg, accuracy_meter.val,
accuracy_meter.avg),
end="\r")
print('\x1b[2K',
'Test[{}]{}:Step {}/{} '
'Loss {:.4f} ({:.4f}) '
'Accuracy {:.4f} ({:.4f})'.format(epoch, dataset_name, step + 1,
len(test_loader),
loss_meter.val, loss_meter.avg,
accuracy_meter.val,
accuracy_meter.avg),
end="\r")
elapsed = time.time() - start
logger.info('Elapsed {:.2f}'.format(elapsed))
logger.info('avg metrics: {}'.format(str(metrics_meter.avg)))
eval_metrics = {"loss": loss_meter.avg, "accuracy": accuracy}
for ef in dataset_config._mapping.get("total_evaluations", []):
ev_func = get_total_evaluation(ef["name"])
eval_metrics = {
**eval_metrics,
**ev_func(metrics_meter,
model=model,
data_loader=test_loader,
config=self.config,
current_dataset_config=dataset_config,
eval_args=ef.get("eval_args", {}))
}
logger.info('total metrics: {}'.format(str(eval_metrics)))
#self.run.info.setdefault("last_metrics", {})[dataset_name] = eval_metrics
# for k, v in eval_metrics.items():
# self.run.log_scalar(dataset_name + "." + k, v, epoch)
if self.config.tensorboard:
writer.add_scalar(dataset_name + '/Loss', loss_meter.avg, epoch)
writer.add_scalar(dataset_name + '/Accuracy', accuracy, epoch)
writer.add_scalar(dataset_name + '/Time', elapsed, epoch)
writer.add_scalars(dataset_name + "/AvgMetrics", metrics_meter.avg,
epoch)
writer.add_scalars(dataset_name + "/TotalMetrics", eval_metrics,
epoch)
return eval_metrics
def train(self, epoch, dataset_name, dataset_config, model=None):
logger.info('Train ({}) epoch {}:'.format(dataset_name, epoch))
if model is None:
model = self.model
scheduler = self.scheduler
optimizer = self.optimizer
optim_config = self.config.optim_config
model_config = self.config.model_config
if self.config.tensorboard:
writer = self.writer
# training mode
model.train()
loss_meter = AverageMeter()
accuracy_meter = AverageMeter()
metrics_meter = DictAverageMeter()
start = time.time()
train_loader = self.data_loaders[dataset_name]
start_loading_time = time.time()
total_loading_time = 0
if optim_config['scheduler'] == 'multistep':
scheduler.step(epoch + 1)
elif optim_config['scheduler'] == 'mycos':
scheduler.step(epoch + 1)
elif optim_config['scheduler'] == 'swa':
scheduler.step(epoch + 1)
elif optim_config['scheduler'] == 'linear':
scheduler.step(epoch)
elif optim_config['scheduler'] == 'drop':
scheduler.step(epoch)
number_of_steps = len(train_loader)
if self.config.maximum_steps_per_epoch and self.config.maximum_steps_per_epoch < number_of_steps:
number_of_steps = self.config.maximum_steps_per_epoch
for step, (data, _, targets) in enumerate(train_loader):
shared_globals.global_step += 1
if optim_config['scheduler'] == 'cosine':
scheduler.step()
if self.config.use_gpu:
data = data.cuda()
targets = targets.cuda()
if self.config.use_mixup and epoch >= int(
self.config.use_mixup) - 1:
# don't forget to use mix up loss
rn_indices, lam = my_mixup(data, targets,
self.config.mixup_alpha,
self.config.get("mixup_mode"))
if self.config.use_gpu:
rn_indices = rn_indices.cuda()
lam = lam.cuda()
data = data * lam.reshape(lam.size(0), 1, 1, 1) \
+ data[rn_indices] * (1 - lam).reshape(lam.size(0), 1, 1, 1)
# data is loaded
total_loading_time += time.time() - start_loading_time
# Model graph to tensor board
if not self.first_batch_done:
self.first_batch_done = True
if self.config.tensorboard and not self.config.tensorboard_no_model_graph:
shared_globals.console.info(
"writing model graph to tensorboard!")
self.writer.add_graph(self.bare_model, data[0:1])
optimizer.zero_grad()
outputs = model(data)
if self.config.use_mixup and epoch >= int(
self.config.use_mixup) - 1:
loss = self.criterion(outputs, targets, targets[rn_indices],
lam, self.config.get("mixup_mode"))
else:
# print("targets", targets)
if model_config.binary_classifier:
targets = targets.float(
) # https://discuss.pytorch.org/t/data-type-mismatch-in-loss-function/34860
# print("targets.float()", targets)
loss = self.criterion(outputs, targets)
loss.backward()
optimizer.step()
if model_config['multi_label']:
preds = (outputs >
model_config['prediction_threshold']).float()
elif model_config.binary_classifier:
if model_config.sigmoid_output:
preds = outputs > 0.5
else:
preds = outputs > 0.
elif model_config.regression:
preds = outputs
else:
_, preds = torch.max(outputs, dim=1)
loss_ = loss.item()
# if data_config['use_mixup']:
# _, targets = targets.max(dim=1)
if model_config.binary_classifier:
targets_binary = targets > 0.5 # this is to account for smoothed labels
# smoothed labels like in [Schlüter 2015] who used [0.02, 0.98] instead of [0, 1]
correct_ = preds.eq(targets_binary).sum().item()
elif model_config.regression:
# in regression accuracy is L1 loss
correct_ = torch.abs(preds - targets).sum().item()
else:
correct_ = preds.eq(targets).sum().item()
if model_config['multi_label']:
num = data.size(0) * model_config['n_classes']
else:
num = data.size(0)
accuracy = correct_ / num
eval_metrics = {}
for ef in dataset_config._mapping.get("evaluations", []):
ev_func = get_evaluation(ef["name"])
if epoch % ef.get("frequency", 1) == 0:
eval_metrics = {
**eval_metrics,
**ev_func(outputs,
targets,
eval_args=ef.get("eval_args", {}))
}
metrics_meter.update(eval_metrics, num)
loss_meter.update(loss_, num)
accuracy_meter.update(accuracy, num)
if self.config.tensorboard:
writer.add_scalar(dataset_name + '/RunningLoss', loss_,
shared_globals.global_step)
writer.add_scalar(dataset_name + '/RunningAccuracy', accuracy,
shared_globals.global_step)
writer.add_scalars(dataset_name + "/RunningMetrics",
eval_metrics, shared_globals.global_step)
if step % (number_of_steps // 10) == 0:
print('\x1b[2K ' + 'Epoch {} Step {}/{} '
'Loss {:.4f} ({:.4f}) '
'Accuracy {:.4f} ({:.4f}) '.format(
epoch, step + 1, number_of_steps, loss_meter.val,
loss_meter.avg, accuracy_meter.val,
accuracy_meter.avg),
end="\r")
if step % 100 == 0:
logger.info('Epoch {} Step {}/{} '
'Loss {:.4f} ({:.4f}) '
'Accuracy {:.4f} ({:.4f})'.format(
epoch,
step,
number_of_steps,
loss_meter.val,
loss_meter.avg,
accuracy_meter.val,
accuracy_meter.avg,
))
# to get the data loading time
start_loading_time = time.time()
if self.config.maximum_steps_per_epoch and step + 1 == self.config.maximum_steps_per_epoch:
break
elapsed = time.time() - start
logger.info('Elapsed {:.2f} (loading: {:.2f} )'.format(
elapsed, total_loading_time))
logger.info('avg metrics: {}'.format(str(metrics_meter.avg)))
print(
'\x1b[2K' + 'Train[{}]{}:Step {}/{} '
'Loss {:.4f} ({:.4f}) '
'Accuracy {:.4f} ({:.4f})'.format(
epoch, dataset_name, step + 1, number_of_steps, loss_meter.val,
loss_meter.avg, accuracy_meter.val, accuracy_meter.avg),
end="\r")
eval_metrics = {"loss": loss_meter.avg, "accuracy": accuracy}
for ef in dataset_config._mapping.get("total_evaluations", []):
ev_func = get_total_evaluation(ef["name"])
eval_metrics = {
**eval_metrics,
**ev_func(metrics_meter,
model=model,
data_loader=train_loader,
config=self.config,
current_dataset_config=dataset_config,
eval_args=ef.get("eval_args", {}))
}
logger.info('total metrics: {}'.format(str(eval_metrics)))
# logging metrics resutls
#self.run.info.setdefault("last_metrics", {})[dataset_name] = eval_metrics
# for k, v in eval_metrics.items():
# self.log_scalar(dataset_name + "." + k, v, epoch)
if self.config.tensorboard:
writer.add_scalar(dataset_name + '/Loss', loss_meter.avg, epoch)
writer.add_scalar(dataset_name + '/Accuracy', accuracy_meter.avg,
epoch)
writer.add_scalar(dataset_name + '/Time', elapsed, epoch)
writer.add_scalars(dataset_name + "/AvgMetrics", metrics_meter.avg,
epoch)
writer.add_scalars(dataset_name + "/TotalMetrics", eval_metrics,
epoch)
if optim_config.get(
'scheduler') and optim_config['scheduler'] != 'none':
lr = scheduler.get_lr()[0]
else:
lr = optim_config['base_lr']
writer.add_scalar(dataset_name + '/LearningRate', lr, epoch)
