def eval_model(model, data_loader, loss_fn, device):
"""
Evaluate model on data and return dictionary with the average metric values
:param nn.Module model:
:param DataLoader data_loader:
:param str device:
:param loss_fn: loss function
:param str device:
:return:
"""
model.eval()
mean_metrics = {
'precision': 0.0,
'recall': 0.0,
'accuracy': 0.0,
'f1': 0.0,
'loss': 0.0
}
num_batches = len(data_loader)
with torch.no_grad():
for batch in data_loader:
conf_matrix, batch_loss = epoch_pass(batch, model, loss_fn, device)
tp, tn, fp, fn = conf_matrix
batch_metrics = classification_metrics(tp.item(), tn.item(),
fp.item(), fn.item())
batch_metrics['loss'] = batch_loss.item()
for metric, metric_value in batch_metrics.items():
mean_metrics[metric] += metric_value
for metric, metric_value in mean_metrics.items():
mean_metrics[metric] /= num_batches
return mean_metrics
def val_one_epoch(sess, ops, dataset, epoch):
"""
One epoch validating
"""
is_training = False
sess.run(ops['val_init_op'])
feed_dict = {ops['is_training_pl']: is_training}
preds = []
targets = []
obj_inds = []
idx = 0
while True:
try:
tower_logits, tower_labels, tower_o_inds = sess.run(
[
ops['tower_logits'],
ops['tower_labels'],
ops['tower_object_inds'],
],
feed_dict=feed_dict)
for logits, labels, inds in zip(tower_logits, tower_labels,
tower_o_inds):
preds += [logits]
targets += [labels]
obj_inds += [inds]
idx += 1
except tf.errors.OutOfRangeError:
break
# Stack all validation predictions
preds = np.vstack(preds)
targets = np.hstack(targets)
obj_inds = np.hstack(obj_inds)
true_num_test = preds.shape[0]
val_preds = np.zeros(
(len(dataset.input_labels['validation']), dataset.num_classes))
val_targets = np.zeros((len(dataset.input_labels['validation'])))
val_preds[obj_inds] = preds
val_targets[obj_inds] = targets
if true_num_test != dataset.num_test:
logger.warning("Validating using {}/{} data, "
"this may be caused by multi-gpu testing".format(
true_num_test, dataset.num_test))
val_preds = val_preds[:true_num_test]
val_targets = val_targets[:true_num_test]
acc, avg_class_acc = classification_metrics(val_preds, val_targets,
dataset.num_classes)
logger.info(f'E{epoch} * Acc {acc:.3%} CAcc {avg_class_acc:.3%}')
wandb.log({"Val Acc": acc, "Val Avg Class Loss": avg_class_acc})
return acc, avg_class_acc
def train_epoch(model, data_loader, device, optimizer, loss_fn, lr_scheduler):
"""
Train model for 1 epoch and return dictionary with the average training metric values
:param nn.Module model:
:param DataLoader data_loader:
:param str device:
:param optimizer:
:param loss_fn: loss function
:param lr_scheduler: schedule of linear decrease in learning rate
:return:
"""
model.train(mode=True)
mean_metrics = {
'precision': 0.0,
'recall': 0.0,
'accuracy': 0.0,
'f1': 0.0,
'loss': 0.0
}
num_batches = len(data_loader)
for batch in data_loader:
optimizer.zero_grad()
conf_matrix, batch_loss = epoch_pass(batch, model, loss_fn, device)
batch_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
optimizer.step()
lr_scheduler.step()
tp, tn, fp, fn = conf_matrix
batch_metrics = classification_metrics(tp.item(), tn.item(), fp.item(),
fn.item())
batch_metrics['loss'] = batch_loss.item()
for metric, metric_value in batch_metrics.items():
mean_metrics[metric] += metric_value
for metric, metric_value in mean_metrics.items():
mean_metrics[metric] /= num_batches
return mean_metrics
def val_vote_one_epoch(sess, ops, dataset, epoch, num_votes=10):
"""
One epoch voting validating
"""
is_training = False
average_preds = np.zeros((dataset.num_test, dataset.num_classes))
original_labels = dataset.input_labels['validation']
feed_dict = {ops['is_training_pl']: is_training}
for v in range(num_votes):
if v == 0:
sess.run(ops['val_init_op'])
else:
sess.run(ops['val_vote_init_op'])
preds = []
targets = []
obj_inds = []
while True:
try:
tower_logits, tower_labels, tower_o_inds = sess.run(
[
ops['tower_logits'], ops['tower_labels'],
ops['tower_object_inds']
],
feed_dict=feed_dict)
for logits, labels, inds in zip(tower_logits, tower_labels,
tower_o_inds):
preds += [logits]
targets += [labels]
obj_inds += [inds]
except tf.errors.OutOfRangeError:
break
# Stack all validation predictions
preds = np.vstack(preds)
targets = np.hstack(targets)
obj_inds = np.hstack(obj_inds)
true_num_test = preds.shape[0]
average_preds[obj_inds] = average_preds[obj_inds] + (
preds - average_preds[obj_inds]) / (v + 1)
if true_num_test != dataset.num_test:
logger.warning("Validating using {}/{} data, "
"this may be caused by multi-gpu testing".format(
true_num_test, dataset.num_test))
vote_preds = average_preds[:true_num_test]
vote_targets = original_labels[:true_num_test]
else:
vote_preds = average_preds
vote_targets = original_labels
if np.any(vote_targets[obj_inds] != targets):
raise ValueError('wrong object indices')
acc, avg_class_acc = classification_metrics(vote_preds, vote_targets,
dataset.num_classes)
logger.info(f'E{epoch} V{v} * Acc {acc:.3%} CAcc {avg_class_acc:.3%}')
return acc, avg_class_acc