def train(train_loader, model, criterion, optimizer, epoch, logger, args=None):
batch_time = logutil.AverageMeter()
data_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
subactivity_error_ratio = logutil.AverageMeter()
# switch to train mode
model.train()
end_time = time.time()
for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths,
activities, sequence_ids) in enumerate(train_loader):
data_time.update(time.time() - end_time)
optimizer.zero_grad()
features = utils.to_variable(features, args.cuda)
labels = utils.to_variable(labels, args.cuda)
probs = utils.to_variable(probs, args.cuda)
total_lengths = torch.autograd.Variable(total_lengths)
ctc_labels = torch.autograd.Variable(
torch.IntTensor(
[item for sublist in ctc_labels for item in sublist]))
ctc_lengths = torch.autograd.Variable(ctc_lengths)
model_outputs = model(features)
_, pred_labels = torch.max(model_outputs, dim=2)
train_loss = criterion(model_outputs, labels, probs, ctc_labels,
total_lengths, ctc_lengths)
# Log
losses.update(train_loss.data[0], torch.sum(total_lengths).data[0])
subact_micro_result = sklearn.metrics.precision_recall_fscore_support(
labels.cpu().data.numpy().flatten().tolist(),
pred_labels.cpu().data.numpy().flatten().tolist(),
labels=range(10),
average='micro')
subactivity_error_ratio.update(1.0 - subact_micro_result[0],
torch.sum(total_lengths).data[0])
train_loss.backward()
optimizer.step()
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
if logger is not None:
logger.log_value('train_epoch_loss', losses.avg)
print(
'Epoch: [{0}] Avg Subactivity Error Ratio {act_err.avg:.3f}; Average Loss {losses.avg:.3f}; Batch Avg Time {b_time.avg:.3f}'
.format(epoch,
act_err=subactivity_error_ratio,
losses=losses,
b_time=batch_time))
def train(train_loader, model, mse_loss, multi_label_loss, optimizer, epoch, logger):
batch_time = logutil.AverageMeter()
data_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
y_true = np.empty((0, action_class_num))
y_score = np.empty((0, action_class_num))
# switch to train mode
model.train()
end_time = time.time()
for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, det_classes, det_boxes, human_num, obj_num) in enumerate(train_loader):
data_time.update(time.time() - end_time)
optimizer.zero_grad()
edge_features = utils.to_variable(edge_features, args.cuda)
node_features = utils.to_variable(node_features, args.cuda)
adj_mat = utils.to_variable(adj_mat, args.cuda)
node_labels = utils.to_variable(node_labels, args.cuda)
pred_adj_mat, pred_node_labels = model(edge_features, node_features, adj_mat, node_labels, human_num, obj_num, args)
det_indices, loss = loss_fn(pred_adj_mat, adj_mat, pred_node_labels, node_labels, mse_loss, multi_label_loss, human_num, obj_num)
# Log and back propagate
if len(det_indices) > 0:
y_true, y_score = evaluation(det_indices, pred_node_labels, node_labels, y_true, y_score)
losses.update(loss.data[0], edge_features.size()[0])
loss.backward()
optimizer.step()
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
if i % args.log_interval == 0:
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t'
'Detected HOIs {y_shape}'
.format(epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, mean_avg_prec=mean_avg_prec, y_shape=y_true.shape))
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
if logger is not None:
logger.log_value('train_epoch_loss', losses.avg)
logger.log_value('train_epoch_map', mean_avg_prec)
print('Epoch: [{0}] Avg Mean Precision {map:.4f}; Average Loss {loss.avg:.4f}; Avg Time x Batch {b_time.avg:.4f}'
.format(epoch, map=mean_avg_prec, loss=losses, b_time=batch_time))
def train(train_loader, model, criterion, optimizer, epoch, logger, args=None):
batch_time = logutil.AverageMeter()
data_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
subactivity_error_ratio = logutil.AverageMeter()
affordance_error_ratio = logutil.AverageMeter()
# switch to train mode
model.train()
end_time = time.time()
for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids,
node_nums) in enumerate(train_loader):
data_time.update(time.time() - end_time)
optimizer.zero_grad()
edge_features = utils.to_variable(edge_features, args.cuda)
node_features = utils.to_variable(node_features, args.cuda)
adj_mat = utils.to_variable(adj_mat, args.cuda)
node_labels = utils.to_variable(node_labels, args.cuda)
pred_adj_mat, pred_node_labels = model(edge_features, node_features,
adj_mat, node_labels, args)
train_loss = criterion(pred_node_labels, node_labels)
# Log
losses.update(train_loss.data[0], edge_features.size(0))
error_rate, total_nodes, predictions, ground_truth = evaluation(
pred_node_labels[:, [0], :], node_labels[:, [0], :])
subactivity_error_ratio.update(error_rate, total_nodes)
error_rate, total_nodes, predictions, ground_truth = evaluation(
pred_node_labels[:, 1:, :], node_labels[:, 1:, :])
affordance_error_ratio.update(error_rate, total_nodes)
train_loss.backward()
optimizer.step()
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
if logger is not None:
logger.log_value('train_epoch_loss', losses.avg)
logger.log_value('train_epoch_subactivity_error_ratio',
subactivity_error_ratio.avg)
logger.log_value('train_epoch_affordance_error_ratio',
affordance_error_ratio.avg)
print(
'Epoch: [{0}] Avg Subactivity Error Ratio {act_err.avg:.3f}; Avg Affordance Error Ratio {aff_err.avg:.3f}; Average Loss {loss.avg:.3f}; Batch Avg Time {b_time.avg:.3f}'
.format(epoch,
act_err=subactivity_error_ratio,
aff_err=affordance_error_ratio,
loss=losses,
b_time=batch_time))
def train(train_loader,
model,
criterion,
optimizer,
epoch,
evaluation,
logger,
args=None):
batch_time = logutil.AverageMeter()
data_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
error_ratio = logutil.AverageMeter()
# switch to train mode
model.train()
end_time = time.time()
for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids,
node_nums) in enumerate(train_loader):
data_time.update(time.time() - end_time)
optimizer.zero_grad()
target = utils.to_variable(adj_mat, args.cuda)
output = model(utils.to_variable(edge_features, args.cuda))
train_loss = criterion(output, target)
# Log
losses.update(train_loss.data[0], edge_features.size(0))
error_ratio.update(
evaluation(output, target).data[0], edge_features.size(0))
# compute gradient and do SGD step
train_loss.backward()
optimizer.step()
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
# if i % args.log_interval == 0 and i > 0:
# print('Epoch: [{0}][{1}/{2}]\t'
# 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
# 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
# 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
# 'Error Ratio {err.val:.4f} ({err.avg:.4f})'
# .format(epoch, i, len(train_loader), batch_time=batch_time,
# data_time=data_time, loss=losses, err=error_ratio))
if logger is not None:
logger.log_value('train_epoch_loss', losses.avg)
logger.log_value('train_epoch_error_ratio', error_ratio.avg)
print(
'Epoch: [{0}] Avg Error Ratio {err.avg:.3f}; Average Loss {loss.avg:.3f}; Batch Avg Time {b_time.avg:.3f}'
.format(epoch, err=error_ratio, loss=losses, b_time=batch_time))
def validate(val_loader,
model,
criterion,
evaluation,
logger=None,
args=None,
test=False):
batch_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
error_ratio = logutil.AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (edge_features, node_features, adj_mat, node_labels, node_roles,
boxes, img_ids, img_names, human_nums, obj_nums,
classes) in enumerate(val_loader):
edge_features = utils.to_variable(edge_features, args.cuda)
edge_features = edge_features.permute(0, 3, 1, 2)
target = utils.to_variable(adj_mat, args.cuda)
output = model(edge_features)
# Logs
losses.update(
criterion(output, target, human_nums, obj_nums).data[0],
edge_features.size(0))
error_ratio.update(
evaluation(output, target, human_nums, obj_nums,
test=test).data[0], edge_features.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_interval == 0 and i > 0:
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error Ratio {err.val:.4f} ({err.avg:.4f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
err=error_ratio))
print(' * Average Error Ratio {err.avg:.3f}; Average Loss {loss.avg:.3f}'.
format(err=error_ratio, loss=losses))
if logger is not None:
logger.log_value('test_epoch_loss', losses.avg)
logger.log_value('test_epoch_error_ratio', error_ratio.avg)
return losses.avg
def validate(val_loader, model, mse_loss, multi_label_loss, logger=None, test=False):
if args.visualize:
result_folder = os.path.join(args.tmp_root, 'results/HICO/detections/', 'top'+str(args.vis_top_k))
if not os.path.exists(result_folder):
os.makedirs(result_folder)
batch_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
y_true = np.empty((0, action_class_num))
y_score = np.empty((0, action_class_num))
# switch to evaluate mode
model.eval()
end = time.time()
for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, det_classes, det_boxes, human_num, obj_num) in enumerate(val_loader):
edge_features = utils.to_variable(edge_features, args.cuda)
node_features = utils.to_variable(node_features, args.cuda)
adj_mat = utils.to_variable(adj_mat, args.cuda)
node_labels = utils.to_variable(node_labels, args.cuda)
pred_adj_mat, pred_node_labels = model(edge_features, node_features, adj_mat, node_labels, human_num, obj_num, args)
det_indices, loss = loss_fn(pred_adj_mat, adj_mat, pred_node_labels, node_labels, mse_loss, multi_label_loss, human_num, obj_num)
# Log
if len(det_indices) > 0:
losses.update(loss.data[0], len(det_indices))
y_true, y_score = evaluation(det_indices, pred_node_labels, node_labels, y_true, y_score, test=test)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_interval == 0 and i > 0:
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t'
'Detected HOIs {y_shape}'
.format(i, len(val_loader), batch_time=batch_time,
loss=losses, mean_avg_prec=mean_avg_prec, y_shape=y_true.shape))
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
print(' * Average Mean Precision {mean_avg_prec:.4f}; Average Loss {loss.avg:.4f}'
.format(mean_avg_prec=mean_avg_prec, loss=losses))
if logger is not None:
logger.log_value('test_epoch_loss', losses.avg)
logger.log_value('train_epoch_map', mean_avg_prec)
return 1.0 - mean_avg_prec
def validate(val_loader, model, args, test=False):
def compute_accuracy(gt_results, results, metric='micro'):
return sklearn.metrics.precision_recall_fscore_support(
gt_results, results, labels=range(10), average=metric)
batch_time = logutil.AverageMeter()
subactivity_acc_ratio = logutil.AverageMeter()
all_gt_seg_predictions = list()
all_seg_predictions = list()
# switch to evaluate mode
model.eval()
end_time = time.time()
for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths,
activities, sequence_ids) in enumerate(val_loader):
features = utils.to_variable(features, args.cuda)
labels = utils.to_variable(labels, args.cuda)
total_lengths = torch.autograd.Variable(total_lengths)
for batch_i in range(features.size()[1]):
for frame in range(pred_duration - 1, total_lengths[batch_i], 10):
model_outputs = model(features[:frame + 1])
_, pred_labels = torch.max(model_outputs, dim=2)
gt_pred_labels = labels[
frame - pred_duration + 1:frame + 1,
batch_i].cpu().data.numpy().flatten().tolist()
pred_labels = pred_labels[-pred_duration:].cpu().data.numpy(
).flatten().tolist()
subact_micro_result = compute_accuracy(gt_pred_labels,
pred_labels)
subactivity_acc_ratio.update(subact_micro_result[0],
pred_duration)
all_gt_seg_predictions.extend(gt_pred_labels)
all_seg_predictions.extend(pred_labels)
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
print(
' * Avg Subactivity Accuracy Ratio {act_err.avg:.3f}; Batch Avg Time {b_time.avg:.3f}'
.format(act_err=subactivity_acc_ratio, b_time=batch_time))
print(
compute_accuracy(all_gt_seg_predictions,
all_seg_predictions,
metric='macro'))
return 1.0 - subactivity_acc_ratio.avg
def visualize(args,
val_loader,
model,
mse_loss,
multi_label_loss,
vcocoeval,
logger=None,
test=False):
if args.visualize:
result_folder = os.path.join(args.tmp_root,
'results/VCOCO/detections/',
'top' + str(args.vis_top_k))
if not os.path.exists(result_folder):
os.makedirs(result_folder)
batch_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
y_true = np.empty((0, action_class_num))
y_score = np.empty((0, action_class_num))
all_results = list()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (edge_features, node_features, part_human_id, adj_mat, node_labels,
node_roles, obj_boxes, part_boxes, human_boxes, img_id, img_name,
human_num, part_num, obj_num, obj_classes, part_classes,
part_adj_mat, img_names) in enumerate(val_loader):
edge_features = utils.to_variable(edge_features, args.cuda)
node_features = utils.to_variable(node_features, args.cuda)
adj_mat = utils.to_variable(adj_mat, args.cuda)
part_adj_mat = utils.to_variable(part_adj_mat, args.cuda)
node_labels = utils.to_variable(node_labels, args.cuda)
node_roles = utils.to_variable(node_roles, args.cuda)
pred_adj_mat, pred_node_labels, pred_node_roles = model(
edge_features, node_features, part_human_id, adj_mat, node_labels,
node_roles, human_num, part_num, obj_num, part_classes, args)
for j in range(len(img_names)):
pickle.dump(
{
'adj_mat': pred_adj_mat[j],
'node_labels': pred_node_labels[j],
'node_roles': pred_node_roles[j]
},
open(os.path.join(args.eval_root, img_names[j] + '.pred'),
'wb'))
if i % 10 == 0:
print('\r%d/%d' % (i, len(val_loader)), end='')
def validate(val_loader, model, criterion, evaluation, logger=None, args=None):
batch_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
error_ratio = logutil.AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids,
node_nums) in enumerate(val_loader):
target = utils.to_variable(adj_mat, args.cuda)
output = model(utils.to_variable(edge_features, args.cuda))
# Logs
losses.update(criterion(output, target).data[0], edge_features.size(0))
error_ratio.update(
evaluation(output, target).data[0], edge_features.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# if i % args.log_interval == 0 and i > 0:
# print('Test: [{0}/{1}]\t'
# 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
# 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
# 'Error Ratio {err.val:.4f} ({err.avg:.4f})'
# .format(i, len(val_loader), batch_time=batch_time,
# loss=losses, err=error_ratio))
print(' * Average Error Ratio {err.avg:.3f}; Average Loss {loss.avg:.3f}'.
format(err=error_ratio, loss=losses))
if logger is not None:
logger.log_value('test_epoch_loss', losses.avg)
logger.log_value('test_epoch_error_ratio', error_ratio.avg)
return error_ratio.avg
def validate_baseline(val_loader, model, args, test=False):
def compute_accuracy(gt_results, results, metric='micro'):
return sklearn.metrics.precision_recall_fscore_support(
gt_results, results, labels=range(10), average=metric)
baseline_acc_ratio = logutil.AverageMeter()
all_baseline_detections = list()
all_gt_detections = list()
# switch to evaluate mode
model.eval()
end_time = time.time()
for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths,
activities, sequence_ids) in enumerate(val_loader):
features = utils.to_variable(features, args.cuda)
labels = utils.to_variable(labels, args.cuda)
total_lengths = torch.autograd.Variable(total_lengths)
# Inference
model_outputs = model(features)
_, pred_labels = torch.max(model_outputs, dim=2)
baseline_detections = pred_labels.cpu().data.numpy().flatten().tolist()
gt_detections = labels.cpu().data.numpy().flatten().tolist()
all_baseline_detections.extend(baseline_detections)
all_gt_detections.extend(gt_detections)
baseline_micro_result = compute_accuracy(gt_detections,
baseline_detections)
baseline_acc_ratio.update(baseline_micro_result[0],
torch.sum(total_lengths).data[0])
for batch_i in range(model_outputs.size()[1]):
model_probs = torch.nn.Softmax(dim=1)(
model_outputs[:int(total_lengths[batch_i]),
batch_i, :].squeeze()).cpu().data.numpy()
np.save(
os.path.join(args.tmp_root, 'nn_output', activities[batch_i],
sequence_ids[batch_i]), model_probs)
print(' * Baseline Accuracy Ratio {base_acc.avg:.3f}; '.format(
base_acc=baseline_acc_ratio))
return 1.0 - baseline_acc_ratio.avg
def validate(val_loader, model, criterion, logger=None, args=None, test=False):
if args.visualize:
result_folder = os.path.join(args.tmp_root,
'results/CAD/figures/prediction/')
if not os.path.exists(result_folder):
os.makedirs(result_folder)
batch_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
error_ratio = logutil.AverageMeter()
subactivity_error_ratio = logutil.AverageMeter()
affordance_error_ratio = logutil.AverageMeter()
subact_predictions = list()
subact_ground_truth = list()
affordance_predictions = list()
affordance_ground_truth = list()
all_sequence_ids = list()
all_node_nums = list()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids,
node_nums) in enumerate(val_loader):
edge_features = utils.to_variable(edge_features, args.cuda)
node_features = utils.to_variable(node_features, args.cuda)
adj_mat = utils.to_variable(adj_mat, args.cuda)
node_labels = utils.to_variable(node_labels, args.cuda)
pred_adj_mat, pred_node_labels = model(edge_features, node_features,
adj_mat, node_labels, args)
# Logs
losses.update(
criterion(pred_node_labels, node_labels).data[0],
edge_features.size(0))
error_rate, total_nodes, predictions, ground_truth = evaluation(
pred_node_labels, node_labels)
error_ratio.update(error_rate, total_nodes)
error_rate, total_nodes, predictions, ground_truth = evaluation(
pred_node_labels[:, [0], :], node_labels[:, [0], :])
subactivity_error_ratio.update(error_rate, total_nodes)
subact_predictions.extend(predictions)
subact_ground_truth.extend(ground_truth)
error_rate, total_nodes, predictions, ground_truth = evaluation(
pred_node_labels[:, 1:, :], node_labels[:, 1:, :])
affordance_error_ratio.update(error_rate, total_nodes)
affordance_predictions.extend(predictions)
affordance_ground_truth.extend(ground_truth)
all_sequence_ids.extend(sequence_ids)
all_node_nums.extend(node_nums)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.visualize:
utils.plot_all_activity_segmentations(all_sequence_ids,
subact_predictions,
subact_ground_truth,
result_folder)
utils.plot_all_affordance_segmentations(all_sequence_ids,
all_node_nums,
affordance_predictions,
affordance_ground_truth,
result_folder)
# Plot confusion matrices
confusion_matrix = sklearn.metrics.confusion_matrix(
subact_ground_truth,
subact_predictions,
labels=range(len(datasets.cad_metadata.subactivities)))
utils.plot_confusion_matrix(confusion_matrix,
datasets.cad_metadata.subactivities,
normalize=True,
title='',
filename=os.path.join(
result_folder,
'confusion_subactivity.pdf'))
confusion_matrix = sklearn.metrics.confusion_matrix(
affordance_ground_truth,
affordance_predictions,
labels=range(len(datasets.cad_metadata.affordances)))
utils.plot_confusion_matrix(confusion_matrix,
datasets.cad_metadata.affordances,
normalize=True,
title='',
filename=os.path.join(
result_folder,
'confusion_affordance.pdf'))
subact_micro_result = sklearn.metrics.precision_recall_fscore_support(
subact_ground_truth,
subact_predictions,
labels=range(10),
average='micro')
subact_macro_result = sklearn.metrics.precision_recall_fscore_support(
subact_ground_truth,
subact_predictions,
labels=range(10),
average='macro')
aff_micro_result = sklearn.metrics.precision_recall_fscore_support(
affordance_ground_truth,
affordance_predictions,
labels=range(12),
average='micro')
aff_macro_result = sklearn.metrics.precision_recall_fscore_support(
affordance_ground_truth,
affordance_predictions,
labels=range(12),
average='macro')
if test:
print('Subactivity prediction micro evaluation:', subact_micro_result)
print('Subactivity prediction macro evaluation:', subact_macro_result)
print('Affordance prediction micro evaluation:', aff_micro_result)
print('Affordance prediction macro evaluation:', aff_macro_result)
print(
' * Avg Subactivity Error Ratio {act_err.avg:.3f}; Avg Affordance Error Ratio {aff_err.avg:.3f}; Average Loss {loss.avg:.3f}'
.format(act_err=subactivity_error_ratio,
aff_err=affordance_error_ratio,
loss=losses))
print(' * Subactivity F1 Score {:.3f}; Affordance F1 Score {:.3f};'.format(
subact_macro_result[2], aff_macro_result[2]))
if logger is not None:
logger.log_value('test_epoch_loss', losses.avg)
logger.log_value('test_epoch_subactivity_error_ratio',
subactivity_error_ratio.avg)
logger.log_value('test_epoch_affordance_error_ratio',
affordance_error_ratio.avg)
logger.log_value('test_epoch_subactivity_f1_prediction',
subact_macro_result[2])
logger.log_value('test_epoch_affordance_f1_prediction',
aff_macro_result[2])
# return error_ratio.avg
# return subactivity_error_ratio.avg+affordance_error_ratio.avg
return 2.0 - (subact_macro_result[2] + aff_macro_result[2])
def validate(args,
val_loader,
model,
mse_loss,
multi_label_loss,
vcocoeval,
logger=None,
test=False,
obj_action_pairs=None):
if args.visualize:
result_folder = os.path.join(args.tmp_root,
'results/VCOCO/detections/',
'top' + str(args.vis_top_k))
if not os.path.exists(result_folder):
os.makedirs(result_folder)
batch_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
y_true = np.empty((0, action_class_num))
y_score = np.empty((0, action_class_num))
all_results = list()
# switch to evaluate mode
model.eval()
th_obj_action_pairs = torch.tensor(
obj_action_pairs).float() #.cuda().float()
end = time.time()
for i, (edge_features, node_features, part_human_id, adj_mat, node_labels,
node_roles, obj_boxes, part_boxes, human_boxes, img_id, img_name,
human_num, part_num, obj_num, obj_classes, part_classes,
part_adj_mat, _) in enumerate(val_loader):
edge_features = utils.to_variable(edge_features, args.cuda)
node_features = utils.to_variable(node_features, args.cuda)
adj_mat = utils.to_variable(adj_mat, args.cuda)
# part_adj_mat = utils.to_variable(part_adj_mat, args.cuda)
node_labels = utils.to_variable(node_labels, args.cuda)
node_roles = utils.to_variable(node_roles, args.cuda)
pred_adj_mat, pred_node_labels, pred_node_roles = model(
edge_features, node_features, part_human_id, adj_mat, node_labels,
node_roles, human_num, part_num, obj_num, part_classes,
obj_classes, args)
if args.NRT:
pred_node_label_lifted, det_indices, loss = loss_fn(
pred_adj_mat,
adj_mat,
pred_node_labels,
node_labels,
pred_node_roles,
node_roles,
human_num,
part_num,
obj_num,
part_human_id,
part_adj_mat,
mse_loss,
multi_label_loss,
obj_action_pairs=th_obj_action_pairs,
obj_classes=obj_classes)
else:
pred_node_label_lifted, det_indices, loss = loss_fn(
pred_adj_mat, adj_mat, pred_node_labels, node_labels,
pred_node_roles, node_roles, human_num, part_num, obj_num,
part_human_id, part_adj_mat, mse_loss, multi_label_loss)
append_results(pred_adj_mat, adj_mat, pred_node_labels, node_labels,
pred_node_roles, node_roles, part_human_id, img_id,
obj_boxes, part_boxes, human_boxes, human_num, part_num,
obj_num, obj_classes, part_classes, all_results)
# TODO: node_classes
if args.NRE:
node_classes = (human_num, obj_classes)
else:
node_classes = None
# Log
if len(det_indices) > 0:
losses.update(loss.item(), len(det_indices))
y_true, y_score = evaluation(det_indices,
pred_node_label_lifted,
node_labels,
y_true,
y_score,
test=test,
obj_action_pairs=obj_action_pairs,
node_classes=node_classes)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_interval == 0:
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
print(
'Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t'
'Detected HOIs {y_shape}'.format(i,
len(val_loader),
batch_time=batch_time,
loss=losses,
mean_avg_prec=mean_avg_prec,
y_shape=y_true.shape))
if args.debug and i == 9:
break
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
if test:
vcoco_evaluation(args, vcocoeval, 'test', all_results)
if args.visualize:
utils.visualize_vcoco_result(args, result_folder, all_results)
else:
pass
# vcoco_evaluation(args, vcocoeval, 'val', all_results)
print(
' * Average Mean Precision {mean_avg_prec:.4f}; Average Loss {loss.avg:.4f}'
.format(mean_avg_prec=mean_avg_prec, loss=losses))
if logger is not None:
logger.log_value('test_epoch_loss', losses.avg)
logger.log_value('test_epoch_map', mean_avg_prec)
return 1.0 - mean_avg_prec
def train(args, train_loader, model, mse_loss, multi_label_loss, optimizer,
epoch, vcocoeval, logger, obj_action_pairs):
batch_time = logutil.AverageMeter()
data_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
y_true = np.empty((0, action_class_num))
y_score = np.empty((0, action_class_num))
all_results = list()
# switch to train mode
model.train()
end_time = time.time()
th_obj_action_pairs = torch.tensor(
obj_action_pairs).float() #.cuda().float()
for i, (edge_features, node_features, part_human_id, adj_mat, node_labels,
node_roles, obj_boxes, part_boxes, human_boxes, img_id, img_name,
human_num, part_num, obj_num, obj_classes, part_classes,
part_adj_mat, _) in enumerate(train_loader):
data_time.update(time.time() - end_time)
optimizer.zero_grad()
edge_features = utils.to_variable(edge_features, args.cuda)
node_features = utils.to_variable(node_features, args.cuda)
adj_mat = utils.to_variable(adj_mat, args.cuda)
# part_adj_mat = utils.to_variable(part_adj_mat, args.cuda)
node_labels = utils.to_variable(node_labels, args.cuda)
node_roles = utils.to_variable(node_roles, args.cuda)
pred_adj_mat, pred_node_labels, pred_node_roles = model(
edge_features, node_features, part_human_id, adj_mat, node_labels,
node_roles, human_num, part_num, obj_num, part_classes,
obj_classes, args)
if args.NRT:
pred_node_label_lifted, det_indices, loss = loss_fn(
pred_adj_mat,
adj_mat,
pred_node_labels,
node_labels,
pred_node_roles,
node_roles,
human_num,
part_num,
obj_num,
part_human_id,
part_adj_mat,
mse_loss,
multi_label_loss,
obj_action_pairs=th_obj_action_pairs,
obj_classes=obj_classes)
else:
pred_node_label_lifted, det_indices, loss = loss_fn(
pred_adj_mat, adj_mat, pred_node_labels, node_labels,
pred_node_roles, node_roles, human_num, part_num, obj_num,
part_human_id, part_adj_mat, mse_loss, multi_label_loss)
append_results(pred_adj_mat, adj_mat, pred_node_labels, node_labels,
pred_node_roles, node_roles, part_human_id, img_id,
obj_boxes, part_boxes, human_boxes, human_num, part_num,
obj_num, obj_classes, part_classes, all_results)
# TODO: node_classes
if args.NRE:
node_classes = (human_num, obj_classes)
else:
node_classes = None
# Log and back propagate
if len(det_indices) > 0:
y_true, y_score = evaluation(det_indices,
pred_node_label_lifted,
node_labels,
y_true,
y_score,
obj_action_pairs=obj_action_pairs,
node_classes=node_classes)
if not isinstance(loss, int):
if torch.isnan(loss):
print(
np.any(
np.isnan(
pred_node_label_lifted.detach().cpu().numpy())),
np.any(np.isnan(loss.detach().cpu().numpy())))
print(
np.any(np.isnan(pred_adj_mat.detach().cpu().numpy())),
np.any(np.isnan(pred_node_labels.detach().cpu().numpy())),
np.any(np.isnan(pred_node_roles.detach().cpu().numpy())))
raise
else:
losses.update(loss.item(), edge_features.size()[0])
loss.backward()
optimizer.step()
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
if i % args.log_interval == 0:
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
print(
'Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t'
'Detected HOIs {y_shape}'.format(epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
mean_avg_prec=mean_avg_prec,
y_shape=y_true.shape))
if args.debug and i == 30:
break
mean_avg_prec = compute_mean_avg_prec(y_true, y_score)
# vcoco_evaluation(args, vcocoeval, 'train', all_results)
if logger is not None:
logger.log_value('train_epoch_loss', losses.avg)
logger.log_value('train_epoch_map', mean_avg_prec)
print(
'Epoch: [{0}] Avg Mean Precision {map:.4f}; Average Loss {loss.avg:.4f}; Avg Time x Batch {b_time.avg:.4f}'
.format(epoch, map=mean_avg_prec, loss=losses, b_time=batch_time))
def train(train_loader, model, criterion, optimizer, epoch, logger, args=None):
batch_time = logutil.AverageMeter()
data_time = logutil.AverageMeter()
losses = logutil.AverageMeter()
subactivity_acc_ratio = logutil.AverageMeter()
# switch to train mode
model.train()
end_time = time.time()
for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths,
activities, sequence_ids) in enumerate(train_loader):
data_time.update(time.time() - end_time)
features = utils.to_variable(features, args.cuda)
labels = utils.to_variable(labels, args.cuda)
probs = utils.to_variable(probs, args.cuda)
total_lengths = torch.autograd.Variable(total_lengths)
for batch_i in range(features.size()[1]):
print 'training epoch [{},{},{}] acc: {:.3f}, loss: {}'.format(
epoch, i, batch_i, subactivity_acc_ratio.avg, losses.avg)
for frame in range(pred_duration - 1, total_lengths[batch_i], 10):
optimizer.zero_grad()
model_outputs = model(features[:frame + 1])
_, pred_labels = torch.max(model_outputs, dim=2)
train_loss = cross_entropy(
model_outputs[-pred_duration:, batch_i],
labels[frame - pred_duration + 1:frame + 1, batch_i])
# Log
losses.update(train_loss.data[0],
torch.sum(total_lengths).data[0])
subact_micro_result = sklearn.metrics.precision_recall_fscore_support(
labels[frame - pred_duration + 1:frame + 1,
batch_i].cpu().data.numpy().flatten().tolist(),
pred_labels[-pred_duration:].cpu().data.numpy().flatten(
).tolist(),
labels=range(10),
average='micro')
subactivity_acc_ratio.update(subact_micro_result[0],
torch.sum(total_lengths).data[0])
train_loss.backward()
optimizer.step()
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
if logger is not None:
logger.log_value('train_epoch_loss', losses.avg)
print(
'Epoch: [{0}] Avg Subactivity Accuracy Ratio {act_err.avg:.3f}; Average Loss {losses.avg:.3f}; Batch Avg Time {b_time.avg:.3f}'
.format(epoch,
act_err=subactivity_acc_ratio,
losses=losses,
b_time=batch_time))
def validate(val_loader, model, args, test=False):
def compute_accuracy(gt_results, results, metric='micro'):
return sklearn.metrics.precision_recall_fscore_support(
gt_results, results, labels=range(10), average=metric)
batch_time = logutil.AverageMeter()
baseline_acc_ratio = logutil.AverageMeter()
subactivity_acc_ratio = logutil.AverageMeter()
seg_pred_acc_ratio = logutil.AverageMeter()
frame_pred_acc_ratio = logutil.AverageMeter()
all_baseline_detections = list()
all_gt_detections = list()
all_detections = list()
all_gt_seg_predictions = list()
all_gt_frame_predictions = list()
all_seg_predictions = list()
all_frame_predictions = list()
# switch to evaluate mode
model.eval()
end_time = time.time()
for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths,
activities, sequence_ids) in enumerate(val_loader):
features = utils.to_variable(features, args.cuda)
labels = utils.to_variable(labels, args.cuda)
total_lengths = torch.autograd.Variable(total_lengths)
# Inference
model_outputs = model(features)
pred_labels, batch_earley_pred_labels, batch_tokens, batch_seg_pos = inference(
model_outputs, activities, sequence_ids, ctc_labels, args)
# Visualize results
for batch_i in range(labels.size()[1]):
vizutil.plot_segmentation(
[
labels[:, batch_i].squeeze(),
pred_labels[:, batch_i].squeeze(),
batch_earley_pred_labels[batch_i]
],
int(total_lengths[batch_i]),
filename=os.path.join(
args.tmp_root, 'visualize', 'segmentation', 'cad',
'{}_{}.pdf'.format(activities[batch_i],
sequence_ids[batch_i])),
border=False,
vmax=len(datasets.cad_metadata.subactivities))
# Evaluation
# Frame-wise detection
baseline_detections = pred_labels.cpu().data.numpy().flatten().tolist()
gt_detections = labels.cpu().data.numpy().flatten().tolist()
detections = [
l for pred_labels in batch_earley_pred_labels
for l in pred_labels.tolist()
]
all_baseline_detections.extend(baseline_detections)
all_gt_detections.extend(gt_detections)
all_detections.extend(detections)
baseline_micro_result = compute_accuracy(gt_detections,
baseline_detections)
subact_micro_result = compute_accuracy(gt_detections, detections)
gt_seg_predictions, gt_frame_predictions, seg_predictions, frame_predictions = predict(
activities, total_lengths, labels, ctc_labels, batch_tokens,
batch_seg_pos)
all_gt_seg_predictions.extend(gt_seg_predictions)
all_gt_frame_predictions.extend(gt_frame_predictions)
all_seg_predictions.extend(seg_predictions)
all_frame_predictions.extend(frame_predictions)
seg_pred_result = compute_accuracy(gt_seg_predictions, seg_predictions)
frame_pred_result = compute_accuracy(gt_frame_predictions,
frame_predictions)
baseline_acc_ratio.update(baseline_micro_result[0],
torch.sum(total_lengths).data[0])
subactivity_acc_ratio.update(subact_micro_result[0],
torch.sum(total_lengths).data[0])
seg_pred_acc_ratio.update(seg_pred_result[0],
torch.sum(total_lengths).data[0])
frame_pred_acc_ratio.update(frame_pred_result[0],
len(all_gt_frame_predictions))
# Measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
print(' * Baseline Accuracy Ratio {base_acc.avg:.3f}; '.format(
base_acc=baseline_acc_ratio))
print(
' * Detection Accuracy Ratio {act_acc.avg:.3f}; Segment Prediction Accuracy Ratio Batch Avg {seg_pred_acc.avg:.3f}; Frame Prediction Accuracy Ratio Batch Avg {frame_pred_acc.avg:.3f}; Time {b_time.avg:.3f}'
.format(act_acc=subactivity_acc_ratio,
seg_pred_acc=seg_pred_acc_ratio,
frame_pred_acc=frame_pred_acc_ratio,
b_time=batch_time))
print(
compute_accuracy(all_gt_detections,
all_baseline_detections,
metric='macro'))
print(compute_accuracy(all_gt_detections, all_detections, metric='macro'))
print(
compute_accuracy(all_gt_seg_predictions,
all_seg_predictions,
metric='macro'))
print(
compute_accuracy(all_gt_frame_predictions,
all_frame_predictions,
metric='macro'))
confusion_matrix = sklearn.metrics.confusion_matrix(
all_gt_detections,
all_detections,
labels=range(len(datasets.cad_metadata.subactivities)))
vizutil.plot_confusion_matrix(confusion_matrix,
datasets.cad_metadata.subactivities[:],
normalize=True,
title='',
filename=os.path.join(
args.tmp_root, 'visualize', 'confusion',
'cad', 'detection.pdf'))
confusion_matrix = sklearn.metrics.confusion_matrix(
all_gt_frame_predictions,
all_frame_predictions,
labels=range(len(datasets.cad_metadata.subactivities)))
vizutil.plot_confusion_matrix(confusion_matrix,
datasets.cad_metadata.subactivities[:],
normalize=True,
title='',
filename=os.path.join(
args.tmp_root, 'visualize', 'confusion',
'cad', 'prediction_frame.pdf'))
confusion_matrix = sklearn.metrics.confusion_matrix(
all_gt_seg_predictions,
all_seg_predictions,
labels=range(len(datasets.cad_metadata.subactivities)))
vizutil.plot_confusion_matrix(confusion_matrix,
datasets.cad_metadata.subactivities[:],
normalize=True,
title='',
filename=os.path.join(
args.tmp_root, 'visualize', 'confusion',
'cad', 'prediction_seg.pdf'))
return 1.0 - subactivity_acc_ratio.avg
