def train(train_loader, model, criterion, optimizer, epoch, logger, args=None): batch_time = logutil.AverageMeter() data_time = logutil.AverageMeter() losses = logutil.AverageMeter() subactivity_error_ratio = logutil.AverageMeter() # switch to train mode model.train() end_time = time.time() for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths, activities, sequence_ids) in enumerate(train_loader): data_time.update(time.time() - end_time) optimizer.zero_grad() features = utils.to_variable(features, args.cuda) labels = utils.to_variable(labels, args.cuda) probs = utils.to_variable(probs, args.cuda) total_lengths = torch.autograd.Variable(total_lengths) ctc_labels = torch.autograd.Variable( torch.IntTensor( [item for sublist in ctc_labels for item in sublist])) ctc_lengths = torch.autograd.Variable(ctc_lengths) model_outputs = model(features) _, pred_labels = torch.max(model_outputs, dim=2) train_loss = criterion(model_outputs, labels, probs, ctc_labels, total_lengths, ctc_lengths) # Log losses.update(train_loss.data[0], torch.sum(total_lengths).data[0]) subact_micro_result = sklearn.metrics.precision_recall_fscore_support( labels.cpu().data.numpy().flatten().tolist(), pred_labels.cpu().data.numpy().flatten().tolist(), labels=range(10), average='micro') subactivity_error_ratio.update(1.0 - subact_micro_result[0], torch.sum(total_lengths).data[0]) train_loss.backward() optimizer.step() # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() if logger is not None: logger.log_value('train_epoch_loss', losses.avg) print( 'Epoch: [{0}] Avg Subactivity Error Ratio {act_err.avg:.3f}; Average Loss {losses.avg:.3f}; Batch Avg Time {b_time.avg:.3f}' .format(epoch, act_err=subactivity_error_ratio, losses=losses, b_time=batch_time))
def train(train_loader, model, mse_loss, multi_label_loss, optimizer, epoch, logger): batch_time = logutil.AverageMeter() data_time = logutil.AverageMeter() losses = logutil.AverageMeter() y_true = np.empty((0, action_class_num)) y_score = np.empty((0, action_class_num)) # switch to train mode model.train() end_time = time.time() for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, det_classes, det_boxes, human_num, obj_num) in enumerate(train_loader): data_time.update(time.time() - end_time) optimizer.zero_grad() edge_features = utils.to_variable(edge_features, args.cuda) node_features = utils.to_variable(node_features, args.cuda) adj_mat = utils.to_variable(adj_mat, args.cuda) node_labels = utils.to_variable(node_labels, args.cuda) pred_adj_mat, pred_node_labels = model(edge_features, node_features, adj_mat, node_labels, human_num, obj_num, args) det_indices, loss = loss_fn(pred_adj_mat, adj_mat, pred_node_labels, node_labels, mse_loss, multi_label_loss, human_num, obj_num) # Log and back propagate if len(det_indices) > 0: y_true, y_score = evaluation(det_indices, pred_node_labels, node_labels, y_true, y_score) losses.update(loss.data[0], edge_features.size()[0]) loss.backward() optimizer.step() # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() if i % args.log_interval == 0: mean_avg_prec = compute_mean_avg_prec(y_true, y_score) print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t' 'Detected HOIs {y_shape}' .format(epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, mean_avg_prec=mean_avg_prec, y_shape=y_true.shape)) mean_avg_prec = compute_mean_avg_prec(y_true, y_score) if logger is not None: logger.log_value('train_epoch_loss', losses.avg) logger.log_value('train_epoch_map', mean_avg_prec) print('Epoch: [{0}] Avg Mean Precision {map:.4f}; Average Loss {loss.avg:.4f}; Avg Time x Batch {b_time.avg:.4f}' .format(epoch, map=mean_avg_prec, loss=losses, b_time=batch_time))
def train(train_loader, model, criterion, optimizer, epoch, logger, args=None): batch_time = logutil.AverageMeter() data_time = logutil.AverageMeter() losses = logutil.AverageMeter() subactivity_error_ratio = logutil.AverageMeter() affordance_error_ratio = logutil.AverageMeter() # switch to train mode model.train() end_time = time.time() for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, node_nums) in enumerate(train_loader): data_time.update(time.time() - end_time) optimizer.zero_grad() edge_features = utils.to_variable(edge_features, args.cuda) node_features = utils.to_variable(node_features, args.cuda) adj_mat = utils.to_variable(adj_mat, args.cuda) node_labels = utils.to_variable(node_labels, args.cuda) pred_adj_mat, pred_node_labels = model(edge_features, node_features, adj_mat, node_labels, args) train_loss = criterion(pred_node_labels, node_labels) # Log losses.update(train_loss.data[0], edge_features.size(0)) error_rate, total_nodes, predictions, ground_truth = evaluation( pred_node_labels[:, [0], :], node_labels[:, [0], :]) subactivity_error_ratio.update(error_rate, total_nodes) error_rate, total_nodes, predictions, ground_truth = evaluation( pred_node_labels[:, 1:, :], node_labels[:, 1:, :]) affordance_error_ratio.update(error_rate, total_nodes) train_loss.backward() optimizer.step() # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() if logger is not None: logger.log_value('train_epoch_loss', losses.avg) logger.log_value('train_epoch_subactivity_error_ratio', subactivity_error_ratio.avg) logger.log_value('train_epoch_affordance_error_ratio', affordance_error_ratio.avg) print( 'Epoch: [{0}] Avg Subactivity Error Ratio {act_err.avg:.3f}; Avg Affordance Error Ratio {aff_err.avg:.3f}; Average Loss {loss.avg:.3f}; Batch Avg Time {b_time.avg:.3f}' .format(epoch, act_err=subactivity_error_ratio, aff_err=affordance_error_ratio, loss=losses, b_time=batch_time))
def train(train_loader, model, criterion, optimizer, epoch, evaluation, logger, args=None): batch_time = logutil.AverageMeter() data_time = logutil.AverageMeter() losses = logutil.AverageMeter() error_ratio = logutil.AverageMeter() # switch to train mode model.train() end_time = time.time() for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, node_nums) in enumerate(train_loader): data_time.update(time.time() - end_time) optimizer.zero_grad() target = utils.to_variable(adj_mat, args.cuda) output = model(utils.to_variable(edge_features, args.cuda)) train_loss = criterion(output, target) # Log losses.update(train_loss.data[0], edge_features.size(0)) error_ratio.update( evaluation(output, target).data[0], edge_features.size(0)) # compute gradient and do SGD step train_loss.backward() optimizer.step() # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() # if i % args.log_interval == 0 and i > 0: # print('Epoch: [{0}][{1}/{2}]\t' # 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' # 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' # 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' # 'Error Ratio {err.val:.4f} ({err.avg:.4f})' # .format(epoch, i, len(train_loader), batch_time=batch_time, # data_time=data_time, loss=losses, err=error_ratio)) if logger is not None: logger.log_value('train_epoch_loss', losses.avg) logger.log_value('train_epoch_error_ratio', error_ratio.avg) print( 'Epoch: [{0}] Avg Error Ratio {err.avg:.3f}; Average Loss {loss.avg:.3f}; Batch Avg Time {b_time.avg:.3f}' .format(epoch, err=error_ratio, loss=losses, b_time=batch_time))
def validate(val_loader, model, criterion, evaluation, logger=None, args=None, test=False): batch_time = logutil.AverageMeter() losses = logutil.AverageMeter() error_ratio = logutil.AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (edge_features, node_features, adj_mat, node_labels, node_roles, boxes, img_ids, img_names, human_nums, obj_nums, classes) in enumerate(val_loader): edge_features = utils.to_variable(edge_features, args.cuda) edge_features = edge_features.permute(0, 3, 1, 2) target = utils.to_variable(adj_mat, args.cuda) output = model(edge_features) # Logs losses.update( criterion(output, target, human_nums, obj_nums).data[0], edge_features.size(0)) error_ratio.update( evaluation(output, target, human_nums, obj_nums, test=test).data[0], edge_features.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_interval == 0 and i > 0: print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Error Ratio {err.val:.4f} ({err.avg:.4f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, err=error_ratio)) print(' * Average Error Ratio {err.avg:.3f}; Average Loss {loss.avg:.3f}'. format(err=error_ratio, loss=losses)) if logger is not None: logger.log_value('test_epoch_loss', losses.avg) logger.log_value('test_epoch_error_ratio', error_ratio.avg) return losses.avg
def validate(val_loader, model, mse_loss, multi_label_loss, logger=None, test=False): if args.visualize: result_folder = os.path.join(args.tmp_root, 'results/HICO/detections/', 'top'+str(args.vis_top_k)) if not os.path.exists(result_folder): os.makedirs(result_folder) batch_time = logutil.AverageMeter() losses = logutil.AverageMeter() y_true = np.empty((0, action_class_num)) y_score = np.empty((0, action_class_num)) # switch to evaluate mode model.eval() end = time.time() for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, det_classes, det_boxes, human_num, obj_num) in enumerate(val_loader): edge_features = utils.to_variable(edge_features, args.cuda) node_features = utils.to_variable(node_features, args.cuda) adj_mat = utils.to_variable(adj_mat, args.cuda) node_labels = utils.to_variable(node_labels, args.cuda) pred_adj_mat, pred_node_labels = model(edge_features, node_features, adj_mat, node_labels, human_num, obj_num, args) det_indices, loss = loss_fn(pred_adj_mat, adj_mat, pred_node_labels, node_labels, mse_loss, multi_label_loss, human_num, obj_num) # Log if len(det_indices) > 0: losses.update(loss.data[0], len(det_indices)) y_true, y_score = evaluation(det_indices, pred_node_labels, node_labels, y_true, y_score, test=test) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_interval == 0 and i > 0: mean_avg_prec = compute_mean_avg_prec(y_true, y_score) print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t' 'Detected HOIs {y_shape}' .format(i, len(val_loader), batch_time=batch_time, loss=losses, mean_avg_prec=mean_avg_prec, y_shape=y_true.shape)) mean_avg_prec = compute_mean_avg_prec(y_true, y_score) print(' * Average Mean Precision {mean_avg_prec:.4f}; Average Loss {loss.avg:.4f}' .format(mean_avg_prec=mean_avg_prec, loss=losses)) if logger is not None: logger.log_value('test_epoch_loss', losses.avg) logger.log_value('train_epoch_map', mean_avg_prec) return 1.0 - mean_avg_prec
def validate(val_loader, model, args, test=False): def compute_accuracy(gt_results, results, metric='micro'): return sklearn.metrics.precision_recall_fscore_support( gt_results, results, labels=range(10), average=metric) batch_time = logutil.AverageMeter() subactivity_acc_ratio = logutil.AverageMeter() all_gt_seg_predictions = list() all_seg_predictions = list() # switch to evaluate mode model.eval() end_time = time.time() for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths, activities, sequence_ids) in enumerate(val_loader): features = utils.to_variable(features, args.cuda) labels = utils.to_variable(labels, args.cuda) total_lengths = torch.autograd.Variable(total_lengths) for batch_i in range(features.size()[1]): for frame in range(pred_duration - 1, total_lengths[batch_i], 10): model_outputs = model(features[:frame + 1]) _, pred_labels = torch.max(model_outputs, dim=2) gt_pred_labels = labels[ frame - pred_duration + 1:frame + 1, batch_i].cpu().data.numpy().flatten().tolist() pred_labels = pred_labels[-pred_duration:].cpu().data.numpy( ).flatten().tolist() subact_micro_result = compute_accuracy(gt_pred_labels, pred_labels) subactivity_acc_ratio.update(subact_micro_result[0], pred_duration) all_gt_seg_predictions.extend(gt_pred_labels) all_seg_predictions.extend(pred_labels) # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() print( ' * Avg Subactivity Accuracy Ratio {act_err.avg:.3f}; Batch Avg Time {b_time.avg:.3f}' .format(act_err=subactivity_acc_ratio, b_time=batch_time)) print( compute_accuracy(all_gt_seg_predictions, all_seg_predictions, metric='macro')) return 1.0 - subactivity_acc_ratio.avg
def visualize(args, val_loader, model, mse_loss, multi_label_loss, vcocoeval, logger=None, test=False): if args.visualize: result_folder = os.path.join(args.tmp_root, 'results/VCOCO/detections/', 'top' + str(args.vis_top_k)) if not os.path.exists(result_folder): os.makedirs(result_folder) batch_time = logutil.AverageMeter() losses = logutil.AverageMeter() y_true = np.empty((0, action_class_num)) y_score = np.empty((0, action_class_num)) all_results = list() # switch to evaluate mode model.eval() end = time.time() for i, (edge_features, node_features, part_human_id, adj_mat, node_labels, node_roles, obj_boxes, part_boxes, human_boxes, img_id, img_name, human_num, part_num, obj_num, obj_classes, part_classes, part_adj_mat, img_names) in enumerate(val_loader): edge_features = utils.to_variable(edge_features, args.cuda) node_features = utils.to_variable(node_features, args.cuda) adj_mat = utils.to_variable(adj_mat, args.cuda) part_adj_mat = utils.to_variable(part_adj_mat, args.cuda) node_labels = utils.to_variable(node_labels, args.cuda) node_roles = utils.to_variable(node_roles, args.cuda) pred_adj_mat, pred_node_labels, pred_node_roles = model( edge_features, node_features, part_human_id, adj_mat, node_labels, node_roles, human_num, part_num, obj_num, part_classes, args) for j in range(len(img_names)): pickle.dump( { 'adj_mat': pred_adj_mat[j], 'node_labels': pred_node_labels[j], 'node_roles': pred_node_roles[j] }, open(os.path.join(args.eval_root, img_names[j] + '.pred'), 'wb')) if i % 10 == 0: print('\r%d/%d' % (i, len(val_loader)), end='')
def validate(val_loader, model, criterion, evaluation, logger=None, args=None): batch_time = logutil.AverageMeter() losses = logutil.AverageMeter() error_ratio = logutil.AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, node_nums) in enumerate(val_loader): target = utils.to_variable(adj_mat, args.cuda) output = model(utils.to_variable(edge_features, args.cuda)) # Logs losses.update(criterion(output, target).data[0], edge_features.size(0)) error_ratio.update( evaluation(output, target).data[0], edge_features.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() # if i % args.log_interval == 0 and i > 0: # print('Test: [{0}/{1}]\t' # 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' # 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' # 'Error Ratio {err.val:.4f} ({err.avg:.4f})' # .format(i, len(val_loader), batch_time=batch_time, # loss=losses, err=error_ratio)) print(' * Average Error Ratio {err.avg:.3f}; Average Loss {loss.avg:.3f}'. format(err=error_ratio, loss=losses)) if logger is not None: logger.log_value('test_epoch_loss', losses.avg) logger.log_value('test_epoch_error_ratio', error_ratio.avg) return error_ratio.avg
def validate_baseline(val_loader, model, args, test=False): def compute_accuracy(gt_results, results, metric='micro'): return sklearn.metrics.precision_recall_fscore_support( gt_results, results, labels=range(10), average=metric) baseline_acc_ratio = logutil.AverageMeter() all_baseline_detections = list() all_gt_detections = list() # switch to evaluate mode model.eval() end_time = time.time() for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths, activities, sequence_ids) in enumerate(val_loader): features = utils.to_variable(features, args.cuda) labels = utils.to_variable(labels, args.cuda) total_lengths = torch.autograd.Variable(total_lengths) # Inference model_outputs = model(features) _, pred_labels = torch.max(model_outputs, dim=2) baseline_detections = pred_labels.cpu().data.numpy().flatten().tolist() gt_detections = labels.cpu().data.numpy().flatten().tolist() all_baseline_detections.extend(baseline_detections) all_gt_detections.extend(gt_detections) baseline_micro_result = compute_accuracy(gt_detections, baseline_detections) baseline_acc_ratio.update(baseline_micro_result[0], torch.sum(total_lengths).data[0]) for batch_i in range(model_outputs.size()[1]): model_probs = torch.nn.Softmax(dim=1)( model_outputs[:int(total_lengths[batch_i]), batch_i, :].squeeze()).cpu().data.numpy() np.save( os.path.join(args.tmp_root, 'nn_output', activities[batch_i], sequence_ids[batch_i]), model_probs) print(' * Baseline Accuracy Ratio {base_acc.avg:.3f}; '.format( base_acc=baseline_acc_ratio)) return 1.0 - baseline_acc_ratio.avg
def validate(val_loader, model, criterion, logger=None, args=None, test=False): if args.visualize: result_folder = os.path.join(args.tmp_root, 'results/CAD/figures/prediction/') if not os.path.exists(result_folder): os.makedirs(result_folder) batch_time = logutil.AverageMeter() losses = logutil.AverageMeter() error_ratio = logutil.AverageMeter() subactivity_error_ratio = logutil.AverageMeter() affordance_error_ratio = logutil.AverageMeter() subact_predictions = list() subact_ground_truth = list() affordance_predictions = list() affordance_ground_truth = list() all_sequence_ids = list() all_node_nums = list() # switch to evaluate mode model.eval() end = time.time() for i, (edge_features, node_features, adj_mat, node_labels, sequence_ids, node_nums) in enumerate(val_loader): edge_features = utils.to_variable(edge_features, args.cuda) node_features = utils.to_variable(node_features, args.cuda) adj_mat = utils.to_variable(adj_mat, args.cuda) node_labels = utils.to_variable(node_labels, args.cuda) pred_adj_mat, pred_node_labels = model(edge_features, node_features, adj_mat, node_labels, args) # Logs losses.update( criterion(pred_node_labels, node_labels).data[0], edge_features.size(0)) error_rate, total_nodes, predictions, ground_truth = evaluation( pred_node_labels, node_labels) error_ratio.update(error_rate, total_nodes) error_rate, total_nodes, predictions, ground_truth = evaluation( pred_node_labels[:, [0], :], node_labels[:, [0], :]) subactivity_error_ratio.update(error_rate, total_nodes) subact_predictions.extend(predictions) subact_ground_truth.extend(ground_truth) error_rate, total_nodes, predictions, ground_truth = evaluation( pred_node_labels[:, 1:, :], node_labels[:, 1:, :]) affordance_error_ratio.update(error_rate, total_nodes) affordance_predictions.extend(predictions) affordance_ground_truth.extend(ground_truth) all_sequence_ids.extend(sequence_ids) all_node_nums.extend(node_nums) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if args.visualize: utils.plot_all_activity_segmentations(all_sequence_ids, subact_predictions, subact_ground_truth, result_folder) utils.plot_all_affordance_segmentations(all_sequence_ids, all_node_nums, affordance_predictions, affordance_ground_truth, result_folder) # Plot confusion matrices confusion_matrix = sklearn.metrics.confusion_matrix( subact_ground_truth, subact_predictions, labels=range(len(datasets.cad_metadata.subactivities))) utils.plot_confusion_matrix(confusion_matrix, datasets.cad_metadata.subactivities, normalize=True, title='', filename=os.path.join( result_folder, 'confusion_subactivity.pdf')) confusion_matrix = sklearn.metrics.confusion_matrix( affordance_ground_truth, affordance_predictions, labels=range(len(datasets.cad_metadata.affordances))) utils.plot_confusion_matrix(confusion_matrix, datasets.cad_metadata.affordances, normalize=True, title='', filename=os.path.join( result_folder, 'confusion_affordance.pdf')) subact_micro_result = sklearn.metrics.precision_recall_fscore_support( subact_ground_truth, subact_predictions, labels=range(10), average='micro') subact_macro_result = sklearn.metrics.precision_recall_fscore_support( subact_ground_truth, subact_predictions, labels=range(10), average='macro') aff_micro_result = sklearn.metrics.precision_recall_fscore_support( affordance_ground_truth, affordance_predictions, labels=range(12), average='micro') aff_macro_result = sklearn.metrics.precision_recall_fscore_support( affordance_ground_truth, affordance_predictions, labels=range(12), average='macro') if test: print('Subactivity prediction micro evaluation:', subact_micro_result) print('Subactivity prediction macro evaluation:', subact_macro_result) print('Affordance prediction micro evaluation:', aff_micro_result) print('Affordance prediction macro evaluation:', aff_macro_result) print( ' * Avg Subactivity Error Ratio {act_err.avg:.3f}; Avg Affordance Error Ratio {aff_err.avg:.3f}; Average Loss {loss.avg:.3f}' .format(act_err=subactivity_error_ratio, aff_err=affordance_error_ratio, loss=losses)) print(' * Subactivity F1 Score {:.3f}; Affordance F1 Score {:.3f};'.format( subact_macro_result[2], aff_macro_result[2])) if logger is not None: logger.log_value('test_epoch_loss', losses.avg) logger.log_value('test_epoch_subactivity_error_ratio', subactivity_error_ratio.avg) logger.log_value('test_epoch_affordance_error_ratio', affordance_error_ratio.avg) logger.log_value('test_epoch_subactivity_f1_prediction', subact_macro_result[2]) logger.log_value('test_epoch_affordance_f1_prediction', aff_macro_result[2]) # return error_ratio.avg # return subactivity_error_ratio.avg+affordance_error_ratio.avg return 2.0 - (subact_macro_result[2] + aff_macro_result[2])
def validate(args, val_loader, model, mse_loss, multi_label_loss, vcocoeval, logger=None, test=False, obj_action_pairs=None): if args.visualize: result_folder = os.path.join(args.tmp_root, 'results/VCOCO/detections/', 'top' + str(args.vis_top_k)) if not os.path.exists(result_folder): os.makedirs(result_folder) batch_time = logutil.AverageMeter() losses = logutil.AverageMeter() y_true = np.empty((0, action_class_num)) y_score = np.empty((0, action_class_num)) all_results = list() # switch to evaluate mode model.eval() th_obj_action_pairs = torch.tensor( obj_action_pairs).float() #.cuda().float() end = time.time() for i, (edge_features, node_features, part_human_id, adj_mat, node_labels, node_roles, obj_boxes, part_boxes, human_boxes, img_id, img_name, human_num, part_num, obj_num, obj_classes, part_classes, part_adj_mat, _) in enumerate(val_loader): edge_features = utils.to_variable(edge_features, args.cuda) node_features = utils.to_variable(node_features, args.cuda) adj_mat = utils.to_variable(adj_mat, args.cuda) # part_adj_mat = utils.to_variable(part_adj_mat, args.cuda) node_labels = utils.to_variable(node_labels, args.cuda) node_roles = utils.to_variable(node_roles, args.cuda) pred_adj_mat, pred_node_labels, pred_node_roles = model( edge_features, node_features, part_human_id, adj_mat, node_labels, node_roles, human_num, part_num, obj_num, part_classes, obj_classes, args) if args.NRT: pred_node_label_lifted, det_indices, loss = loss_fn( pred_adj_mat, adj_mat, pred_node_labels, node_labels, pred_node_roles, node_roles, human_num, part_num, obj_num, part_human_id, part_adj_mat, mse_loss, multi_label_loss, obj_action_pairs=th_obj_action_pairs, obj_classes=obj_classes) else: pred_node_label_lifted, det_indices, loss = loss_fn( pred_adj_mat, adj_mat, pred_node_labels, node_labels, pred_node_roles, node_roles, human_num, part_num, obj_num, part_human_id, part_adj_mat, mse_loss, multi_label_loss) append_results(pred_adj_mat, adj_mat, pred_node_labels, node_labels, pred_node_roles, node_roles, part_human_id, img_id, obj_boxes, part_boxes, human_boxes, human_num, part_num, obj_num, obj_classes, part_classes, all_results) # TODO: node_classes if args.NRE: node_classes = (human_num, obj_classes) else: node_classes = None # Log if len(det_indices) > 0: losses.update(loss.item(), len(det_indices)) y_true, y_score = evaluation(det_indices, pred_node_label_lifted, node_labels, y_true, y_score, test=test, obj_action_pairs=obj_action_pairs, node_classes=node_classes) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_interval == 0: mean_avg_prec = compute_mean_avg_prec(y_true, y_score) print( 'Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t' 'Detected HOIs {y_shape}'.format(i, len(val_loader), batch_time=batch_time, loss=losses, mean_avg_prec=mean_avg_prec, y_shape=y_true.shape)) if args.debug and i == 9: break mean_avg_prec = compute_mean_avg_prec(y_true, y_score) if test: vcoco_evaluation(args, vcocoeval, 'test', all_results) if args.visualize: utils.visualize_vcoco_result(args, result_folder, all_results) else: pass # vcoco_evaluation(args, vcocoeval, 'val', all_results) print( ' * Average Mean Precision {mean_avg_prec:.4f}; Average Loss {loss.avg:.4f}' .format(mean_avg_prec=mean_avg_prec, loss=losses)) if logger is not None: logger.log_value('test_epoch_loss', losses.avg) logger.log_value('test_epoch_map', mean_avg_prec) return 1.0 - mean_avg_prec
def train(args, train_loader, model, mse_loss, multi_label_loss, optimizer, epoch, vcocoeval, logger, obj_action_pairs): batch_time = logutil.AverageMeter() data_time = logutil.AverageMeter() losses = logutil.AverageMeter() y_true = np.empty((0, action_class_num)) y_score = np.empty((0, action_class_num)) all_results = list() # switch to train mode model.train() end_time = time.time() th_obj_action_pairs = torch.tensor( obj_action_pairs).float() #.cuda().float() for i, (edge_features, node_features, part_human_id, adj_mat, node_labels, node_roles, obj_boxes, part_boxes, human_boxes, img_id, img_name, human_num, part_num, obj_num, obj_classes, part_classes, part_adj_mat, _) in enumerate(train_loader): data_time.update(time.time() - end_time) optimizer.zero_grad() edge_features = utils.to_variable(edge_features, args.cuda) node_features = utils.to_variable(node_features, args.cuda) adj_mat = utils.to_variable(adj_mat, args.cuda) # part_adj_mat = utils.to_variable(part_adj_mat, args.cuda) node_labels = utils.to_variable(node_labels, args.cuda) node_roles = utils.to_variable(node_roles, args.cuda) pred_adj_mat, pred_node_labels, pred_node_roles = model( edge_features, node_features, part_human_id, adj_mat, node_labels, node_roles, human_num, part_num, obj_num, part_classes, obj_classes, args) if args.NRT: pred_node_label_lifted, det_indices, loss = loss_fn( pred_adj_mat, adj_mat, pred_node_labels, node_labels, pred_node_roles, node_roles, human_num, part_num, obj_num, part_human_id, part_adj_mat, mse_loss, multi_label_loss, obj_action_pairs=th_obj_action_pairs, obj_classes=obj_classes) else: pred_node_label_lifted, det_indices, loss = loss_fn( pred_adj_mat, adj_mat, pred_node_labels, node_labels, pred_node_roles, node_roles, human_num, part_num, obj_num, part_human_id, part_adj_mat, mse_loss, multi_label_loss) append_results(pred_adj_mat, adj_mat, pred_node_labels, node_labels, pred_node_roles, node_roles, part_human_id, img_id, obj_boxes, part_boxes, human_boxes, human_num, part_num, obj_num, obj_classes, part_classes, all_results) # TODO: node_classes if args.NRE: node_classes = (human_num, obj_classes) else: node_classes = None # Log and back propagate if len(det_indices) > 0: y_true, y_score = evaluation(det_indices, pred_node_label_lifted, node_labels, y_true, y_score, obj_action_pairs=obj_action_pairs, node_classes=node_classes) if not isinstance(loss, int): if torch.isnan(loss): print( np.any( np.isnan( pred_node_label_lifted.detach().cpu().numpy())), np.any(np.isnan(loss.detach().cpu().numpy()))) print( np.any(np.isnan(pred_adj_mat.detach().cpu().numpy())), np.any(np.isnan(pred_node_labels.detach().cpu().numpy())), np.any(np.isnan(pred_node_roles.detach().cpu().numpy()))) raise else: losses.update(loss.item(), edge_features.size()[0]) loss.backward() optimizer.step() # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() if i % args.log_interval == 0: mean_avg_prec = compute_mean_avg_prec(y_true, y_score) print( 'Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Mean Avg Precision {mean_avg_prec:.4f} ({mean_avg_prec:.4f})\t' 'Detected HOIs {y_shape}'.format(epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, mean_avg_prec=mean_avg_prec, y_shape=y_true.shape)) if args.debug and i == 30: break mean_avg_prec = compute_mean_avg_prec(y_true, y_score) # vcoco_evaluation(args, vcocoeval, 'train', all_results) if logger is not None: logger.log_value('train_epoch_loss', losses.avg) logger.log_value('train_epoch_map', mean_avg_prec) print( 'Epoch: [{0}] Avg Mean Precision {map:.4f}; Average Loss {loss.avg:.4f}; Avg Time x Batch {b_time.avg:.4f}' .format(epoch, map=mean_avg_prec, loss=losses, b_time=batch_time))
def train(train_loader, model, criterion, optimizer, epoch, logger, args=None): batch_time = logutil.AverageMeter() data_time = logutil.AverageMeter() losses = logutil.AverageMeter() subactivity_acc_ratio = logutil.AverageMeter() # switch to train mode model.train() end_time = time.time() for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths, activities, sequence_ids) in enumerate(train_loader): data_time.update(time.time() - end_time) features = utils.to_variable(features, args.cuda) labels = utils.to_variable(labels, args.cuda) probs = utils.to_variable(probs, args.cuda) total_lengths = torch.autograd.Variable(total_lengths) for batch_i in range(features.size()[1]): print 'training epoch [{},{},{}] acc: {:.3f}, loss: {}'.format( epoch, i, batch_i, subactivity_acc_ratio.avg, losses.avg) for frame in range(pred_duration - 1, total_lengths[batch_i], 10): optimizer.zero_grad() model_outputs = model(features[:frame + 1]) _, pred_labels = torch.max(model_outputs, dim=2) train_loss = cross_entropy( model_outputs[-pred_duration:, batch_i], labels[frame - pred_duration + 1:frame + 1, batch_i]) # Log losses.update(train_loss.data[0], torch.sum(total_lengths).data[0]) subact_micro_result = sklearn.metrics.precision_recall_fscore_support( labels[frame - pred_duration + 1:frame + 1, batch_i].cpu().data.numpy().flatten().tolist(), pred_labels[-pred_duration:].cpu().data.numpy().flatten( ).tolist(), labels=range(10), average='micro') subactivity_acc_ratio.update(subact_micro_result[0], torch.sum(total_lengths).data[0]) train_loss.backward() optimizer.step() # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() if logger is not None: logger.log_value('train_epoch_loss', losses.avg) print( 'Epoch: [{0}] Avg Subactivity Accuracy Ratio {act_err.avg:.3f}; Average Loss {losses.avg:.3f}; Batch Avg Time {b_time.avg:.3f}' .format(epoch, act_err=subactivity_acc_ratio, losses=losses, b_time=batch_time))
def validate(val_loader, model, args, test=False): def compute_accuracy(gt_results, results, metric='micro'): return sklearn.metrics.precision_recall_fscore_support( gt_results, results, labels=range(10), average=metric) batch_time = logutil.AverageMeter() baseline_acc_ratio = logutil.AverageMeter() subactivity_acc_ratio = logutil.AverageMeter() seg_pred_acc_ratio = logutil.AverageMeter() frame_pred_acc_ratio = logutil.AverageMeter() all_baseline_detections = list() all_gt_detections = list() all_detections = list() all_gt_seg_predictions = list() all_gt_frame_predictions = list() all_seg_predictions = list() all_frame_predictions = list() # switch to evaluate mode model.eval() end_time = time.time() for i, (features, labels, probs, total_lengths, ctc_labels, ctc_lengths, activities, sequence_ids) in enumerate(val_loader): features = utils.to_variable(features, args.cuda) labels = utils.to_variable(labels, args.cuda) total_lengths = torch.autograd.Variable(total_lengths) # Inference model_outputs = model(features) pred_labels, batch_earley_pred_labels, batch_tokens, batch_seg_pos = inference( model_outputs, activities, sequence_ids, ctc_labels, args) # Visualize results for batch_i in range(labels.size()[1]): vizutil.plot_segmentation( [ labels[:, batch_i].squeeze(), pred_labels[:, batch_i].squeeze(), batch_earley_pred_labels[batch_i] ], int(total_lengths[batch_i]), filename=os.path.join( args.tmp_root, 'visualize', 'segmentation', 'cad', '{}_{}.pdf'.format(activities[batch_i], sequence_ids[batch_i])), border=False, vmax=len(datasets.cad_metadata.subactivities)) # Evaluation # Frame-wise detection baseline_detections = pred_labels.cpu().data.numpy().flatten().tolist() gt_detections = labels.cpu().data.numpy().flatten().tolist() detections = [ l for pred_labels in batch_earley_pred_labels for l in pred_labels.tolist() ] all_baseline_detections.extend(baseline_detections) all_gt_detections.extend(gt_detections) all_detections.extend(detections) baseline_micro_result = compute_accuracy(gt_detections, baseline_detections) subact_micro_result = compute_accuracy(gt_detections, detections) gt_seg_predictions, gt_frame_predictions, seg_predictions, frame_predictions = predict( activities, total_lengths, labels, ctc_labels, batch_tokens, batch_seg_pos) all_gt_seg_predictions.extend(gt_seg_predictions) all_gt_frame_predictions.extend(gt_frame_predictions) all_seg_predictions.extend(seg_predictions) all_frame_predictions.extend(frame_predictions) seg_pred_result = compute_accuracy(gt_seg_predictions, seg_predictions) frame_pred_result = compute_accuracy(gt_frame_predictions, frame_predictions) baseline_acc_ratio.update(baseline_micro_result[0], torch.sum(total_lengths).data[0]) subactivity_acc_ratio.update(subact_micro_result[0], torch.sum(total_lengths).data[0]) seg_pred_acc_ratio.update(seg_pred_result[0], torch.sum(total_lengths).data[0]) frame_pred_acc_ratio.update(frame_pred_result[0], len(all_gt_frame_predictions)) # Measure elapsed time batch_time.update(time.time() - end_time) end_time = time.time() print(' * Baseline Accuracy Ratio {base_acc.avg:.3f}; '.format( base_acc=baseline_acc_ratio)) print( ' * Detection Accuracy Ratio {act_acc.avg:.3f}; Segment Prediction Accuracy Ratio Batch Avg {seg_pred_acc.avg:.3f}; Frame Prediction Accuracy Ratio Batch Avg {frame_pred_acc.avg:.3f}; Time {b_time.avg:.3f}' .format(act_acc=subactivity_acc_ratio, seg_pred_acc=seg_pred_acc_ratio, frame_pred_acc=frame_pred_acc_ratio, b_time=batch_time)) print( compute_accuracy(all_gt_detections, all_baseline_detections, metric='macro')) print(compute_accuracy(all_gt_detections, all_detections, metric='macro')) print( compute_accuracy(all_gt_seg_predictions, all_seg_predictions, metric='macro')) print( compute_accuracy(all_gt_frame_predictions, all_frame_predictions, metric='macro')) confusion_matrix = sklearn.metrics.confusion_matrix( all_gt_detections, all_detections, labels=range(len(datasets.cad_metadata.subactivities))) vizutil.plot_confusion_matrix(confusion_matrix, datasets.cad_metadata.subactivities[:], normalize=True, title='', filename=os.path.join( args.tmp_root, 'visualize', 'confusion', 'cad', 'detection.pdf')) confusion_matrix = sklearn.metrics.confusion_matrix( all_gt_frame_predictions, all_frame_predictions, labels=range(len(datasets.cad_metadata.subactivities))) vizutil.plot_confusion_matrix(confusion_matrix, datasets.cad_metadata.subactivities[:], normalize=True, title='', filename=os.path.join( args.tmp_root, 'visualize', 'confusion', 'cad', 'prediction_frame.pdf')) confusion_matrix = sklearn.metrics.confusion_matrix( all_gt_seg_predictions, all_seg_predictions, labels=range(len(datasets.cad_metadata.subactivities))) vizutil.plot_confusion_matrix(confusion_matrix, datasets.cad_metadata.subactivities[:], normalize=True, title='', filename=os.path.join( args.tmp_root, 'visualize', 'confusion', 'cad', 'prediction_seg.pdf')) return 1.0 - subactivity_acc_ratio.avg