def train(train_loader, target_net, optimizer, epoch): batch_time = network.AverageMeter() data_time = network.AverageMeter() train_loss = network.AverageMeter() train_loss_box_rpn = network.AverageMeter() train_loss_entropy_rpn = network.AverageMeter() target_net.train() end = time.time() for i, (im_data, im_info, gt_boxes) in enumerate(train_loader): # measure the data loading time data_time.update(time.time() - end) # Forward pass target_net(im_data, im_info.numpy(), gt_boxes.numpy()[0]) # record loss loss = target_net.loss # total loss train_loss.update(loss.data[0], im_data.size(0)) train_loss_box_rpn.update(target_net.loss_box.data[0], im_data.size(0)) train_loss_entropy_rpn.update(target_net.cross_entropy.data[0], im_data.size(0)) # backward optimizer.zero_grad() loss.backward() if not args.disable_clip_gradient: network.clip_gradient(target_net, 10.) optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if (i + 1) % args.log_interval == 0: print( 'Epoch: [{0}][{1}/{2}]\t' 'Batch_Time: {batch_time.avg:.3f}s\t' 'lr: {lr: f}\t' 'Loss: {loss.avg:.4f}\n' '\t[rpn]: ' 'cls_loss_rpn: {cls_loss_rpn.avg:.3f}\t' 'reg_loss_rpn: {reg_loss_rpn.avg:.3f}\n'.format( epoch, i + 1, len(train_loader), batch_time=batch_time, lr=args.lr, data_time=data_time, loss=train_loss, cls_loss_rpn=train_loss_entropy_rpn, reg_loss_rpn=train_loss_box_rpn))
def test(test_loader, target_net): box_num = np.array([0, 0]) correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0]) print('========== Testing =======') target_net.eval() batch_time = network.AverageMeter() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(test_loader): correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0]) # Forward pass object_rois, region_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy(), gt_regions.numpy())[1:] box_num[0] += object_rois.size(0) box_num[1] += region_rois.size(0) correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects[0].numpy(), 50) correct_cnt_t[1], total_cnt_t[1] = check_recall(region_rois, gt_regions[0].numpy(), 50) correct_cnt += correct_cnt_t total_cnt += total_cnt_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % 100 == 0 and i > 0: print('[{0}/{10}] Time: {1:2.3f}s/img).' '\t[object] Avg: {2:2.2f} Boxes/im, Top-50 recall: {3:2.3f} ({4:d}/{5:d})' '\t[region] Avg: {6:2.2f} Boxes/im, Top-50 recall: {7:2.3f} ({8:d}/{9:d})'.format( i + 1, batch_time.avg, box_num[0] / float(i + 1), correct_cnt[0] / float(total_cnt[0]) * 100, correct_cnt[0], total_cnt[0], box_num[1] / float(i + 1), correct_cnt[1] / float(total_cnt[1]) * 100, correct_cnt[1], total_cnt[1], len(test_loader))) recall = correct_cnt / total_cnt.astype(np.float) print('====== Done Testing ====') return recall
def evaluate(test_loader, target_net, object_classes, score_thresh=0.00, overlap_thresh=0.5, nms_thresh=0.5): print(object_classes) box_num, correct_cnt, total_cnt = 0, 0, 0 # ipdb.set_trace() # store cls_scores, cls_tp, cls_gt_num of each cls and every image all_cls_gt_num = [0] * (len(object_classes) - 1) all_cls_scores = [np.array([])] * (len(object_classes) - 1) all_cls_tp = [np.array([])] * (len(object_classes) - 1) print('========== Evaluate =======') target_net.eval() batch_time = network.AverageMeter() end = time.time() for i, (im_data, im_info, gt_boxes, gt_relationships) in enumerate(test_loader): # get every class scores, tf array and gt number classes_scores, classes_tf, classes_gt_num, object_rois = \ image_eval(target_net, im_data, im_info, gt_boxes.numpy()[0], object_classes, max_per_image=100, score_thresh=score_thresh, overlap_thresh=overlap_thresh, nms_thresh=nms_thresh) for j in range(len(object_classes) - 1): all_cls_scores[j] = np.append(all_cls_scores[j], classes_scores[j]) all_cls_tp[j] = np.append(all_cls_tp[j], classes_tf[j]) all_cls_gt_num[j] += classes_gt_num[j] box_num += object_rois.size(0) correct_cnt_t, total_cnt_t = check_recall(object_rois, gt_boxes.numpy()[0], 64, thresh=0.5) correct_cnt += correct_cnt_t total_cnt += total_cnt_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % args.log_interval == 0: print( '[{0}/{6}] Time: {1:2.3f}s/img).' '\t[object] Avg: {2:2.2f} Boxes/im, Top-64 recall: {3:2.3f} ({4:d}/{5:d})' .format(i + 1, batch_time.avg, box_num / float(i + 1), correct_cnt / float(total_cnt) * 100, correct_cnt, total_cnt, len(test_loader))) all_aps = [] for k, cls in enumerate(object_classes[1:]): # sort scores of all images cls_ap = cls_eval(all_cls_scores[k], all_cls_tp[k], all_cls_gt_num[k]) all_aps += [cls_ap] print('AP for {} = {:.4f}'.format(cls, cls_ap)) mean_ap = np.mean(all_aps) print('Mean AP = {:.4f}'.format(mean_ap)) print('====== Done Testing ====') return mean_ap
def test(test_loader, target_net): box_num, correct_cnt, total_cnt = 0, 0, 0 print '========== Testing =======' target_net.eval() batch_time = network.AverageMeter() end = time.time() for i, (im_data, im_info, gt_boxes, gt_relationship) in enumerate(test_loader): # Forward pass features, object_rois, scores = target_net.rpn(im_data, im_info.numpy(), gt_boxes.numpy()[0]) box_num += object_rois.size(0) correct_cnt_t, total_cnt_t = check_recall(object_rois, gt_boxes.numpy()[0], 64) correct_cnt += correct_cnt_t total_cnt += total_cnt_t batch_time.update(time.time()-end) end = time.time() if (i+1)%100 == 0 and i > 0: print('[{0}/{6}] Time: {1:2.3f}s/img).' '\t[object] Avg: {2:2.2f} Boxes/im, Top-64 recall: {3:2.3f} ({4:d}/{5:d})'.format( i+1, batch_time.avg, box_num/float(i+1), correct_cnt/float(total_cnt)*100, correct_cnt, total_cnt, len(test_loader))) recall = correct_cnt/float(total_cnt) print '====== Done Testing ====' return recall
def test(test_loader, net, top_Ns): """test the network, print the number of correct predictions Args: test_loader: torch.utils.data.DataLoader(train_set) net: Hierarchical_Descriptive_Model() top_Ns: [50, 100] Res: Recall: rel_cnt_correct / rel_cnt """ global args print '========== Testing =======' net.eval() rel_cnt = 0. rel_cnt_correct = np.zeros(len(top_Ns)) precision_correct, precision_total, recall_correct, recall_total = 0, 0, 0, 0 batch_time = network.AverageMeter() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(test_loader): # Forward pass total_cnt_t, rel_cnt_correct_t = net.evaluate( im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0], top_Ns=top_Ns, nms=True) rel_cnt += total_cnt_t rel_cnt_correct += rel_cnt_correct_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % 500 == 0 and i > 0: for idx, top_N in enumerate(top_Ns): print '[%d/%d][Evaluation] Top-%d Recall: %2.3f%%' % ( i + 1, len(test_loader), top_N, rel_cnt_correct[idx] / float(rel_cnt) * 100) recall = rel_cnt_correct / rel_cnt print '====== Done Testing ====' return recall
def test(test_loader, net, top_Ns): global args print '========== Testing =======' net.eval() # For efficiency inference languge_state = net.use_language_loss region_reg_state = net.use_region_reg net.use_language_loss = False net.use_region_reg = False rel_cnt = 0. rel_cnt_correct = np.zeros(len(top_Ns)) batch_time = network.AverageMeter() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(test_loader): # Forward pass total_cnt_t, rel_cnt_correct_t = net.evaluate( im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_regions.numpy()[0], top_Ns=top_Ns, nms=True) rel_cnt += total_cnt_t rel_cnt_correct += rel_cnt_correct_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % 500 == 0 and i > 0: for idx, top_N in enumerate(top_Ns): print '[%d/%d][Evaluation] Top-%d Recall: %2.3f%%' % ( i + 1, len(test_loader), top_N, rel_cnt_correct[idx] / float(rel_cnt) * 100) recall = rel_cnt_correct / rel_cnt print '====== Done Testing ====' # Restore the related states net.use_language_loss = languge_state net.use_region_reg = region_reg_state return recall
def test(test_loader, target_net): box_num = 0 correct_cnt, total_cnt = 0, 0 precision_correct, precision_total, recall_correct, recall_total = 0, 0, 0, 0 print '========== Testing =======' target_net.eval() batch_time = network.AverageMeter() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(test_loader): # Forward pass object_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy())[1] box_num += object_rois.size(0) # all the objects found by RPN # check RPN precision and recall # precision = correct objects/ all the objects found # recall = correct objects/ gt_objects precision_correct_t, precision_total_t, recall_correct_t, recall_total_t = checker(object_rois, gt_objects[0].numpy(), 0.3) precision_correct += precision_correct_t precision_total += precision_total_t recall_correct += recall_correct_t recall_total += recall_total_t correct_cnt_t, total_cnt_t = check_recall(object_rois, gt_objects[0].numpy(), 50) correct_cnt += correct_cnt_t total_cnt += total_cnt_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % 100 == 0 and i > 0: print('[{0}/{6}] Time: {1:2.3f}s/img).' '\t[object] Avg: {2:2.2f} Boxes/im, Top-50 recall: {3:2.3f} ({4:d}/{5:d})'.format( i + 1, batch_time.avg, box_num / float(i + 1), correct_cnt / float(total_cnt) * 100, correct_cnt, total_cnt, len(test_loader))) recall = correct_cnt / float(total_cnt) RPN_precision = precision_correct / float(precision_total) RPN_recall = recall_correct / float(recall_total) print '====== Done Testing ====' return recall, RPN_precision, RPN_recall
def train(train_loader, target_net, optimizer, epoch): global args # Overall loss logger global overall_train_loss global overall_train_rpn_loss global overall_train_region_caption_loss batch_time = network.AverageMeter() data_time = network.AverageMeter() # Total loss train_loss = network.AverageMeter() # object related loss train_obj_cls_loss = network.AverageMeter() train_obj_box_loss = network.AverageMeter() # relationship cls loss train_pred_cls_loss = network.AverageMeter() # region captioning related loss train_region_caption_loss = network.AverageMeter() train_region_box_loss = network.AverageMeter() train_region_objectiveness_loss = network.AverageMeter() # RPN loss train_rpn_loss = network.AverageMeter() # object accuracy_obj = network.AccuracyMeter() accuracy_pred = network.AccuracyMeter() accuracy_reg = network.AccuracyMeter() target_net.train() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(train_loader): # measure the data loading time data_time.update(time.time() - end) target_net(im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_regions.numpy()[0]) # Determine the loss function if args.train_all: loss = target_net.loss + target_net.rpn.loss elif args.finetune_language_model: loss = target_net.loss_region_box + target_net.region_caption_loss else: loss = target_net.loss loss += target_net.objectiveness_loss train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0)) train_obj_cls_loss.update( target_net.cross_entropy_object.data.cpu().numpy()[0], im_data.size(0)) train_obj_box_loss.update( target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0)) train_pred_cls_loss.update( target_net.cross_entropy_predicate.data.cpu().numpy()[0], im_data.size(0)) train_region_caption_loss.update( target_net.region_caption_loss.data.cpu().numpy()[0], im_data.size(0)) train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0)) overall_train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0)) overall_train_rpn_loss.update( target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0)) overall_train_region_caption_loss.update( target_net.region_caption_loss.data.cpu().numpy()[0], im_data.size(0)) accuracy_obj.update(target_net.tp, target_net.tf, target_net.fg_cnt, target_net.bg_cnt) accuracy_pred.update(target_net.tp_pred, target_net.tf_pred, target_net.fg_cnt_pred, target_net.bg_cnt_pred) accuracy_reg.update(target_net.tp_reg, target_net.tf_reg, target_net.fg_cnt_reg, target_net.bg_cnt_reg) if args.region_bbox_reg: train_region_box_loss.update( target_net.loss_region_box.data.cpu().numpy()[0], im_data.size(0)) train_region_objectiveness_loss.update( target_net.objectiveness_loss.data.cpu().numpy()[0], im_data.size(0)) optimizer.zero_grad() loss.backward() if args.enable_clip_gradient: network.clip_gradient(target_net, 10.) optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() # Logging the training loss if (i + 1) % args.log_interval == 0: print( 'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n' '\tBatch_Time: {batch_time.avg: .3f}s\t' 'FRCNN Loss: {loss.avg: .4f}\t' 'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch, i + 1, len(train_loader), batch_time=batch_time, lr=args.lr, loss=train_loss, rpn_loss=train_rpn_loss, solver=args.solver)) print('\t[Loss]\tobj_cls_loss: %.4f\tobj_box_loss: %.4f' % (train_obj_cls_loss.avg, train_obj_box_loss.avg)), print('\tpred_cls_loss: %.4f,' % (train_pred_cls_loss.avg)), if not args.disable_language_model: print('\tcaption_loss: %.4f,' % (train_region_caption_loss.avg)), if args.region_bbox_reg: print('\tregion_box_loss: %.4f, ' % (train_region_box_loss.avg)), print('\tregion_objectness_loss: %.4f' % (train_region_objectiveness_loss.avg)), print('\n\t[object]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' % (accuracy_obj.ture_pos * 100., accuracy_obj.true_neg * 100., accuracy_obj.foreground, accuracy_obj.background)) print('\t[predicate]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' % (accuracy_pred.ture_pos * 100., accuracy_pred.true_neg * 100., accuracy_pred.foreground, accuracy_pred.background)) print('\t[region]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' % (accuracy_reg.ture_pos * 100., accuracy_reg.true_neg * 100., accuracy_reg.foreground, accuracy_reg.background)) # logging to tensor board log_value('FRCNN loss', overall_train_loss.avg, overall_train_loss.count) log_value('RPN_loss loss', overall_train_rpn_loss.avg, overall_train_rpn_loss.count) log_value('caption loss', overall_train_region_caption_loss.avg, overall_train_region_caption_loss.count)
action='store_true', help='Set to disable the update of other parameters') parser.add_argument( '--optimizer', type=int, default=0, help= 'which optimizer used for optimize language model [0: SGD | 1: Adam | 2: Adagrad]' ) parser.add_argument('--evaluate', action='store_true', help='Only use the testing mode') args = parser.parse_args() # Overall loss logger overall_train_loss = network.AverageMeter() overall_train_rpn_loss = network.AverageMeter() overall_train_region_caption_loss = network.AverageMeter() optimizer_select = 0 def main(): global args, optimizer_select # To set the model name automatically print args lr = args.lr args = get_model_name(args) print 'Model name: {}'.format(args.model_name) # To set the random seed
def test(test_loader, target_net): box_num = np.array([0, 0]) correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0]) print '========== Testing =======' target_net.eval() batch_time = network.AverageMeter() # cover_cnt = 0 # # cover_gt_cnt = 0 # fg_cover = 0 # fg_object = 0 # cover_gt = 0 # object_gt = 0 # num = 0 end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_boxes_relationship) in enumerate(test_loader): # num += 1 # if num > 320: # break correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0]) # Forward pass object_rois, relationship_rois, scores_object, scores_relationship = target_net( im_data, im_info.numpy(), gt_objects.numpy()[0], gt_boxes_relationship.numpy()[0])[1:] # TODO: add rules # subject_id, object_id, relationship_cover = compare_rel_rois( # object_rois, relationship_rois, scores_object, scores_relationship, topN_covers=2048, thresh=0.5) # # cover_gt_num = check_recall(relationship_cover, gt_boxes_relationship[0].numpy(), # top_N=relationship_cover.size()[0]) # cover_cnt += cover_gt_num[0] # cover_obj_check = check_obj_rel_recall(gt_objects[0].numpy(), gt_relationships[0].numpy(), # gt_boxes_relationship[0].numpy(), relationship_cover, # subject_id.cpu().numpy(), object_id.cpu().numpy(), # object_rois, cover_thresh=0.4, object_thresh=0.4, log=num) # cover_gt_cnt += cover_obj_check[0] # fg_cover += cover_obj_check[1] # fg_object += cover_obj_check[2] # cover_gt += cover_obj_check[3] # object_gt += cover_obj_check[4] # box_num[0] += object_rois.size(0) box_num[1] += relationship_rois.size(0) correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects[0].numpy(), 256, thresh=0.5) correct_cnt_t[1], total_cnt_t[1] = check_recall( relationship_rois, gt_boxes_relationship[0].numpy(), 256, thresh=0.5) correct_cnt += correct_cnt_t total_cnt += total_cnt_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % 100 == 0 and i > 0: print( '[{0}/{10}] Time: {1:2.3f}s/img).' '\t[object] Avg: {2:2.2f} Boxes/im, Top-256 recall: {3:2.3f} ({4:d}/{5:d})' '\t[relationship] Avg: {6:2.2f} Boxes/im, Top-256 recall: {7:2.3f} ({8:d}/{9:d})' .format(i + 1, batch_time.avg, box_num[0] / float(i + 1), correct_cnt[0] / float(total_cnt[0]) * 100, correct_cnt[0], total_cnt[0], box_num[1] / float(i + 1), correct_cnt[1] / float(total_cnt[1]) * 100, correct_cnt[1], total_cnt[1], len(test_loader))) # print('relationship_cover number: {0}' # '\tcover vs gt_relationship_boxes average recall: {1:.3f}' # '\tcover & sub & obj vs gt_relationship_boxes average recall: {2:.3f}').format( # relationship_cover.size()[0], cover_cnt/float(total_cnt[1])*100, cover_gt_cnt/float(total_cnt[1])*100) # print('average fg_cover: {0:.2f}' # '\taverage fg_object: {1:.2f}' # '\taverage cover_gt: {2:.2f}' # '\taverage object_gt: {3:.2f}').format( # fg_cover / float(i), fg_object / float(i), cover_gt / float(i), object_gt / float(i)) recall = correct_cnt / total_cnt.astype(np.float) print '====== Done Testing ====' return recall
def test(test_loader, target_net): box_num = np.array([0, 0]) correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0]) print '========== Testing =======' target_net.eval() batch_time = network.AverageMeter() cover_cnt = 0 cover_gt_cnt = 0 fg_cover = 0 fg_object = 0 cover_gt = 0 object_gt = 0 num = 0 end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_boxes_relationship) in enumerate(test_loader): # num += 1 # if num > 20: # break correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0]) # Forward pass object_rois, relationship_rois, scores_object, scores_relationship = target_net( im_data, im_info.numpy(), gt_objects.numpy(), gt_boxes_relationship.numpy())[1:] # TODO: add rules # img_shape = im_info[0][:2] # object_rois = object_rois[:, 1:5] # relationship_rois = enlarge_rois_clip(relationship_rois[:, 1:5], 1.2, img_shape) # obj_in_predicate(object_rois, relationship_rois, 9) # subject_id, object_id, relationship_cover: Variable # ipdb.set_trace() subject_id, object_id, relationship_cover = compare_rel_rois( object_rois.data.cpu().numpy(), relationship_rois.data.cpu().numpy(), scores_object, scores_relationship, topN_obj=object_rois.size(0), topN_rel=relationship_rois.size(0), obj_rel_thresh=0.6, max_objects=18, topN_covers=2016, cover_thresh=0.6) cover_obj_check = check_obj_rel_recall( gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_boxes_relationship.numpy()[0], relationship_cover, object_rois.data.cpu().numpy(), # all_rois_phrase, all_rois, subject_id, object_id, # subject_inds, object_inds, cover_thresh=0.5, object_thresh=0.5, log=False) cover_gt_cnt += cover_obj_check[0] fg_cover += cover_obj_check[1] fg_object += cover_obj_check[2] cover_gt += cover_obj_check[3] object_gt += cover_obj_check[4] # print('relationship_cover size', relationship_cover.size()) # unique_obj = np.unique(np.append(subject_id.cpu().numpy(), object_id.cpu().numpy())) # print('max', np.max(unique_obj)) # print(unique_obj.size) cover_gt_num = check_recall(relationship_rois, gt_boxes_relationship.numpy()[0], top_N=relationship_rois.size(0), thresh=0.5) cover_cnt += cover_gt_num[0] # all_rois = object_rois.data.cpu().numpy() # zeros = np.zeros((gt_objects.numpy().shape[1], 1), dtype=gt_objects.numpy().dtype) # # add gt_obj to predict_rois # all_rois = np.vstack( # (all_rois, np.hstack((zeros, gt_objects.numpy()[0][:, :4]))) # ) # all_rois_phrase = relationship_rois.data.cpu().numpy() # zeros = np.zeros((gt_boxes_relationship.numpy().shape[1], 1), dtype=gt_boxes_relationship.numpy()[0].dtype) # all_rois_phrase = np.vstack( # (all_rois_phrase, np.hstack((zeros, gt_boxes_relationship.numpy()[0][:, :4]))) # ) # scores_object = np.append(scores_object, np.ones(gt_objects.size()[1], dtype=scores_object.dtype)) # scores_relationship = np.append(scores_relationship, np.ones(gt_boxes_relationship.size()[1], dtype=scores_relationship.dtype)) # subject_id, object_id, relationship_cover = compare_rel_rois( # all_rois, all_rois_phrase, scores_object, scores_relationship, # topN_obj=all_rois.shape[0], topN_rel=all_rois_phrase.shape[0], # obj_rel_thresh=0.6, max_objects=18, topN_covers=2048, cover_thresh=0.6) # all_rois_phrase_all = relationship_cover # zeros = np.zeros((gt_boxes_relationship.numpy().shape[1], 1), dtype=gt_boxes_relationship.numpy()[0].dtype) # all_rois_phrase_all = np.vstack( # (all_rois_phrase_all, np.hstack((zeros, gt_boxes_relationship.numpy()[0][:, :4]))) # ) # gt_rel_sub_idx, gt_rel_obj_idx = np.where(gt_relationships.numpy()[0] > 0) # gt_rel_sub_idx, gt_rel_obj_idx = gt_rel_sub_idx + object_rois.size()[0], gt_rel_obj_idx + object_rois.size()[0] # subject_inds = np.append(subject_id, gt_rel_sub_idx) # object_inds = np.append(object_id, gt_rel_obj_idx) # test # object_labels, object_rois_pro, bbox_targets_object, bbox_inside_weights_object, bbox_outside_weights_object, \ # region_labels, region_rois_pro, bbox_targets_region, bbox_inside_weights_region, bbox_outside_weights_region, \ # mat_object = proposal_target_layer(object_rois.data.cpu().numpy(), relationship_cover.data.cpu().numpy(), # subject_id.cpu().numpy(), object_id.cpu().numpy(), # gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_boxes_relationship.numpy()[0], # n_classes_obj=151, n_classes_pred=51, is_training=False, graph_generation=False) # print('object_labels', object_labels[:20]) # print(object_labels.shape) # print('bbox_targets_object', bbox_targets_object[:10, :60]) # print(bbox_targets_object.shape) # print('bbox_inside_weights_object', bbox_inside_weights_object[:10, :60]) # print(bbox_inside_weights_object.shape) # print('region \n') # print('region_labels', region_labels[:20]) # print(region_labels.shape) # print('region_rois_pro', region_rois_pro[:10]) # print(region_rois_pro.shape) # print('bbox_targets_region', bbox_targets_region[:10]) # print(bbox_targets_region.shape) # print('bbox_inside_weights_region', bbox_inside_weights_region[:10, :60]) # print(bbox_inside_weights_region.shape) # print('object_rois_pro', object_rois_pro[:10]) # print('object_rois_pro shape', object_rois_pro.shape) # print('region_rois_pro', region_rois_pro[:10]) # print(region_rois_pro.shape) # print('mat \n') # print(mat_object[:32, 0, :32]) # print(mat_object.shape) # print(mat_object[:32, 0, :32].T) box_num[0] += object_rois.size(0) box_num[1] += relationship_rois.size(0) correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects.numpy()[0], 200, thresh=0.5) correct_cnt_t[1], total_cnt_t[1] = check_recall( relationship_rois, gt_boxes_relationship.numpy()[0], 128, thresh=0.6) correct_cnt += correct_cnt_t total_cnt += total_cnt_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % 100 == 0 and i > 0: print( '([{0}/{10}] Time: {1:2.3f}s/img).\n' '[object] Avg: {2:2.2f} Boxes/im, Top-256 recall: {3:2.3f} ({4:d}/{5:d})\n' '[relationship] Avg: {6:2.2f} Boxes/im, Top-128 recall: {7:2.3f} ({8:d}/{9:d})' .format(i + 1, batch_time.avg, box_num[0] / float(i + 1), correct_cnt[0] / float(total_cnt[0]) * 100, correct_cnt[0], total_cnt[0], box_num[1] / float(i + 1), correct_cnt[1] / float(total_cnt[1]) * 100, correct_cnt[1], total_cnt[1], len(test_loader))) print( '[relationship_cover number]: {0}\n' '[cover vs gt_relationship_boxes average recall]: {1:.3f}\n' '[cover & sub & obj vs gt_relationship_boxes average recall]: {2:.3f}' ).format(relationship_cover.shape[0], cover_cnt / float(total_cnt[1]) * 100, cover_gt_cnt / float(total_cnt[1]) * 100) print('average fg_cover: {0:.2f}' '\taverage fg_object: {1:.2f}' '\taverage cover_gt: {2:.2f}' '\taverage object_gt: {3:.2f}').format( fg_cover / float(i), fg_object / float(i), cover_gt / float(i), object_gt / float(i)) recall = correct_cnt / total_cnt.astype(np.float) print '====== Done Testing ====' return recall
def train(train_loader, target_net, optimizer, epoch): global args # Overall loss logger # global overall_train_loss # global overall_train_rpn_loss # global overall_train_region_caption_loss batch_time = network.AverageMeter() data_time = network.AverageMeter() # Total loss train_loss = network.AverageMeter() train_rpn_loss = network.AverageMeter() # object related loss train_rcnn_loss = network.AverageMeter() # pre_mps_obj_cls_loss train_post_mps_obj_cls_loss = network.AverageMeter() # train_obj_box_loss = network.AverageMeter() # relationship cls loss train_pre_mps_pred_cls_loss = network.AverageMeter() train_post_mps_pred_cls_loss = network.AverageMeter() # train_pred_box_loss = network.AverageMeter() # accuracy accuracy_obj_pre_mps = network.AccuracyMeter() accuracy_pred_pre_mps = network.AccuracyMeter() accuracy_obj_post_mps = network.AccuracyMeter() accuracy_pred_post_mps = network.AccuracyMeter() target_net.train() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships) in enumerate(train_loader): data_time.update(time.time() - end) t0 = time.time() target_net(im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0]) if TIME_IT: t1 = time.time() print('forward time %.3fs') % (t1 - t0) # Determine the loss function if args.train_all: loss = target_net.loss + target_net.rcnn.loss + target_net.rcnn.rpn.loss else: loss = target_net.loss train_loss.update(loss.data.cpu().numpy()[0], im_data.size(0)) train_rcnn_loss.update(target_net.rcnn.loss.data.cpu().numpy()[0], im_data.size(0)) train_post_mps_obj_cls_loss.update( target_net.post_mps_cross_entropy_object.data.cpu().numpy()[0], im_data.size(0)) # train_obj_box_loss.update(target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0)) train_pre_mps_pred_cls_loss.update( target_net.pre_mps_cross_entropy_predicate.data.cpu().numpy()[0], im_data.size(0)) train_post_mps_pred_cls_loss.update( target_net.post_mps_cross_entropy_predicate.data.cpu().numpy()[0], im_data.size(0)) train_rpn_loss.update(target_net.rcnn.rpn.loss.data.cpu().numpy()[0], im_data.size(0)) # overall_train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0)) # overall_train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0)) accuracy_obj_pre_mps.update(target_net.pre_mps_tp_obj, target_net.pre_mps_tf_obj, target_net.pre_mps_fg_cnt_obj, target_net.pre_mps_bg_cnt_obj) accuracy_pred_pre_mps.update(target_net.pre_mps_tp_pred, target_net.pre_mps_tf_pred, target_net.pre_mps_fg_cnt_pred, target_net.pre_mps_bg_cnt_pred) accuracy_obj_post_mps.update(target_net.post_mps_tp_obj, target_net.post_mps_tf_obj, target_net.post_mps_fg_cnt_obj, target_net.post_mps_bg_cnt_obj) accuracy_pred_post_mps.update(target_net.post_mps_tp_pred, target_net.post_mps_tf_pred, target_net.post_mps_fg_cnt_pred, target_net.post_mps_bg_cnt_pred) t2 = time.time() optimizer.zero_grad() loss.backward() if args.enable_clip_gradient: network.clip_gradient(target_net, 10.) optimizer.step() if TIME_IT: t3 = time.time() print('backward time %.3fs') % (t3 - t2) # measure elapsed time batch_time.update(time.time() - end) end = time.time() # Logging the training loss if (i + 1) % args.log_interval == 0: print( 'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n' '\tBatch_Time: {batch_time.avg: .3f}s\t' 'TOTAL Loss: {loss.avg: .4f}\t' 'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch, i + 1, len(train_loader), batch_time=batch_time, lr=args.lr, loss=train_loss, rpn_loss=train_rpn_loss, solver=args.solver)) print( '[pre mps][Loss]\tRCNN_loss: %.4f\t pre_mps_pred_cls_loss: %.4f' % (train_rcnn_loss.avg, train_pre_mps_pred_cls_loss.avg)) print( '[Accuracy]\t pre_mps_tp: %.2f, \tpre_mps_tf: %.2f, \tfg/bg=(%d/%d)' % (accuracy_obj_pre_mps.ture_pos * 100., accuracy_obj_pre_mps.true_neg * 100., accuracy_obj_pre_mps.foreground, accuracy_obj_pre_mps.background)) print( '[post mps][Loss]\tpost_mps_obj_cls_loss: %.4f\t post_mps_pred_cls_loss: %.4f' % (train_post_mps_obj_cls_loss.avg, train_post_mps_pred_cls_loss.avg)) print( '[Accuracy]\t post_mps_tp: %.2f, \tpost_mps_tf: %.2f, \tfg/bg=(%d/%d)\n' % (accuracy_obj_post_mps.ture_pos * 100., accuracy_obj_post_mps.true_neg * 100., accuracy_obj_post_mps.foreground, accuracy_obj_post_mps.background))
type=int, default=0, help= 'which optimizer used for optimize model [0: SGD | 1: Adam | 2: Adagrad]') parser.add_argument('--evaluate', action='store_true', help='Only use the testing mode') parser.add_argument('--use_rpn_scores', action='store_true', help='Use rpn scores to help to predict') parser.add_argument('--use_gt_boxes', action='store_true', help='Use GT boxes') args = parser.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # Overall loss logger overall_train_loss = network.AverageMeter() overall_train_rpn_loss = network.AverageMeter() optimizer_select = 2 # normal_test = False def main(): global args, optimizer_select # To set the model name automatically print args lr = args.lr args = get_model_name(args) print 'Model name: {}'.format(args.model_name) # To set the random seed
def train(train_loader, target_net, optimizer, epoch): global args # ----------------------------- # initialization of loggers # ----------------------------- # Overall loss logger global overall_train_loss global overall_train_rpn_loss loss_list = list() batch_time = network.AverageMeter() data_time = network.AverageMeter() # Total loss train_loss = network.AverageMeter() # object related loss train_s_cls_loss = network.AverageMeter() train_o_cls_loss = network.AverageMeter() train_s_box_loss = network.AverageMeter() train_o_box_loss = network.AverageMeter() # relationship cls loss train_r_cls_loss = network.AverageMeter() # RPN loss train_rpn_loss = network.AverageMeter() # object accuracy_s = network.AccuracyMeter() accuracy_o = network.AccuracyMeter() accuracy_r = network.AccuracyMeter() # ----------------------------- # initialization of loggers ends # ----------------------------- target_net.train() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(train_loader): # measure the data loading time data_time.update(time.time() - end) target_net(im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0]) if target_net.bad_img_flag: target_net.bad_img_flag = False continue # Determine the loss function if args.train_all: loss = target_net.loss + target_net.rpn.loss else: loss = target_net.loss # ----------------------------- # update logger # ----------------------------- train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0)) train_s_cls_loss.update( target_net.cross_entropy_s.data.cpu().numpy()[0], im_data.size(0)) train_o_cls_loss.update( target_net.cross_entropy_o.data.cpu().numpy()[0], im_data.size(0)) train_s_box_loss.update(target_net.loss_s_box.data.cpu().numpy()[0], im_data.size(0)) train_o_box_loss.update(target_net.loss_o_box.data.cpu().numpy()[0], im_data.size(0)) train_r_cls_loss.update( target_net.cross_entropy_r.data.cpu().numpy()[0], im_data.size(0)) train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0)) overall_train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0)) overall_train_rpn_loss.update( target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0)) accuracy_s.update(target_net.tp_s, target_net.tf_s, target_net.fg_cnt_s, target_net.bg_cnt_s) accuracy_o.update(target_net.tp_o, target_net.tf_o, target_net.fg_cnt_o, target_net.bg_cnt_o) accuracy_r.update(target_net.tp_r, target_net.tf_r, target_net.fg_cnt_r, target_net.bg_cnt_r) # ----------------------------- # end # ----------------------------- optimizer.zero_grad() loss.backward() if args.enable_clip_gradient: network.clip_gradient(target_net, 10.) optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() # ----------------------------- # print loss # ----------------------------- # Logging the training loss if (i + 1) % args.log_interval == 0: loss_list.append(train_loss.avg) print( '\nEpoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n' '\tBatch_Time: {batch_time.avg: .3f}s\t' 'FRCNN Loss: {loss.avg: .4f}\t' 'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch, i + 1, len(train_loader), batch_time=batch_time, lr=args.lr, loss=train_loss, rpn_loss=train_rpn_loss, solver=args.solver)) print('\t[Loss]\ts_cls_loss: %.4f\ts_box_loss: %.4f' % (train_s_cls_loss.avg, train_s_box_loss.avg)), print('\tr_cls_loss: %.4f,' % (train_r_cls_loss.avg)), print('\t[Loss]\to_cls_loss: %.4f\to_box_loss: %.4f' % (train_o_cls_loss.avg, train_o_box_loss.avg)), print('\n\t[s]\ttp: %.2f, \tfg=%d' % (accuracy_s.ture_pos * 100., accuracy_s.foreground)) print('\t[r]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' % (accuracy_r.ture_pos * 100., accuracy_r.true_neg * 100., accuracy_r.foreground, accuracy_r.background)) print('\t[o]\ttp: %.2f, \tfg=%d' % (accuracy_o.ture_pos * 100., accuracy_o.foreground)) # ----------------------------- # end # ----------------------------- # logging to tensor board log_value('FRCNN loss', overall_train_loss.avg, overall_train_loss.count) log_value('RPN_loss loss', overall_train_rpn_loss.avg, overall_train_rpn_loss.count) return loss_list
def train(train_loader, target_net, optimizer, epoch): batch_time = network.AverageMeter() data_time = network.AverageMeter() train_loss = network.AverageMeter() train_loss_box_rpn = network.AverageMeter() train_loss_entropy_rpn = network.AverageMeter() train_loss_box_det = network.AverageMeter() train_loss_entropy_det = network.AverageMeter() tp, tf, fg, bg = 0., 0., 0, 0 target_net.train() end = time.time() for i, (im_data, im_info, gt_boxes, gt_relationships) in enumerate(train_loader): # measure the data loading time data_time.update(time.time() - end) # Forward pass target_net(im_data, im_info.numpy(), gt_boxes.numpy()[0]) # record loss loss = target_net.loss + target_net.rpn.loss # total loss train_loss.update(loss.data[0], im_data.size(0)) train_loss_box_det.update(target_net.loss_box.data[0], im_data.size(0)) train_loss_entropy_det.update(target_net.cross_entropy.data[0], im_data.size(0)) train_loss_box_rpn.update(target_net.rpn.loss_box.data[0], im_data.size(0)) train_loss_entropy_rpn.update(target_net.rpn.cross_entropy.data[0], im_data.size(0)) tp += float(target_net.tp) tf += float(target_net.tf) fg += target_net.fg_cnt bg += target_net.bg_cnt # backward optimizer.zero_grad() loss.backward() if args.clip_gradient: network.clip_gradient(target_net, 10.) optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if (i + 1) % args.log_interval == 0: print( 'Epoch: [{0}][{1}/{2}]\t' 'Batch_Time: {batch_time.avg:.3f}s\t' 'lr: {lr: f}\t' 'Loss: {loss.avg:.4f}\n' '\t[rpn]: ' 'cls_loss_rpn: {cls_loss_rpn.avg:.3f}\t' 'reg_loss_rpn: {reg_loss_rpn.avg:.3f}\n' '\t[rcnn]: ' 'cls_loss_rcnn: {cls_loss_rcnn.avg:.3f}\t' 'reg_loss_rcnn: {reg_loss_rcnn.avg:.3f}'.format( epoch, i + 1, len(train_loader), batch_time=batch_time, lr=args.lr, data_time=data_time, loss=train_loss, cls_loss_rpn=train_loss_entropy_rpn, cls_loss_rcnn=train_loss_entropy_det, reg_loss_rpn=train_loss_box_rpn, reg_loss_rcnn=train_loss_box_det)) print('\tTP: %.2f%%, TF: %.2f%%, fg/bg=(%d/%d)' % (tp / fg * 100., tf / bg * 100., fg, bg))
def test(test_loader, target_net): box_num = np.array([0, 0]) correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0]) print '========== Testing =======' target_net.eval() batch_time = network.AverageMeter() cover_cnt = 0 cnt_gt = 0 object_cnt = 0 num = 0 end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_boxes_relationship) in enumerate(test_loader): # num += 1 # if num > 320: # break correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0]) # Forward pass object_rois, relationship_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy(), gt_boxes_relationship.numpy())[1:] # all_rois = object_rois.data.cpu().numpy() # zeros = np.zeros((gt_objects.numpy().shape[1], 1), dtype=gt_objects.numpy().dtype) # # add gt_obj to predict_rois # all_rois = np.vstack( # (all_rois, np.hstack((zeros, gt_objects.numpy()[0][:, :4]))) # ) # object_rois = network.np_to_variable(all_rois, is_cuda=True) # TODO: add rules # img_shape = im_info[0][:2] # object_rois = object_rois[:, 1:5] # relationship_rois = enlarge_rois_clip(relationship_rois[:, 1:5], 1.2, img_shape) # obj_in_predicate(object_rois, relationship_rois, 9) object_cnt_t, cover_cnt_t, cnt_gt_t = check_msdn_rpn( object_rois.data.cpu().numpy(), gt_objects.numpy()[0], gt_relationships.numpy()[0]) object_cnt += object_cnt_t cover_cnt += cover_cnt_t cnt_gt += cnt_gt_t # gt_num = gt_boxes_relationship[0].size()[0] box_num[0] += object_rois.size(0) box_num[1] += relationship_rois.size(0) correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects[0].numpy(), object_rois.size(0), thres=0.5) correct_cnt_t[1], total_cnt_t[1] = check_recall(relationship_rois, gt_boxes_relationship[0].numpy(), 64, thres=0.5) correct_cnt += correct_cnt_t total_cnt += total_cnt_t batch_time.update(time.time()-end) end = time.time() if (i+1)%100 == 0 and i > 0: print('[{0}/{10}] Time: {1:2.3f}s/img).' '\t[object] Avg: {2:2.2f} Boxes/im, Top-2000 recall: {3:2.3f} ({4:d}/{5:d})' '\t[relationship] Avg: {6:2.2f} Boxes/im, Top-500 recall: {7:2.3f} ({8:d}/{9:d})'.format( i+1, batch_time.avg, box_num[0]/float(i+1), correct_cnt[0]/float(total_cnt[0])*100, correct_cnt[0], total_cnt[0], box_num[1]/float(i+1), correct_cnt[1]/float(total_cnt[1])*100, correct_cnt[1], total_cnt[1], len(test_loader))) print('fg object number:{0:d}' '\tcover number:{1:d}' '\tcover & sub & obj vs gt_relationship_boxes average recall: {2:.3f}').format( object_cnt/i, cover_cnt / i, cover_cnt / float(cnt_gt) * 100) # print('\n') recall = correct_cnt/total_cnt.astype(np.float) print '====== Done Testing ====' return recall
def test(test_loader, net, top_Ns, object_classes): global args print '========== Testing ========' net.eval() # For efficiency inference # net.use_region_reg = True rel_cnt = 0. rel_cnt_correct = np.zeros(len(top_Ns)) box_num, obj_correct_cnt, obj_total_cnt = 0, 0, 0 batch_time = network.AverageMeter() end = time.time() all_cls_gt_num = [0] * (len(object_classes) - 1) all_cls_scores = [np.array([])] * (len(object_classes) - 1) all_cls_tp = [np.array([])] * (len(object_classes) - 1) for i, (im_data, im_info, gt_objects, gt_relationships) in enumerate(test_loader): # Forward pass # pdb.set_trace() total_cnt_t, rel_cnt_correct_t, object_rois, classes_scores, classes_tf, classes_gt_num = net.evaluate( im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0], top_Ns=top_Ns, use_gt_boxes=args.use_gt_boxes, nms=False, nms_thresh=0.4, thresh=0.5, use_rpn_scores=args.use_rpn_scores) box_num += object_rois.size(0) obj_correct_cnt_t, obj_total_cnt_t = check_recall( object_rois / im_info[0][2], gt_objects.numpy()[0], 64) obj_correct_cnt += obj_correct_cnt_t obj_total_cnt += obj_total_cnt_t rel_cnt += total_cnt_t rel_cnt_correct += rel_cnt_correct_t for j in range(len(object_classes) - 1): all_cls_scores[j] = np.append(all_cls_scores[j], classes_scores[j]) all_cls_tp[j] = np.append(all_cls_tp[j], classes_tf[j]) all_cls_gt_num[j] += classes_gt_num[j] batch_time.update(time.time() - end) end = time.time() if (i + 1) % args.log_interval == 0 and i > 0: print( 'Time: {0:2.3f}s/img.' '\t[object] Avg: {1:2.2f} Boxes/im, Top-64 recall: {2:2.3f} ({3:d}/{4:d})' .format(batch_time.avg, box_num / float(i + 1), obj_correct_cnt / float(obj_total_cnt) * 100, obj_correct_cnt, obj_total_cnt)) for idx, top_N in enumerate(top_Ns): print '[%d/%d][Evaluation] Top-%d Recall: %2.3f%%' % ( i + 1, len(test_loader), top_N, rel_cnt_correct[idx] / float(rel_cnt) * 100) all_aps = [] for k, cls in enumerate(object_classes[1:]): # sort scores of all images cls_ap = cls_eval(all_cls_scores[k], all_cls_tp[k], all_cls_gt_num[k]) all_aps += [cls_ap] print('AP for {} = {:.4f}'.format(cls, cls_ap)) print('Mean AP = {:.4f}'.format(np.mean(all_aps))) recall = rel_cnt_correct / rel_cnt print '====== Done Testing ====' return recall
def test(test_loader, net, top_Ns): global args print('========== Testing =======') net.eval() # For efficiency inference languge_state = net.use_language_loss region_reg_state = net.use_region_reg net.use_language_loss = False net.use_region_reg = False rel_cnt = 0. rel_cnt_correct = np.zeros(len(top_Ns)) batch_time = network.AverageMeter() end = time.time() for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(tqdm(test_loader)): # Forward pass if args.mode == 'sgdet': total_cnt_t, rel_cnt_correct_t = net.evaluate( im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_regions.numpy()[0], top_Ns=top_Ns, nms=True, freq_baseline=args.frequency_baseline) elif args.mode == 'sgcls': total_cnt_t, rel_cnt_correct_t = net.evaluate( im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_regions.numpy()[0], top_Ns=top_Ns, nms=False, use_gt_boxes=True, use_gt_regions=True, freq_baseline=args.frequency_baseline) elif args.mode == 'predcls': total_cnt_t, rel_cnt_correct_t = net.evaluate( im_data, im_info, gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_regions.numpy()[0], top_Ns=top_Ns, nms=False, use_gt_boxes=True, use_gt_regions=False, only_predicate=True, freq_baseline=args.frequency_baseline, ) rel_cnt += total_cnt_t rel_cnt_correct += rel_cnt_correct_t batch_time.update(time.time() - end) end = time.time() if (i + 1) % 100 == 0 and i > 0: for idx, top_N in enumerate(top_Ns): print('\n[%d/%d][Evaluation] Top-%d Recall: %2.3f%%\n' % (i + 1, len(test_loader), top_N, rel_cnt_correct[idx] / float(rel_cnt) * 100), flush=True) recall = rel_cnt_correct / rel_cnt print('====== Done Testing ====', flush=True) # Restore the related states net.use_language_loss = languge_state net.use_region_reg = region_reg_state return recall