def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print('========== Testing =======')
target_net.eval()
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(test_loader):
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, region_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy(), gt_regions.numpy())[1:]
box_num[0] += object_rois.size(0)
box_num[1] += region_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects[0].numpy(), 50)
correct_cnt_t[1], total_cnt_t[1] = check_recall(region_rois, gt_regions[0].numpy(), 50)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
print('[{0}/{10}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-50 recall: {3:2.3f} ({4:d}/{5:d})'
'\t[region] Avg: {6:2.2f} Boxes/im, Top-50 recall: {7:2.3f} ({8:d}/{9:d})'.format(
i + 1, batch_time.avg,
box_num[0] / float(i + 1), correct_cnt[0] / float(total_cnt[0]) * 100, correct_cnt[0], total_cnt[0],
box_num[1] / float(i + 1), correct_cnt[1] / float(total_cnt[1]) * 100, correct_cnt[1], total_cnt[1],
len(test_loader)))
recall = correct_cnt / total_cnt.astype(np.float)
print('====== Done Testing ====')
return recall
def test(test_loader, target_net):
box_num, correct_cnt, total_cnt = 0, 0, 0
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_boxes, gt_relationship) in enumerate(test_loader):
# Forward pass
features, object_rois, scores = target_net.rpn(im_data, im_info.numpy(), gt_boxes.numpy()[0])
box_num += object_rois.size(0)
correct_cnt_t, total_cnt_t = check_recall(object_rois, gt_boxes.numpy()[0], 64)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time()-end)
end = time.time()
if (i+1)%100 == 0 and i > 0:
print('[{0}/{6}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-64 recall: {3:2.3f} ({4:d}/{5:d})'.format(
i+1, batch_time.avg, box_num/float(i+1), correct_cnt/float(total_cnt)*100,
correct_cnt, total_cnt, len(test_loader)))
recall = correct_cnt/float(total_cnt)
print '====== Done Testing ===='
return recall
def evaluate(test_loader,
target_net,
object_classes,
score_thresh=0.00,
overlap_thresh=0.5,
nms_thresh=0.5):
print(object_classes)
box_num, correct_cnt, total_cnt = 0, 0, 0
# ipdb.set_trace()
# store cls_scores, cls_tp, cls_gt_num of each cls and every image
all_cls_gt_num = [0] * (len(object_classes) - 1)
all_cls_scores = [np.array([])] * (len(object_classes) - 1)
all_cls_tp = [np.array([])] * (len(object_classes) - 1)
print('========== Evaluate =======')
target_net.eval()
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_boxes,
gt_relationships) in enumerate(test_loader):
# get every class scores, tf array and gt number
classes_scores, classes_tf, classes_gt_num, object_rois = \
image_eval(target_net, im_data, im_info, gt_boxes.numpy()[0], object_classes,
max_per_image=100, score_thresh=score_thresh, overlap_thresh=overlap_thresh, nms_thresh=nms_thresh)
for j in range(len(object_classes) - 1):
all_cls_scores[j] = np.append(all_cls_scores[j], classes_scores[j])
all_cls_tp[j] = np.append(all_cls_tp[j], classes_tf[j])
all_cls_gt_num[j] += classes_gt_num[j]
box_num += object_rois.size(0)
correct_cnt_t, total_cnt_t = check_recall(object_rois,
gt_boxes.numpy()[0],
64,
thresh=0.5)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0:
print(
'[{0}/{6}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-64 recall: {3:2.3f} ({4:d}/{5:d})'
.format(i + 1, batch_time.avg, box_num / float(i + 1),
correct_cnt / float(total_cnt) * 100, correct_cnt,
total_cnt, len(test_loader)))
all_aps = []
for k, cls in enumerate(object_classes[1:]):
# sort scores of all images
cls_ap = cls_eval(all_cls_scores[k], all_cls_tp[k], all_cls_gt_num[k])
all_aps += [cls_ap]
print('AP for {} = {:.4f}'.format(cls, cls_ap))
mean_ap = np.mean(all_aps)
print('Mean AP = {:.4f}'.format(mean_ap))
print('====== Done Testing ====')
return mean_ap
def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
cover_cnt = 0
cnt_gt = 0
object_cnt = 0
num = 0
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships, gt_boxes_relationship) in enumerate(test_loader):
# num += 1
# if num > 320:
# break
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, relationship_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy(),
gt_boxes_relationship.numpy())[1:]
# all_rois = object_rois.data.cpu().numpy()
# zeros = np.zeros((gt_objects.numpy().shape[1], 1), dtype=gt_objects.numpy().dtype)
# # add gt_obj to predict_rois
# all_rois = np.vstack(
# (all_rois, np.hstack((zeros, gt_objects.numpy()[0][:, :4])))
# )
# object_rois = network.np_to_variable(all_rois, is_cuda=True)
# TODO: add rules
# img_shape = im_info[0][:2]
# object_rois = object_rois[:, 1:5]
# relationship_rois = enlarge_rois_clip(relationship_rois[:, 1:5], 1.2, img_shape)
# obj_in_predicate(object_rois, relationship_rois, 9)
object_cnt_t, cover_cnt_t, cnt_gt_t = check_msdn_rpn(
object_rois.data.cpu().numpy(), gt_objects.numpy()[0], gt_relationships.numpy()[0])
object_cnt += object_cnt_t
cover_cnt += cover_cnt_t
cnt_gt += cnt_gt_t
# gt_num = gt_boxes_relationship[0].size()[0]
box_num[0] += object_rois.size(0)
box_num[1] += relationship_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects[0].numpy(), object_rois.size(0), thres=0.5)
correct_cnt_t[1], total_cnt_t[1] = check_recall(relationship_rois, gt_boxes_relationship[0].numpy(), 64, thres=0.5)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time()-end)
end = time.time()
if (i+1)%100 == 0 and i > 0:
print('[{0}/{10}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-2000 recall: {3:2.3f} ({4:d}/{5:d})'
'\t[relationship] Avg: {6:2.2f} Boxes/im, Top-500 recall: {7:2.3f} ({8:d}/{9:d})'.format(
i+1, batch_time.avg,
box_num[0]/float(i+1), correct_cnt[0]/float(total_cnt[0])*100, correct_cnt[0], total_cnt[0],
box_num[1]/float(i+1), correct_cnt[1]/float(total_cnt[1])*100, correct_cnt[1], total_cnt[1],
len(test_loader)))
print('fg object number:{0:d}'
'\tcover number:{1:d}'
'\tcover & sub & obj vs gt_relationship_boxes average recall: {2:.3f}').format(
object_cnt/i, cover_cnt / i,
cover_cnt / float(cnt_gt) * 100)
# print('\n')
recall = correct_cnt/total_cnt.astype(np.float)
print '====== Done Testing ===='
return recall
def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
# cover_cnt = 0
#
# cover_gt_cnt = 0
# fg_cover = 0
# fg_object = 0
# cover_gt = 0
# object_gt = 0
# num = 0
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_boxes_relationship) in enumerate(test_loader):
# num += 1
# if num > 320:
# break
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, relationship_rois, scores_object, scores_relationship = target_net(
im_data, im_info.numpy(),
gt_objects.numpy()[0],
gt_boxes_relationship.numpy()[0])[1:]
# TODO: add rules
# subject_id, object_id, relationship_cover = compare_rel_rois(
# object_rois, relationship_rois, scores_object, scores_relationship, topN_covers=2048, thresh=0.5)
#
# cover_gt_num = check_recall(relationship_cover, gt_boxes_relationship[0].numpy(),
# top_N=relationship_cover.size()[0])
# cover_cnt += cover_gt_num[0]
# cover_obj_check = check_obj_rel_recall(gt_objects[0].numpy(), gt_relationships[0].numpy(),
# gt_boxes_relationship[0].numpy(), relationship_cover,
# subject_id.cpu().numpy(), object_id.cpu().numpy(),
# object_rois, cover_thresh=0.4, object_thresh=0.4, log=num)
# cover_gt_cnt += cover_obj_check[0]
# fg_cover += cover_obj_check[1]
# fg_object += cover_obj_check[2]
# cover_gt += cover_obj_check[3]
# object_gt += cover_obj_check[4]
#
box_num[0] += object_rois.size(0)
box_num[1] += relationship_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois,
gt_objects[0].numpy(),
256,
thresh=0.5)
correct_cnt_t[1], total_cnt_t[1] = check_recall(
relationship_rois,
gt_boxes_relationship[0].numpy(),
256,
thresh=0.5)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
print(
'[{0}/{10}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-256 recall: {3:2.3f} ({4:d}/{5:d})'
'\t[relationship] Avg: {6:2.2f} Boxes/im, Top-256 recall: {7:2.3f} ({8:d}/{9:d})'
.format(i + 1, batch_time.avg, box_num[0] / float(i + 1),
correct_cnt[0] / float(total_cnt[0]) * 100,
correct_cnt[0], total_cnt[0],
box_num[1] / float(i + 1),
correct_cnt[1] / float(total_cnt[1]) * 100,
correct_cnt[1], total_cnt[1], len(test_loader)))
# print('relationship_cover number: {0}'
# '\tcover vs gt_relationship_boxes average recall: {1:.3f}'
# '\tcover & sub & obj vs gt_relationship_boxes average recall: {2:.3f}').format(
# relationship_cover.size()[0], cover_cnt/float(total_cnt[1])*100, cover_gt_cnt/float(total_cnt[1])*100)
# print('average fg_cover: {0:.2f}'
# '\taverage fg_object: {1:.2f}'
# '\taverage cover_gt: {2:.2f}'
# '\taverage object_gt: {3:.2f}').format(
# fg_cover / float(i), fg_object / float(i), cover_gt / float(i), object_gt / float(i))
recall = correct_cnt / total_cnt.astype(np.float)
print '====== Done Testing ===='
return recall
def test(test_loader, net, top_Ns, object_classes):
global args
print '========== Testing ========'
net.eval()
# For efficiency inference
# net.use_region_reg = True
rel_cnt = 0.
rel_cnt_correct = np.zeros(len(top_Ns))
box_num, obj_correct_cnt, obj_total_cnt = 0, 0, 0
batch_time = network.AverageMeter()
end = time.time()
all_cls_gt_num = [0] * (len(object_classes) - 1)
all_cls_scores = [np.array([])] * (len(object_classes) - 1)
all_cls_tp = [np.array([])] * (len(object_classes) - 1)
for i, (im_data, im_info, gt_objects,
gt_relationships) in enumerate(test_loader):
# Forward pass
# pdb.set_trace()
total_cnt_t, rel_cnt_correct_t, object_rois, classes_scores, classes_tf, classes_gt_num = net.evaluate(
im_data,
im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
top_Ns=top_Ns,
use_gt_boxes=args.use_gt_boxes,
nms=False,
nms_thresh=0.4,
thresh=0.5,
use_rpn_scores=args.use_rpn_scores)
box_num += object_rois.size(0)
obj_correct_cnt_t, obj_total_cnt_t = check_recall(
object_rois / im_info[0][2],
gt_objects.numpy()[0], 64)
obj_correct_cnt += obj_correct_cnt_t
obj_total_cnt += obj_total_cnt_t
rel_cnt += total_cnt_t
rel_cnt_correct += rel_cnt_correct_t
for j in range(len(object_classes) - 1):
all_cls_scores[j] = np.append(all_cls_scores[j], classes_scores[j])
all_cls_tp[j] = np.append(all_cls_tp[j], classes_tf[j])
all_cls_gt_num[j] += classes_gt_num[j]
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0 and i > 0:
print(
'Time: {0:2.3f}s/img.'
'\t[object] Avg: {1:2.2f} Boxes/im, Top-64 recall: {2:2.3f} ({3:d}/{4:d})'
.format(batch_time.avg, box_num / float(i + 1),
obj_correct_cnt / float(obj_total_cnt) * 100,
obj_correct_cnt, obj_total_cnt))
for idx, top_N in enumerate(top_Ns):
print '[%d/%d][Evaluation] Top-%d Recall: %2.3f%%' % (
i + 1, len(test_loader), top_N,
rel_cnt_correct[idx] / float(rel_cnt) * 100)
all_aps = []
for k, cls in enumerate(object_classes[1:]):
# sort scores of all images
cls_ap = cls_eval(all_cls_scores[k], all_cls_tp[k], all_cls_gt_num[k])
all_aps += [cls_ap]
print('AP for {} = {:.4f}'.format(cls, cls_ap))
print('Mean AP = {:.4f}'.format(np.mean(all_aps)))
recall = rel_cnt_correct / rel_cnt
print '====== Done Testing ===='
return recall
def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
cover_cnt = 0
cover_gt_cnt = 0
fg_cover = 0
fg_object = 0
cover_gt = 0
object_gt = 0
num = 0
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_boxes_relationship) in enumerate(test_loader):
# num += 1
# if num > 20:
# break
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, relationship_rois, scores_object, scores_relationship = target_net(
im_data, im_info.numpy(), gt_objects.numpy(),
gt_boxes_relationship.numpy())[1:]
# TODO: add rules
# img_shape = im_info[0][:2]
# object_rois = object_rois[:, 1:5]
# relationship_rois = enlarge_rois_clip(relationship_rois[:, 1:5], 1.2, img_shape)
# obj_in_predicate(object_rois, relationship_rois, 9)
# subject_id, object_id, relationship_cover: Variable
# ipdb.set_trace()
subject_id, object_id, relationship_cover = compare_rel_rois(
object_rois.data.cpu().numpy(),
relationship_rois.data.cpu().numpy(),
scores_object,
scores_relationship,
topN_obj=object_rois.size(0),
topN_rel=relationship_rois.size(0),
obj_rel_thresh=0.6,
max_objects=18,
topN_covers=2016,
cover_thresh=0.6)
cover_obj_check = check_obj_rel_recall(
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_boxes_relationship.numpy()[0],
relationship_cover,
object_rois.data.cpu().numpy(),
# all_rois_phrase, all_rois,
subject_id,
object_id,
# subject_inds, object_inds,
cover_thresh=0.5,
object_thresh=0.5,
log=False)
cover_gt_cnt += cover_obj_check[0]
fg_cover += cover_obj_check[1]
fg_object += cover_obj_check[2]
cover_gt += cover_obj_check[3]
object_gt += cover_obj_check[4]
# print('relationship_cover size', relationship_cover.size())
# unique_obj = np.unique(np.append(subject_id.cpu().numpy(), object_id.cpu().numpy()))
# print('max', np.max(unique_obj))
# print(unique_obj.size)
cover_gt_num = check_recall(relationship_rois,
gt_boxes_relationship.numpy()[0],
top_N=relationship_rois.size(0),
thresh=0.5)
cover_cnt += cover_gt_num[0]
# all_rois = object_rois.data.cpu().numpy()
# zeros = np.zeros((gt_objects.numpy().shape[1], 1), dtype=gt_objects.numpy().dtype)
# # add gt_obj to predict_rois
# all_rois = np.vstack(
# (all_rois, np.hstack((zeros, gt_objects.numpy()[0][:, :4])))
# )
# all_rois_phrase = relationship_rois.data.cpu().numpy()
# zeros = np.zeros((gt_boxes_relationship.numpy().shape[1], 1), dtype=gt_boxes_relationship.numpy()[0].dtype)
# all_rois_phrase = np.vstack(
# (all_rois_phrase, np.hstack((zeros, gt_boxes_relationship.numpy()[0][:, :4])))
# )
# scores_object = np.append(scores_object, np.ones(gt_objects.size()[1], dtype=scores_object.dtype))
# scores_relationship = np.append(scores_relationship, np.ones(gt_boxes_relationship.size()[1], dtype=scores_relationship.dtype))
# subject_id, object_id, relationship_cover = compare_rel_rois(
# all_rois, all_rois_phrase, scores_object, scores_relationship,
# topN_obj=all_rois.shape[0], topN_rel=all_rois_phrase.shape[0],
# obj_rel_thresh=0.6, max_objects=18, topN_covers=2048, cover_thresh=0.6)
# all_rois_phrase_all = relationship_cover
# zeros = np.zeros((gt_boxes_relationship.numpy().shape[1], 1), dtype=gt_boxes_relationship.numpy()[0].dtype)
# all_rois_phrase_all = np.vstack(
# (all_rois_phrase_all, np.hstack((zeros, gt_boxes_relationship.numpy()[0][:, :4])))
# )
# gt_rel_sub_idx, gt_rel_obj_idx = np.where(gt_relationships.numpy()[0] > 0)
# gt_rel_sub_idx, gt_rel_obj_idx = gt_rel_sub_idx + object_rois.size()[0], gt_rel_obj_idx + object_rois.size()[0]
# subject_inds = np.append(subject_id, gt_rel_sub_idx)
# object_inds = np.append(object_id, gt_rel_obj_idx)
# test
# object_labels, object_rois_pro, bbox_targets_object, bbox_inside_weights_object, bbox_outside_weights_object, \
# region_labels, region_rois_pro, bbox_targets_region, bbox_inside_weights_region, bbox_outside_weights_region, \
# mat_object = proposal_target_layer(object_rois.data.cpu().numpy(), relationship_cover.data.cpu().numpy(),
# subject_id.cpu().numpy(), object_id.cpu().numpy(),
# gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_boxes_relationship.numpy()[0],
# n_classes_obj=151, n_classes_pred=51, is_training=False, graph_generation=False)
# print('object_labels', object_labels[:20])
# print(object_labels.shape)
# print('bbox_targets_object', bbox_targets_object[:10, :60])
# print(bbox_targets_object.shape)
# print('bbox_inside_weights_object', bbox_inside_weights_object[:10, :60])
# print(bbox_inside_weights_object.shape)
# print('region \n')
# print('region_labels', region_labels[:20])
# print(region_labels.shape)
# print('region_rois_pro', region_rois_pro[:10])
# print(region_rois_pro.shape)
# print('bbox_targets_region', bbox_targets_region[:10])
# print(bbox_targets_region.shape)
# print('bbox_inside_weights_region', bbox_inside_weights_region[:10, :60])
# print(bbox_inside_weights_region.shape)
# print('object_rois_pro', object_rois_pro[:10])
# print('object_rois_pro shape', object_rois_pro.shape)
# print('region_rois_pro', region_rois_pro[:10])
# print(region_rois_pro.shape)
# print('mat \n')
# print(mat_object[:32, 0, :32])
# print(mat_object.shape)
# print(mat_object[:32, 0, :32].T)
box_num[0] += object_rois.size(0)
box_num[1] += relationship_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois,
gt_objects.numpy()[0],
200,
thresh=0.5)
correct_cnt_t[1], total_cnt_t[1] = check_recall(
relationship_rois,
gt_boxes_relationship.numpy()[0],
128,
thresh=0.6)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
print(
'([{0}/{10}] Time: {1:2.3f}s/img).\n'
'[object] Avg: {2:2.2f} Boxes/im, Top-256 recall: {3:2.3f} ({4:d}/{5:d})\n'
'[relationship] Avg: {6:2.2f} Boxes/im, Top-128 recall: {7:2.3f} ({8:d}/{9:d})'
.format(i + 1, batch_time.avg, box_num[0] / float(i + 1),
correct_cnt[0] / float(total_cnt[0]) * 100,
correct_cnt[0], total_cnt[0],
box_num[1] / float(i + 1),
correct_cnt[1] / float(total_cnt[1]) * 100,
correct_cnt[1], total_cnt[1], len(test_loader)))
print(
'[relationship_cover number]: {0}\n'
'[cover vs gt_relationship_boxes average recall]: {1:.3f}\n'
'[cover & sub & obj vs gt_relationship_boxes average recall]: {2:.3f}'
).format(relationship_cover.shape[0],
cover_cnt / float(total_cnt[1]) * 100,
cover_gt_cnt / float(total_cnt[1]) * 100)
print('average fg_cover: {0:.2f}'
'\taverage fg_object: {1:.2f}'
'\taverage cover_gt: {2:.2f}'
'\taverage object_gt: {3:.2f}').format(
fg_cover / float(i), fg_object / float(i),
cover_gt / float(i), object_gt / float(i))
recall = correct_cnt / total_cnt.astype(np.float)
print '====== Done Testing ===='
return recall