def train(train_loader, target_net, optimizer, epoch):
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
train_loss = network.AverageMeter()
train_loss_box_rpn = network.AverageMeter()
train_loss_entropy_rpn = network.AverageMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_boxes) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
# Forward pass
target_net(im_data, im_info.numpy(), gt_boxes.numpy()[0])
# record loss
loss = target_net.loss
# total loss
train_loss.update(loss.data[0], im_data.size(0))
train_loss_box_rpn.update(target_net.loss_box.data[0], im_data.size(0))
train_loss_entropy_rpn.update(target_net.cross_entropy.data[0],
im_data.size(0))
# backward
optimizer.zero_grad()
loss.backward()
if not args.disable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}]\t'
'Batch_Time: {batch_time.avg:.3f}s\t'
'lr: {lr: f}\t'
'Loss: {loss.avg:.4f}\n'
'\t[rpn]: '
'cls_loss_rpn: {cls_loss_rpn.avg:.3f}\t'
'reg_loss_rpn: {reg_loss_rpn.avg:.3f}\n'.format(
epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
data_time=data_time,
loss=train_loss,
cls_loss_rpn=train_loss_entropy_rpn,
reg_loss_rpn=train_loss_box_rpn))
def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print('========== Testing =======')
target_net.eval()
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(test_loader):
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, region_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy(), gt_regions.numpy())[1:]
box_num[0] += object_rois.size(0)
box_num[1] += region_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects[0].numpy(), 50)
correct_cnt_t[1], total_cnt_t[1] = check_recall(region_rois, gt_regions[0].numpy(), 50)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
print('[{0}/{10}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-50 recall: {3:2.3f} ({4:d}/{5:d})'
'\t[region] Avg: {6:2.2f} Boxes/im, Top-50 recall: {7:2.3f} ({8:d}/{9:d})'.format(
i + 1, batch_time.avg,
box_num[0] / float(i + 1), correct_cnt[0] / float(total_cnt[0]) * 100, correct_cnt[0], total_cnt[0],
box_num[1] / float(i + 1), correct_cnt[1] / float(total_cnt[1]) * 100, correct_cnt[1], total_cnt[1],
len(test_loader)))
recall = correct_cnt / total_cnt.astype(np.float)
print('====== Done Testing ====')
return recall
def evaluate(test_loader,
target_net,
object_classes,
score_thresh=0.00,
overlap_thresh=0.5,
nms_thresh=0.5):
print(object_classes)
box_num, correct_cnt, total_cnt = 0, 0, 0
# ipdb.set_trace()
# store cls_scores, cls_tp, cls_gt_num of each cls and every image
all_cls_gt_num = [0] * (len(object_classes) - 1)
all_cls_scores = [np.array([])] * (len(object_classes) - 1)
all_cls_tp = [np.array([])] * (len(object_classes) - 1)
print('========== Evaluate =======')
target_net.eval()
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_boxes,
gt_relationships) in enumerate(test_loader):
# get every class scores, tf array and gt number
classes_scores, classes_tf, classes_gt_num, object_rois = \
image_eval(target_net, im_data, im_info, gt_boxes.numpy()[0], object_classes,
max_per_image=100, score_thresh=score_thresh, overlap_thresh=overlap_thresh, nms_thresh=nms_thresh)
for j in range(len(object_classes) - 1):
all_cls_scores[j] = np.append(all_cls_scores[j], classes_scores[j])
all_cls_tp[j] = np.append(all_cls_tp[j], classes_tf[j])
all_cls_gt_num[j] += classes_gt_num[j]
box_num += object_rois.size(0)
correct_cnt_t, total_cnt_t = check_recall(object_rois,
gt_boxes.numpy()[0],
64,
thresh=0.5)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0:
print(
'[{0}/{6}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-64 recall: {3:2.3f} ({4:d}/{5:d})'
.format(i + 1, batch_time.avg, box_num / float(i + 1),
correct_cnt / float(total_cnt) * 100, correct_cnt,
total_cnt, len(test_loader)))
all_aps = []
for k, cls in enumerate(object_classes[1:]):
# sort scores of all images
cls_ap = cls_eval(all_cls_scores[k], all_cls_tp[k], all_cls_gt_num[k])
all_aps += [cls_ap]
print('AP for {} = {:.4f}'.format(cls, cls_ap))
mean_ap = np.mean(all_aps)
print('Mean AP = {:.4f}'.format(mean_ap))
print('====== Done Testing ====')
return mean_ap
def test(test_loader, target_net):
box_num, correct_cnt, total_cnt = 0, 0, 0
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_boxes, gt_relationship) in enumerate(test_loader):
# Forward pass
features, object_rois, scores = target_net.rpn(im_data, im_info.numpy(), gt_boxes.numpy()[0])
box_num += object_rois.size(0)
correct_cnt_t, total_cnt_t = check_recall(object_rois, gt_boxes.numpy()[0], 64)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time()-end)
end = time.time()
if (i+1)%100 == 0 and i > 0:
print('[{0}/{6}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-64 recall: {3:2.3f} ({4:d}/{5:d})'.format(
i+1, batch_time.avg, box_num/float(i+1), correct_cnt/float(total_cnt)*100,
correct_cnt, total_cnt, len(test_loader)))
recall = correct_cnt/float(total_cnt)
print '====== Done Testing ===='
return recall
def test(test_loader, net, top_Ns):
"""test the network, print the number of correct predictions
Args:
test_loader: torch.utils.data.DataLoader(train_set)
net: Hierarchical_Descriptive_Model()
top_Ns: [50, 100]
Res:
Recall: rel_cnt_correct / rel_cnt
"""
global args
print '========== Testing ======='
net.eval()
rel_cnt = 0.
rel_cnt_correct = np.zeros(len(top_Ns))
precision_correct, precision_total, recall_correct, recall_total = 0, 0, 0, 0
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(test_loader):
# Forward pass
total_cnt_t, rel_cnt_correct_t = net.evaluate(
im_data,
im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
top_Ns=top_Ns,
nms=True)
rel_cnt += total_cnt_t
rel_cnt_correct += rel_cnt_correct_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 500 == 0 and i > 0:
for idx, top_N in enumerate(top_Ns):
print '[%d/%d][Evaluation] Top-%d Recall: %2.3f%%' % (
i + 1, len(test_loader), top_N,
rel_cnt_correct[idx] / float(rel_cnt) * 100)
recall = rel_cnt_correct / rel_cnt
print '====== Done Testing ===='
return recall
def test(test_loader, net, top_Ns):
global args
print '========== Testing ======='
net.eval()
# For efficiency inference
languge_state = net.use_language_loss
region_reg_state = net.use_region_reg
net.use_language_loss = False
net.use_region_reg = False
rel_cnt = 0.
rel_cnt_correct = np.zeros(len(top_Ns))
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(test_loader):
# Forward pass
total_cnt_t, rel_cnt_correct_t = net.evaluate(
im_data,
im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_regions.numpy()[0],
top_Ns=top_Ns,
nms=True)
rel_cnt += total_cnt_t
rel_cnt_correct += rel_cnt_correct_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 500 == 0 and i > 0:
for idx, top_N in enumerate(top_Ns):
print '[%d/%d][Evaluation] Top-%d Recall: %2.3f%%' % (
i + 1, len(test_loader), top_N,
rel_cnt_correct[idx] / float(rel_cnt) * 100)
recall = rel_cnt_correct / rel_cnt
print '====== Done Testing ===='
# Restore the related states
net.use_language_loss = languge_state
net.use_region_reg = region_reg_state
return recall
def test(test_loader, target_net):
box_num = 0
correct_cnt, total_cnt = 0, 0
precision_correct, precision_total, recall_correct, recall_total = 0, 0, 0, 0
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(test_loader):
# Forward pass
object_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy())[1]
box_num += object_rois.size(0) # all the objects found by RPN
# check RPN precision and recall
# precision = correct objects/ all the objects found # recall = correct objects/ gt_objects
precision_correct_t, precision_total_t, recall_correct_t, recall_total_t = checker(object_rois, gt_objects[0].numpy(), 0.3)
precision_correct += precision_correct_t
precision_total += precision_total_t
recall_correct += recall_correct_t
recall_total += recall_total_t
correct_cnt_t, total_cnt_t = check_recall(object_rois, gt_objects[0].numpy(), 50)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
print('[{0}/{6}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-50 recall: {3:2.3f} ({4:d}/{5:d})'.format(
i + 1, batch_time.avg,
box_num / float(i + 1), correct_cnt / float(total_cnt) * 100, correct_cnt, total_cnt,
len(test_loader)))
recall = correct_cnt / float(total_cnt)
RPN_precision = precision_correct / float(precision_total)
RPN_recall = recall_correct / float(recall_total)
print '====== Done Testing ===='
return recall, RPN_precision, RPN_recall
def train(train_loader, target_net, optimizer, epoch):
global args
# Overall loss logger
global overall_train_loss
global overall_train_rpn_loss
global overall_train_region_caption_loss
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
# object related loss
train_obj_cls_loss = network.AverageMeter()
train_obj_box_loss = network.AverageMeter()
# relationship cls loss
train_pred_cls_loss = network.AverageMeter()
# region captioning related loss
train_region_caption_loss = network.AverageMeter()
train_region_box_loss = network.AverageMeter()
train_region_objectiveness_loss = network.AverageMeter()
# RPN loss
train_rpn_loss = network.AverageMeter()
# object
accuracy_obj = network.AccuracyMeter()
accuracy_pred = network.AccuracyMeter()
accuracy_reg = network.AccuracyMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_regions.numpy()[0])
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rpn.loss
elif args.finetune_language_model:
loss = target_net.loss_region_box + target_net.region_caption_loss
else:
loss = target_net.loss
loss += target_net.objectiveness_loss
train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
train_obj_cls_loss.update(
target_net.cross_entropy_object.data.cpu().numpy()[0],
im_data.size(0))
train_obj_box_loss.update(
target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0))
train_pred_cls_loss.update(
target_net.cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_region_caption_loss.update(
target_net.region_caption_loss.data.cpu().numpy()[0],
im_data.size(0))
train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_rpn_loss.update(
target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
overall_train_region_caption_loss.update(
target_net.region_caption_loss.data.cpu().numpy()[0],
im_data.size(0))
accuracy_obj.update(target_net.tp, target_net.tf, target_net.fg_cnt,
target_net.bg_cnt)
accuracy_pred.update(target_net.tp_pred, target_net.tf_pred,
target_net.fg_cnt_pred, target_net.bg_cnt_pred)
accuracy_reg.update(target_net.tp_reg, target_net.tf_reg,
target_net.fg_cnt_reg, target_net.bg_cnt_reg)
if args.region_bbox_reg:
train_region_box_loss.update(
target_net.loss_region_box.data.cpu().numpy()[0],
im_data.size(0))
train_region_objectiveness_loss.update(
target_net.objectiveness_loss.data.cpu().numpy()[0],
im_data.size(0))
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# Logging the training loss
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'FRCNN Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print('\t[Loss]\tobj_cls_loss: %.4f\tobj_box_loss: %.4f' %
(train_obj_cls_loss.avg, train_obj_box_loss.avg)),
print('\tpred_cls_loss: %.4f,' % (train_pred_cls_loss.avg)),
if not args.disable_language_model:
print('\tcaption_loss: %.4f,' %
(train_region_caption_loss.avg)),
if args.region_bbox_reg:
print('\tregion_box_loss: %.4f, ' %
(train_region_box_loss.avg)),
print('\tregion_objectness_loss: %.4f' %
(train_region_objectiveness_loss.avg)),
print('\n\t[object]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_obj.ture_pos * 100., accuracy_obj.true_neg * 100.,
accuracy_obj.foreground, accuracy_obj.background))
print('\t[predicate]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_pred.ture_pos * 100., accuracy_pred.true_neg *
100., accuracy_pred.foreground, accuracy_pred.background))
print('\t[region]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_reg.ture_pos * 100., accuracy_reg.true_neg * 100.,
accuracy_reg.foreground, accuracy_reg.background))
# logging to tensor board
log_value('FRCNN loss', overall_train_loss.avg,
overall_train_loss.count)
log_value('RPN_loss loss', overall_train_rpn_loss.avg,
overall_train_rpn_loss.count)
log_value('caption loss', overall_train_region_caption_loss.avg,
overall_train_region_caption_loss.count)
action='store_true',
help='Set to disable the update of other parameters')
parser.add_argument(
'--optimizer',
type=int,
default=0,
help=
'which optimizer used for optimize language model [0: SGD | 1: Adam | 2: Adagrad]'
)
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
args = parser.parse_args()
# Overall loss logger
overall_train_loss = network.AverageMeter()
overall_train_rpn_loss = network.AverageMeter()
overall_train_region_caption_loss = network.AverageMeter()
optimizer_select = 0
def main():
global args, optimizer_select
# To set the model name automatically
print args
lr = args.lr
args = get_model_name(args)
print 'Model name: {}'.format(args.model_name)
# To set the random seed
def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
# cover_cnt = 0
#
# cover_gt_cnt = 0
# fg_cover = 0
# fg_object = 0
# cover_gt = 0
# object_gt = 0
# num = 0
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_boxes_relationship) in enumerate(test_loader):
# num += 1
# if num > 320:
# break
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, relationship_rois, scores_object, scores_relationship = target_net(
im_data, im_info.numpy(),
gt_objects.numpy()[0],
gt_boxes_relationship.numpy()[0])[1:]
# TODO: add rules
# subject_id, object_id, relationship_cover = compare_rel_rois(
# object_rois, relationship_rois, scores_object, scores_relationship, topN_covers=2048, thresh=0.5)
#
# cover_gt_num = check_recall(relationship_cover, gt_boxes_relationship[0].numpy(),
# top_N=relationship_cover.size()[0])
# cover_cnt += cover_gt_num[0]
# cover_obj_check = check_obj_rel_recall(gt_objects[0].numpy(), gt_relationships[0].numpy(),
# gt_boxes_relationship[0].numpy(), relationship_cover,
# subject_id.cpu().numpy(), object_id.cpu().numpy(),
# object_rois, cover_thresh=0.4, object_thresh=0.4, log=num)
# cover_gt_cnt += cover_obj_check[0]
# fg_cover += cover_obj_check[1]
# fg_object += cover_obj_check[2]
# cover_gt += cover_obj_check[3]
# object_gt += cover_obj_check[4]
#
box_num[0] += object_rois.size(0)
box_num[1] += relationship_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois,
gt_objects[0].numpy(),
256,
thresh=0.5)
correct_cnt_t[1], total_cnt_t[1] = check_recall(
relationship_rois,
gt_boxes_relationship[0].numpy(),
256,
thresh=0.5)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
print(
'[{0}/{10}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-256 recall: {3:2.3f} ({4:d}/{5:d})'
'\t[relationship] Avg: {6:2.2f} Boxes/im, Top-256 recall: {7:2.3f} ({8:d}/{9:d})'
.format(i + 1, batch_time.avg, box_num[0] / float(i + 1),
correct_cnt[0] / float(total_cnt[0]) * 100,
correct_cnt[0], total_cnt[0],
box_num[1] / float(i + 1),
correct_cnt[1] / float(total_cnt[1]) * 100,
correct_cnt[1], total_cnt[1], len(test_loader)))
# print('relationship_cover number: {0}'
# '\tcover vs gt_relationship_boxes average recall: {1:.3f}'
# '\tcover & sub & obj vs gt_relationship_boxes average recall: {2:.3f}').format(
# relationship_cover.size()[0], cover_cnt/float(total_cnt[1])*100, cover_gt_cnt/float(total_cnt[1])*100)
# print('average fg_cover: {0:.2f}'
# '\taverage fg_object: {1:.2f}'
# '\taverage cover_gt: {2:.2f}'
# '\taverage object_gt: {3:.2f}').format(
# fg_cover / float(i), fg_object / float(i), cover_gt / float(i), object_gt / float(i))
recall = correct_cnt / total_cnt.astype(np.float)
print '====== Done Testing ===='
return recall
def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
cover_cnt = 0
cover_gt_cnt = 0
fg_cover = 0
fg_object = 0
cover_gt = 0
object_gt = 0
num = 0
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_boxes_relationship) in enumerate(test_loader):
# num += 1
# if num > 20:
# break
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, relationship_rois, scores_object, scores_relationship = target_net(
im_data, im_info.numpy(), gt_objects.numpy(),
gt_boxes_relationship.numpy())[1:]
# TODO: add rules
# img_shape = im_info[0][:2]
# object_rois = object_rois[:, 1:5]
# relationship_rois = enlarge_rois_clip(relationship_rois[:, 1:5], 1.2, img_shape)
# obj_in_predicate(object_rois, relationship_rois, 9)
# subject_id, object_id, relationship_cover: Variable
# ipdb.set_trace()
subject_id, object_id, relationship_cover = compare_rel_rois(
object_rois.data.cpu().numpy(),
relationship_rois.data.cpu().numpy(),
scores_object,
scores_relationship,
topN_obj=object_rois.size(0),
topN_rel=relationship_rois.size(0),
obj_rel_thresh=0.6,
max_objects=18,
topN_covers=2016,
cover_thresh=0.6)
cover_obj_check = check_obj_rel_recall(
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_boxes_relationship.numpy()[0],
relationship_cover,
object_rois.data.cpu().numpy(),
# all_rois_phrase, all_rois,
subject_id,
object_id,
# subject_inds, object_inds,
cover_thresh=0.5,
object_thresh=0.5,
log=False)
cover_gt_cnt += cover_obj_check[0]
fg_cover += cover_obj_check[1]
fg_object += cover_obj_check[2]
cover_gt += cover_obj_check[3]
object_gt += cover_obj_check[4]
# print('relationship_cover size', relationship_cover.size())
# unique_obj = np.unique(np.append(subject_id.cpu().numpy(), object_id.cpu().numpy()))
# print('max', np.max(unique_obj))
# print(unique_obj.size)
cover_gt_num = check_recall(relationship_rois,
gt_boxes_relationship.numpy()[0],
top_N=relationship_rois.size(0),
thresh=0.5)
cover_cnt += cover_gt_num[0]
# all_rois = object_rois.data.cpu().numpy()
# zeros = np.zeros((gt_objects.numpy().shape[1], 1), dtype=gt_objects.numpy().dtype)
# # add gt_obj to predict_rois
# all_rois = np.vstack(
# (all_rois, np.hstack((zeros, gt_objects.numpy()[0][:, :4])))
# )
# all_rois_phrase = relationship_rois.data.cpu().numpy()
# zeros = np.zeros((gt_boxes_relationship.numpy().shape[1], 1), dtype=gt_boxes_relationship.numpy()[0].dtype)
# all_rois_phrase = np.vstack(
# (all_rois_phrase, np.hstack((zeros, gt_boxes_relationship.numpy()[0][:, :4])))
# )
# scores_object = np.append(scores_object, np.ones(gt_objects.size()[1], dtype=scores_object.dtype))
# scores_relationship = np.append(scores_relationship, np.ones(gt_boxes_relationship.size()[1], dtype=scores_relationship.dtype))
# subject_id, object_id, relationship_cover = compare_rel_rois(
# all_rois, all_rois_phrase, scores_object, scores_relationship,
# topN_obj=all_rois.shape[0], topN_rel=all_rois_phrase.shape[0],
# obj_rel_thresh=0.6, max_objects=18, topN_covers=2048, cover_thresh=0.6)
# all_rois_phrase_all = relationship_cover
# zeros = np.zeros((gt_boxes_relationship.numpy().shape[1], 1), dtype=gt_boxes_relationship.numpy()[0].dtype)
# all_rois_phrase_all = np.vstack(
# (all_rois_phrase_all, np.hstack((zeros, gt_boxes_relationship.numpy()[0][:, :4])))
# )
# gt_rel_sub_idx, gt_rel_obj_idx = np.where(gt_relationships.numpy()[0] > 0)
# gt_rel_sub_idx, gt_rel_obj_idx = gt_rel_sub_idx + object_rois.size()[0], gt_rel_obj_idx + object_rois.size()[0]
# subject_inds = np.append(subject_id, gt_rel_sub_idx)
# object_inds = np.append(object_id, gt_rel_obj_idx)
# test
# object_labels, object_rois_pro, bbox_targets_object, bbox_inside_weights_object, bbox_outside_weights_object, \
# region_labels, region_rois_pro, bbox_targets_region, bbox_inside_weights_region, bbox_outside_weights_region, \
# mat_object = proposal_target_layer(object_rois.data.cpu().numpy(), relationship_cover.data.cpu().numpy(),
# subject_id.cpu().numpy(), object_id.cpu().numpy(),
# gt_objects.numpy()[0], gt_relationships.numpy()[0], gt_boxes_relationship.numpy()[0],
# n_classes_obj=151, n_classes_pred=51, is_training=False, graph_generation=False)
# print('object_labels', object_labels[:20])
# print(object_labels.shape)
# print('bbox_targets_object', bbox_targets_object[:10, :60])
# print(bbox_targets_object.shape)
# print('bbox_inside_weights_object', bbox_inside_weights_object[:10, :60])
# print(bbox_inside_weights_object.shape)
# print('region \n')
# print('region_labels', region_labels[:20])
# print(region_labels.shape)
# print('region_rois_pro', region_rois_pro[:10])
# print(region_rois_pro.shape)
# print('bbox_targets_region', bbox_targets_region[:10])
# print(bbox_targets_region.shape)
# print('bbox_inside_weights_region', bbox_inside_weights_region[:10, :60])
# print(bbox_inside_weights_region.shape)
# print('object_rois_pro', object_rois_pro[:10])
# print('object_rois_pro shape', object_rois_pro.shape)
# print('region_rois_pro', region_rois_pro[:10])
# print(region_rois_pro.shape)
# print('mat \n')
# print(mat_object[:32, 0, :32])
# print(mat_object.shape)
# print(mat_object[:32, 0, :32].T)
box_num[0] += object_rois.size(0)
box_num[1] += relationship_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois,
gt_objects.numpy()[0],
200,
thresh=0.5)
correct_cnt_t[1], total_cnt_t[1] = check_recall(
relationship_rois,
gt_boxes_relationship.numpy()[0],
128,
thresh=0.6)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
print(
'([{0}/{10}] Time: {1:2.3f}s/img).\n'
'[object] Avg: {2:2.2f} Boxes/im, Top-256 recall: {3:2.3f} ({4:d}/{5:d})\n'
'[relationship] Avg: {6:2.2f} Boxes/im, Top-128 recall: {7:2.3f} ({8:d}/{9:d})'
.format(i + 1, batch_time.avg, box_num[0] / float(i + 1),
correct_cnt[0] / float(total_cnt[0]) * 100,
correct_cnt[0], total_cnt[0],
box_num[1] / float(i + 1),
correct_cnt[1] / float(total_cnt[1]) * 100,
correct_cnt[1], total_cnt[1], len(test_loader)))
print(
'[relationship_cover number]: {0}\n'
'[cover vs gt_relationship_boxes average recall]: {1:.3f}\n'
'[cover & sub & obj vs gt_relationship_boxes average recall]: {2:.3f}'
).format(relationship_cover.shape[0],
cover_cnt / float(total_cnt[1]) * 100,
cover_gt_cnt / float(total_cnt[1]) * 100)
print('average fg_cover: {0:.2f}'
'\taverage fg_object: {1:.2f}'
'\taverage cover_gt: {2:.2f}'
'\taverage object_gt: {3:.2f}').format(
fg_cover / float(i), fg_object / float(i),
cover_gt / float(i), object_gt / float(i))
recall = correct_cnt / total_cnt.astype(np.float)
print '====== Done Testing ===='
return recall
def train(train_loader, target_net, optimizer, epoch):
global args
# Overall loss logger
# global overall_train_loss
# global overall_train_rpn_loss
# global overall_train_region_caption_loss
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
train_rpn_loss = network.AverageMeter()
# object related loss
train_rcnn_loss = network.AverageMeter() # pre_mps_obj_cls_loss
train_post_mps_obj_cls_loss = network.AverageMeter()
# train_obj_box_loss = network.AverageMeter()
# relationship cls loss
train_pre_mps_pred_cls_loss = network.AverageMeter()
train_post_mps_pred_cls_loss = network.AverageMeter()
# train_pred_box_loss = network.AverageMeter()
# accuracy
accuracy_obj_pre_mps = network.AccuracyMeter()
accuracy_pred_pre_mps = network.AccuracyMeter()
accuracy_obj_post_mps = network.AccuracyMeter()
accuracy_pred_post_mps = network.AccuracyMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects,
gt_relationships) in enumerate(train_loader):
data_time.update(time.time() - end)
t0 = time.time()
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0])
if TIME_IT:
t1 = time.time()
print('forward time %.3fs') % (t1 - t0)
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rcnn.loss + target_net.rcnn.rpn.loss
else:
loss = target_net.loss
train_loss.update(loss.data.cpu().numpy()[0], im_data.size(0))
train_rcnn_loss.update(target_net.rcnn.loss.data.cpu().numpy()[0],
im_data.size(0))
train_post_mps_obj_cls_loss.update(
target_net.post_mps_cross_entropy_object.data.cpu().numpy()[0],
im_data.size(0))
# train_obj_box_loss.update(target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0))
train_pre_mps_pred_cls_loss.update(
target_net.pre_mps_cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_post_mps_pred_cls_loss.update(
target_net.post_mps_cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_rpn_loss.update(target_net.rcnn.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
# overall_train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0))
# overall_train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
accuracy_obj_pre_mps.update(target_net.pre_mps_tp_obj,
target_net.pre_mps_tf_obj,
target_net.pre_mps_fg_cnt_obj,
target_net.pre_mps_bg_cnt_obj)
accuracy_pred_pre_mps.update(target_net.pre_mps_tp_pred,
target_net.pre_mps_tf_pred,
target_net.pre_mps_fg_cnt_pred,
target_net.pre_mps_bg_cnt_pred)
accuracy_obj_post_mps.update(target_net.post_mps_tp_obj,
target_net.post_mps_tf_obj,
target_net.post_mps_fg_cnt_obj,
target_net.post_mps_bg_cnt_obj)
accuracy_pred_post_mps.update(target_net.post_mps_tp_pred,
target_net.post_mps_tf_pred,
target_net.post_mps_fg_cnt_pred,
target_net.post_mps_bg_cnt_pred)
t2 = time.time()
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
if TIME_IT:
t3 = time.time()
print('backward time %.3fs') % (t3 - t2)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# Logging the training loss
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'TOTAL Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print(
'[pre mps][Loss]\tRCNN_loss: %.4f\t pre_mps_pred_cls_loss: %.4f'
% (train_rcnn_loss.avg, train_pre_mps_pred_cls_loss.avg))
print(
'[Accuracy]\t pre_mps_tp: %.2f, \tpre_mps_tf: %.2f, \tfg/bg=(%d/%d)'
% (accuracy_obj_pre_mps.ture_pos * 100.,
accuracy_obj_pre_mps.true_neg * 100.,
accuracy_obj_pre_mps.foreground,
accuracy_obj_pre_mps.background))
print(
'[post mps][Loss]\tpost_mps_obj_cls_loss: %.4f\t post_mps_pred_cls_loss: %.4f'
% (train_post_mps_obj_cls_loss.avg,
train_post_mps_pred_cls_loss.avg))
print(
'[Accuracy]\t post_mps_tp: %.2f, \tpost_mps_tf: %.2f, \tfg/bg=(%d/%d)\n'
% (accuracy_obj_post_mps.ture_pos * 100.,
accuracy_obj_post_mps.true_neg * 100.,
accuracy_obj_post_mps.foreground,
accuracy_obj_post_mps.background))
type=int,
default=0,
help=
'which optimizer used for optimize model [0: SGD | 1: Adam | 2: Adagrad]')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--use_rpn_scores',
action='store_true',
help='Use rpn scores to help to predict')
parser.add_argument('--use_gt_boxes', action='store_true', help='Use GT boxes')
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# Overall loss logger
overall_train_loss = network.AverageMeter()
overall_train_rpn_loss = network.AverageMeter()
optimizer_select = 2
# normal_test = False
def main():
global args, optimizer_select
# To set the model name automatically
print args
lr = args.lr
args = get_model_name(args)
print 'Model name: {}'.format(args.model_name)
# To set the random seed
def train(train_loader, target_net, optimizer, epoch):
global args
# -----------------------------
# initialization of loggers
# -----------------------------
# Overall loss logger
global overall_train_loss
global overall_train_rpn_loss
loss_list = list()
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
# object related loss
train_s_cls_loss = network.AverageMeter()
train_o_cls_loss = network.AverageMeter()
train_s_box_loss = network.AverageMeter()
train_o_box_loss = network.AverageMeter()
# relationship cls loss
train_r_cls_loss = network.AverageMeter()
# RPN loss
train_rpn_loss = network.AverageMeter()
# object
accuracy_s = network.AccuracyMeter()
accuracy_o = network.AccuracyMeter()
accuracy_r = network.AccuracyMeter()
# -----------------------------
# initialization of loggers ends
# -----------------------------
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0])
if target_net.bad_img_flag:
target_net.bad_img_flag = False
continue
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rpn.loss
else:
loss = target_net.loss
# -----------------------------
# update logger
# -----------------------------
train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
train_s_cls_loss.update(
target_net.cross_entropy_s.data.cpu().numpy()[0], im_data.size(0))
train_o_cls_loss.update(
target_net.cross_entropy_o.data.cpu().numpy()[0], im_data.size(0))
train_s_box_loss.update(target_net.loss_s_box.data.cpu().numpy()[0],
im_data.size(0))
train_o_box_loss.update(target_net.loss_o_box.data.cpu().numpy()[0],
im_data.size(0))
train_r_cls_loss.update(
target_net.cross_entropy_r.data.cpu().numpy()[0], im_data.size(0))
train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_rpn_loss.update(
target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
accuracy_s.update(target_net.tp_s, target_net.tf_s,
target_net.fg_cnt_s, target_net.bg_cnt_s)
accuracy_o.update(target_net.tp_o, target_net.tf_o,
target_net.fg_cnt_o, target_net.bg_cnt_o)
accuracy_r.update(target_net.tp_r, target_net.tf_r,
target_net.fg_cnt_r, target_net.bg_cnt_r)
# -----------------------------
# end
# -----------------------------
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# -----------------------------
# print loss
# -----------------------------
# Logging the training loss
if (i + 1) % args.log_interval == 0:
loss_list.append(train_loss.avg)
print(
'\nEpoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'FRCNN Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print('\t[Loss]\ts_cls_loss: %.4f\ts_box_loss: %.4f' %
(train_s_cls_loss.avg, train_s_box_loss.avg)),
print('\tr_cls_loss: %.4f,' % (train_r_cls_loss.avg)),
print('\t[Loss]\to_cls_loss: %.4f\to_box_loss: %.4f' %
(train_o_cls_loss.avg, train_o_box_loss.avg)),
print('\n\t[s]\ttp: %.2f, \tfg=%d' %
(accuracy_s.ture_pos * 100., accuracy_s.foreground))
print('\t[r]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_r.ture_pos * 100., accuracy_r.true_neg * 100.,
accuracy_r.foreground, accuracy_r.background))
print('\t[o]\ttp: %.2f, \tfg=%d' %
(accuracy_o.ture_pos * 100., accuracy_o.foreground))
# -----------------------------
# end
# -----------------------------
# logging to tensor board
log_value('FRCNN loss', overall_train_loss.avg,
overall_train_loss.count)
log_value('RPN_loss loss', overall_train_rpn_loss.avg,
overall_train_rpn_loss.count)
return loss_list
def train(train_loader, target_net, optimizer, epoch):
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
train_loss = network.AverageMeter()
train_loss_box_rpn = network.AverageMeter()
train_loss_entropy_rpn = network.AverageMeter()
train_loss_box_det = network.AverageMeter()
train_loss_entropy_det = network.AverageMeter()
tp, tf, fg, bg = 0., 0., 0, 0
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_boxes,
gt_relationships) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
# Forward pass
target_net(im_data, im_info.numpy(), gt_boxes.numpy()[0])
# record loss
loss = target_net.loss + target_net.rpn.loss
# total loss
train_loss.update(loss.data[0], im_data.size(0))
train_loss_box_det.update(target_net.loss_box.data[0], im_data.size(0))
train_loss_entropy_det.update(target_net.cross_entropy.data[0],
im_data.size(0))
train_loss_box_rpn.update(target_net.rpn.loss_box.data[0],
im_data.size(0))
train_loss_entropy_rpn.update(target_net.rpn.cross_entropy.data[0],
im_data.size(0))
tp += float(target_net.tp)
tf += float(target_net.tf)
fg += target_net.fg_cnt
bg += target_net.bg_cnt
# backward
optimizer.zero_grad()
loss.backward()
if args.clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}]\t'
'Batch_Time: {batch_time.avg:.3f}s\t'
'lr: {lr: f}\t'
'Loss: {loss.avg:.4f}\n'
'\t[rpn]: '
'cls_loss_rpn: {cls_loss_rpn.avg:.3f}\t'
'reg_loss_rpn: {reg_loss_rpn.avg:.3f}\n'
'\t[rcnn]: '
'cls_loss_rcnn: {cls_loss_rcnn.avg:.3f}\t'
'reg_loss_rcnn: {reg_loss_rcnn.avg:.3f}'.format(
epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
data_time=data_time,
loss=train_loss,
cls_loss_rpn=train_loss_entropy_rpn,
cls_loss_rcnn=train_loss_entropy_det,
reg_loss_rpn=train_loss_box_rpn,
reg_loss_rcnn=train_loss_box_det))
print('\tTP: %.2f%%, TF: %.2f%%, fg/bg=(%d/%d)' %
(tp / fg * 100., tf / bg * 100., fg, bg))
def test(test_loader, target_net):
box_num = np.array([0, 0])
correct_cnt, total_cnt = np.array([0, 0]), np.array([0, 0])
print '========== Testing ======='
target_net.eval()
batch_time = network.AverageMeter()
cover_cnt = 0
cnt_gt = 0
object_cnt = 0
num = 0
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships, gt_boxes_relationship) in enumerate(test_loader):
# num += 1
# if num > 320:
# break
correct_cnt_t, total_cnt_t = np.array([0, 0]), np.array([0, 0])
# Forward pass
object_rois, relationship_rois = target_net(im_data, im_info.numpy(), gt_objects.numpy(),
gt_boxes_relationship.numpy())[1:]
# all_rois = object_rois.data.cpu().numpy()
# zeros = np.zeros((gt_objects.numpy().shape[1], 1), dtype=gt_objects.numpy().dtype)
# # add gt_obj to predict_rois
# all_rois = np.vstack(
# (all_rois, np.hstack((zeros, gt_objects.numpy()[0][:, :4])))
# )
# object_rois = network.np_to_variable(all_rois, is_cuda=True)
# TODO: add rules
# img_shape = im_info[0][:2]
# object_rois = object_rois[:, 1:5]
# relationship_rois = enlarge_rois_clip(relationship_rois[:, 1:5], 1.2, img_shape)
# obj_in_predicate(object_rois, relationship_rois, 9)
object_cnt_t, cover_cnt_t, cnt_gt_t = check_msdn_rpn(
object_rois.data.cpu().numpy(), gt_objects.numpy()[0], gt_relationships.numpy()[0])
object_cnt += object_cnt_t
cover_cnt += cover_cnt_t
cnt_gt += cnt_gt_t
# gt_num = gt_boxes_relationship[0].size()[0]
box_num[0] += object_rois.size(0)
box_num[1] += relationship_rois.size(0)
correct_cnt_t[0], total_cnt_t[0] = check_recall(object_rois, gt_objects[0].numpy(), object_rois.size(0), thres=0.5)
correct_cnt_t[1], total_cnt_t[1] = check_recall(relationship_rois, gt_boxes_relationship[0].numpy(), 64, thres=0.5)
correct_cnt += correct_cnt_t
total_cnt += total_cnt_t
batch_time.update(time.time()-end)
end = time.time()
if (i+1)%100 == 0 and i > 0:
print('[{0}/{10}] Time: {1:2.3f}s/img).'
'\t[object] Avg: {2:2.2f} Boxes/im, Top-2000 recall: {3:2.3f} ({4:d}/{5:d})'
'\t[relationship] Avg: {6:2.2f} Boxes/im, Top-500 recall: {7:2.3f} ({8:d}/{9:d})'.format(
i+1, batch_time.avg,
box_num[0]/float(i+1), correct_cnt[0]/float(total_cnt[0])*100, correct_cnt[0], total_cnt[0],
box_num[1]/float(i+1), correct_cnt[1]/float(total_cnt[1])*100, correct_cnt[1], total_cnt[1],
len(test_loader)))
print('fg object number:{0:d}'
'\tcover number:{1:d}'
'\tcover & sub & obj vs gt_relationship_boxes average recall: {2:.3f}').format(
object_cnt/i, cover_cnt / i,
cover_cnt / float(cnt_gt) * 100)
# print('\n')
recall = correct_cnt/total_cnt.astype(np.float)
print '====== Done Testing ===='
return recall
def test(test_loader, net, top_Ns, object_classes):
global args
print '========== Testing ========'
net.eval()
# For efficiency inference
# net.use_region_reg = True
rel_cnt = 0.
rel_cnt_correct = np.zeros(len(top_Ns))
box_num, obj_correct_cnt, obj_total_cnt = 0, 0, 0
batch_time = network.AverageMeter()
end = time.time()
all_cls_gt_num = [0] * (len(object_classes) - 1)
all_cls_scores = [np.array([])] * (len(object_classes) - 1)
all_cls_tp = [np.array([])] * (len(object_classes) - 1)
for i, (im_data, im_info, gt_objects,
gt_relationships) in enumerate(test_loader):
# Forward pass
# pdb.set_trace()
total_cnt_t, rel_cnt_correct_t, object_rois, classes_scores, classes_tf, classes_gt_num = net.evaluate(
im_data,
im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
top_Ns=top_Ns,
use_gt_boxes=args.use_gt_boxes,
nms=False,
nms_thresh=0.4,
thresh=0.5,
use_rpn_scores=args.use_rpn_scores)
box_num += object_rois.size(0)
obj_correct_cnt_t, obj_total_cnt_t = check_recall(
object_rois / im_info[0][2],
gt_objects.numpy()[0], 64)
obj_correct_cnt += obj_correct_cnt_t
obj_total_cnt += obj_total_cnt_t
rel_cnt += total_cnt_t
rel_cnt_correct += rel_cnt_correct_t
for j in range(len(object_classes) - 1):
all_cls_scores[j] = np.append(all_cls_scores[j], classes_scores[j])
all_cls_tp[j] = np.append(all_cls_tp[j], classes_tf[j])
all_cls_gt_num[j] += classes_gt_num[j]
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0 and i > 0:
print(
'Time: {0:2.3f}s/img.'
'\t[object] Avg: {1:2.2f} Boxes/im, Top-64 recall: {2:2.3f} ({3:d}/{4:d})'
.format(batch_time.avg, box_num / float(i + 1),
obj_correct_cnt / float(obj_total_cnt) * 100,
obj_correct_cnt, obj_total_cnt))
for idx, top_N in enumerate(top_Ns):
print '[%d/%d][Evaluation] Top-%d Recall: %2.3f%%' % (
i + 1, len(test_loader), top_N,
rel_cnt_correct[idx] / float(rel_cnt) * 100)
all_aps = []
for k, cls in enumerate(object_classes[1:]):
# sort scores of all images
cls_ap = cls_eval(all_cls_scores[k], all_cls_tp[k], all_cls_gt_num[k])
all_aps += [cls_ap]
print('AP for {} = {:.4f}'.format(cls, cls_ap))
print('Mean AP = {:.4f}'.format(np.mean(all_aps)))
recall = rel_cnt_correct / rel_cnt
print '====== Done Testing ===='
return recall
def test(test_loader, net, top_Ns):
global args
print('========== Testing =======')
net.eval()
# For efficiency inference
languge_state = net.use_language_loss
region_reg_state = net.use_region_reg
net.use_language_loss = False
net.use_region_reg = False
rel_cnt = 0.
rel_cnt_correct = np.zeros(len(top_Ns))
batch_time = network.AverageMeter()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(tqdm(test_loader)):
# Forward pass
if args.mode == 'sgdet':
total_cnt_t, rel_cnt_correct_t = net.evaluate(
im_data,
im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_regions.numpy()[0],
top_Ns=top_Ns,
nms=True,
freq_baseline=args.frequency_baseline)
elif args.mode == 'sgcls':
total_cnt_t, rel_cnt_correct_t = net.evaluate(
im_data,
im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_regions.numpy()[0],
top_Ns=top_Ns,
nms=False,
use_gt_boxes=True,
use_gt_regions=True,
freq_baseline=args.frequency_baseline)
elif args.mode == 'predcls':
total_cnt_t, rel_cnt_correct_t = net.evaluate(
im_data,
im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_regions.numpy()[0],
top_Ns=top_Ns,
nms=False,
use_gt_boxes=True,
use_gt_regions=False,
only_predicate=True,
freq_baseline=args.frequency_baseline,
)
rel_cnt += total_cnt_t
rel_cnt_correct += rel_cnt_correct_t
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 100 == 0 and i > 0:
for idx, top_N in enumerate(top_Ns):
print('\n[%d/%d][Evaluation] Top-%d Recall: %2.3f%%\n' %
(i + 1, len(test_loader), top_N,
rel_cnt_correct[idx] / float(rel_cnt) * 100),
flush=True)
recall = rel_cnt_correct / rel_cnt
print('====== Done Testing ====', flush=True)
# Restore the related states
net.use_language_loss = languge_state
net.use_region_reg = region_reg_state
return recall
