def train(train_loader, target_net, optimizer, epoch):
global args
# Overall loss logger
# global overall_train_loss
# global overall_train_rpn_loss
# global overall_train_region_caption_loss
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
train_rpn_loss = network.AverageMeter()
# object related loss
train_rcnn_loss = network.AverageMeter() # pre_mps_obj_cls_loss
train_post_mps_obj_cls_loss = network.AverageMeter()
# train_obj_box_loss = network.AverageMeter()
# relationship cls loss
train_pre_mps_pred_cls_loss = network.AverageMeter()
train_post_mps_pred_cls_loss = network.AverageMeter()
# train_pred_box_loss = network.AverageMeter()
# accuracy
accuracy_obj_pre_mps = network.AccuracyMeter()
accuracy_pred_pre_mps = network.AccuracyMeter()
accuracy_obj_post_mps = network.AccuracyMeter()
accuracy_pred_post_mps = network.AccuracyMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects,
gt_relationships) in enumerate(train_loader):
data_time.update(time.time() - end)
t0 = time.time()
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0])
if TIME_IT:
t1 = time.time()
print('forward time %.3fs') % (t1 - t0)
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rcnn.loss + target_net.rcnn.rpn.loss
else:
loss = target_net.loss
train_loss.update(loss.data.cpu().numpy()[0], im_data.size(0))
train_rcnn_loss.update(target_net.rcnn.loss.data.cpu().numpy()[0],
im_data.size(0))
train_post_mps_obj_cls_loss.update(
target_net.post_mps_cross_entropy_object.data.cpu().numpy()[0],
im_data.size(0))
# train_obj_box_loss.update(target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0))
train_pre_mps_pred_cls_loss.update(
target_net.pre_mps_cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_post_mps_pred_cls_loss.update(
target_net.post_mps_cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_rpn_loss.update(target_net.rcnn.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
# overall_train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0))
# overall_train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
accuracy_obj_pre_mps.update(target_net.pre_mps_tp_obj,
target_net.pre_mps_tf_obj,
target_net.pre_mps_fg_cnt_obj,
target_net.pre_mps_bg_cnt_obj)
accuracy_pred_pre_mps.update(target_net.pre_mps_tp_pred,
target_net.pre_mps_tf_pred,
target_net.pre_mps_fg_cnt_pred,
target_net.pre_mps_bg_cnt_pred)
accuracy_obj_post_mps.update(target_net.post_mps_tp_obj,
target_net.post_mps_tf_obj,
target_net.post_mps_fg_cnt_obj,
target_net.post_mps_bg_cnt_obj)
accuracy_pred_post_mps.update(target_net.post_mps_tp_pred,
target_net.post_mps_tf_pred,
target_net.post_mps_fg_cnt_pred,
target_net.post_mps_bg_cnt_pred)
t2 = time.time()
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
if TIME_IT:
t3 = time.time()
print('backward time %.3fs') % (t3 - t2)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# Logging the training loss
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'TOTAL Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print(
'[pre mps][Loss]\tRCNN_loss: %.4f\t pre_mps_pred_cls_loss: %.4f'
% (train_rcnn_loss.avg, train_pre_mps_pred_cls_loss.avg))
print(
'[Accuracy]\t pre_mps_tp: %.2f, \tpre_mps_tf: %.2f, \tfg/bg=(%d/%d)'
% (accuracy_obj_pre_mps.ture_pos * 100.,
accuracy_obj_pre_mps.true_neg * 100.,
accuracy_obj_pre_mps.foreground,
accuracy_obj_pre_mps.background))
print(
'[post mps][Loss]\tpost_mps_obj_cls_loss: %.4f\t post_mps_pred_cls_loss: %.4f'
% (train_post_mps_obj_cls_loss.avg,
train_post_mps_pred_cls_loss.avg))
print(
'[Accuracy]\t post_mps_tp: %.2f, \tpost_mps_tf: %.2f, \tfg/bg=(%d/%d)\n'
% (accuracy_obj_post_mps.ture_pos * 100.,
accuracy_obj_post_mps.true_neg * 100.,
accuracy_obj_post_mps.foreground,
accuracy_obj_post_mps.background))
def train(train_loader, target_net, optimizer, epoch):
global args
# Overall loss logger
global overall_train_loss
global overall_train_rpn_loss
global overall_train_region_caption_loss
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
# object related loss
train_obj_cls_loss = network.AverageMeter()
train_obj_box_loss = network.AverageMeter()
# relationship cls loss
train_pred_cls_loss = network.AverageMeter()
# region captioning related loss
train_region_caption_loss = network.AverageMeter()
train_region_box_loss = network.AverageMeter()
train_region_objectiveness_loss = network.AverageMeter()
# RPN loss
train_rpn_loss = network.AverageMeter()
# object
accuracy_obj = network.AccuracyMeter()
accuracy_pred = network.AccuracyMeter()
accuracy_reg = network.AccuracyMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_regions.numpy()[0])
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rpn.loss
elif args.finetune_language_model:
loss = target_net.loss_region_box + target_net.region_caption_loss
else:
loss = target_net.loss
loss += target_net.objectiveness_loss
train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
train_obj_cls_loss.update(
target_net.cross_entropy_object.data.cpu().numpy()[0],
im_data.size(0))
train_obj_box_loss.update(
target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0))
train_pred_cls_loss.update(
target_net.cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_region_caption_loss.update(
target_net.region_caption_loss.data.cpu().numpy()[0],
im_data.size(0))
train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_rpn_loss.update(
target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
overall_train_region_caption_loss.update(
target_net.region_caption_loss.data.cpu().numpy()[0],
im_data.size(0))
accuracy_obj.update(target_net.tp, target_net.tf, target_net.fg_cnt,
target_net.bg_cnt)
accuracy_pred.update(target_net.tp_pred, target_net.tf_pred,
target_net.fg_cnt_pred, target_net.bg_cnt_pred)
accuracy_reg.update(target_net.tp_reg, target_net.tf_reg,
target_net.fg_cnt_reg, target_net.bg_cnt_reg)
if args.region_bbox_reg:
train_region_box_loss.update(
target_net.loss_region_box.data.cpu().numpy()[0],
im_data.size(0))
train_region_objectiveness_loss.update(
target_net.objectiveness_loss.data.cpu().numpy()[0],
im_data.size(0))
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# Logging the training loss
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'FRCNN Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print('\t[Loss]\tobj_cls_loss: %.4f\tobj_box_loss: %.4f' %
(train_obj_cls_loss.avg, train_obj_box_loss.avg)),
print('\tpred_cls_loss: %.4f,' % (train_pred_cls_loss.avg)),
if not args.disable_language_model:
print('\tcaption_loss: %.4f,' %
(train_region_caption_loss.avg)),
if args.region_bbox_reg:
print('\tregion_box_loss: %.4f, ' %
(train_region_box_loss.avg)),
print('\tregion_objectness_loss: %.4f' %
(train_region_objectiveness_loss.avg)),
print('\n\t[object]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_obj.ture_pos * 100., accuracy_obj.true_neg * 100.,
accuracy_obj.foreground, accuracy_obj.background))
print('\t[predicate]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_pred.ture_pos * 100., accuracy_pred.true_neg *
100., accuracy_pred.foreground, accuracy_pred.background))
print('\t[region]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_reg.ture_pos * 100., accuracy_reg.true_neg * 100.,
accuracy_reg.foreground, accuracy_reg.background))
# logging to tensor board
log_value('FRCNN loss', overall_train_loss.avg,
overall_train_loss.count)
log_value('RPN_loss loss', overall_train_rpn_loss.avg,
overall_train_rpn_loss.count)
log_value('caption loss', overall_train_region_caption_loss.avg,
overall_train_region_caption_loss.count)
def train(train_loader, target_net, optimizer, epoch):
global args
# -----------------------------
# initialization of loggers
# -----------------------------
# Overall loss logger
global overall_train_loss
global overall_train_rpn_loss
loss_list = list()
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
# object related loss
train_s_cls_loss = network.AverageMeter()
train_o_cls_loss = network.AverageMeter()
train_s_box_loss = network.AverageMeter()
train_o_box_loss = network.AverageMeter()
# relationship cls loss
train_r_cls_loss = network.AverageMeter()
# RPN loss
train_rpn_loss = network.AverageMeter()
# object
accuracy_s = network.AccuracyMeter()
accuracy_o = network.AccuracyMeter()
accuracy_r = network.AccuracyMeter()
# -----------------------------
# initialization of loggers ends
# -----------------------------
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0])
if target_net.bad_img_flag:
target_net.bad_img_flag = False
continue
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rpn.loss
else:
loss = target_net.loss
# -----------------------------
# update logger
# -----------------------------
train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
train_s_cls_loss.update(
target_net.cross_entropy_s.data.cpu().numpy()[0], im_data.size(0))
train_o_cls_loss.update(
target_net.cross_entropy_o.data.cpu().numpy()[0], im_data.size(0))
train_s_box_loss.update(target_net.loss_s_box.data.cpu().numpy()[0],
im_data.size(0))
train_o_box_loss.update(target_net.loss_o_box.data.cpu().numpy()[0],
im_data.size(0))
train_r_cls_loss.update(
target_net.cross_entropy_r.data.cpu().numpy()[0], im_data.size(0))
train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_rpn_loss.update(
target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
accuracy_s.update(target_net.tp_s, target_net.tf_s,
target_net.fg_cnt_s, target_net.bg_cnt_s)
accuracy_o.update(target_net.tp_o, target_net.tf_o,
target_net.fg_cnt_o, target_net.bg_cnt_o)
accuracy_r.update(target_net.tp_r, target_net.tf_r,
target_net.fg_cnt_r, target_net.bg_cnt_r)
# -----------------------------
# end
# -----------------------------
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# -----------------------------
# print loss
# -----------------------------
# Logging the training loss
if (i + 1) % args.log_interval == 0:
loss_list.append(train_loss.avg)
print(
'\nEpoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'FRCNN Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print('\t[Loss]\ts_cls_loss: %.4f\ts_box_loss: %.4f' %
(train_s_cls_loss.avg, train_s_box_loss.avg)),
print('\tr_cls_loss: %.4f,' % (train_r_cls_loss.avg)),
print('\t[Loss]\to_cls_loss: %.4f\to_box_loss: %.4f' %
(train_o_cls_loss.avg, train_o_box_loss.avg)),
print('\n\t[s]\ttp: %.2f, \tfg=%d' %
(accuracy_s.ture_pos * 100., accuracy_s.foreground))
print('\t[r]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_r.ture_pos * 100., accuracy_r.true_neg * 100.,
accuracy_r.foreground, accuracy_r.background))
print('\t[o]\ttp: %.2f, \tfg=%d' %
(accuracy_o.ture_pos * 100., accuracy_o.foreground))
# -----------------------------
# end
# -----------------------------
# logging to tensor board
log_value('FRCNN loss', overall_train_loss.avg,
overall_train_loss.count)
log_value('RPN_loss loss', overall_train_rpn_loss.avg,
overall_train_rpn_loss.count)
return loss_list