def train(train_loader, target_net, optimizer, epoch):
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
train_loss = network.AverageMeter()
train_loss_obj_box = network.AverageMeter()
train_loss_obj_entropy = network.AverageMeter()
train_loss_reg_box = network.AverageMeter()
train_loss_reg_entropy = network.AverageMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships, gt_regions) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
# Forward pass
target_net(im_data, im_info.numpy(), gt_objects.numpy()[0], gt_regions.numpy()[0])
# record loss
loss = target_net.loss
# total loss
train_loss.update(loss.data[0], im_data.size(0))
# object bbox reg
train_loss_obj_box.update(target_net.loss_box.data[0], im_data.size(0))
# object score
train_loss_obj_entropy.update(target_net.cross_entropy.data[0], im_data.size(0))
# region bbox reg
train_loss_reg_box.update(target_net.loss_box_region.data[0], im_data.size(0))
# region score
train_loss_reg_entropy.update(target_net.cross_entropy_region.data[0], im_data.size(0))
# backward
optimizer.zero_grad()
loss.backward()
if not args.disable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Batch_Time: {batch_time.avg:.3f}s\t'
'lr: {lr: f}\t'
'Loss: {loss.avg:.4f}\n'
'\t[object]: '
'cls_loss: {cls_loss_object.avg:.3f}\t'
'reg_loss: {reg_loss_object.avg:.3f}\n'
'\t[region]: '
'cls_loss: {cls_loss_region.avg:.3f}\t'
'reg_loss: {reg_loss_region.avg:.3f}\t'.format(
epoch, i + 1, len(train_loader), batch_time=batch_time, lr=args.lr,
data_time=data_time, loss=train_loss,
cls_loss_object=train_loss_obj_entropy, reg_loss_object=train_loss_obj_box,
cls_loss_region=train_loss_reg_entropy, reg_loss_region=train_loss_reg_box))
net(im_data, im_info, gt_boxes, gt_ishard, dontcare_areas)
loss = net.loss + net.rpn.loss
if _DEBUG:
tp += float(net.tp)
tf += float(net.tf)
fg += net.fg_cnt
bg += net.bg_cnt
train_loss += loss.data[0]
step_cnt += 1
# backward
optimizer.zero_grad()
loss.backward()
network.clip_gradient(net, 10.)
optimizer.step()
if step % disp_interval == 0:
duration = t.toc(average=False)
fps = step_cnt / duration
log_text = 'step %d, image: %s, loss: %.4f, fps: %.2f (%.2fs per batch)' % (
step, blobs['im_name'], train_loss / step_cnt, fps, 1./fps)
log_print(log_text, color='green', attrs=['bold'])
if _DEBUG:
log_print('\tTP: %.2f%%, TF: %.2f%%, fg/bg=(%d/%d)' % (tp/fg*100., tf/bg*100., fg/step_cnt, bg/step_cnt))
log_print('\trpn_cls: %.4f, rpn_box: %.4f, rcnn_cls: %.4f, rcnn_box: %.4f' % (
net.rpn.cross_entropy.data.cpu().numpy()[0], net.rpn.loss_box.data.cpu().numpy()[0],
net.cross_entropy.data.cpu().numpy()[0], net.loss_box.data.cpu().numpy()[0])
def train(train_loader, target_net, optimizer, epoch):
global args
# Overall loss logger
global overall_train_loss
global overall_train_rpn_loss
global overall_train_region_caption_loss
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
# object related loss
train_obj_cls_loss = network.AverageMeter()
train_obj_box_loss = network.AverageMeter()
# relationship cls loss
train_pred_cls_loss = network.AverageMeter()
# region captioning related loss
train_region_caption_loss = network.AverageMeter()
train_region_box_loss = network.AverageMeter()
train_region_objectiveness_loss = network.AverageMeter()
# RPN loss
train_rpn_loss = network.AverageMeter()
# object
accuracy_obj = network.AccuracyMeter()
accuracy_pred = network.AccuracyMeter()
accuracy_reg = network.AccuracyMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0],
gt_regions.numpy()[0])
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rpn.loss
elif args.finetune_language_model:
loss = target_net.loss_region_box + target_net.region_caption_loss
else:
loss = target_net.loss
loss += target_net.objectiveness_loss
train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
train_obj_cls_loss.update(
target_net.cross_entropy_object.data.cpu().numpy()[0],
im_data.size(0))
train_obj_box_loss.update(
target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0))
train_pred_cls_loss.update(
target_net.cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_region_caption_loss.update(
target_net.region_caption_loss.data.cpu().numpy()[0],
im_data.size(0))
train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_rpn_loss.update(
target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
overall_train_region_caption_loss.update(
target_net.region_caption_loss.data.cpu().numpy()[0],
im_data.size(0))
accuracy_obj.update(target_net.tp, target_net.tf, target_net.fg_cnt,
target_net.bg_cnt)
accuracy_pred.update(target_net.tp_pred, target_net.tf_pred,
target_net.fg_cnt_pred, target_net.bg_cnt_pred)
accuracy_reg.update(target_net.tp_reg, target_net.tf_reg,
target_net.fg_cnt_reg, target_net.bg_cnt_reg)
if args.region_bbox_reg:
train_region_box_loss.update(
target_net.loss_region_box.data.cpu().numpy()[0],
im_data.size(0))
train_region_objectiveness_loss.update(
target_net.objectiveness_loss.data.cpu().numpy()[0],
im_data.size(0))
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# Logging the training loss
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'FRCNN Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print('\t[Loss]\tobj_cls_loss: %.4f\tobj_box_loss: %.4f' %
(train_obj_cls_loss.avg, train_obj_box_loss.avg)),
print('\tpred_cls_loss: %.4f,' % (train_pred_cls_loss.avg)),
if not args.disable_language_model:
print('\tcaption_loss: %.4f,' %
(train_region_caption_loss.avg)),
if args.region_bbox_reg:
print('\tregion_box_loss: %.4f, ' %
(train_region_box_loss.avg)),
print('\tregion_objectness_loss: %.4f' %
(train_region_objectiveness_loss.avg)),
print('\n\t[object]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_obj.ture_pos * 100., accuracy_obj.true_neg * 100.,
accuracy_obj.foreground, accuracy_obj.background))
print('\t[predicate]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_pred.ture_pos * 100., accuracy_pred.true_neg *
100., accuracy_pred.foreground, accuracy_pred.background))
print('\t[region]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_reg.ture_pos * 100., accuracy_reg.true_neg * 100.,
accuracy_reg.foreground, accuracy_reg.background))
# logging to tensor board
log_value('FRCNN loss', overall_train_loss.avg,
overall_train_loss.count)
log_value('RPN_loss loss', overall_train_rpn_loss.avg,
overall_train_rpn_loss.count)
log_value('caption loss', overall_train_region_caption_loss.avg,
overall_train_region_caption_loss.count)
def train(train_loader, target_net, optimizer, epoch):
global args
# Overall loss logger
# global overall_train_loss
# global overall_train_rpn_loss
# global overall_train_region_caption_loss
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
train_rpn_loss = network.AverageMeter()
# object related loss
train_rcnn_loss = network.AverageMeter() # pre_mps_obj_cls_loss
train_post_mps_obj_cls_loss = network.AverageMeter()
# train_obj_box_loss = network.AverageMeter()
# relationship cls loss
train_pre_mps_pred_cls_loss = network.AverageMeter()
train_post_mps_pred_cls_loss = network.AverageMeter()
# train_pred_box_loss = network.AverageMeter()
# accuracy
accuracy_obj_pre_mps = network.AccuracyMeter()
accuracy_pred_pre_mps = network.AccuracyMeter()
accuracy_obj_post_mps = network.AccuracyMeter()
accuracy_pred_post_mps = network.AccuracyMeter()
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects,
gt_relationships) in enumerate(train_loader):
data_time.update(time.time() - end)
t0 = time.time()
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0])
if TIME_IT:
t1 = time.time()
print('forward time %.3fs') % (t1 - t0)
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rcnn.loss + target_net.rcnn.rpn.loss
else:
loss = target_net.loss
train_loss.update(loss.data.cpu().numpy()[0], im_data.size(0))
train_rcnn_loss.update(target_net.rcnn.loss.data.cpu().numpy()[0],
im_data.size(0))
train_post_mps_obj_cls_loss.update(
target_net.post_mps_cross_entropy_object.data.cpu().numpy()[0],
im_data.size(0))
# train_obj_box_loss.update(target_net.loss_obj_box.data.cpu().numpy()[0], im_data.size(0))
train_pre_mps_pred_cls_loss.update(
target_net.pre_mps_cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_post_mps_pred_cls_loss.update(
target_net.post_mps_cross_entropy_predicate.data.cpu().numpy()[0],
im_data.size(0))
train_rpn_loss.update(target_net.rcnn.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
# overall_train_loss.update(target_net.loss.data.cpu().numpy()[0], im_data.size(0))
# overall_train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
accuracy_obj_pre_mps.update(target_net.pre_mps_tp_obj,
target_net.pre_mps_tf_obj,
target_net.pre_mps_fg_cnt_obj,
target_net.pre_mps_bg_cnt_obj)
accuracy_pred_pre_mps.update(target_net.pre_mps_tp_pred,
target_net.pre_mps_tf_pred,
target_net.pre_mps_fg_cnt_pred,
target_net.pre_mps_bg_cnt_pred)
accuracy_obj_post_mps.update(target_net.post_mps_tp_obj,
target_net.post_mps_tf_obj,
target_net.post_mps_fg_cnt_obj,
target_net.post_mps_bg_cnt_obj)
accuracy_pred_post_mps.update(target_net.post_mps_tp_pred,
target_net.post_mps_tf_pred,
target_net.post_mps_fg_cnt_pred,
target_net.post_mps_bg_cnt_pred)
t2 = time.time()
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
if TIME_IT:
t3 = time.time()
print('backward time %.3fs') % (t3 - t2)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# Logging the training loss
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'TOTAL Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print(
'[pre mps][Loss]\tRCNN_loss: %.4f\t pre_mps_pred_cls_loss: %.4f'
% (train_rcnn_loss.avg, train_pre_mps_pred_cls_loss.avg))
print(
'[Accuracy]\t pre_mps_tp: %.2f, \tpre_mps_tf: %.2f, \tfg/bg=(%d/%d)'
% (accuracy_obj_pre_mps.ture_pos * 100.,
accuracy_obj_pre_mps.true_neg * 100.,
accuracy_obj_pre_mps.foreground,
accuracy_obj_pre_mps.background))
print(
'[post mps][Loss]\tpost_mps_obj_cls_loss: %.4f\t post_mps_pred_cls_loss: %.4f'
% (train_post_mps_obj_cls_loss.avg,
train_post_mps_pred_cls_loss.avg))
print(
'[Accuracy]\t post_mps_tp: %.2f, \tpost_mps_tf: %.2f, \tfg/bg=(%d/%d)\n'
% (accuracy_obj_post_mps.ture_pos * 100.,
accuracy_obj_post_mps.true_neg * 100.,
accuracy_obj_post_mps.foreground,
accuracy_obj_post_mps.background))
def train(train_loader, target_net, optimizer, epoch):
global args
# -----------------------------
# initialization of loggers
# -----------------------------
# Overall loss logger
global overall_train_loss
global overall_train_rpn_loss
loss_list = list()
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
# Total loss
train_loss = network.AverageMeter()
# object related loss
train_s_cls_loss = network.AverageMeter()
train_o_cls_loss = network.AverageMeter()
train_s_box_loss = network.AverageMeter()
train_o_box_loss = network.AverageMeter()
# relationship cls loss
train_r_cls_loss = network.AverageMeter()
# RPN loss
train_rpn_loss = network.AverageMeter()
# object
accuracy_s = network.AccuracyMeter()
accuracy_o = network.AccuracyMeter()
accuracy_r = network.AccuracyMeter()
# -----------------------------
# initialization of loggers ends
# -----------------------------
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_objects, gt_relationships,
gt_regions) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
target_net(im_data, im_info,
gt_objects.numpy()[0],
gt_relationships.numpy()[0])
if target_net.bad_img_flag:
target_net.bad_img_flag = False
continue
# Determine the loss function
if args.train_all:
loss = target_net.loss + target_net.rpn.loss
else:
loss = target_net.loss
# -----------------------------
# update logger
# -----------------------------
train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
train_s_cls_loss.update(
target_net.cross_entropy_s.data.cpu().numpy()[0], im_data.size(0))
train_o_cls_loss.update(
target_net.cross_entropy_o.data.cpu().numpy()[0], im_data.size(0))
train_s_box_loss.update(target_net.loss_s_box.data.cpu().numpy()[0],
im_data.size(0))
train_o_box_loss.update(target_net.loss_o_box.data.cpu().numpy()[0],
im_data.size(0))
train_r_cls_loss.update(
target_net.cross_entropy_r.data.cpu().numpy()[0], im_data.size(0))
train_rpn_loss.update(target_net.rpn.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_loss.update(target_net.loss.data.cpu().numpy()[0],
im_data.size(0))
overall_train_rpn_loss.update(
target_net.rpn.loss.data.cpu().numpy()[0], im_data.size(0))
accuracy_s.update(target_net.tp_s, target_net.tf_s,
target_net.fg_cnt_s, target_net.bg_cnt_s)
accuracy_o.update(target_net.tp_o, target_net.tf_o,
target_net.fg_cnt_o, target_net.bg_cnt_o)
accuracy_r.update(target_net.tp_r, target_net.tf_r,
target_net.fg_cnt_r, target_net.bg_cnt_r)
# -----------------------------
# end
# -----------------------------
optimizer.zero_grad()
loss.backward()
if args.enable_clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# -----------------------------
# print loss
# -----------------------------
# Logging the training loss
if (i + 1) % args.log_interval == 0:
loss_list.append(train_loss.avg)
print(
'\nEpoch: [{0}][{1}/{2}] [lr: {lr}] [Solver: {solver}]\n'
'\tBatch_Time: {batch_time.avg: .3f}s\t'
'FRCNN Loss: {loss.avg: .4f}\t'
'RPN Loss: {rpn_loss.avg: .4f}'.format(epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
loss=train_loss,
rpn_loss=train_rpn_loss,
solver=args.solver))
print('\t[Loss]\ts_cls_loss: %.4f\ts_box_loss: %.4f' %
(train_s_cls_loss.avg, train_s_box_loss.avg)),
print('\tr_cls_loss: %.4f,' % (train_r_cls_loss.avg)),
print('\t[Loss]\to_cls_loss: %.4f\to_box_loss: %.4f' %
(train_o_cls_loss.avg, train_o_box_loss.avg)),
print('\n\t[s]\ttp: %.2f, \tfg=%d' %
(accuracy_s.ture_pos * 100., accuracy_s.foreground))
print('\t[r]\ttp: %.2f, \ttf: %.2f, \tfg/bg=(%d/%d)' %
(accuracy_r.ture_pos * 100., accuracy_r.true_neg * 100.,
accuracy_r.foreground, accuracy_r.background))
print('\t[o]\ttp: %.2f, \tfg=%d' %
(accuracy_o.ture_pos * 100., accuracy_o.foreground))
# -----------------------------
# end
# -----------------------------
# logging to tensor board
log_value('FRCNN loss', overall_train_loss.avg,
overall_train_loss.count)
log_value('RPN_loss loss', overall_train_rpn_loss.avg,
overall_train_rpn_loss.count)
return loss_list
def train(train_loader, target_net, optimizer, epoch):
batch_time = network.AverageMeter()
data_time = network.AverageMeter()
train_loss = network.AverageMeter()
train_loss_box_rpn = network.AverageMeter()
train_loss_entropy_rpn = network.AverageMeter()
train_loss_box_det = network.AverageMeter()
train_loss_entropy_det = network.AverageMeter()
tp, tf, fg, bg = 0., 0., 0, 0
target_net.train()
end = time.time()
for i, (im_data, im_info, gt_boxes,
gt_relationships) in enumerate(train_loader):
# measure the data loading time
data_time.update(time.time() - end)
# Forward pass
target_net(im_data, im_info.numpy(), gt_boxes.numpy()[0])
# record loss
loss = target_net.loss + target_net.rpn.loss
# total loss
train_loss.update(loss.data[0], im_data.size(0))
train_loss_box_det.update(target_net.loss_box.data[0], im_data.size(0))
train_loss_entropy_det.update(target_net.cross_entropy.data[0],
im_data.size(0))
train_loss_box_rpn.update(target_net.rpn.loss_box.data[0],
im_data.size(0))
train_loss_entropy_rpn.update(target_net.rpn.cross_entropy.data[0],
im_data.size(0))
tp += float(target_net.tp)
tf += float(target_net.tf)
fg += target_net.fg_cnt
bg += target_net.bg_cnt
# backward
optimizer.zero_grad()
loss.backward()
if args.clip_gradient:
network.clip_gradient(target_net, 10.)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.log_interval == 0:
print(
'Epoch: [{0}][{1}/{2}]\t'
'Batch_Time: {batch_time.avg:.3f}s\t'
'lr: {lr: f}\t'
'Loss: {loss.avg:.4f}\n'
'\t[rpn]: '
'cls_loss_rpn: {cls_loss_rpn.avg:.3f}\t'
'reg_loss_rpn: {reg_loss_rpn.avg:.3f}\n'
'\t[rcnn]: '
'cls_loss_rcnn: {cls_loss_rcnn.avg:.3f}\t'
'reg_loss_rcnn: {reg_loss_rcnn.avg:.3f}'.format(
epoch,
i + 1,
len(train_loader),
batch_time=batch_time,
lr=args.lr,
data_time=data_time,
loss=train_loss,
cls_loss_rpn=train_loss_entropy_rpn,
cls_loss_rcnn=train_loss_entropy_det,
reg_loss_rpn=train_loss_box_rpn,
reg_loss_rcnn=train_loss_box_det))
print('\tTP: %.2f%%, TF: %.2f%%, fg/bg=(%d/%d)' %
(tp / fg * 100., tf / bg * 100., fg, bg))
