def train():
with tf.Graph().as_default():
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'ship', 'spacenet', 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.py_func(back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
with tf.name_scope('draw_gtboxes'):
gtboxes_in_img = draw_box_with_color(img_batch, tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1],
text=tf.shape(gtboxes_and_label_batch)[1])
# ***********************************************************************************************
# * shared CNN *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * rpn *
# * Note: here the rpn is Feature Pyramid Networks *
# ***********************************************************************************************
rpn = build_rpn.RPN(net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=gtboxes_and_label,
is_training=True,
share_head=cfgs.SHARED_HEADS,
share_net=share_net,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
anchor_angles=cfgs.ANCHOR_ANGLES,
scale_factors=cfgs.SCALE_FACTORS, # this parameter will affect the performance
base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
anchor_stride=cfgs.ANCHOR_STRIDE,
top_k_nms=cfgs.RPN_TOP_K_NMS,
kernel_size=cfgs.KERNEL_SIZE,
use_angles_condition=False,
anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD,
nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
# iou>=0.7 is positive box, iou< 0.3 is negative
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=cfgs.IS_FILTER_OUTSIDE_BOXES, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
scope='')
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ]
rpn_location_loss, rpn_classification_loss, rpn_predict_boxes, rpn_predict_scores = rpn.rpn_losses()
rpn_total_loss = rpn_classification_loss + rpn_location_loss
with tf.name_scope('draw_proposals'):
# score > 0.6 is object
rpn_object_boxes_indices = tf.reshape(tf.where(tf.greater(rpn_proposals_scores, cfgs.FINAL_SCORE_THRESHOLD)),
[-1])
rpn_object_boxes = tf.gather(rpn_proposals_boxes, rpn_object_boxes_indices)
rpn_object_soxres = tf.gather(rpn_proposals_scores, rpn_object_boxes_indices)
rpn_proposals_objcet_boxes_in_img = draw_boxes_with_scores(img_batch,
rpn_object_boxes,
scores=rpn_object_soxres)
# rpn_proposals_objcet_boxes_in_img = draw_box_with_color(img_batch, rpn_object_boxes,
# text=tf.shape(rpn_object_boxes)[0])
rpn_proposals_boxes_in_img = draw_box_with_color(img_batch, rpn_proposals_boxes,
text=tf.shape(rpn_proposals_boxes)[0])
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
stop_gradient_for_proposals=False,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=gtboxes_and_label,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS,
nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD,
use_angle_condition=False,
level=cfgs.LEVEL,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD,
# iou>0.5 is positive, iou<0.5 is negative
boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD,
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=True)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss()
fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss
with tf.name_scope('draw_boxes_with_categories'):
fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories(img_batch=img_batch,
boxes=fast_rcnn_decode_boxes,
labels=detection_category,
scores=fast_rcnn_score)
# train
total_loss = slim.losses.get_total_loss()
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(global_step,
boundaries=[np.int64(70000), np.int64(120000)],
values=[cfgs.LR, cfgs.LR/10, cfgs.LR/100])
# optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
if cfgs.RPN_TRAIN:
train_op = slim.learning.create_train_op(rpn_total_loss, optimizer, global_step)
else:
train_op = slim.learning.create_train_op(total_loss, optimizer, global_step)
# train_op = optimizer.minimize(second_classification_loss, global_step)
# ***********************************************************************************************
# * Summary *
# ***********************************************************************************************
# ground truth and predict
tf.summary.image('img/gtboxes', gtboxes_in_img)
tf.summary.image('img/fast_rcnn_predict', fast_rcnn_predict_boxes_in_imgs)
# rpn loss and image
tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss)
tf.summary.scalar('rpn/rpn_classification_loss', rpn_classification_loss)
tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_location_loss', fast_rcnn_location_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss', fast_rcnn_classification_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss', fast_rcnn_total_loss)
tf.summary.scalar('loss/total_loss', total_loss)
tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img)
tf.summary.image('rpn/rpn_object_boxes', rpn_proposals_objcet_boxes_in_img)
# learning_rate
tf.summary.scalar('learning_rate', lr)
summary_op = tf.summary.merge_all()
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer()
)
restorer, restore_ckpt = restore_model.get_restorer()
saver = tf.train.Saver(max_to_keep=10)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
tools.mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
start = time.time()
_global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \
_rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \
_fast_rcnn_total_loss, _total_loss, _= \
sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss,
rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss,
fast_rcnn_total_loss, total_loss, train_op])
end = time.time()
if step % 10 == 0:
print(""" {}: step{} image_name:{} |\t
rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} |
fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} |
total_loss:{} |\t per_cost_time:{}s""" \
.format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss,
_rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss,
_fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss,
(end - start)))
if (step % 50 == 0) and (step % 10000 != 0): # 50
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, _global_step)
summary_writer.flush()
if (step > 0 and step % 10000 == 0) or (step == cfgs.MAX_ITERATION - 1):
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, _global_step)
summary_writer.flush()
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.mkdir(save_dir)
save_ckpt = os.path.join(save_dir, 'voc_'+str(_global_step)+'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def eval_dict_convert(img_num):
with tf.Graph().as_default():
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=None,
is_training=False,
share_head=True,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
gtboxes_and_label=None,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.
FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
use_dropout=False,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
gtbox_dict = {}
predict_dict = {}
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label_batch, _fast_rcnn_decode_boxes, \
_fast_rcnn_score, _detection_category \
= sess.run([img_name_batch, img_batch, gtboxes_and_label_batch, fast_rcnn_decode_boxes,
fast_rcnn_score, detection_category])
end = time.time()
# gtboxes convert dict
gtbox_dict[str(_img_name_batch[0])] = []
predict_dict[str(_img_name_batch[0])] = []
for j, box in enumerate(_gtboxes_and_label_batch[0]):
bbox_dict = {}
bbox_dict['bbox'] = np.array(
_gtboxes_and_label_batch[0][j, :-1], np.float64)
bbox_dict['name'] = LABEl_NAME_MAP[int(
_gtboxes_and_label_batch[0][j, -1])]
gtbox_dict[str(_img_name_batch[0])].append(bbox_dict)
for label in NAME_LABEL_MAP.keys():
if label == 'back_ground':
continue
else:
temp_dict = {}
temp_dict['name'] = label
ind = np.where(
_detection_category == NAME_LABEL_MAP[label])[0]
temp_boxes = _fast_rcnn_decode_boxes[ind]
temp_score = np.reshape(_fast_rcnn_score[ind], [-1, 1])
temp_dict['bbox'] = np.array(
np.concatenate([temp_boxes, temp_score], axis=1),
np.float64)
predict_dict[str(_img_name_batch[0])].append(temp_dict)
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
fw1 = open('gtboxes_dict.pkl', 'w')
fw2 = open('predict_dict.pkl', 'w')
pickle.dump(gtbox_dict, fw1)
pickle.dump(predict_dict, fw2)
fw1.close()
fw2.close()
coord.request_stop()
coord.join(threads)
def test(img_num):
with tf.Graph().as_default():
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label, head = get_head(
tf.squeeze(gtboxes_and_label_batch, 0))
gtboxes_and_label = tf.py_func(back_forward_convert,
inp=[gtboxes_and_label],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
head_quadrant = tf.py_func(get_head_quadrant,
inp=[head, gtboxes_and_label],
Tout=tf.float32)
head_quadrant = tf.reshape(head_quadrant, [-1, 1])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=None,
is_training=False,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
img_batch=img_batch,
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
gtboxes_and_label_minAreaRectangle=
gtboxes_and_label_minAreaRectangle,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
# iou>0.5 is positive, iou<0.5 is negative
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False,
level=cfgs.LEVEL,
head_quadrant=head_quadrant)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \
fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, fast_rcnn_head_quadrant, \
num_of_objects_rotate, detection_category_rotate = fast_rcnn.fast_rcnn_predict()
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, _head_quadrant, \
_fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \
_fast_rcnn_score_rotate, _fast_rcnn_head_quadrant, _detection_category_rotate \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle, head_quadrant,
fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate,
fast_rcnn_score_rotate, fast_rcnn_head_quadrant, detection_category_rotate])
end = time.time()
_img_batch = np.squeeze(_img_batch, axis=0)
_img_batch_fpn_horizonal = help_utils.draw_box_cv(
_img_batch,
boxes=_fast_rcnn_decode_boxes,
labels=_detection_category,
scores=_fast_rcnn_score)
_img_batch_fpn_rotate = help_utils.draw_rotate_box_cv(
_img_batch,
boxes=_fast_rcnn_decode_boxes_rotate,
labels=_detection_category_rotate,
scores=_fast_rcnn_score_rotate,
head=np.argmax(_fast_rcnn_head_quadrant, axis=1))
mkdir(cfgs.TEST_SAVE_PATH)
cv2.imwrite(
cfgs.TEST_SAVE_PATH +
'/{}_horizontal_fpn.jpg'.format(str(_img_name_batch[0])),
_img_batch_fpn_horizonal)
cv2.imwrite(
cfgs.TEST_SAVE_PATH +
'/{}_rotate_fpn.jpg'.format(str(_img_name_batch[0])),
_img_batch_fpn_rotate)
temp_label_horizontal = np.reshape(_gtboxes_and_label[:, -1:],
[
-1,
]).astype(np.int64)
temp_label_rotate = np.reshape(_gtboxes_and_label[:, -1:], [
-1,
]).astype(np.int64)
_img_batch_gt_horizontal = help_utils.draw_box_cv(
_img_batch,
boxes=_gtboxes_and_label_minAreaRectangle[:, :-1],
labels=temp_label_horizontal,
scores=None)
_img_batch_gt_rotate = help_utils.draw_rotate_box_cv(
_img_batch,
boxes=_gtboxes_and_label[:, :-1],
labels=temp_label_rotate,
scores=None,
head=np.reshape(_head_quadrant, [
-1,
]))
cv2.imwrite(
cfgs.TEST_SAVE_PATH +
'/{}_horizontal_gt.jpg'.format(str(_img_name_batch[0])),
_img_batch_gt_horizontal)
cv2.imwrite(
cfgs.TEST_SAVE_PATH +
'/{}_rotate_gt.jpg'.format(str(_img_name_batch[0])),
_img_batch_gt_rotate)
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
coord.request_stop()
coord.join(threads)
def train():
# Step 1:
# clw note：传递网络名称如resnet_v1，是否训练is_training，以及每个位置含有anchor box的个数，
# 构建基本的网络
faster_rcnn = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=True)
with tf.name_scope(
'get_batch'
): # clw note：tf.name_scope 主要结合 tf.Variable() 来使用，方便参数命名管理。
# clw note：从文件队列、内存队列中读取、组合，得到该batch的内容
# 主要包括每个批次（目前仅支持批次数目即batch_size=1，也就是这里每次只读出1张图片）
# 对应的变量包括：图片名称、图片矩阵、ground truth坐标及对应的label，图片中包含的目标数
# 这些变量的组成结构均为 [批次数目，相应批次中每一幅图片的相关信息]
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5])
# clw note：样本个数m不知道，但是对单个样本都有gtboxes的4个坐标，加上1个label共5个值；使用-1来自动计算样本个数
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY)
# Step 2：
# clw note：Faster R-CNN网络的搭建！
# 先看一下下面这个函数arg_scope的声明
# @tf_contextlib.contextmanager
# def arg_scope(list_ops_or_scope, **kwargs): 功能是给list_ops中的内容设置默认值，即list中所有元素都用**kargs的参数设置。
# 有函数修饰符@tf_contextlib.contextmanager修饰arg_scope函数：@之后一般接一个可调用对象为其执行一系列辅助操作，
# 我们来看一个demo：
#########################################
# import time
# def my_time(func):
# print(time.ctime())
# return func()
#
# @my_time # 从这里可以看出@time 等价于 time(xxx()),但是这种写法你得考虑python代码的执行顺序
# def xxx():
# print('Hello world!')
#
# 运行结果：
# Wed Jul 26 23:01:21 2017
# Hello world!
##########################################
# 在这个例子中，xxx函数实现我们的主要功能，打印Hello world，但我们想给xxx函数添加一些辅助操作，让它同时打印出时间，于是我们用
# 函数修饰符 @ my_time完成这个目标。整个例子的执行流程为调用my_time可调用对象，它接受xxx函数作为参数，先打印时间，再执行xxx函数
# 详见：https://www.cnblogs.com/zzy-tf/p/9356883.html
# 来看另一个demo：
##########################################
# with slim.arg_scope(
# [slim.conv2d, slim.max_pool2d, slim.avg_pool2d],stride = 1, padding = 'VALID'):
# net = slim.conv2d(inputs, 32, [3, 3], stride = 2, scope = 'Conv2d_1a_3x3')
# net = slim.conv2d(net, 32, [3, 3], scope = 'Conv2d_2a_3x3')
# net = slim.conv2d(net, 64, [3, 3], padding = 'SAME', scope = 'Conv2d_2b_3x3')
# 所以，在使用过程中可以直接slim.conv2d( )等函数设置默认参数。例如在下面的代码中，不做单独声明的情况下，
# slim.conv2d, slim.max_pool2d, slim.avg_pool2d三个函数默认的步长都设为1，padding模式都是'VALID'的。
# 当然也可以在调用时进行单独声明，只不过一个一个写很麻烦，不如统一给个默认值。
# 这种参数设置方式在构建网络模型时，尤其是较深的网络时，可以节省时间。
with slim.arg_scope([
slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose,
slim.separable_conv2d, slim.fully_connected
],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
biases_initializer=tf.constant_initializer(
0.0)): # list as many types of layers as possible,
# even if they are not used now
# build_whole_detection_network功能：构建整体网络架构，包含backbone，RPN网络，Pooling层，以及后续网络。
# return：网络的最后的预测框，预测的类别信息，预测的概率，以及整体网络和RPN网络的损失，所有的损失被写入到一个字典中。
final_bbox, final_scores, final_category, loss_dict = faster_rcnn.build_whole_detection_network(
input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label)
# ----------------------------------------------------------------------------------------------------build loss
# weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
# weight_decay_loss = tf.add_n(tf.losses.get_regularization_losses())
rpn_location_loss = loss_dict['rpn_loc_loss']
rpn_cls_loss = loss_dict['rpn_cls_loss']
rpn_total_loss = rpn_location_loss + rpn_cls_loss
fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss']
fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss']
fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss
# clw note：根据论文的公式，最后将RPN网络的（分类，回归）误差与Fast-RCNN的(分类，回归）误差相加后作为总的误差进行训练即可。
total_loss = rpn_total_loss + fastrcnn_total_loss
# ____________________________________________________________________________________________________build loss
# ---------------------------------------------------------------------------------------------------add summary
tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss)
tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss)
tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss', fastrcnn_loc_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss)
tf.summary.scalar('LOSS/total_loss', total_loss)
# tf.summary.scalar('LOSS/regular_weights', weight_decay_loss)
gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories(
img_batch=img_batch,
boxes=gtboxes_and_label[:, :-1],
labels=gtboxes_and_label[:, -1])
if cfgs.ADD_BOX_IN_TENSORBOARD:
detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores(
img_batch=img_batch,
boxes=final_bbox,
labels=final_category,
scores=final_scores)
tf.summary.image('Compare/final_detection', detections_in_img)
tf.summary.image('Compare/gtboxes', gtboxes_in_img)
# ___________________________________________________________________________________________________add summary
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int64(cfgs.DECAY_STEP[0]),
np.int64(cfgs.DECAY_STEP[1])
],
values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(
lr, momentum=cfgs.MOMENTUM) # clw note：选择优化器，可以尝试其他选择，
# 也可以尝试tf.train.AdamOptimizer(1e-4).minimize(total_loss)
# ---------------------------------------------------------------------------------------------compute gradients
# clw note：对于上面优化器没有使用minimize()的几点说明，
# 使用minimize()操作，该操作不仅可以计算出梯度，而且还可以将梯度作用在变量上。
# 如果想按照自己的方式处理梯度，可以按照以下步骤：
# 1、使用compute_gradients()计算梯度，其实下面的get_gradients()方法就是optimizer.compute_gradients(loss)
# 2、使用自己的方式进一步处理梯度
# 3、使用apply_gradients()应用处理过后的梯度；
gradients = faster_rcnn.get_gradients(optimizer, total_loss)
# enlarge_gradients for bias
if cfgs.MUTILPY_BIAS_GRADIENT:
gradients = faster_rcnn.enlarge_gradients_for_bias(gradients)
if cfgs.GRADIENT_CLIPPING_BY_NORM: # clw note：clip_by_norm是指对梯度进行裁剪，通过控制梯度的最大范式，防止梯度爆炸的问题，是一种比较常用的梯度规约的方式
with tf.name_scope('clip_gradients_YJR'):
gradients = slim.learning.clip_gradient_norms(
gradients, cfgs.GRADIENT_CLIPPING_BY_NORM)
# _____________________________________________________________________________________________compute gradients
# train_op
train_op = optimizer.apply_gradients(grads_and_vars=gradients,
global_step=global_step)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = faster_rcnn.get_restorer()
saver = tf.train.Saver(max_to_keep=30)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
# tools.mkdir(summary_path)
if not os.path.exists(summary_path):
os.makedirs(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
_, global_stepnp = sess.run([train_op, global_step])
else:
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
_, global_stepnp, img_name, rpnLocLoss, rpnClsLoss, rpnTotalLoss, \
fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss = \
sess.run(
[train_op, global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss,
fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss, total_loss])
end = time.time()
print(""" {}: step{} image_name:{} |\t
rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} |
fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} |
total_loss:{} |\t per_cost_time:{}s""" \
.format(training_time, global_stepnp, str(img_name[0]), rpnLocLoss, rpnClsLoss,
rpnTotalLoss, fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss,
(end - start)))
else:
if step % cfgs.SMRY_ITER == 0:
_, global_stepnp, summary_str = sess.run(
[train_op, global_step, summary_op])
summary_writer.add_summary(summary_str, global_stepnp)
summary_writer.flush()
if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE
== 0) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_ckpt = os.path.join(
save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def train():
# get Network Class
faster_rcnn = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=True)
# get batch
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
# convert gtboxes_and_labels
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
# draw ground-truth
with tf.name_scope('draw_gtboxes'):
gtboxes_in_img = draw_box_with_color_rotate(
img_batch,
tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1],
text=tf.shape(gtboxes_and_label)[0])
# default regularizers
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY)
# list as many types of layers as possible, even if they are not used now
with slim.arg_scope([
slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose,
slim.separable_conv2d, slim.fully_connected
],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
biases_initializer=tf.constant_initializer(0.0)):
# build network
final_boxes, final_scores, final_category, loss_dict = \
faster_rcnn.build_whole_detection_network(input_img_batch=img_batch,
gtboxes_batch=gtboxes_and_label)
# draw detections
dets_in_img = draw_boxes_with_categories_and_scores_rotate(
img_batch=img_batch,
boxes=final_boxes,
labels=final_category,
scores=final_scores)
# ----------------------------------------------------------------------------------------------------build loss
# weight decay loss
weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
# rpn losses
rpn_location_loss = loss_dict['rpn_loc_loss']
rpn_cls_loss = loss_dict['rpn_cls_loss']
rpn_total_loss = rpn_location_loss + rpn_cls_loss
# fastrcnn losses
fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss']
fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss']
fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss
# total loss
total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss
# ____________________________________________________________________________________________________build loss
# ---------------------------------------------------------------------------------------------------add summary
tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss)
tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss)
tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss', fastrcnn_loc_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss)
tf.summary.scalar('LOSS/total_loss', total_loss)
tf.summary.scalar('LOSS/regular_weights', weight_decay_loss)
tf.summary.image('img/gtboxes', gtboxes_in_img)
tf.summary.image('img/dets', dets_in_img)
# ___________________________________________________________________________________________________add summary
# create global step
global_step = slim.get_or_create_global_step()
# learning rate setting
lr = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int64(cfgs.DECAY_STEP[0]),
np.int64(cfgs.DECAY_STEP[1])
],
values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.])
tf.summary.scalar('lr', lr)
# optimizer with lr
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
# ---------------------------------------------------------------------------------------------compute gradients
# compute gradients
gradients = faster_rcnn.get_gradients(optimizer, total_loss)
# enlarge gradients for bias
if cfgs.MUTILPY_BIAS_GRADIENT:
gradients = faster_rcnn.enlarge_gradients_for_bias(gradients)
# clipping gradients
if cfgs.GRADIENT_CLIPPING_BY_NORM:
with tf.name_scope('clip_gradients'):
gradients = slim.learning.clip_gradient_norms(
gradients, cfgs.GRADIENT_CLIPPING_BY_NORM)
# _____________________________________________________________________________________________compute gradients
# train_op, ie. apply gradients
train_op = optimizer.apply_gradients(grads_and_vars=gradients,
global_step=global_step)
# summary_op
summary_op = tf.summary.merge_all()
# init_op
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
# restorer and Saver
restorer, restore_ckpt = faster_rcnn.get_restorer()
saver = tf.train.Saver(max_to_keep=10)
# GPU Config
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# start a session
with tf.Session(config=config) as sess:
# init
sess.run(init_op)
# restore
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
# Session对象是支持多线程的, 可以在同一个会话（Session）中创建多个线程，并行执行;
# 创建线程协调器(管理器), 并启动(Tensor/训练数据)入队线程,
# 入队具体使用多少个线程在read_tfrecord.py->next_batch->tf.train.batch中定义;
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
# construct summary writer
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
tools.mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
# train MAX_ITERATION steps
for step in range(cfgs.MAX_ITERATION):
# time of this step
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
# no show & no summary
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
# global_step is added to run, global_stepnp will be used in Save
_, global_stepnp = sess.run([train_op, global_step])
else:
# show
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
# middle infos are added to run, losses and img_name_batch
_global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \
_rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \
_fast_rcnn_total_loss, _total_loss, _ = \
sess.run([global_step, img_name_batch, rpn_location_loss, rpn_cls_loss,
rpn_total_loss, fastrcnn_loc_loss, fastrcnn_cls_loss,
fastrcnn_total_loss, total_loss, train_op])
# # show boxes, scores, category infos
# final_boxes_r, _final_scores_r, _final_category_r = sess.run([final_boxes_r, final_scores_r, final_category_r])
# print('*'*100)
# print(_final_boxes_r)
# print(_final_scores_r)
# print(_final_category_r)
end = time.time()
# print
print(""" {}: step {}: image_name:{} |\t
rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t
rpn_total_loss:{} |
fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t
fast_rcnn_total_loss:{} |\t
total_loss:{} |\t pre_cost_time:{}s""" \
.format(training_time, _global_step, str(_img_name_batch[0]),
_rpn_location_loss, _rpn_classification_loss, _rpn_total_loss,
_fast_rcnn_location_loss, _fast_rcnn_classification_loss, _fast_rcnn_total_loss,
_total_loss, (end - start)))
# summary
else:
if step % cfgs.SMRY_ITER == 0:
# summary_op is added to run
_, global_stepnp, summary_str = sess.run(
[train_op, global_step, summary_op])
# add summary
summary_writer.add_summary(summary_str, global_stepnp)
summary_writer.flush()
# save
if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE
== 0) or (step == cfgs.MAX_ITERATION - 1):
# mkdir
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# save checkpoint
save_ckpt = os.path.join(
save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
# 协调器发出终止所有线程的命令
coord.request_stop()
# 把开启的线程加入主线程，等待threads结束
coord.join(threads)
def test(args):
with tf.Graph().as_default():
# img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8)
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=None,
is_training=False,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ]
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
gtboxes_and_label=None,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD,
use_dropout=False,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
# train
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer()
)
restorer, restore_ckpt = restore_model.get_restorer(checkpoint_path=args.weights)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
print('start!')
for _ in range(args.img_num):
_img_name_batch, _img_batch, _gtboxes_and_label_batch, _fast_rcnn_decode_boxes, \
_fast_rcnn_score, _detection_category \
= sess.run([img_name_batch, img_batch, gtboxes_and_label_batch, fast_rcnn_decode_boxes,
fast_rcnn_score, detection_category])
_img_batch = np.squeeze(_img_batch, axis=0)
_img_batch_fpn = help_utils.draw_box_cv(_img_batch,
boxes=_fast_rcnn_decode_boxes,
labels=_detection_category,
scores=_fast_rcnn_score)
mkdir(cfgs.TEST_SAVE_PATH)
cv2.imwrite(cfgs.TEST_SAVE_PATH +
'/{}_fpn.jpg'.format(str(_img_name_batch[0])), _img_batch_fpn)
_gtboxes_and_label_batch = np.squeeze(_gtboxes_and_label_batch, axis=0)
temp_label = np.reshape(_gtboxes_and_label_batch[:, -1:], [-1, ]).astype(np.int64)
_img_batch_gt = help_utils.draw_box_cv(_img_batch,
boxes=_gtboxes_and_label_batch[:, :-1],
labels=temp_label,
scores=None)
cv2.imwrite(cfgs.TEST_SAVE_PATH +
'/{}_gt.jpg'.format(str(_img_name_batch[0])), _img_batch_gt)
coord.request_stop()
coord.join(threads)
print('end!')
def train():
with tf.Graph().as_default():
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch = \
next_batch(data_tfrecord_location=cfgs.DATASET_DIR,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
longside_len=cfgs.LONG_SIDE_LEN,
is_training=True)
image_height, image_width = tf.shape(img_batch)[1], tf.shape(
img_batch)[2]
# with tf.name_scope('draw_gtboxes'):
# gtboxes_in_img = draw_box_with_color(img_batch, tf.reshape(gtboxes_and_label_batch, [-1, 5])[:, :-1],
# text=tf.shape(gtboxes_and_label_batch)[1])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# fpn
feature_pyramid = build_fpn.build_feature_pyramid(
share_net) # [P2,P3,P4,P5,P6]
# ***********************************************************************************************
# * rpn *
# ***********************************************************************************************
rpn = build_rpn.RPN(feature_pyramid=feature_pyramid,
image_height=image_height,
image_width=image_width,
gtboxes_and_label=gtboxes_and_label_batch,
is_training=True)
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
is_training=True)
rpn_location_loss, rpn_classification_loss = rpn.rpn_losses()
rpn_total_loss = rpn_classification_loss + rpn_location_loss
# ***********************************************************************************************
# * Fast RCNN Head *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FAST_RCNN(
feature_pyramid=feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
gtboxes_and_label=gtboxes_and_label_batch,
origin_image=img_batch,
is_training=True,
image_height=image_height,
image_width=image_width)
detections = fast_rcnn.head_detection()
if cfgs.DEBUG:
print_tensors(rpn_proposals_scores[0, :50], "scores") # 前50
print_tensors(rpn_proposals_boxes[0, :50, :], "bbox")
rpn_proposals_vision = draw_boxes_with_scores(
img_batch[0, :, :, :], rpn_proposals_boxes[0, :50, :],
rpn_proposals_scores[0, :50])
fast_rcnn_vision = draw_boxes_with_categories_and_scores(
img_batch[0, :, :, :], detections[0, :, :4],
detections[0, :, 4], detections[0, :, 5], cfgs.LABEL_TO_NAME)
tf.summary.image("rpn_proposals_vision", rpn_proposals_vision)
tf.summary.image("fast_rcnn_vision", fast_rcnn_vision)
fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.head_loss(
)
fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss
# train
with tf.name_scope("regularization_losses"):
regularization_list = [
tf.nn.l2_loss(w.read_value()) *
cfgs.WEIGHT_DECAY[cfgs.NET_NAME] /
tf.cast(tf.size(w.read_value()), tf.float32)
for w in tf.trainable_variables()
if 'gamma' not in w.name and 'beta' not in w.name
]
regularization_loss = tf.add_n(regularization_list)
total_loss = regularization_loss + fast_rcnn_total_loss + rpn_total_loss
total_loss = tf.cond(tf.is_nan(total_loss), lambda: 0.0,
lambda: total_loss)
global_step = tf.train.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[np.int64(10000), np.int64(20000)],
values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100])
# * Summary *
# ***********************************************************************************************
# rpn loss and image
tf.summary.scalar('rpn_location_loss',
rpn_location_loss,
family="rpn_loss")
tf.summary.scalar('rpn_classification_loss',
rpn_classification_loss,
family="rpn_loss")
tf.summary.scalar('rpn_total_loss', rpn_total_loss, family="rpn_loss")
tf.summary.scalar('fast_rcnn_location_loss',
fast_rcnn_location_loss,
family="head_loss")
tf.summary.scalar('fast_rcnn_classification_loss',
fast_rcnn_classification_loss,
family="head_loss")
tf.summary.scalar('fast_rcnn_total_loss',
fast_rcnn_total_loss,
family="head_loss")
tf.summary.scalar("regularization_loss", regularization_loss)
tf.summary.scalar('total_loss', total_loss)
tf.summary.scalar('learning_rate', lr)
# optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
#
# train_op = slim.learning.create_train_op(total_loss, optimizer, global_step)
with tf.name_scope("optimizer"):
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
optimizer = tf.contrib.estimator.TowerOptimizer(optimizer)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies([tf.group(*update_ops)]):
grads = optimizer.compute_gradients(total_loss)
# clip gradients
grads = tf.contrib.training.clip_gradient_norms(
grads, cfgs.CLIP_GRADIENT_NORM)
train_op = optimizer.apply_gradients(grads, global_step)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
# restorer, restore_ckpt = restore_model.get_restorer(test=False,checkpoint_path=cfgs.TRAINED_MODEL_DIR)
saver = tf.train.Saver(max_to_keep=3)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
# if not restorer is None:
# restorer.restore(sess, restore_ckpt)
# print('restore model successfully')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = cfgs.SUMMARY_PATH
summary_writer = tf.summary.FileWriter(summary_path,
graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
start = time.time()
_global_step,_rpn_location_loss, _rpn_classification_loss, \
_rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \
_fast_rcnn_total_loss, _total_loss, _ = \
sess.run([global_step,rpn_location_loss, rpn_classification_loss,
rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss,
fast_rcnn_total_loss,total_loss, train_op])
end = time.time()
if step % 50 == 0:
print("""{}: step{}
rpn_loc_loss:{:.4f} | rpn_cla_loss:{:.4f} | rpn_total_loss:{:.4f}
fast_rcnn_loc_loss:{:.4f} | fast_rcnn_cla_loss:{:.4f} | fast_rcnn_total_loss:{:.4f}
| total_loss:{:.4f} | pre_cost_time:{:.4f}s""".
format(training_time, _global_step,
_rpn_location_loss, _rpn_classification_loss,
_rpn_total_loss, _fast_rcnn_location_loss,
_fast_rcnn_classification_loss,
_fast_rcnn_total_loss, _total_loss,
(end - start)))
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, _global_step)
summary_writer.flush()
if (step > 0
and step % 5000 == 0) or (step
== cfgs.MAX_ITERATION - 1):
save_dir = cfgs.TRAINED_MODEL_DIR
save_ckpt = os.path.join(
save_dir, 'sarship' + '{}_'.format(cfgs.NET_NAME) +
str(_global_step) + 'model.ckpt')
saver.save(sess, save_ckpt)
print('Weights have been saved to {}.'.format(save_ckpt))
print('Training done')
coord.request_stop()
coord.join(threads)
def test(img_num):
with tf.Graph().as_default():
# img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8)
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=gtboxes_and_label,
is_training=False,
share_head=False,
share_net=share_net,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
anchor_angles=cfgs.ANCHOR_ANGLES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
anchor_stride=cfgs.ANCHOR_STRIDE,
top_k_nms=cfgs.RPN_TOP_K_NMS,
kernel_size=cfgs.KERNEL_SIZE,
use_angles_condition=True,
anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD,
nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
scope='')
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ]
# _, _, rpn_top_k_boxes, rpn_top_k_scores = rpn.rpn_losses()
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
stop_gradient_for_proposals=False,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS,
nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD,
use_angle_condition=False,
level=cfgs.LEVEL,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD,
boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD,
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
# test
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer()
)
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \
_fast_rcnn_score, _detection_category \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes,
fast_rcnn_score, detection_category])
end = time.time()
_img_batch = np.squeeze(_img_batch, axis=0)
_img_batch_fpn = help_utils.draw_box_cv(_img_batch,
boxes=_fast_rcnn_decode_boxes,
labels=_detection_category,
scores=_fast_rcnn_score)
cv2.imwrite(cfgs.TEST_SAVE_PATH + '/{}_fpn.jpg'.format(str(_img_name_batch[0]).split('.tif')[0]),
_img_batch_fpn)
temp_label = np.reshape(_gtboxes_and_label[:, -1:], [-1, ]).astype(np.int64)
_img_batch_gt = help_utils.draw_box_cv(_img_batch,
boxes=_gtboxes_and_label[:, :-1],
labels=temp_label,
scores=None)
cv2.imwrite(cfgs.TEST_SAVE_PATH + '/{}_gt.jpg'.format(str(_img_name_batch[0]).split('.tif')[0]),
_img_batch_gt)
view_bar('{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i+1, img_num)
coord.request_stop()
coord.join(threads)
def eval_dict_convert(img_num, mode):
with tf.Graph().as_default():
# img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8)
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_minAreaRectangle = get_horizon_minAreaRectangle(gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(gtboxes_and_label_minAreaRectangle, [-1, 5])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=gtboxes_and_label,
is_training=False,
share_head=False,
share_net=share_net,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
anchor_angles=cfgs.ANCHOR_ANGLES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
anchor_stride=cfgs.ANCHOR_STRIDE,
top_k_nms=cfgs.RPN_TOP_K_NMS,
kernel_size=cfgs.KERNEL_SIZE,
use_angles_condition=False,
anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD,
nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
# iou>=0.7 is positive box, iou< 0.3 is negative
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=cfgs.IS_FILTER_OUTSIDE_BOXES, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
scope='')
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ]
_, _, rpn_predict_boxes, rpn_predict_scores = rpn.rpn_losses()
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
stop_gradient_for_proposals=False,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS,
nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD,
use_angle_condition=False,
level=cfgs.LEVEL,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD,
boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD,
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
##############################################################################################
if cfgs.NEED_AUXILIARY:
predict_boxes = tf.concat([fast_rcnn_decode_boxes, rpn_predict_boxes], axis=0)
predict_scores = tf.concat([fast_rcnn_score, rpn_predict_scores - 0.2], axis=0)
rpn_predict_label = tf.ones([tf.shape(rpn_predict_scores)[0], ], tf.int64)
labels = tf.concat([detection_category, rpn_predict_label], axis=0)
# valid_indices = nms_rotate.nms_rotate(decode_boxes=predict_boxes,
# scores=predict_scores,
# iou_threshold=0.15,
# max_output_size=30,
# use_angle_condition=False,
# angle_threshold=15,
# use_gpu=True)
valid_indices = tf.py_func(nms_rotate.nms_rotate_cpu,
inp=[predict_boxes, predict_scores,
tf.constant(0.15, tf.float32), tf.constant(30, tf.float32)],
Tout=tf.int64)
fast_rcnn_decode_boxes = tf.gather(predict_boxes, valid_indices)
fast_rcnn_score = tf.gather(predict_scores, valid_indices)
detection_category = tf.gather(labels, valid_indices)
##############################################################################################
if mode == 0:
fast_rcnn_decode_boxes = get_horizon_minAreaRectangle(fast_rcnn_decode_boxes, False)
# train
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer()
)
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
gtboxes_dict = {}
predict_dict = {}
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \
_gtboxes_and_label_minAreaRectangle, _fast_rcnn_score, _detection_category \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes,
gtboxes_and_label_minAreaRectangle, fast_rcnn_score, detection_category])
end = time.time()
# gtboxes convert dict
gtboxes_dict[str(_img_name_batch[0])] = []
predict_dict[str(_img_name_batch[0])] = []
# for j, box in enumerate(_gtboxes_and_label):
# bbox_dict = {}
# bbox_dict['bbox'] = np.array(_gtboxes_and_label[j, :-1], np.float64)
# bbox_dict['name'] = LABEl_NAME_MAP[int(_gtboxes_and_label[j, -1])]
# gtbox_dict[str(_img_name_batch[0])].append(bbox_dict)
#
# for label in NAME_LABEL_MAP.keys():
# if label == 'back_ground':
# continue
# else:
# temp_dict = {}
# temp_dict['name'] = label
#
# ind = np.where(_detection_category == NAME_LABEL_MAP[label])[0]
# temp_boxes = _fast_rcnn_decode_boxes[ind]
# temp_score = np.reshape(_fast_rcnn_score[ind], [-1, 1])
# temp_dict['bbox'] = np.array(np.concatenate([temp_boxes, temp_score], axis=1), np.float64)
# predict_dict[str(_img_name_batch[0])].append(temp_dict)
if mode == 0:
gtboxes_list, predict_list = \
make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes,
_fast_rcnn_score, _detection_category)
else:
gtboxes_list, predict_list = \
make_dict_packle(_gtboxes_and_label, _fast_rcnn_decode_boxes,
_fast_rcnn_score, _detection_category)
gtboxes_dict[str(_img_name_batch[0])].extend(gtboxes_list)
predict_dict[str(_img_name_batch[0])].extend(predict_list)
view_bar('{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num)
fw1 = open('gtboxes_dict.pkl', 'w')
fw2 = open('predict_dict.pkl', 'w')
pickle.dump(gtboxes_dict, fw1)
pickle.dump(predict_dict, fw2)
fw1.close()
fw2.close()
coord.request_stop()
coord.join(threads)
def train():
retinanet = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=True)
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5])
if cfgs.NET_NAME in ['resnet152_v1d', 'resnet101_v1d', 'resnet50_v1d']:
img_batch = img_batch / tf.constant([cfgs.PIXEL_STD])
final_bbox, final_scores, final_category, loss_dict = retinanet.build_whole_detection_network(
input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label)
# ----------------------------------------------------------------------------------------------------build loss
weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
cls_loss = loss_dict['cls_loss']
reg_loss = loss_dict['reg_loss']
total_loss = cls_loss + reg_loss + weight_decay_loss
# ---------------------------------------------------------------------------------------------------add summary
tf.summary.scalar('RETINANET_LOSS/cls_loss', cls_loss)
tf.summary.scalar('RETINANET_LOSS/reg_loss', reg_loss)
tf.summary.scalar('LOSS/total_loss', total_loss)
tf.summary.scalar('LOSS/regular_weights', weight_decay_loss)
gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories(
img_batch=img_batch,
boxes=gtboxes_and_label[:, :-1],
labels=gtboxes_and_label[:, -1])
if cfgs.ADD_BOX_IN_TENSORBOARD:
detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores(
img_batch=img_batch,
boxes=final_bbox,
labels=final_category,
scores=final_scores)
tf.summary.image('Compare/final_detection', detections_in_img)
tf.summary.image('Compare/gtboxes', gtboxes_in_img)
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int64(cfgs.DECAY_STEP[0]),
np.int64(cfgs.DECAY_STEP[1])
],
values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
# ---------------------------------------------------------------------------------------------compute gradients
gradients = retinanet.get_gradients(optimizer, total_loss)
# enlarge_gradients for bias
if cfgs.MUTILPY_BIAS_GRADIENT:
gradients = retinanet.enlarge_gradients_for_bias(gradients)
if cfgs.GRADIENT_CLIPPING_BY_NORM:
with tf.name_scope('clip_gradients'):
gradients = slim.learning.clip_gradient_norms(
gradients, cfgs.GRADIENT_CLIPPING_BY_NORM)
# train_op
train_op = optimizer.apply_gradients(grads_and_vars=gradients,
global_step=global_step)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = retinanet.get_restorer()
saver = tf.train.Saver(max_to_keep=30)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
if not os.path.exists(summary_path):
os.makedirs(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
_, global_stepnp = sess.run([train_op, global_step])
else:
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
_, global_stepnp, img_name, reg_loss_, cls_loss_, total_loss_ = \
sess.run(
[train_op, global_step, img_name_batch, reg_loss, cls_loss, total_loss])
end = time.time()
print(""" {}: step{} image_name:{} |\t
reg_loss:{} |\t cls_loss:{} |\t total_loss:{} |per_cost_time:{}s""" \
.format(training_time, global_stepnp, str(img_name[0]), reg_loss_, cls_loss_, total_loss_,
(end - start)))
else:
if step % cfgs.SMRY_ITER == 0:
_, global_stepnp, summary_str = sess.run(
[train_op, global_step, summary_op])
summary_writer.add_summary(summary_str, global_stepnp)
summary_writer.flush()
if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE
== 0) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_ckpt = os.path.join(
save_dir, '{}_'.format(cfgs.DATASET_NAME) +
str(global_stepnp) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def train():
with tf.Graph().as_default():
tf.set_random_seed(1234)
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=True,
is_val=False)
with tf.name_scope('draw_gtboxes'):
gtboxes_in_img = draw_box_with_color(
img_batch,
tf.reshape(gtboxes_and_label_batch, [-1, 5])[:, :-1],
text=tf.shape(gtboxes_and_label_batch)[1])
# *********************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * rpn *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0),
is_training=True,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.
RPN_IOU_NEGATIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
rpn_location_loss, rpn_classification_loss = rpn.rpn_losses()
rpn_total_loss = rpn_classification_loss + rpn_location_loss
with tf.name_scope('draw_proposals'):
# score > 0.5 is object
rpn_object_boxes_indices = tf.reshape(
tf.where(tf.greater(rpn_proposals_scores, 0.5)), [-1])
rpn_object_boxes = tf.gather(rpn_proposals_boxes,
rpn_object_boxes_indices)
rpn_proposals_objcet_boxes_in_img = draw_box_with_color(
img_batch,
rpn_object_boxes,
text=tf.shape(rpn_object_boxes)[0])
rpn_proposals_boxes_in_img = draw_box_with_color(
img_batch,
rpn_proposals_boxes,
text=tf.shape(rpn_proposals_boxes)[0])
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0),
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.
FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative
use_dropout=False,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=True,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss(
)
fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss
with tf.name_scope('draw_boxes_with_categories'):
fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories(
img_batch=img_batch,
boxes=fast_rcnn_decode_boxes,
labels=detection_category,
scores=fast_rcnn_score)
# train
total_loss = slim.losses.get_total_loss()
global_step = slim.get_or_create_global_step() #返回并创建全局步长张量
#
# lr = tf.train.piecewise_constant(global_step,
# boundaries=[np.int64(10000), np.int64(20000)],
# values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100])
lr = tf.train.exponential_decay(cfgs.LR,
global_step,
decay_steps=5000,
decay_rate=1 / 2.,
staircase=True)
# lr = tf.train.piecewise_constant(global_step,
# boundaries=[np.int64(30000), np.int64(40000)],
# values=[lr, cfgs.LR/100, cfgs.LR/1000])
tf.summary.scalar('learning_rate', lr)
# optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
optimizer = tf.train.AdamOptimizer(lr,
beta1=cfgs.MOMENTUM,
beta2=0.999,
epsilon=1e-8,
use_locking=False,
name='Adam')
# optimizer = tf.train.RMSPropOptimizer(lr, decay=0.9, epsilon=1e-6, name='RMSProp')
#创建一个计算梯度并返回损失的Operation
train_op = slim.learning.create_train_op(
total_loss, optimizer, global_step) # rpn_total_loss,
# train_op = optimizer.minimize(second_classification_loss, global_step)
# ***********************************************************************************************
# * Summary *
# ***********************************************************************************************
# ground truth and predict
tf.summary.image('img/gtboxes', gtboxes_in_img)
tf.summary.image('img/faster_rcnn_predict',
fast_rcnn_predict_boxes_in_imgs)
# rpn loss and image
tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss)
tf.summary.scalar('rpn/rpn_classification_loss',
rpn_classification_loss)
tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_location_loss',
fast_rcnn_location_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss',
fast_rcnn_classification_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss',
fast_rcnn_total_loss)
tf.summary.scalar('loss/total_loss', total_loss)
# #
# tf.summary.image('C2', _concact_features(share_net['resnet_v1_50/block1/unit_2/bottleneck_v1'][:, :, :, 0:16]), 1)
# tf.summary.image('C3', _concact_features(share_net['resnet_v1_50/block2/unit_3/bottleneck_v1'][:, :, :, 0:16]), 1)
# tf.summary.image('C4', _concact_features(share_net['resnet_v1_50/block3/unit_5/bottleneck_v1'][:, :, :, 0:16]), 1)
# tf.summary.image('C5', _concact_features(share_net['resnet_v1_50/block4'][:, :, :, 0:16]), 1)
# tf.summary.image('P2', _concact_features(rpn.feature_pyramid['P2'][:, :, :, 0:16]),1)
# tf.summary.image('P3', _concact_features(rpn.feature_pyramid['P3'][:, :, :, 0:16]),1)
# tf.summary.image('P4', _concact_features(rpn.feature_pyramid['P4'][:, :, :, 0:16]),1)
# tf.summary.image('P5', _concact_features(rpn.feature_pyramid['P5'][:, :, :, 0:16]), 1)
# tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img)
# tf.summary.image('rpn/rpn_object_boxes', rpn_proposals_objcet_boxes_in_img)
# learning_rate
# tf.summary.scalar('learning_rate', lr)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
saver = tf.train.Saver(max_to_keep=16)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
if cfgs.NET_NAME == 'pvanet':
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
start = 0
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
start = int("".join(
list(restore_ckpt.split('/')[-1])[4:8])) + 1
else:
# read_npy.load_initial_weights(sess)
read_npy.load_ckpt_weights(sess)
else:
sess.run(init_op)
# print(sess.run('resnet_v1_50/block4/unit_3/bottleneck_v1/conv3/BatchNorm/moving_variance'))
# print(sess.run('vgg_16/block4/unit_3/bottleneck_v1/conv3/BatchNorm/moving_variance'))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
start = 0
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
# start = int("".join(list(restore_ckpt.split('/')[-1])[4:8]))+1
summary_path = os.path.join(FLAGS.summary_path, cfgs.VERSION)
mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path,
graph=sess.graph)
df = pd.DataFrame(
[],
columns=['Recall', 'Precision', 'mAP', 'F1_score'],
index=[])
for step in range(0, cfgs.MAX_ITERATION):
# print(img_name_batch.eval())
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
start = time.time()
_global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \
_rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \
_fast_rcnn_total_loss, _total_loss, _ = \
sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss,
rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss,
fast_rcnn_total_loss, total_loss, train_op])
end = time.time()
# if step == 100:
# save_dir = os.path.join(FLAGS.trained_checkpoint, cfgs.VERSION)
# mkdir(save_dir)
#
# save_ckpt = os.path.join(save_dir, 'voc_' + str(_global_step) + 'model.ckpt')
# saver.save(sess, save_ckpt)
# print(' weights had been saved')
# if step == 500:
# save_dir = os.path.join(FLAGS.trained_checkpoint, cfgs.VERSION)
# mkdir(save_dir)
#
# save_ckpt = os.path.join(save_dir, 'voc_' + str(_global_step) + 'model.ckpt')
# saver.save(sess, save_ckpt)
# print(' weights had been saved')
if step % 50 == 0:
print(""" {}: step{} image_name:{} |\t
rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} |
fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} |
total_loss:{} |\t pre_cost_time:{}s""" \
.format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss,
_rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss,
_fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss,
(end - start)))
# print(""" {}: step{} image_name:{} |\t
# rpn_loc_loss:{} |\t
# fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} |
# total_loss:{} |\t pre_cost_time:{}s""" \
# .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss,
# _fast_rcnn_location_loss,
# _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss,
# (end - start)))
if step % 250 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, _global_step)
summary_writer.flush()
if (step > 0 and step % 2000 == 0) or (
step > 0 and
(step == 1000)) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(FLAGS.trained_checkpoint,
cfgs.VERSION)
mkdir(save_dir)
save_ckpt = os.path.join(
save_dir, 'voc_' + str(_global_step) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
#保存验证集信息
if (step > 0
and step % 2000 == 0) or (step
== cfgs.MAX_ITERATION - 1):
save_excel = os.path.abspath(
'../'
) + r'/Loss/' + cfgs.NET_NAME + r'_' + cfgs.VERSION
mkdir(save_excel)
new_index = np.append(df.index, [str(step)])
df2 = pd.DataFrame(
[valval.val(is_val=True)],
columns=['Recall', 'Precision', 'mAP', 'F1_score'])
df = df.append(df2)
df.index = new_index
df.to_excel(save_excel + r'/validation.xls')
print('validation result had been saved')
coord.request_stop()
coord.join(threads)
def train():
with tf.Graph().as_default():
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=True)
with tf.name_scope('draw_gtboxes'):
gtboxes_in_img = draw_box_with_color(img_batch, tf.reshape(gtboxes_and_label_batch, [-1, 5])[:, :-1],
text=tf.shape(gtboxes_and_label_batch)[1])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * rpn *
# ***********************************************************************************************
rpn = build_rpn.RPN(net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0),
is_training=True,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
# iou>=0.7 is positive box, iou< 0.3 is negative
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ]
rpn_location_loss, rpn_classification_loss = rpn.rpn_losses()
rpn_total_loss = 10*rpn_classification_loss + rpn_location_loss
with tf.name_scope('draw_proposals'):
# score > 0.5 is object
rpn_object_boxes_indices = tf.reshape(tf.where(tf.greater(rpn_proposals_scores, 0.5)), [-1])
rpn_object_boxes = tf.gather(rpn_proposals_boxes, rpn_object_boxes_indices)
rpn_proposals_objcet_boxes_in_img = draw_box_with_color(img_batch, rpn_object_boxes,
text=tf.shape(rpn_object_boxes)[0])
rpn_proposals_boxes_in_img = draw_box_with_color(img_batch, rpn_proposals_boxes,
text=tf.shape(rpn_proposals_boxes)[0])
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0),
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
# iou>0.5 is positive, iou<0.5 is negative
fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD,
use_dropout=False,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=True,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
fast_rcnn_location_l, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss()
fast_rcnn_total_loss = fast_rcnn_location_l + 10*fast_rcnn_classification_loss
with tf.name_scope('draw_boxes_with_categories'):
fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories(img_batch=img_batch,
boxes=fast_rcnn_decode_boxes,
labels=detection_category,
scores=fast_rcnn_score)
# train
added_loss = rpn_total_loss + fast_rcnn_total_loss
total_loss = tf.losses.get_total_loss()
global_step = tf.train.get_or_create_global_step()
lr = tf.train.piecewise_constant(global_step,
boundaries=[np.int64(20000), np.int64(40000)],
values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
# optimizer = tf.train.AdamOptimizer(lr)
# train_op = slim.learning.create_train_op(total_loss, optimizer, global_step) # rpn_total_loss,
train_op = slim.learning.create_train_op(added_loss, optimizer, global_step) # rpn_total_loss,
# train_op = optimizer.minimize(second_classification_loss, global_step)
# **********************************-*************************************************************
# * Summary *
# ***********************************************************************************************
# ground truth and predict
tf.summary.image('img/gtboxes', gtboxes_in_img)
tf.summary.image('img/faster_rcnn_predict', fast_rcnn_predict_boxes_in_imgs)
# rpn loss and image
tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss)
tf.summary.scalar('rpn/rpn_classification_loss', rpn_classification_loss)
tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_location_lossum', fast_rcnn_location_l)
tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss', fast_rcnn_classification_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss', fast_rcnn_total_loss)
tf.summary.scalar('loss/added_loss', added_loss)
tf.summary.scalar('loss/total_loss', total_loss)
tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img)
tf.summary.image('rpn/rpn_object_boxes', rpn_proposals_objcet_boxes_in_img)
# learning_rate
tf.summary.scalar('learning_rate', lr)
summary_op = tf.compat.v1.summary.merge_all()
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer()
)
restorer, restore_ckpt = restore_model.get_restorer(test=False)
saver = tf.train.Saver(max_to_keep=5)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model', restore_ckpt)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join('output/{}'.format(cfgs.DATASET_NAME),
FLAGS.summary_path, cfgs.VERSION)
mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
start = time.time()
# gtboxes = tf.cast(
# tf.reshape(fast_rcnn.gtboxes_and_label[:, :-1], [-1, 4]), tf.float32)
# ious = iou.iou_calculate(fast_rcnn.fast_rcnn_all_level_proposals, gtboxes)
# _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \
# _rpn_total_loss, _fast_rcnn_location_l, _fast_rcnn_classification_loss, \
# _fast_rcnn_total_loss, _added_loss, _total_loss, rcnn_minibatch, find_samples,\
# gtbox1, iou1, _ = \
# sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss,
# rpn_total_loss, fast_rcnn_location_l, fast_rcnn_classification_loss,
# fast_rcnn_total_loss, added_loss, total_loss,
# fast_rcnn.fast_rcnn_minibatch(fast_rcnn.fast_rcnn_all_level_proposals),
# fast_rcnn.fast_rcnn_find_positive_negative_samples(fast_rcnn.fast_rcnn_all_level_proposals),
# gtboxes, ious,
# train_op])
_global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \
_rpn_total_loss, _fast_rcnn_location_l, _fast_rcnn_classification_loss, \
_fast_rcnn_total_loss, _added_loss, _total_loss, \
_ = \
sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss,
rpn_total_loss, fast_rcnn_location_l, fast_rcnn_classification_loss,
fast_rcnn_total_loss, added_loss, total_loss,
train_op])
# _global_step, _img_name_batch, _added_loss, _total_loss, \
# _ = \
# sess.run([global_step, img_name_batch, added_loss, total_loss,
# train_op])
end = time.time()
# a, b, c ,d = rcnn_minibatch
# aa, bb, cc = find_samples
# q = []
# for i in cc:
# if i not in q:
# q.append(i)
# qq = []
# for i in c:
# if i not in qq:
# qq.append(i)
# minibatch_encode_boxes = tf.gather(fast_rcnn.fast_rcnn_encode_boxes,a)
# minibatch_reference_boxes = tf.gather(fast_rcnn.fast_rcnn_all_level_proposals, a)
# minibatch_encode_gtboxes = \
# encode_and_decode.encode_boxes(
# unencode_boxes=b,
# reference_boxes=minibatch_reference_boxes,
# scale_factors=fast_rcnn.scale_factors
# )
# # [minibatch_size, num_classes*4]
# minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes, [1, fast_rcnn.num_classes])
#
# class_weights_list = []
# category_list = tf.unstack(d, axis=1)
# for i in range(1, fast_rcnn.num_classes + 1):
# tmp_class_weights = tf.ones(
# shape=[tf.shape(minibatch_encode_boxes)[0], 4], dtype=tf.float32)
# tmp_class_weights = tmp_class_weights * tf.expand_dims(category_list[i], axis=1)
# class_weights_list.append(tmp_class_weights)
# class_weights = tf.concat(class_weights_list, axis=1) # [minibatch_size, num_classes*4]
#
# loc_loss1 = losses.l1_smooth_losses(predict_boxes=minibatch_encode_boxes,
# gtboxes=minibatch_encode_gtboxes,
# object_weights=c,
# classes_weights=class_weights)
# print('loction-loss:', sess.run(loc_loss1))
if step % 50 == 0:
print("""{}: step{} image_name:{}
rpn_loc_loss:{:.4f} | rpn_cla_loss:{:.4f} | rpn_total_loss:{:.4f}
fast_rcnn_loc_loss:{:.10f} | fast_rcnn_cla_loss:{:.10f} | fast_rcnn_total_loss:{:.4f}
added_loss:{:.4f} | total_loss:{:.4f} | pre_cost_time:{:.4f}s"""
.format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss,
_rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_l,
_fast_rcnn_classification_loss, _fast_rcnn_total_loss, _added_loss, _total_loss,
(end - start)))
# if step % 100 == 0:
# print("""{}: step{} image_name:{}
# added_loss:{:.4f} | total_loss:{:.4f} | pre_cost_time:{:.4f}s"""
# .format(training_time, _global_step, str(_img_name_batch[0]), _added_loss, _total_loss, (end - start)
# ))
# print(q, '\n', qq)
# print(gtbox1)
# print(iou1)
if step % 500000 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, _global_step)
summary_writer.flush()
if (step > 0 and step % 10000 == 0) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join('output/{}'.format(cfgs.DATASET_NAME),
FLAGS.trained_checkpoint, cfgs.VERSION)
mkdir(save_dir)
save_ckpt = os.path.join(save_dir, '{}_'.format(
cfgs.DATASET_NAME)+str(_global_step)+'model.ckpt')
saver.save(sess, save_ckpt)
print('Weights have been saved to {}.'.format(save_ckpt))
print('Training done.')
coord.request_stop()
coord.join(threads)
def test(img_num):
with tf.Graph().as_default():
# img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8)
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=gtboxes_and_label,
is_training=False,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
anchor_angles=cfgs.ANCHOR_ANGLES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
anchor_stride=cfgs.ANCHOR_STRIDE,
pool_stride=cfgs.POOL_STRIDE,
top_k_nms=cfgs.RPN_TOP_K_NMS,
kernel_size=cfgs.KERNEL_SIZE,
use_angles_condition=True,
anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD,
nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
scope='')
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
_, _, rpn_top_k_boxes, rpn_top_k_scores = rpn.rpn_losses()
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
stop_gradient_for_proposals=False,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS,
nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD,
use_angle_condition=False,
level=cfgs.LEVEL,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD,
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
##############################################################################################
if cfgs.NEED_AUXILIARY:
predict_boxes = tf.concat(
[fast_rcnn_decode_boxes, rpn_top_k_boxes], axis=0)
predict_scores = tf.concat(
[fast_rcnn_score, rpn_top_k_scores - 0.2], axis=0)
rpn_top_k_label = tf.ones([
tf.shape(rpn_top_k_scores)[0],
], tf.int64)
labels = tf.concat([detection_category, rpn_top_k_label], axis=0)
# valid_indices = nms_rotate.nms_rotate(decode_boxes=predict_boxes,
# scores=predict_scores,
# iou_threshold=0.15,
# max_output_size=30,
# use_angle_condition=False,
# angle_threshold=15,
# use_gpu=True)
valid_indices = tf.py_func(nms_rotate.nms_rotate_cpu,
inp=[
predict_boxes, predict_scores,
tf.constant(0.15, tf.float32),
tf.constant(30, tf.float32)
],
Tout=tf.int64)
fast_rcnn_decode_boxes = tf.gather(predict_boxes, valid_indices)
fast_rcnn_score = tf.gather(predict_scores, valid_indices)
detection_category = tf.gather(labels, valid_indices)
##############################################################################################
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \
_fast_rcnn_score, _detection_category \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes,
fast_rcnn_score, detection_category])
end = time.time()
_img_batch = np.squeeze(_img_batch, axis=0)
_img_batch_fpn = help_utils.draw_box_cv(
_img_batch,
boxes=_fast_rcnn_decode_boxes,
labels=_detection_category,
scores=_fast_rcnn_score)
mkdir(cfgs.TEST_SAVE_PATH)
cv2.imwrite(
cfgs.TEST_SAVE_PATH + '/{}_fpn.jpg'.format(
str(_img_name_batch[0]).split('.tif')[0]),
_img_batch_fpn)
# _gtboxes_and_label_batch = np.squeeze(_gtboxes_and_label_batch, axis=0)
temp_label = np.reshape(_gtboxes_and_label[:, -1:], [
-1,
]).astype(np.int64)
_img_batch_gt = help_utils.draw_box_cv(
_img_batch,
boxes=_gtboxes_and_label[:, :-1],
labels=temp_label,
scores=None)
cv2.imwrite(
cfgs.TEST_SAVE_PATH + '/{}_gt.jpg'.format(
str(_img_name_batch[0]).split('.tif')[0]),
_img_batch_gt)
#view_bar('{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i+1, img_num)
print('{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)) + str(i + 1) +
'/' + str(img_num))
coord.request_stop()
coord.join(threads)
def eval_text(img_num, mode):
with tf.Graph().as_default():
faster_rcnn = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=False)
# 通过readtfrecord来读取文件
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
final_boxes_h, final_scores_h, final_category_h, \
final_boxes_r, final_scores_r, final_category_r= faster_rcnn.build_whole_detection_network(
input_img_batch=img_batch,
gtboxes_r_batch=gtboxes_and_label,
gtboxes_h_batch=gtboxes_and_label_minAreaRectangle)
if mode == 0:
final_boxes_r = get_horizen_minAreaRectangle(final_boxes_r, False)
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = faster_rcnn.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
gtboxes_horizontal_dict = {}
predict_horizontal_dict = {}
gtboxes_rotate_dict = {}
predict_rotate_dict = {}
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \
_final_boxes_h, _final_scores_h, _final_category_h, _final_boxes_r, \
_final_scores_r, _final_category_r \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle,
final_boxes_h, final_scores_h, final_category_h, final_boxes_r,
final_scores_r, final_category_r])
end = time.time()
# det_detections_h = draw_box_in_img.draw_box_cv(np.squeeze(_img_batch, 0),
# boxes=_final_boxes_h,
# labels=_final_category_h,
# scores=final_scores_h)
# det_detections_r = draw_box_in_img.draw_rotate_box_cv(np.squeeze(_img_batch, 0),
# boxes=_final_boxes_r,
# labels=_final_category_r,
# scores= _final_scores_r)
# save_dir = os.path.join('/home/sws/code/Tfrecord/', cfgs.VERSION)
# tools.mkdir(save_dir)
# cv2.imwrite(save_dir + '/' + _img_name_batch[0] + '_h.jpg',
# det_detections_h)
# cv2.imwrite(save_dir + '/' + _img_name_batch[0] + '_r.jpg',
# det_detections_r)
# gtboxes convert dict
gtboxes_horizontal_dict[str(_img_name_batch[0])] = []
predict_horizontal_dict[str(_img_name_batch[0])] = []
gtboxes_rotate_dict[str(_img_name_batch[0])] = []
predict_rotate_dict[str(_img_name_batch[0])] = []
gtbox_horizontal_list, predict_horizontal_list = \
make_dict_packle(_gtboxes_and_label_minAreaRectangle, _final_boxes_h,
_final_scores_h, _final_category_h)
# if mode == 0:
# gtbox_rotate_list, predict_rotate_list = \
# make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes_rotate,
# _fast_rcnn_score_rotate, _detection_category_rotate)
if mode == 0:
gtbox_rotate_list, predict_rotate_list = \
make_dict_packle(_gtboxes_and_label_minAreaRectangle, _final_boxes_r,
_final_scores_r, _final_category_r)
else:
gtbox_rotate_list, predict_rotate_list = \
make_dict_packle(_gtboxes_and_label, _final_boxes_r,
_final_scores_r, _final_category_r)
gtboxes_horizontal_dict[str(
_img_name_batch[0])].extend(gtbox_horizontal_list)
predict_horizontal_dict[str(
_img_name_batch[0])].extend(predict_horizontal_list)
gtboxes_rotate_dict[str(
_img_name_batch[0])].extend(gtbox_rotate_list)
predict_rotate_dict[str(
_img_name_batch[0])].extend(predict_rotate_list)
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
print('\n')
fw1 = open('gtboxes_horizontal_dict.pkl', 'w')
fw2 = open('predict_horizontal_dict.pkl', 'w')
fw3 = open('gtboxes_rotate_dict.pkl', 'w')
fw4 = open('predict_rotate_dict.pkl', 'w')
pickle.dump(gtboxes_horizontal_dict, fw1)
pickle.dump(predict_horizontal_dict, fw2)
pickle.dump(gtboxes_rotate_dict, fw3)
pickle.dump(predict_rotate_dict, fw4)
fw1.close()
fw2.close()
fw3.close()
fw4.close()
coord.request_stop()
coord.join(threads)
def train():
with tf.Graph().as_default(), tf.device('/cpu:0'):
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int64(cfgs.DECAY_STEP[0]),
np.int64(cfgs.DECAY_STEP[1])
],
values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
img_batch = tf.reshape(img_batch, [1, 720, 1280, 3])
batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue(
[img_name_batch, img_batch, gtboxes_and_label, num_objects_batch],
capacity=2 * len(cfgs.GPU_GROUP))
tower_grads = []
for i in range(len(cfgs.GPU_GROUP)):
with tf.device('/gpu:%d' % i):
with tf.name_scope('Gpu%d' % i) as scope:
img_name_, img_, gtboxes_and_label_, num_objects_ = batch_queue.dequeue(
)
loss, faster_rcnn, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss,\
fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss = \
tower_loss(scope, img_name_, img_, gtboxes_and_label_, num_objects_)
tf.get_variable_scope().reuse_variables()
grads = optimizer.compute_gradients(loss)
if cfgs.MUTILPY_BIAS_GRADIENT:
grads = faster_rcnn.enlarge_gradients_for_bias(grads)
if cfgs.GRADIENT_CLIPPING_BY_NORM:
with tf.name_scope('clip_gradients'):
grads = slim.learning.clip_gradient_norms(
grads, cfgs.GRADIENT_CLIPPING_BY_NORM)
tower_grads.append(grads)
grads = average_gradients(tower_grads)
train_op = optimizer.apply_gradients(grads, global_step)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = faster_rcnn.get_restorer()
saver = tf.train.Saver(max_to_keep=10)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
tools.mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path,
graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
_, global_stepnp = sess.run([train_op, global_step])
else:
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
_, global_stepnp, img_name, rpnLocLoss, rpnClsLoss, rpnTotalLoss, \
fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss = \
sess.run(
[train_op, global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss,
fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss, loss])
end = time.time()
print(""" {}: step{} image_name:{} |\t
rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} |
fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} |
total_loss:{} |\t per_cost_time:{}s""" \
.format(training_time, global_stepnp, str(img_name[0]), rpnLocLoss, rpnClsLoss,
rpnTotalLoss, fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss,
(end - start)))
else:
if step % cfgs.SMRY_ITER == 0:
_, global_stepnp, summary_str = sess.run(
[train_op, global_step, summary_op])
summary_writer.add_summary(summary_str,
global_stepnp)
summary_writer.flush()
if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE
== 0) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.mkdir(save_dir)
save_ckpt = os.path.join(
save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def tower_loss(scope):
with tf.name_scope(scope):
faster_rcnn = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=True)
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5])
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(
cfgs.WEIGHT_DECAY)
# list as many types of layers as possible, even if they are not used now
with slim.arg_scope([
slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose,
slim.separable_conv2d, slim.fully_connected
],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
biases_initializer=tf.constant_initializer(0.0)):
final_bbox, final_scores, final_category, loss_dict = faster_rcnn.build_whole_detection_network(
input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label)
# ----------------------------------------------------------------------------------------------------build loss
weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
rpn_location_loss = loss_dict['rpn_loc_loss']
rpn_cls_loss = loss_dict['rpn_cls_loss']
rpn_total_loss = rpn_location_loss + rpn_cls_loss
fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss']
fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss']
fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss
total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss
# ____________________________________________________________________________________________________build loss
# ---------------------------------------------------------------------------------------------------add summary
tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss)
tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss)
tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss',
fastrcnn_loc_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss)
tf.summary.scalar('LOSS/total_loss', total_loss)
tf.summary.scalar('LOSS/regular_weights', weight_decay_loss)
gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories(
img_batch=img_batch,
boxes=gtboxes_and_label[:, :-1],
labels=gtboxes_and_label[:, -1])
if cfgs.ADD_BOX_IN_TENSORBOARD:
detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores(
img_batch=img_batch,
boxes=final_bbox,
labels=final_category,
scores=final_scores)
tf.summary.image('Compare/final_detection', detections_in_img)
tf.summary.image('Compare/gtboxes', gtboxes_in_img)
return total_loss, faster_rcnn, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss,\
fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss
def eval_dict_convert(img_num, mode):
with tf.Graph().as_default():
# img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8)
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_minAreaRectangle = get_horizon_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=gtboxes_and_label,
is_training=False,
share_head=False,
share_net=share_net,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
anchor_angles=cfgs.ANCHOR_ANGLES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
anchor_stride=cfgs.ANCHOR_STRIDE,
top_k_nms=cfgs.RPN_TOP_K_NMS,
kernel_size=cfgs.KERNEL_SIZE,
use_angles_condition=False,
anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD,
nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
# iou>=0.7 is positive box, iou< 0.3 is negative
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=cfgs.
IS_FILTER_OUTSIDE_BOXES, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
scope='')
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
_, _, rpn_predict_boxes, rpn_predict_scores = rpn.rpn_losses()
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
img_batch=img_batch,
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
stop_gradient_for_proposals=False,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS,
nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD,
use_angle_condition=False,
level=cfgs.LEVEL,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD,
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
##############################################################################################
if cfgs.NEED_AUXILIARY:
predict_boxes = tf.concat(
[fast_rcnn_decode_boxes, rpn_predict_boxes], axis=0)
predict_scores = tf.concat(
[fast_rcnn_score, rpn_predict_scores - 0.2], axis=0)
rpn_predict_label = tf.ones([
tf.shape(rpn_predict_scores)[0],
], tf.int64)
labels = tf.concat([detection_category, rpn_predict_label], axis=0)
# valid_indices = nms_rotate.nms_rotate(decode_boxes=predict_boxes,
# scores=predict_scores,
# iou_threshold=0.15,
# max_output_size=30,
# use_angle_condition=False,
# angle_threshold=15,
# use_gpu=True)
valid_indices = tf.py_func(nms_rotate.nms_rotate_cpu,
inp=[
predict_boxes, predict_scores,
tf.constant(0.15, tf.float32),
tf.constant(30, tf.float32)
],
Tout=tf.int64)
fast_rcnn_decode_boxes = tf.gather(predict_boxes, valid_indices)
fast_rcnn_score = tf.gather(predict_scores, valid_indices)
detection_category = tf.gather(labels, valid_indices)
##############################################################################################
if mode == 0:
fast_rcnn_decode_boxes = get_horizon_minAreaRectangle(
fast_rcnn_decode_boxes, False)
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
gtboxes_dict = {}
predict_dict = {}
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \
_gtboxes_and_label_minAreaRectangle, _fast_rcnn_score, _detection_category \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes,
gtboxes_and_label_minAreaRectangle, fast_rcnn_score, detection_category])
end = time.time()
# gtboxes convert dict
gtboxes_dict[str(_img_name_batch[0])] = []
predict_dict[str(_img_name_batch[0])] = []
# for j, box in enumerate(_gtboxes_and_label):
# bbox_dict = {}
# bbox_dict['bbox'] = np.array(_gtboxes_and_label[j, :-1], np.float64)
# bbox_dict['name'] = LABEl_NAME_MAP[int(_gtboxes_and_label[j, -1])]
# gtbox_dict[str(_img_name_batch[0])].append(bbox_dict)
#
# for label in NAME_LABEL_MAP.keys():
# if label == 'back_ground':
# continue
# else:
# temp_dict = {}
# temp_dict['name'] = label
#
# ind = np.where(_detection_category == NAME_LABEL_MAP[label])[0]
# temp_boxes = _fast_rcnn_decode_boxes[ind]
# temp_score = np.reshape(_fast_rcnn_score[ind], [-1, 1])
# temp_dict['bbox'] = np.array(np.concatenate([temp_boxes, temp_score], axis=1), np.float64)
# predict_dict[str(_img_name_batch[0])].append(temp_dict)
if mode == 0:
gtboxes_list, predict_list = \
make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes,
_fast_rcnn_score, _detection_category)
else:
gtboxes_list, predict_list = \
make_dict_packle(_gtboxes_and_label, _fast_rcnn_decode_boxes,
_fast_rcnn_score, _detection_category)
gtboxes_dict[str(_img_name_batch[0])].extend(gtboxes_list)
predict_dict[str(_img_name_batch[0])].extend(predict_list)
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
fw1 = open('gtboxes_dict.pkl', 'w')
fw2 = open('predict_dict.pkl', 'w')
pickle.dump(gtboxes_dict, fw1)
pickle.dump(predict_dict, fw2)
fw1.close()
fw2.close()
coord.request_stop()
coord.join(threads)
def train():
with tf.Graph().as_default():
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
with tf.name_scope('draw_gtboxes'):
gtboxes_in_img = draw_box_with_color(
img_batch,
tf.reshape(gtboxes_and_label_minAreaRectangle,
[-1, 5])[:, :-1],
text=tf.shape(gtboxes_and_label_minAreaRectangle)[0])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * rpn *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=gtboxes_and_label_minAreaRectangle,
is_training=True,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.
RPN_IOU_NEGATIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
rpn_location_loss, rpn_classification_loss = rpn.rpn_losses()
rpn_total_loss = rpn_classification_loss + rpn_location_loss
with tf.name_scope('draw_proposals'):
# score > 0.5 is object
rpn_object_boxes_indices = tf.reshape(
tf.where(tf.greater(rpn_proposals_scores, 0.5)), [-1])
rpn_object_boxes = tf.gather(rpn_proposals_boxes,
rpn_object_boxes_indices)
rpn_proposals_objcet_boxes_in_img = draw_box_with_color(
img_batch,
rpn_object_boxes,
text=tf.shape(rpn_object_boxes)[0])
rpn_proposals_boxes_in_img = draw_box_with_color(
img_batch,
rpn_proposals_boxes,
text=tf.shape(rpn_proposals_boxes)[0])
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=gtboxes_and_label,
gtboxes_and_label_minAreaRectangle=
gtboxes_and_label_minAreaRectangle,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.
FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=True,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \
fast_rcnn.fast_rcnn_predict()
fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss(
)
fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss
with tf.name_scope('draw_boxes_with_categories'):
fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories(
img_batch=img_batch,
boxes=fast_rcnn_decode_boxes,
labels=detection_category,
scores=fast_rcnn_score)
# train
total_loss = slim.losses.get_total_loss()
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[np.int64(20000), np.int64(40000)],
values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
train_op = slim.learning.create_train_op(
total_loss, optimizer, global_step) # rpn_total_loss,
# train_op = optimizer.minimize(second_classification_loss, global_step)
# ***********************************************************************************************
# * Summary *
# ***********************************************************************************************
# ground truth and predict
tf.summary.image('img/gtboxes', gtboxes_in_img)
tf.summary.image('img/faster_rcnn_predict',
fast_rcnn_predict_boxes_in_imgs)
# rpn loss and image
tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss)
tf.summary.scalar('rpn/rpn_classification_loss',
rpn_classification_loss)
tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_location_loss',
fast_rcnn_location_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss',
fast_rcnn_classification_loss)
tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss',
fast_rcnn_total_loss)
tf.summary.scalar('loss/total_loss', total_loss)
tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img)
tf.summary.image('rpn/rpn_object_boxes',
rpn_proposals_objcet_boxes_in_img)
# learning_rate
tf.summary.scalar('learning_rate', lr)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
saver = tf.train.Saver(max_to_keep=10)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join(FLAGS.summary_path, cfgs.VERSION)
mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path,
graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
start = time.time()
_global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \
_rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \
_fast_rcnn_total_loss, _total_loss, _ = \
sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss,
rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss,
fast_rcnn_total_loss, total_loss, train_op])
end = time.time()
if step % 10 == 0:
print(""" {}: step{} image_name:{} |\t
rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} |
fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} |
total_loss:{} |\t pre_cost_time:{}s""" \
.format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss,
_rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss,
_fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss,
(end - start)))
if step % 50 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, _global_step)
summary_writer.flush()
if (step > 0
and step % 1000 == 0) or (step
== cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(FLAGS.trained_checkpoint,
cfgs.VERSION)
mkdir(save_dir)
save_ckpt = os.path.join(
save_dir, 'voc_' + str(_global_step) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def test_rotate():
with tf.Graph().as_default():
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label, head = get_head(
tf.squeeze(gtboxes_and_label_batch, 0))
gtboxes_and_label = tf.py_func(back_forward_convert,
inp=[gtboxes_and_label],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
head_quadrant = tf.py_func(get_head_quadrant,
inp=[head, gtboxes_and_label],
Tout=tf.float32)
head_quadrant = tf.reshape(head_quadrant, [-1, 1])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
with tf.name_scope('draw_gtboxes'):
gtboxes_in_img = draw_box_with_color(
img_batch,
tf.reshape(gtboxes_and_label_minAreaRectangle,
[-1, 5])[:, :-1],
text=tf.shape(gtboxes_and_label_minAreaRectangle)[0])
gtboxes_rotate_in_img = draw_box_with_color_rotate(
img_batch,
tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1],
text=tf.shape(gtboxes_and_label)[0],
head=head_quadrant)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
for i in range(650):
img_gtboxes, img_gtboxes_rotate, img_name = sess.run(
[gtboxes_in_img, gtboxes_rotate_in_img, img_name_batch])
img_gtboxes = np.squeeze(img_gtboxes, axis=0)
img_gtboxes_rotate = np.squeeze(img_gtboxes_rotate, axis=0)
print(i)
cv2.imwrite(
cfgs.INFERENCE_SAVE_PATH +
'/{}_horizontal_fpn.jpg'.format(str(img_name[0])),
img_gtboxes)
cv2.imwrite(
cfgs.INFERENCE_SAVE_PATH +
'/{}_rotate_fpn.jpg'.format(str(img_name[0])),
img_gtboxes_rotate)
coord.request_stop()
coord.join(threads)
def train():
faster_rcnn = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=True)
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
mask_batch = tf.py_func(
image_preprocess.get_mask,
[tf.squeeze(img_batch, 0),
tf.squeeze(gtboxes_and_label_batch, 0)], [tf.float32])
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_AreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_AreaRectangle = tf.reshape(
gtboxes_and_label_AreaRectangle, [-1, 5])
with tf.name_scope('draw_gtboxes'):
gtboxes_in_img = draw_box_with_color(
img_batch,
tf.reshape(gtboxes_and_label_AreaRectangle, [-1, 5])[:, :-1],
text=tf.shape(gtboxes_and_label_AreaRectangle)[0])
gtboxes_rotate_in_img = draw_box_with_color_rotate(
img_batch,
tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1],
text=tf.shape(gtboxes_and_label)[0])
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY)
# list as many types of layers as possible, even if they are not used now
with slim.arg_scope([
slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose,
slim.separable_conv2d, slim.fully_connected
],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
biases_initializer=tf.constant_initializer(0.0)):
final_boxes_h, final_scores_h, final_category_h, \
final_boxes_r, final_scores_r, final_category_r, loss_dict = faster_rcnn.build_whole_detection_network(
input_img_batch=img_batch,
gtboxes_r_batch=gtboxes_and_label,
gtboxes_h_batch=gtboxes_and_label_AreaRectangle,
mask_batch=mask_batch[0])
dets_in_img = draw_boxes_with_categories_and_scores(
img_batch=img_batch,
boxes=final_boxes_h,
labels=final_category_h,
scores=final_scores_h)
dets_rotate_in_img = draw_boxes_with_categories_and_scores_rotate(
img_batch=img_batch,
boxes=final_boxes_r,
labels=final_category_r,
scores=final_scores_r)
# ----------------------------------------------------------------------------------------------------build loss
weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
rpn_location_loss = loss_dict['rpn_loc_loss']
rpn_cls_loss = loss_dict['rpn_cls_loss']
rpn_total_loss = rpn_location_loss + rpn_cls_loss
fastrcnn_cls_loss_h = loss_dict['fastrcnn_cls_loss_h']
fastrcnn_loc_loss_h = loss_dict['fastrcnn_loc_loss_h']
fastrcnn_cls_loss_r = loss_dict['fastrcnn_cls_loss_r']
fastrcnn_loc_loss_r = loss_dict['fastrcnn_loc_loss_r']
fastrcnn_total_loss = fastrcnn_cls_loss_h + fastrcnn_loc_loss_h + fastrcnn_cls_loss_r + fastrcnn_loc_loss_r
if cfgs.ADD_ATTENTION:
attention_loss = loss_dict['attention_loss']
total_loss = rpn_total_loss + fastrcnn_total_loss + attention_loss + weight_decay_loss
else:
attention_loss = tf.convert_to_tensor(0)
total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss
# ---------------------------------------------------------------------------------------------------add summary
tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss)
tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss)
tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss_h', fastrcnn_cls_loss_h)
tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss_h',
fastrcnn_loc_loss_h)
tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss_r', fastrcnn_cls_loss_r)
tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss_r',
fastrcnn_loc_loss_r)
tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss)
if cfgs.ADD_ATTENTION:
tf.summary.scalar('ATTENTION_LOSS/attention_loss', attention_loss)
tf.summary.scalar('LOSS/total_loss', total_loss)
tf.summary.scalar('LOSS/regular_weights', weight_decay_loss)
tf.summary.image('img/gtboxes', gtboxes_in_img)
tf.summary.image('img/gtboxes_rotate', gtboxes_rotate_in_img)
tf.summary.image('img/mask', mask_batch)
tf.summary.image('img/dets', dets_in_img)
tf.summary.image('img/dets_rotate', dets_rotate_in_img)
# ---------------------------------------------------------------------------------------------add summary
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int64(cfgs.DECAY_STEP[0]),
np.int64(cfgs.DECAY_STEP[1])
],
values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
# ---------------------------------------------------------------------------------------------compute gradients
gradients = faster_rcnn.get_gradients(optimizer, total_loss)
# enlarge_gradients for bias
if cfgs.MUTILPY_BIAS_GRADIENT:
gradients = faster_rcnn.enlarge_gradients_for_bias(gradients)
if cfgs.GRADIENT_CLIPPING_BY_NORM:
with tf.name_scope('clip_gradients'):
gradients = slim.learning.clip_gradient_norms(
gradients, cfgs.GRADIENT_CLIPPING_BY_NORM)
# ---------------------------------------------------------------------------------------------compute gradients
# train_op
train_op = optimizer.apply_gradients(grads_and_vars=gradients,
global_step=global_step)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = faster_rcnn.get_restorer()
saver = tf.train.Saver(max_to_keep=10)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
#
with tf.Session(config=config) as sess:
# with tf.Session() as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
tools.mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
_, global_stepnp = sess.run([train_op, global_step])
else:
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
_global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \
_rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \
_fast_rcnn_location_rotate_loss, _fast_rcnn_classification_rotate_loss, \
_fast_rcnn_total_loss, _attention_loss, _total_loss, _ = \
sess.run([global_step, img_name_batch, rpn_location_loss, rpn_cls_loss,
rpn_total_loss, fastrcnn_loc_loss_h, fastrcnn_cls_loss_h,
fastrcnn_loc_loss_r, fastrcnn_cls_loss_r, fastrcnn_total_loss,
attention_loss, total_loss, train_op])
end = time.time()
print(""" {}: step{} image_name:{} |\t
rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t
rpn_total_loss:{} |
fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t
fast_rcnn_loc_rotate_loss:{} |\t fast_rcnn_cla_rotate_loss:{} |\t
fast_rcnn_total_loss:{} |\t attention_loss:{} |\t
total_loss:{} |\t pre_cost_time:{}s""" \
.format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss,
_rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss,
_fast_rcnn_classification_loss, _fast_rcnn_location_rotate_loss,
_fast_rcnn_classification_rotate_loss, _fast_rcnn_total_loss,
_attention_loss, _total_loss, (end - start)))
else:
if step % cfgs.SMRY_ITER == 0:
_, global_stepnp, summary_str = sess.run(
[train_op, global_step, summary_op])
summary_writer.add_summary(summary_str, global_stepnp)
summary_writer.flush()
if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE
== 0) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_ckpt = os.path.join(
save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def eval_ship(img_num):
with tf.Graph().as_default():
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label, head = get_head(
tf.squeeze(gtboxes_and_label_batch, 0))
gtboxes_and_label = tf.py_func(back_forward_convert,
inp=[gtboxes_and_label],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
head_quadrant = tf.py_func(get_head_quadrant,
inp=[head, gtboxes_and_label],
Tout=tf.float32)
head_quadrant = tf.reshape(head_quadrant, [-1, 1])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=None,
is_training=False,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
img_batch=img_batch,
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
gtboxes_and_label_minAreaRectangle=
gtboxes_and_label_minAreaRectangle,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
# iou>0.5 is positive, iou<0.5 is negative
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False,
level=cfgs.LEVEL,
head_quadrant=head_quadrant)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \
fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, fast_rcnn_head_quadrant, \
num_of_objects_rotate, detection_category_rotate = fast_rcnn.fast_rcnn_predict()
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
gtboxes_horizontal_dict = {}
gtboxes_rotate_dict = {}
all_boxes_h = []
all_boxes_r = []
all_img_names = []
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \
_fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \
_fast_rcnn_score_rotate, _detection_category_rotate \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle,
fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate,
fast_rcnn_score_rotate, detection_category_rotate])
end = time.time()
# gtboxes convert dict
gtboxes_horizontal_dict[str(_img_name_batch[0])] = []
gtboxes_rotate_dict[str(_img_name_batch[0])] = []
gtbox_horizontal_list, gtbox_rotate_list = make_dict_packle(
_gtboxes_and_label, _gtboxes_and_label_minAreaRectangle)
xmin, ymin, xmax, ymax = _fast_rcnn_decode_boxes[:,
1], _fast_rcnn_decode_boxes[:,
0], _fast_rcnn_decode_boxes[:,
3], _fast_rcnn_decode_boxes[:,
2]
x_c, y_c, w, h, theta = _fast_rcnn_decode_boxes_rotate[:, 1], _fast_rcnn_decode_boxes_rotate[:, 0], _fast_rcnn_decode_boxes_rotate[:, 3], \
_fast_rcnn_decode_boxes_rotate[:, 2], _fast_rcnn_decode_boxes_rotate[:, 4]
boxes_h = np.transpose(np.stack([xmin, ymin, xmax, ymax]))
boxes_r = np.transpose(np.stack([x_c, y_c, w, h, theta]))
dets_h = np.hstack((_detection_category.reshape(-1, 1),
_fast_rcnn_score.reshape(-1, 1), boxes_h))
dets_r = np.hstack(
(_detection_category_rotate.reshape(-1, 1),
_fast_rcnn_score_rotate.reshape(-1, 1), boxes_r))
all_boxes_h.append(dets_h)
all_boxes_r.append(dets_r)
all_img_names.append(str(_img_name_batch[0]))
gtboxes_horizontal_dict[str(
_img_name_batch[0])].extend(gtbox_horizontal_list)
gtboxes_rotate_dict[str(
_img_name_batch[0])].extend(gtbox_rotate_list)
print(str(_img_name_batch[0]))
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
write_voc_results_file(all_boxes_h, all_img_names,
cfgs.EVALUATE_R_DIR, 0)
write_voc_results_file(all_boxes_r, all_img_names,
cfgs.EVALUATE_R_DIR, 1)
fw1 = open('gtboxes_horizontal_dict.pkl', 'wb')
fw2 = open('gtboxes_rotate_dict.pkl', 'wb')
pickle.dump(gtboxes_horizontal_dict, fw1)
pickle.dump(gtboxes_rotate_dict, fw2)
fw1.close()
fw2.close()
coord.request_stop()
coord.join(threads)
def eval_ship(img_num, mode):
with tf.Graph().as_default():
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=None,
is_training=False,
share_head=False,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn1.FastRCNN(
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
gtboxes_and_label_minAreaRectangle=
gtboxes_and_label_minAreaRectangle,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
# iou>0.5 is positive, iou<0.5 is negative
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \
fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, num_of_objects_rotate, detection_category_rotate = \
fast_rcnn.fast_rcnn_predict()
if mode == 0:
fast_rcnn_decode_boxes_rotate = get_horizen_minAreaRectangle(
fast_rcnn_decode_boxes_rotate, False)
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
with tf.Session() as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
gtboxes_horizontal_dict = {}
predict_horizontal_dict = {}
gtboxes_rotate_dict = {}
predict_rotate_dict = {}
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \
_fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \
_fast_rcnn_score_rotate, _detection_category_rotate \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle,
fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate,
fast_rcnn_score_rotate, detection_category_rotate])
end = time.time()
# gtboxes convert dict
gtboxes_horizontal_dict[str(_img_name_batch[0])] = []
predict_horizontal_dict[str(_img_name_batch[0])] = []
gtboxes_rotate_dict[str(_img_name_batch[0])] = []
predict_rotate_dict[str(_img_name_batch[0])] = []
gtbox_horizontal_list, predict_horizontal_list = \
make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes,
_fast_rcnn_score, _detection_category)
if mode == 0:
gtbox_rotate_list, predict_rotate_list = \
make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes_rotate,
_fast_rcnn_score_rotate, _detection_category_rotate)
else:
gtbox_rotate_list, predict_rotate_list = \
make_dict_packle(_gtboxes_and_label, _fast_rcnn_decode_boxes_rotate,
_fast_rcnn_score_rotate, _detection_category_rotate)
gtboxes_horizontal_dict[str(
_img_name_batch[0])].extend(gtbox_horizontal_list)
predict_horizontal_dict[str(
_img_name_batch[0])].extend(predict_horizontal_list)
gtboxes_rotate_dict[str(
_img_name_batch[0])].extend(gtbox_rotate_list)
predict_rotate_dict[str(
_img_name_batch[0])].extend(predict_rotate_list)
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
fw1 = open('gtboxes_horizontal_dict.pkl', 'w')
fw2 = open('predict_horizontal_dict.pkl', 'w')
fw3 = open('gtboxes_rotate_dict.pkl', 'w')
fw4 = open('predict_rotate_dict.pkl', 'w')
pickle.dump(gtboxes_horizontal_dict, fw1)
pickle.dump(predict_horizontal_dict, fw2)
pickle.dump(gtboxes_rotate_dict, fw3)
pickle.dump(predict_rotate_dict, fw4)
fw1.close()
fw2.close()
fw3.close()
fw4.close()
coord.request_stop()
coord.join(threads)
def test(img_num):
with tf.Graph().as_default():
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=None,
is_training=False,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn1.FastRCNN(
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
gtboxes_and_label_minAreaRectangle=
gtboxes_and_label_minAreaRectangle,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
# iou>0.5 is positive, iou<0.5 is negative
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \
fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, num_of_objects_rotate, detection_category_rotate,_, _, _, _ = \
fast_rcnn.fast_rcnn_predict()
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
TrueDetection = 0
FalseAlarm = 0
TotalObjects = 0
TotalDetections = 0
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \
_fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \
_fast_rcnn_score_rotate, _detection_category_rotate \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle,
fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate,
fast_rcnn_score_rotate, detection_category_rotate])
end = time.time()
_img_batch = np.squeeze(_img_batch, axis=0)
_img_batch_fpn_horizonal = help_utils.draw_box_cv(
_img_batch,
boxes=_fast_rcnn_decode_boxes,
labels=_detection_category,
scores=_fast_rcnn_score)
_img_batch_fpn_rotate = help_utils.draw_rotate_box_cv(
_img_batch,
boxes=_fast_rcnn_decode_boxes_rotate,
labels=_detection_category_rotate,
scores=_fast_rcnn_score_rotate)
mkdir(cfgs.TEST_SAVE_PATH)
cv2.imwrite(
cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) +
'_horizontal_fpn.jpg', _img_batch_fpn_horizonal)
cv2.imwrite(
cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) +
'_rotate_fpn.jpg', _img_batch_fpn_rotate)
temp_label_horizontal = np.reshape(_gtboxes_and_label[:, -1:],
[
-1,
]).astype(np.int64)
temp_label_rotate = np.reshape(_gtboxes_and_label[:, -1:], [
-1,
]).astype(np.int64)
_img_batch_gt_horizontal = help_utils.draw_box_cv(
_img_batch,
boxes=_gtboxes_and_label_minAreaRectangle[:, :-1],
labels=temp_label_horizontal,
scores=None)
_img_batch_gt_rotate = help_utils.draw_rotate_box_cv(
_img_batch,
boxes=_gtboxes_and_label[:, :-1],
labels=temp_label_rotate,
scores=None)
cv2.imwrite(
cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) +
'_horizontal_gt.jpg', _img_batch_gt_horizontal)
cv2.imwrite(
cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) +
'_rotate_gt.jpg', _img_batch_gt_rotate)
Current_TrueDetection = 0
lock_list = []
for m in range(len(_gtboxes_and_label)):
for n, box in enumerate(_fast_rcnn_decode_boxes_rotate):
if n in lock_list:
continue
y_c, x_c, h, w, theta = box[0], box[1], box[2], box[
3], box[4]
r1 = ((x_c, y_c), (w, h), theta)
gt_box = _gtboxes_and_label[m]
gt_yc, gt_xc, gt_h, gt_w, gt_theta = gt_box[0], gt_box[
1], gt_box[2], gt_box[3], gt_box[4]
r2 = ((gt_xc, gt_yc), (gt_w, gt_h), gt_theta)
int_pts = cv2.rotatedRectangleIntersection(r1, r2)[0]
if int_pts > 0.5:
Current_TrueDetection += 1
lock_list.append(n)
TrueDetection += Current_TrueDetection
if (len(_fast_rcnn_decode_boxes_rotate) > 0):
FalseAlarm += len(
_fast_rcnn_score_rotate) - Current_TrueDetection
TotalDetections += len(_fast_rcnn_score_rotate)
TotalObjects += len(_gtboxes_and_label)
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
Pd = float(TrueDetection) / float(TotalDetections)
Pf = float(FalseAlarm) / float(TotalDetections)
F1 = 2 * float(Pd * (1 - Pf)) / float(Pd + (1 - Pf))
Op = float(TrueDetection) / float(TotalObjects)
print("\nACC: " + str(Pd))
print("FalseAlarm: " + str(Pf))
print("F1: " + str(F1))
print("Op: " + str(TrueDetection) + "/" + str(TotalObjects) +
"=" + str(Op))
coord.request_stop()
coord.join(threads)
def train():
faster_rcnn = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=True)
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5])
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY)
# list as many types of layers as possible, even if they are not used now
with slim.arg_scope([slim.conv2d, slim.conv2d_in_plane, \
slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
biases_initializer=tf.constant_initializer(0.0)):
final_bbox, final_scores, final_category, loss_dict = faster_rcnn.build_whole_detection_network(
input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label)
# ----------------------------------------------------------------------------------------------------build loss
weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
rpn_location_loss = loss_dict['rpn_loc_loss']
rpn_cls_loss = loss_dict['rpn_cls_loss']
rpn_total_loss = rpn_location_loss + rpn_cls_loss
fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss']
fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss']
fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss
total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss
# ____________________________________________________________________________________________________build loss
# ---------------------------------------------------------------------------------------------------add summary
tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss)
tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss)
tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss', fastrcnn_loc_loss)
tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss)
tf.summary.scalar('LOSS/total_loss', total_loss)
tf.summary.scalar('LOSS/regular_weights', weight_decay_loss)
gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories(
img_batch=img_batch,
boxes=gtboxes_and_label[:, :-1],
labels=gtboxes_and_label[:, -1])
if cfgs.ADD_BOX_IN_TENSORBOARD:
detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores(
img_batch=img_batch,
boxes=final_bbox,
labels=final_category,
scores=final_scores)
tf.summary.image('Compare/final_detection', detections_in_img)
tf.summary.image('Compare/gtboxes', gtboxes_in_img)
# ___________________________________________________________________________________________________add summary
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int64(cfgs.DECAY_STEP[0]),
np.int64(cfgs.DECAY_STEP[1])
],
values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
# optimizer = tf.train.AdamOptimizer(lr)
# ---------------------------------------------------------------------------------------------compute gradients
gradients = faster_rcnn.get_gradients(optimizer, total_loss)
# enlarge_gradients for bias
if cfgs.MUTILPY_BIAS_GRADIENT:
gradients = faster_rcnn.enlarge_gradients_for_bias(gradients)
if cfgs.GRADIENT_CLIPPING_BY_NORM:
with tf.name_scope('clip_gradients_YJR'):
gradients = slim.learning.clip_gradient_norms(
gradients, cfgs.GRADIENT_CLIPPING_BY_NORM)
# _____________________________________________________________________________________________compute gradients
# train_op
train_op = optimizer.apply_gradients(grads_and_vars=gradients,
global_step=global_step)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = faster_rcnn.get_restorer()
saver = tf.train.Saver(max_to_keep=30)
# Create session
#allow_soft_placement=True自动将无法放到GPU上的操作放回到CPU
config = tf.ConfigProto(allow_soft_placement=True)
#让GPU按需分配，不一定占用某个GPU的全部内存
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
#日志保存地址
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
tools.mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
timer = Timer()
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
_, global_stepnp = sess.run([train_op, global_step])
else:
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
_, global_stepnp, img_name, rpnLocLoss, rpnClsLoss, rpnTotalLoss, \
fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss = \
sess.run(
[train_op, global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss,
fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss, total_loss])
end = time.time()
print('{}: step:{}/{}iter'.format(training_time,
global_stepnp,
cfgs.MAX_ITERATION))
print('>>> rpn_loc_loss:{:.4f} | rpn_cla_loss:{:.4f} | rpn_total_loss:{:.4f}\n'
'>>> fast_rcnn_loc_loss:{:.4f} | fast_rcnn_cla_loss:{:.4f} | fast_rcnn_total_loss:{:.4f}\n'
'>>> single_time:{:.4f}s | total_loss:{:.4f} '\
.format(rpnLocLoss, rpnClsLoss,rpnTotalLoss, fastrcnnLocLoss, fastrcnnClsLoss,
fastrcnnTotalLoss,(end-start), totalLoss))
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
else:
if step % cfgs.SMRY_ITER == 0:
_, global_stepnp, summary_str = sess.run(
[train_op, global_step, summary_op])
summary_writer.add_summary(summary_str, global_stepnp)
summary_writer.flush()
if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE
== 0) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.mkdir(save_dir)
save_ckpt = os.path.join(
save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def train():
faster_rcnn = build_whole_network.DetectionNetwork(
base_network_name=cfgs.NET_NAME, is_training=True)
with tf.name_scope('get_batch'):
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco'
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.IMG_SHORT_SIDE_LEN,
is_training=True)
gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5])
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY)
# list as many types of layers as possible, even if they are not used now
with slim.arg_scope([
slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose,
slim.separable_conv2d, slim.fully_connected
],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
biases_initializer=tf.constant_initializer(0.0)):
# result_dict, losses_dict = faster_rcnn.build_whole_detection_network(input_img_batch=img_batch,
# gtboxes_batch=gtboxes_and_label)
result_dict, losses_dict = faster_rcnn.build_whole_detection_network(
input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label)
# ----------------------------------------------------------------------------------------------------build loss
weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
# weight_decay_loss = tf.add_n(tf.losses.get_regularization_losses())
bbox_loss_m1 = losses_dict['bbox_loss_m1']
cls_loss_m1 = losses_dict['cls_loss_m1']
total_loss_m1 = bbox_loss_m1 + cls_loss_m1
bbox_loss_m2 = losses_dict['bbox_loss_m2']
cls_loss_m2 = losses_dict['cls_loss_m2']
total_loss_m2 = bbox_loss_m2 + cls_loss_m2
bbox_loss_m3 = losses_dict['bbox_loss_m3']
cls_loss_m3 = losses_dict['cls_loss_m3']
total_loss_m3 = bbox_loss_m3 + cls_loss_m3
total_loss = total_loss_m1 + total_loss_m2 + total_loss_m3 + weight_decay_loss
# ---------------------------------------------------------------------------------------------------add summary
tf.summary.scalar('SSH_M1_LOSS/cls_loss_m1', cls_loss_m1)
tf.summary.scalar('SSH_M1_LOSS/bbox_loss_m1', bbox_loss_m1)
tf.summary.scalar('SSH_M1_LOSS/total_loss_m1', total_loss_m1)
tf.summary.scalar('SSH_M2_LOSS/cls_loss_m2', cls_loss_m2)
tf.summary.scalar('SSH_M2_LOSS/bbox_loss_m2', bbox_loss_m2)
tf.summary.scalar('SSH_M2_LOSS/total_loss_m2', total_loss_m2)
tf.summary.scalar('SSH_M3_LOSS/cls_loss_m3', cls_loss_m3)
tf.summary.scalar('SSH_M3_LOSS/bbox_loss_m3', bbox_loss_m3)
tf.summary.scalar('SSH_M3_LOSS/total_loss_m3', total_loss_m3)
tf.summary.scalar('LOSS/total_loss', total_loss)
tf.summary.scalar('LOSS/regular_weights', weight_decay_loss)
gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories(
img_batch=img_batch,
boxes=gtboxes_and_label[:, :-1],
labels=gtboxes_and_label[:, -1])
if cfgs.ADD_BOX_IN_TENSORBOARD:
detections_in_img_m1 = \
show_box_in_tensor.draw_boxes_with_categories_and_scores(img_batch=img_batch,
boxes=result_dict['final_bbox_m1'],
labels=result_dict['final_category_m1'],
scores=result_dict['final_scores_m1'])
tf.summary.image('Compare/final_detection_m1', detections_in_img_m1)
detections_in_img_m2 = \
show_box_in_tensor.draw_boxes_with_categories_and_scores(img_batch=img_batch,
boxes=result_dict['final_bbox_m2'],
labels=result_dict['final_category_m2'],
scores=result_dict['final_scores_m2'])
tf.summary.image('Compare/final_detection_m2', detections_in_img_m2)
detections_in_img_m3 = \
show_box_in_tensor.draw_boxes_with_categories_and_scores(img_batch=img_batch,
boxes=result_dict['final_bbox_m3'],
labels=result_dict['final_category_m3'],
scores=result_dict['final_scores_m3'])
tf.summary.image('Compare/final_detection_m3', detections_in_img_m3)
tf.summary.image('Compare/gtboxes', gtboxes_in_img)
global_step = slim.get_or_create_global_step()
lr = tf.train.piecewise_constant(
global_step,
boundaries=[
np.int64(cfgs.DECAY_STEP[0]),
np.int64(cfgs.DECAY_STEP[1])
],
values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.])
tf.summary.scalar('lr', lr)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
# ---------------------------------------------------------------------------------------------compute gradients
gradients = faster_rcnn.get_gradients(optimizer, total_loss)
# enlarge_gradients for bias
if cfgs.MUTILPY_BIAS_GRADIENT:
gradients = faster_rcnn.enlarge_gradients_for_bias(gradients)
if cfgs.GRADIENT_CLIPPING_BY_NORM:
with tf.name_scope('clip_gradients'):
gradients = slim.learning.clip_gradient_norms(
gradients, cfgs.GRADIENT_CLIPPING_BY_NORM)
# train_op
train_op = optimizer.apply_gradients(grads_and_vars=gradients,
global_step=global_step)
summary_op = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = faster_rcnn.get_restorer()
saver = tf.train.Saver(max_to_keep=30)
config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
if not os.path.exists(summary_path):
os.makedirs(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
for step in range(cfgs.MAX_ITERATION):
training_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
# start = time.time()
# _, global_stepnp, img_name, totalLoss, summary_str = \
# sess.run(
# [train_op, global_step, img_name_batch, total_loss, summary_op])
#
# end = time.time()
#
# print(""" {}: step{} image_name:{} |\t total_loss:{} |\t per_cost_time:{}s""" \
# .format(training_time, global_stepnp, str(img_name[0]), totalLoss,
# (end - start)))
# summary_writer.add_summary(summary_str, global_stepnp)
# summary_writer.flush()
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
_, global_stepnp = sess.run([train_op, global_step])
else:
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
_, global_stepnp, img_name, totalLoss = \
sess.run(
[train_op, global_step, img_name_batch, total_loss])
end = time.time()
print(""" {}: step{} image_name:{} |\t total_loss:{} |\t per_cost_time:{}s""" \
.format(training_time, global_stepnp, str(img_name[0]), totalLoss,
(end - start)))
else:
if step % cfgs.SMRY_ITER == 0:
_, global_stepnp, summary_str = sess.run(
[train_op, global_step, summary_op])
summary_writer.add_summary(summary_str, global_stepnp)
summary_writer.flush()
if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE
== 0) or (step == cfgs.MAX_ITERATION - 1):
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_ckpt = os.path.join(
save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
