def train(): with tf.Graph().as_default(): with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'ship', 'spacenet', 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.py_func(back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) with tf.name_scope('draw_gtboxes'): gtboxes_in_img = draw_box_with_color(img_batch, tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1], text=tf.shape(gtboxes_and_label_batch)[1]) # *********************************************************************************************** # * shared CNN * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * rpn * # * Note: here the rpn is Feature Pyramid Networks * # *********************************************************************************************** rpn = build_rpn.RPN(net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=gtboxes_and_label, is_training=True, share_head=cfgs.SHARED_HEADS, share_net=share_net, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, anchor_angles=cfgs.ANCHOR_ANGLES, scale_factors=cfgs.SCALE_FACTORS, # this parameter will affect the performance base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, anchor_stride=cfgs.ANCHOR_STRIDE, top_k_nms=cfgs.RPN_TOP_K_NMS, kernel_size=cfgs.KERNEL_SIZE, use_angles_condition=False, anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD, nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=cfgs.IS_FILTER_OUTSIDE_BOXES, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], scope='') rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ] rpn_location_loss, rpn_classification_loss, rpn_predict_boxes, rpn_predict_scores = rpn.rpn_losses() rpn_total_loss = rpn_classification_loss + rpn_location_loss with tf.name_scope('draw_proposals'): # score > 0.6 is object rpn_object_boxes_indices = tf.reshape(tf.where(tf.greater(rpn_proposals_scores, cfgs.FINAL_SCORE_THRESHOLD)), [-1]) rpn_object_boxes = tf.gather(rpn_proposals_boxes, rpn_object_boxes_indices) rpn_object_soxres = tf.gather(rpn_proposals_scores, rpn_object_boxes_indices) rpn_proposals_objcet_boxes_in_img = draw_boxes_with_scores(img_batch, rpn_object_boxes, scores=rpn_object_soxres) # rpn_proposals_objcet_boxes_in_img = draw_box_with_color(img_batch, rpn_object_boxes, # text=tf.shape(rpn_object_boxes)[0]) rpn_proposals_boxes_in_img = draw_box_with_color(img_batch, rpn_proposals_boxes, text=tf.shape(rpn_proposals_boxes)[0]) # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, stop_gradient_for_proposals=False, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=gtboxes_and_label, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS, nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD, use_angle_condition=False, level=cfgs.LEVEL, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD, use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=True) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss() fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss with tf.name_scope('draw_boxes_with_categories'): fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories(img_batch=img_batch, boxes=fast_rcnn_decode_boxes, labels=detection_category, scores=fast_rcnn_score) # train total_loss = slim.losses.get_total_loss() global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant(global_step, boundaries=[np.int64(70000), np.int64(120000)], values=[cfgs.LR, cfgs.LR/10, cfgs.LR/100]) # optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) if cfgs.RPN_TRAIN: train_op = slim.learning.create_train_op(rpn_total_loss, optimizer, global_step) else: train_op = slim.learning.create_train_op(total_loss, optimizer, global_step) # train_op = optimizer.minimize(second_classification_loss, global_step) # *********************************************************************************************** # * Summary * # *********************************************************************************************** # ground truth and predict tf.summary.image('img/gtboxes', gtboxes_in_img) tf.summary.image('img/fast_rcnn_predict', fast_rcnn_predict_boxes_in_imgs) # rpn loss and image tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss) tf.summary.scalar('rpn/rpn_classification_loss', rpn_classification_loss) tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_location_loss', fast_rcnn_location_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss', fast_rcnn_classification_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss', fast_rcnn_total_loss) tf.summary.scalar('loss/total_loss', total_loss) tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img) tf.summary.image('rpn/rpn_object_boxes', rpn_proposals_objcet_boxes_in_img) # learning_rate tf.summary.scalar('learning_rate', lr) summary_op = tf.summary.merge_all() init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer() ) restorer, restore_ckpt = restore_model.get_restorer() saver = tf.train.Saver(max_to_keep=10) config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) tools.mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) start = time.time() _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \ _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \ _fast_rcnn_total_loss, _total_loss, _= \ sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss, rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss, fast_rcnn_total_loss, total_loss, train_op]) end = time.time() if step % 10 == 0: print(""" {}: step{} image_name:{} |\t rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} | fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} | total_loss:{} |\t per_cost_time:{}s""" \ .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss, _rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss, (end - start))) if (step % 50 == 0) and (step % 10000 != 0): # 50 summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, _global_step) summary_writer.flush() if (step > 0 and step % 10000 == 0) or (step == cfgs.MAX_ITERATION - 1): summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, _global_step) summary_writer.flush() save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.mkdir(save_dir) save_ckpt = os.path.join(save_dir, 'voc_'+str(_global_step)+'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)
def eval_dict_convert(img_num): with tf.Graph().as_default(): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=None, is_training=False, share_head=True, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN( img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, scale_factors=cfgs.SCALE_FACTORS, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, gtboxes_and_label=None, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs. FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, use_dropout=False, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False, level=cfgs.LEVEL) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) gtbox_dict = {} predict_dict = {} for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label_batch, _fast_rcnn_decode_boxes, \ _fast_rcnn_score, _detection_category \ = sess.run([img_name_batch, img_batch, gtboxes_and_label_batch, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category]) end = time.time() # gtboxes convert dict gtbox_dict[str(_img_name_batch[0])] = [] predict_dict[str(_img_name_batch[0])] = [] for j, box in enumerate(_gtboxes_and_label_batch[0]): bbox_dict = {} bbox_dict['bbox'] = np.array( _gtboxes_and_label_batch[0][j, :-1], np.float64) bbox_dict['name'] = LABEl_NAME_MAP[int( _gtboxes_and_label_batch[0][j, -1])] gtbox_dict[str(_img_name_batch[0])].append(bbox_dict) for label in NAME_LABEL_MAP.keys(): if label == 'back_ground': continue else: temp_dict = {} temp_dict['name'] = label ind = np.where( _detection_category == NAME_LABEL_MAP[label])[0] temp_boxes = _fast_rcnn_decode_boxes[ind] temp_score = np.reshape(_fast_rcnn_score[ind], [-1, 1]) temp_dict['bbox'] = np.array( np.concatenate([temp_boxes, temp_score], axis=1), np.float64) predict_dict[str(_img_name_batch[0])].append(temp_dict) view_bar( '{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) fw1 = open('gtboxes_dict.pkl', 'w') fw2 = open('predict_dict.pkl', 'w') pickle.dump(gtbox_dict, fw1) pickle.dump(predict_dict, fw2) fw1.close() fw2.close() coord.request_stop() coord.join(threads)
def test(img_num): with tf.Graph().as_default(): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) gtboxes_and_label, head = get_head( tf.squeeze(gtboxes_and_label_batch, 0)) gtboxes_and_label = tf.py_func(back_forward_convert, inp=[gtboxes_and_label], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) head_quadrant = tf.py_func(get_head_quadrant, inp=[head, gtboxes_and_label], Tout=tf.float32) head_quadrant = tf.reshape(head_quadrant, [-1, 1]) gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=None, is_training=False, share_head=cfgs.SHARE_HEAD, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN( feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), img_batch=img_batch, roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, gtboxes_and_label_minAreaRectangle= gtboxes_and_label_minAreaRectangle, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False, level=cfgs.LEVEL, head_quadrant=head_quadrant) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \ fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, fast_rcnn_head_quadrant, \ num_of_objects_rotate, detection_category_rotate = fast_rcnn.fast_rcnn_predict() # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, _head_quadrant, \ _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \ _fast_rcnn_score_rotate, _fast_rcnn_head_quadrant, _detection_category_rotate \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle, head_quadrant, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, fast_rcnn_head_quadrant, detection_category_rotate]) end = time.time() _img_batch = np.squeeze(_img_batch, axis=0) _img_batch_fpn_horizonal = help_utils.draw_box_cv( _img_batch, boxes=_fast_rcnn_decode_boxes, labels=_detection_category, scores=_fast_rcnn_score) _img_batch_fpn_rotate = help_utils.draw_rotate_box_cv( _img_batch, boxes=_fast_rcnn_decode_boxes_rotate, labels=_detection_category_rotate, scores=_fast_rcnn_score_rotate, head=np.argmax(_fast_rcnn_head_quadrant, axis=1)) mkdir(cfgs.TEST_SAVE_PATH) cv2.imwrite( cfgs.TEST_SAVE_PATH + '/{}_horizontal_fpn.jpg'.format(str(_img_name_batch[0])), _img_batch_fpn_horizonal) cv2.imwrite( cfgs.TEST_SAVE_PATH + '/{}_rotate_fpn.jpg'.format(str(_img_name_batch[0])), _img_batch_fpn_rotate) temp_label_horizontal = np.reshape(_gtboxes_and_label[:, -1:], [ -1, ]).astype(np.int64) temp_label_rotate = np.reshape(_gtboxes_and_label[:, -1:], [ -1, ]).astype(np.int64) _img_batch_gt_horizontal = help_utils.draw_box_cv( _img_batch, boxes=_gtboxes_and_label_minAreaRectangle[:, :-1], labels=temp_label_horizontal, scores=None) _img_batch_gt_rotate = help_utils.draw_rotate_box_cv( _img_batch, boxes=_gtboxes_and_label[:, :-1], labels=temp_label_rotate, scores=None, head=np.reshape(_head_quadrant, [ -1, ])) cv2.imwrite( cfgs.TEST_SAVE_PATH + '/{}_horizontal_gt.jpg'.format(str(_img_name_batch[0])), _img_batch_gt_horizontal) cv2.imwrite( cfgs.TEST_SAVE_PATH + '/{}_rotate_gt.jpg'.format(str(_img_name_batch[0])), _img_batch_gt_rotate) view_bar( '{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) coord.request_stop() coord.join(threads)
def train(): # Step 1: # clw note:传递网络名称如resnet_v1,是否训练is_training,以及每个位置含有anchor box的个数, # 构建基本的网络 faster_rcnn = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=True) with tf.name_scope( 'get_batch' ): # clw note:tf.name_scope 主要结合 tf.Variable() 来使用,方便参数命名管理。 # clw note:从文件队列、内存队列中读取、组合,得到该batch的内容 # 主要包括每个批次(目前仅支持批次数目即batch_size=1,也就是这里每次只读出1张图片) # 对应的变量包括:图片名称、图片矩阵、ground truth坐标及对应的label,图片中包含的目标数 # 这些变量的组成结构均为 [批次数目,相应批次中每一幅图片的相关信息] img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5]) # clw note:样本个数m不知道,但是对单个样本都有gtboxes的4个坐标,加上1个label共5个值;使用-1来自动计算样本个数 biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY) # Step 2: # clw note:Faster R-CNN网络的搭建! # 先看一下下面这个函数arg_scope的声明 # @tf_contextlib.contextmanager # def arg_scope(list_ops_or_scope, **kwargs): 功能是给list_ops中的内容设置默认值,即list中所有元素都用**kargs的参数设置。 # 有函数修饰符@tf_contextlib.contextmanager修饰arg_scope函数:@之后一般接一个可调用对象为其执行一系列辅助操作, # 我们来看一个demo: ######################################### # import time # def my_time(func): # print(time.ctime()) # return func() # # @my_time # 从这里可以看出@time 等价于 time(xxx()),但是这种写法你得考虑python代码的执行顺序 # def xxx(): # print('Hello world!') # # 运行结果: # Wed Jul 26 23:01:21 2017 # Hello world! ########################################## # 在这个例子中,xxx函数实现我们的主要功能,打印Hello world,但我们想给xxx函数添加一些辅助操作,让它同时打印出时间,于是我们用 # 函数修饰符 @ my_time完成这个目标。整个例子的执行流程为调用my_time可调用对象,它接受xxx函数作为参数,先打印时间,再执行xxx函数 # 详见:https://www.cnblogs.com/zzy-tf/p/9356883.html # 来看另一个demo: ########################################## # with slim.arg_scope( # [slim.conv2d, slim.max_pool2d, slim.avg_pool2d],stride = 1, padding = 'VALID'): # net = slim.conv2d(inputs, 32, [3, 3], stride = 2, scope = 'Conv2d_1a_3x3') # net = slim.conv2d(net, 32, [3, 3], scope = 'Conv2d_2a_3x3') # net = slim.conv2d(net, 64, [3, 3], padding = 'SAME', scope = 'Conv2d_2b_3x3') # 所以,在使用过程中可以直接slim.conv2d( )等函数设置默认参数。例如在下面的代码中,不做单独声明的情况下, # slim.conv2d, slim.max_pool2d, slim.avg_pool2d三个函数默认的步长都设为1,padding模式都是'VALID'的。 # 当然也可以在调用时进行单独声明,只不过一个一个写很麻烦,不如统一给个默认值。 # 这种参数设置方式在构建网络模型时,尤其是较深的网络时,可以节省时间。 with slim.arg_scope([ slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected ], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, biases_initializer=tf.constant_initializer( 0.0)): # list as many types of layers as possible, # even if they are not used now # build_whole_detection_network功能:构建整体网络架构,包含backbone,RPN网络,Pooling层,以及后续网络。 # return:网络的最后的预测框,预测的类别信息,预测的概率,以及整体网络和RPN网络的损失,所有的损失被写入到一个字典中。 final_bbox, final_scores, final_category, loss_dict = faster_rcnn.build_whole_detection_network( input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label) # ----------------------------------------------------------------------------------------------------build loss # weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses()) # weight_decay_loss = tf.add_n(tf.losses.get_regularization_losses()) rpn_location_loss = loss_dict['rpn_loc_loss'] rpn_cls_loss = loss_dict['rpn_cls_loss'] rpn_total_loss = rpn_location_loss + rpn_cls_loss fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss'] fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss'] fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss # clw note:根据论文的公式,最后将RPN网络的(分类,回归)误差与Fast-RCNN的(分类,回归)误差相加后作为总的误差进行训练即可。 total_loss = rpn_total_loss + fastrcnn_total_loss # ____________________________________________________________________________________________________build loss # ---------------------------------------------------------------------------------------------------add summary tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss) tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss) tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss', fastrcnn_loc_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss) tf.summary.scalar('LOSS/total_loss', total_loss) # tf.summary.scalar('LOSS/regular_weights', weight_decay_loss) gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories( img_batch=img_batch, boxes=gtboxes_and_label[:, :-1], labels=gtboxes_and_label[:, -1]) if cfgs.ADD_BOX_IN_TENSORBOARD: detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores( img_batch=img_batch, boxes=final_bbox, labels=final_category, scores=final_scores) tf.summary.image('Compare/final_detection', detections_in_img) tf.summary.image('Compare/gtboxes', gtboxes_in_img) # ___________________________________________________________________________________________________add summary global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[ np.int64(cfgs.DECAY_STEP[0]), np.int64(cfgs.DECAY_STEP[1]) ], values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer( lr, momentum=cfgs.MOMENTUM) # clw note:选择优化器,可以尝试其他选择, # 也可以尝试tf.train.AdamOptimizer(1e-4).minimize(total_loss) # ---------------------------------------------------------------------------------------------compute gradients # clw note:对于上面优化器没有使用minimize()的几点说明, # 使用minimize()操作,该操作不仅可以计算出梯度,而且还可以将梯度作用在变量上。 # 如果想按照自己的方式处理梯度,可以按照以下步骤: # 1、使用compute_gradients()计算梯度,其实下面的get_gradients()方法就是optimizer.compute_gradients(loss) # 2、使用自己的方式进一步处理梯度 # 3、使用apply_gradients()应用处理过后的梯度; gradients = faster_rcnn.get_gradients(optimizer, total_loss) # enlarge_gradients for bias if cfgs.MUTILPY_BIAS_GRADIENT: gradients = faster_rcnn.enlarge_gradients_for_bias(gradients) if cfgs.GRADIENT_CLIPPING_BY_NORM: # clw note:clip_by_norm是指对梯度进行裁剪,通过控制梯度的最大范式,防止梯度爆炸的问题,是一种比较常用的梯度规约的方式 with tf.name_scope('clip_gradients_YJR'): gradients = slim.learning.clip_gradient_norms( gradients, cfgs.GRADIENT_CLIPPING_BY_NORM) # _____________________________________________________________________________________________compute gradients # train_op train_op = optimizer.apply_gradients(grads_and_vars=gradients, global_step=global_step) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = faster_rcnn.get_restorer() saver = tf.train.Saver(max_to_keep=30) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) # tools.mkdir(summary_path) if not os.path.exists(summary_path): os.makedirs(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0: _, global_stepnp = sess.run([train_op, global_step]) else: if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0: start = time.time() _, global_stepnp, img_name, rpnLocLoss, rpnClsLoss, rpnTotalLoss, \ fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss = \ sess.run( [train_op, global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss, fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss, total_loss]) end = time.time() print(""" {}: step{} image_name:{} |\t rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} | fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} | total_loss:{} |\t per_cost_time:{}s""" \ .format(training_time, global_stepnp, str(img_name[0]), rpnLocLoss, rpnClsLoss, rpnTotalLoss, fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss, (end - start))) else: if step % cfgs.SMRY_ITER == 0: _, global_stepnp, summary_str = sess.run( [train_op, global_step, summary_op]) summary_writer.add_summary(summary_str, global_stepnp) summary_writer.flush() if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.makedirs(save_dir) save_ckpt = os.path.join( save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)
def train(): # get Network Class faster_rcnn = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=True) # get batch with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) # convert gtboxes_and_labels gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) # draw ground-truth with tf.name_scope('draw_gtboxes'): gtboxes_in_img = draw_box_with_color_rotate( img_batch, tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1], text=tf.shape(gtboxes_and_label)[0]) # default regularizers biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY) # list as many types of layers as possible, even if they are not used now with slim.arg_scope([ slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected ], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, biases_initializer=tf.constant_initializer(0.0)): # build network final_boxes, final_scores, final_category, loss_dict = \ faster_rcnn.build_whole_detection_network(input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label) # draw detections dets_in_img = draw_boxes_with_categories_and_scores_rotate( img_batch=img_batch, boxes=final_boxes, labels=final_category, scores=final_scores) # ----------------------------------------------------------------------------------------------------build loss # weight decay loss weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses()) # rpn losses rpn_location_loss = loss_dict['rpn_loc_loss'] rpn_cls_loss = loss_dict['rpn_cls_loss'] rpn_total_loss = rpn_location_loss + rpn_cls_loss # fastrcnn losses fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss'] fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss'] fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss # total loss total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss # ____________________________________________________________________________________________________build loss # ---------------------------------------------------------------------------------------------------add summary tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss) tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss) tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss', fastrcnn_loc_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss) tf.summary.scalar('LOSS/total_loss', total_loss) tf.summary.scalar('LOSS/regular_weights', weight_decay_loss) tf.summary.image('img/gtboxes', gtboxes_in_img) tf.summary.image('img/dets', dets_in_img) # ___________________________________________________________________________________________________add summary # create global step global_step = slim.get_or_create_global_step() # learning rate setting lr = tf.train.piecewise_constant( global_step, boundaries=[ np.int64(cfgs.DECAY_STEP[0]), np.int64(cfgs.DECAY_STEP[1]) ], values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.]) tf.summary.scalar('lr', lr) # optimizer with lr optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) # ---------------------------------------------------------------------------------------------compute gradients # compute gradients gradients = faster_rcnn.get_gradients(optimizer, total_loss) # enlarge gradients for bias if cfgs.MUTILPY_BIAS_GRADIENT: gradients = faster_rcnn.enlarge_gradients_for_bias(gradients) # clipping gradients if cfgs.GRADIENT_CLIPPING_BY_NORM: with tf.name_scope('clip_gradients'): gradients = slim.learning.clip_gradient_norms( gradients, cfgs.GRADIENT_CLIPPING_BY_NORM) # _____________________________________________________________________________________________compute gradients # train_op, ie. apply gradients train_op = optimizer.apply_gradients(grads_and_vars=gradients, global_step=global_step) # summary_op summary_op = tf.summary.merge_all() # init_op init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) # restorer and Saver restorer, restore_ckpt = faster_rcnn.get_restorer() saver = tf.train.Saver(max_to_keep=10) # GPU Config config = tf.ConfigProto() config.gpu_options.allow_growth = True # start a session with tf.Session(config=config) as sess: # init sess.run(init_op) # restore if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') # Session对象是支持多线程的, 可以在同一个会话(Session)中创建多个线程,并行执行; # 创建线程协调器(管理器), 并启动(Tensor/训练数据)入队线程, # 入队具体使用多少个线程在read_tfrecord.py->next_batch->tf.train.batch中定义; coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) # construct summary writer summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) tools.mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) # train MAX_ITERATION steps for step in range(cfgs.MAX_ITERATION): # time of this step training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) # no show & no summary if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0: # global_step is added to run, global_stepnp will be used in Save _, global_stepnp = sess.run([train_op, global_step]) else: # show if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0: start = time.time() # middle infos are added to run, losses and img_name_batch _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \ _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \ _fast_rcnn_total_loss, _total_loss, _ = \ sess.run([global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss, fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss, total_loss, train_op]) # # show boxes, scores, category infos # final_boxes_r, _final_scores_r, _final_category_r = sess.run([final_boxes_r, final_scores_r, final_category_r]) # print('*'*100) # print(_final_boxes_r) # print(_final_scores_r) # print(_final_category_r) end = time.time() # print print(""" {}: step {}: image_name:{} |\t rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} | fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} |\t total_loss:{} |\t pre_cost_time:{}s""" \ .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss, _rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss, (end - start))) # summary else: if step % cfgs.SMRY_ITER == 0: # summary_op is added to run _, global_stepnp, summary_str = sess.run( [train_op, global_step, summary_op]) # add summary summary_writer.add_summary(summary_str, global_stepnp) summary_writer.flush() # save if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE == 0) or (step == cfgs.MAX_ITERATION - 1): # mkdir save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.makedirs(save_dir) # save checkpoint save_ckpt = os.path.join( save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') # 协调器发出终止所有线程的命令 coord.request_stop() # 把开启的线程加入主线程,等待threads结束 coord.join(threads)
def test(args): with tf.Graph().as_default(): # img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8) img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN(net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=None, is_training=False, share_head=cfgs.SHARE_HEAD, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ] # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, scale_factors=cfgs.SCALE_FACTORS, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, gtboxes_and_label=None, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD, use_dropout=False, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False, level=cfgs.LEVEL) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() # train init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer() ) restorer, restore_ckpt = restore_model.get_restorer(checkpoint_path=args.weights) config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) print('start!') for _ in range(args.img_num): _img_name_batch, _img_batch, _gtboxes_and_label_batch, _fast_rcnn_decode_boxes, \ _fast_rcnn_score, _detection_category \ = sess.run([img_name_batch, img_batch, gtboxes_and_label_batch, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category]) _img_batch = np.squeeze(_img_batch, axis=0) _img_batch_fpn = help_utils.draw_box_cv(_img_batch, boxes=_fast_rcnn_decode_boxes, labels=_detection_category, scores=_fast_rcnn_score) mkdir(cfgs.TEST_SAVE_PATH) cv2.imwrite(cfgs.TEST_SAVE_PATH + '/{}_fpn.jpg'.format(str(_img_name_batch[0])), _img_batch_fpn) _gtboxes_and_label_batch = np.squeeze(_gtboxes_and_label_batch, axis=0) temp_label = np.reshape(_gtboxes_and_label_batch[:, -1:], [-1, ]).astype(np.int64) _img_batch_gt = help_utils.draw_box_cv(_img_batch, boxes=_gtboxes_and_label_batch[:, :-1], labels=temp_label, scores=None) cv2.imwrite(cfgs.TEST_SAVE_PATH + '/{}_gt.jpg'.format(str(_img_name_batch[0])), _img_batch_gt) coord.request_stop() coord.join(threads) print('end!')
def train(): with tf.Graph().as_default(): with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch = \ next_batch(data_tfrecord_location=cfgs.DATASET_DIR, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, longside_len=cfgs.LONG_SIDE_LEN, is_training=True) image_height, image_width = tf.shape(img_batch)[1], tf.shape( img_batch)[2] # with tf.name_scope('draw_gtboxes'): # gtboxes_in_img = draw_box_with_color(img_batch, tf.reshape(gtboxes_and_label_batch, [-1, 5])[:, :-1], # text=tf.shape(gtboxes_and_label_batch)[1]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # fpn feature_pyramid = build_fpn.build_feature_pyramid( share_net) # [P2,P3,P4,P5,P6] # *********************************************************************************************** # * rpn * # *********************************************************************************************** rpn = build_rpn.RPN(feature_pyramid=feature_pyramid, image_height=image_height, image_width=image_width, gtboxes_and_label=gtboxes_and_label_batch, is_training=True) rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( is_training=True) rpn_location_loss, rpn_classification_loss = rpn.rpn_losses() rpn_total_loss = rpn_classification_loss + rpn_location_loss # *********************************************************************************************** # * Fast RCNN Head * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FAST_RCNN( feature_pyramid=feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, gtboxes_and_label=gtboxes_and_label_batch, origin_image=img_batch, is_training=True, image_height=image_height, image_width=image_width) detections = fast_rcnn.head_detection() if cfgs.DEBUG: print_tensors(rpn_proposals_scores[0, :50], "scores") # 前50 print_tensors(rpn_proposals_boxes[0, :50, :], "bbox") rpn_proposals_vision = draw_boxes_with_scores( img_batch[0, :, :, :], rpn_proposals_boxes[0, :50, :], rpn_proposals_scores[0, :50]) fast_rcnn_vision = draw_boxes_with_categories_and_scores( img_batch[0, :, :, :], detections[0, :, :4], detections[0, :, 4], detections[0, :, 5], cfgs.LABEL_TO_NAME) tf.summary.image("rpn_proposals_vision", rpn_proposals_vision) tf.summary.image("fast_rcnn_vision", fast_rcnn_vision) fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.head_loss( ) fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss # train with tf.name_scope("regularization_losses"): regularization_list = [ tf.nn.l2_loss(w.read_value()) * cfgs.WEIGHT_DECAY[cfgs.NET_NAME] / tf.cast(tf.size(w.read_value()), tf.float32) for w in tf.trainable_variables() if 'gamma' not in w.name and 'beta' not in w.name ] regularization_loss = tf.add_n(regularization_list) total_loss = regularization_loss + fast_rcnn_total_loss + rpn_total_loss total_loss = tf.cond(tf.is_nan(total_loss), lambda: 0.0, lambda: total_loss) global_step = tf.train.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[np.int64(10000), np.int64(20000)], values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100]) # * Summary * # *********************************************************************************************** # rpn loss and image tf.summary.scalar('rpn_location_loss', rpn_location_loss, family="rpn_loss") tf.summary.scalar('rpn_classification_loss', rpn_classification_loss, family="rpn_loss") tf.summary.scalar('rpn_total_loss', rpn_total_loss, family="rpn_loss") tf.summary.scalar('fast_rcnn_location_loss', fast_rcnn_location_loss, family="head_loss") tf.summary.scalar('fast_rcnn_classification_loss', fast_rcnn_classification_loss, family="head_loss") tf.summary.scalar('fast_rcnn_total_loss', fast_rcnn_total_loss, family="head_loss") tf.summary.scalar("regularization_loss", regularization_loss) tf.summary.scalar('total_loss', total_loss) tf.summary.scalar('learning_rate', lr) # optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) # # train_op = slim.learning.create_train_op(total_loss, optimizer, global_step) with tf.name_scope("optimizer"): optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) optimizer = tf.contrib.estimator.TowerOptimizer(optimizer) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies([tf.group(*update_ops)]): grads = optimizer.compute_gradients(total_loss) # clip gradients grads = tf.contrib.training.clip_gradient_norms( grads, cfgs.CLIP_GRADIENT_NORM) train_op = optimizer.apply_gradients(grads, global_step) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) # restorer, restore_ckpt = restore_model.get_restorer(test=False,checkpoint_path=cfgs.TRAINED_MODEL_DIR) saver = tf.train.Saver(max_to_keep=3) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) # if not restorer is None: # restorer.restore(sess, restore_ckpt) # print('restore model successfully') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = cfgs.SUMMARY_PATH summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) start = time.time() _global_step,_rpn_location_loss, _rpn_classification_loss, \ _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \ _fast_rcnn_total_loss, _total_loss, _ = \ sess.run([global_step,rpn_location_loss, rpn_classification_loss, rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss, fast_rcnn_total_loss,total_loss, train_op]) end = time.time() if step % 50 == 0: print("""{}: step{} rpn_loc_loss:{:.4f} | rpn_cla_loss:{:.4f} | rpn_total_loss:{:.4f} fast_rcnn_loc_loss:{:.4f} | fast_rcnn_cla_loss:{:.4f} | fast_rcnn_total_loss:{:.4f} | total_loss:{:.4f} | pre_cost_time:{:.4f}s""". format(training_time, _global_step, _rpn_location_loss, _rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss, (end - start))) summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, _global_step) summary_writer.flush() if (step > 0 and step % 5000 == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = cfgs.TRAINED_MODEL_DIR save_ckpt = os.path.join( save_dir, 'sarship' + '{}_'.format(cfgs.NET_NAME) + str(_global_step) + 'model.ckpt') saver.save(sess, save_ckpt) print('Weights have been saved to {}.'.format(save_ckpt)) print('Training done') coord.request_stop() coord.join(threads)
def test(img_num): with tf.Graph().as_default(): # img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8) img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) gtboxes_and_label = tf.py_func(back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN(net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=gtboxes_and_label, is_training=False, share_head=False, share_net=share_net, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, anchor_angles=cfgs.ANCHOR_ANGLES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, anchor_stride=cfgs.ANCHOR_STRIDE, top_k_nms=cfgs.RPN_TOP_K_NMS, kernel_size=cfgs.KERNEL_SIZE, use_angles_condition=True, anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD, nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], scope='') # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ] # _, _, rpn_top_k_boxes, rpn_top_k_scores = rpn.rpn_losses() # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, stop_gradient_for_proposals=False, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS, nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD, use_angle_condition=False, level=cfgs.LEVEL, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD, boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD, use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() # test init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer() ) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \ _fast_rcnn_score, _detection_category \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category]) end = time.time() _img_batch = np.squeeze(_img_batch, axis=0) _img_batch_fpn = help_utils.draw_box_cv(_img_batch, boxes=_fast_rcnn_decode_boxes, labels=_detection_category, scores=_fast_rcnn_score) cv2.imwrite(cfgs.TEST_SAVE_PATH + '/{}_fpn.jpg'.format(str(_img_name_batch[0]).split('.tif')[0]), _img_batch_fpn) temp_label = np.reshape(_gtboxes_and_label[:, -1:], [-1, ]).astype(np.int64) _img_batch_gt = help_utils.draw_box_cv(_img_batch, boxes=_gtboxes_and_label[:, :-1], labels=temp_label, scores=None) cv2.imwrite(cfgs.TEST_SAVE_PATH + '/{}_gt.jpg'.format(str(_img_name_batch[0]).split('.tif')[0]), _img_batch_gt) view_bar('{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i+1, img_num) coord.request_stop() coord.join(threads)
def eval_dict_convert(img_num, mode): with tf.Graph().as_default(): # img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8) img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) gtboxes_and_label = tf.py_func(back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) gtboxes_and_label_minAreaRectangle = get_horizon_minAreaRectangle(gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape(gtboxes_and_label_minAreaRectangle, [-1, 5]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN(net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=gtboxes_and_label, is_training=False, share_head=False, share_net=share_net, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, anchor_angles=cfgs.ANCHOR_ANGLES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, anchor_stride=cfgs.ANCHOR_STRIDE, top_k_nms=cfgs.RPN_TOP_K_NMS, kernel_size=cfgs.KERNEL_SIZE, use_angles_condition=False, anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD, nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=cfgs.IS_FILTER_OUTSIDE_BOXES, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], scope='') rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ] _, _, rpn_predict_boxes, rpn_predict_scores = rpn.rpn_losses() # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, stop_gradient_for_proposals=False, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS, nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD, use_angle_condition=False, level=cfgs.LEVEL, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD, boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD, use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() ############################################################################################## if cfgs.NEED_AUXILIARY: predict_boxes = tf.concat([fast_rcnn_decode_boxes, rpn_predict_boxes], axis=0) predict_scores = tf.concat([fast_rcnn_score, rpn_predict_scores - 0.2], axis=0) rpn_predict_label = tf.ones([tf.shape(rpn_predict_scores)[0], ], tf.int64) labels = tf.concat([detection_category, rpn_predict_label], axis=0) # valid_indices = nms_rotate.nms_rotate(decode_boxes=predict_boxes, # scores=predict_scores, # iou_threshold=0.15, # max_output_size=30, # use_angle_condition=False, # angle_threshold=15, # use_gpu=True) valid_indices = tf.py_func(nms_rotate.nms_rotate_cpu, inp=[predict_boxes, predict_scores, tf.constant(0.15, tf.float32), tf.constant(30, tf.float32)], Tout=tf.int64) fast_rcnn_decode_boxes = tf.gather(predict_boxes, valid_indices) fast_rcnn_score = tf.gather(predict_scores, valid_indices) detection_category = tf.gather(labels, valid_indices) ############################################################################################## if mode == 0: fast_rcnn_decode_boxes = get_horizon_minAreaRectangle(fast_rcnn_decode_boxes, False) # train init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer() ) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) gtboxes_dict = {} predict_dict = {} for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \ _gtboxes_and_label_minAreaRectangle, _fast_rcnn_score, _detection_category \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes, gtboxes_and_label_minAreaRectangle, fast_rcnn_score, detection_category]) end = time.time() # gtboxes convert dict gtboxes_dict[str(_img_name_batch[0])] = [] predict_dict[str(_img_name_batch[0])] = [] # for j, box in enumerate(_gtboxes_and_label): # bbox_dict = {} # bbox_dict['bbox'] = np.array(_gtboxes_and_label[j, :-1], np.float64) # bbox_dict['name'] = LABEl_NAME_MAP[int(_gtboxes_and_label[j, -1])] # gtbox_dict[str(_img_name_batch[0])].append(bbox_dict) # # for label in NAME_LABEL_MAP.keys(): # if label == 'back_ground': # continue # else: # temp_dict = {} # temp_dict['name'] = label # # ind = np.where(_detection_category == NAME_LABEL_MAP[label])[0] # temp_boxes = _fast_rcnn_decode_boxes[ind] # temp_score = np.reshape(_fast_rcnn_score[ind], [-1, 1]) # temp_dict['bbox'] = np.array(np.concatenate([temp_boxes, temp_score], axis=1), np.float64) # predict_dict[str(_img_name_batch[0])].append(temp_dict) if mode == 0: gtboxes_list, predict_list = \ make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category) else: gtboxes_list, predict_list = \ make_dict_packle(_gtboxes_and_label, _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category) gtboxes_dict[str(_img_name_batch[0])].extend(gtboxes_list) predict_dict[str(_img_name_batch[0])].extend(predict_list) view_bar('{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) fw1 = open('gtboxes_dict.pkl', 'w') fw2 = open('predict_dict.pkl', 'w') pickle.dump(gtboxes_dict, fw1) pickle.dump(predict_dict, fw2) fw1.close() fw2.close() coord.request_stop() coord.join(threads)
def train(): retinanet = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=True) with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5]) if cfgs.NET_NAME in ['resnet152_v1d', 'resnet101_v1d', 'resnet50_v1d']: img_batch = img_batch / tf.constant([cfgs.PIXEL_STD]) final_bbox, final_scores, final_category, loss_dict = retinanet.build_whole_detection_network( input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label) # ----------------------------------------------------------------------------------------------------build loss weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses()) cls_loss = loss_dict['cls_loss'] reg_loss = loss_dict['reg_loss'] total_loss = cls_loss + reg_loss + weight_decay_loss # ---------------------------------------------------------------------------------------------------add summary tf.summary.scalar('RETINANET_LOSS/cls_loss', cls_loss) tf.summary.scalar('RETINANET_LOSS/reg_loss', reg_loss) tf.summary.scalar('LOSS/total_loss', total_loss) tf.summary.scalar('LOSS/regular_weights', weight_decay_loss) gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories( img_batch=img_batch, boxes=gtboxes_and_label[:, :-1], labels=gtboxes_and_label[:, -1]) if cfgs.ADD_BOX_IN_TENSORBOARD: detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores( img_batch=img_batch, boxes=final_bbox, labels=final_category, scores=final_scores) tf.summary.image('Compare/final_detection', detections_in_img) tf.summary.image('Compare/gtboxes', gtboxes_in_img) global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[ np.int64(cfgs.DECAY_STEP[0]), np.int64(cfgs.DECAY_STEP[1]) ], values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) # ---------------------------------------------------------------------------------------------compute gradients gradients = retinanet.get_gradients(optimizer, total_loss) # enlarge_gradients for bias if cfgs.MUTILPY_BIAS_GRADIENT: gradients = retinanet.enlarge_gradients_for_bias(gradients) if cfgs.GRADIENT_CLIPPING_BY_NORM: with tf.name_scope('clip_gradients'): gradients = slim.learning.clip_gradient_norms( gradients, cfgs.GRADIENT_CLIPPING_BY_NORM) # train_op train_op = optimizer.apply_gradients(grads_and_vars=gradients, global_step=global_step) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = retinanet.get_restorer() saver = tf.train.Saver(max_to_keep=30) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) if not os.path.exists(summary_path): os.makedirs(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0: _, global_stepnp = sess.run([train_op, global_step]) else: if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0: start = time.time() _, global_stepnp, img_name, reg_loss_, cls_loss_, total_loss_ = \ sess.run( [train_op, global_step, img_name_batch, reg_loss, cls_loss, total_loss]) end = time.time() print(""" {}: step{} image_name:{} |\t reg_loss:{} |\t cls_loss:{} |\t total_loss:{} |per_cost_time:{}s""" \ .format(training_time, global_stepnp, str(img_name[0]), reg_loss_, cls_loss_, total_loss_, (end - start))) else: if step % cfgs.SMRY_ITER == 0: _, global_stepnp, summary_str = sess.run( [train_op, global_step, summary_op]) summary_writer.add_summary(summary_str, global_stepnp) summary_writer.flush() if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.makedirs(save_dir) save_ckpt = os.path.join( save_dir, '{}_'.format(cfgs.DATASET_NAME) + str(global_stepnp) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)
def train(): with tf.Graph().as_default(): tf.set_random_seed(1234) with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=True, is_val=False) with tf.name_scope('draw_gtboxes'): gtboxes_in_img = draw_box_with_color( img_batch, tf.reshape(gtboxes_and_label_batch, [-1, 5])[:, :-1], text=tf.shape(gtboxes_and_label_batch)[1]) # ********************************************************************* # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * rpn * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0), is_training=True, share_head=cfgs.SHARE_HEAD, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs. RPN_IOU_NEGATIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] rpn_location_loss, rpn_classification_loss = rpn.rpn_losses() rpn_total_loss = rpn_classification_loss + rpn_location_loss with tf.name_scope('draw_proposals'): # score > 0.5 is object rpn_object_boxes_indices = tf.reshape( tf.where(tf.greater(rpn_proposals_scores, 0.5)), [-1]) rpn_object_boxes = tf.gather(rpn_proposals_boxes, rpn_object_boxes_indices) rpn_proposals_objcet_boxes_in_img = draw_box_with_color( img_batch, rpn_object_boxes, text=tf.shape(rpn_object_boxes)[0]) rpn_proposals_boxes_in_img = draw_box_with_color( img_batch, rpn_proposals_boxes, text=tf.shape(rpn_proposals_boxes)[0]) # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN( img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0), fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs. FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative use_dropout=False, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=True, level=cfgs.LEVEL) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss( ) fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss with tf.name_scope('draw_boxes_with_categories'): fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories( img_batch=img_batch, boxes=fast_rcnn_decode_boxes, labels=detection_category, scores=fast_rcnn_score) # train total_loss = slim.losses.get_total_loss() global_step = slim.get_or_create_global_step() #返回并创建全局步长张量 # # lr = tf.train.piecewise_constant(global_step, # boundaries=[np.int64(10000), np.int64(20000)], # values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100]) lr = tf.train.exponential_decay(cfgs.LR, global_step, decay_steps=5000, decay_rate=1 / 2., staircase=True) # lr = tf.train.piecewise_constant(global_step, # boundaries=[np.int64(30000), np.int64(40000)], # values=[lr, cfgs.LR/100, cfgs.LR/1000]) tf.summary.scalar('learning_rate', lr) # optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) optimizer = tf.train.AdamOptimizer(lr, beta1=cfgs.MOMENTUM, beta2=0.999, epsilon=1e-8, use_locking=False, name='Adam') # optimizer = tf.train.RMSPropOptimizer(lr, decay=0.9, epsilon=1e-6, name='RMSProp') #创建一个计算梯度并返回损失的Operation train_op = slim.learning.create_train_op( total_loss, optimizer, global_step) # rpn_total_loss, # train_op = optimizer.minimize(second_classification_loss, global_step) # *********************************************************************************************** # * Summary * # *********************************************************************************************** # ground truth and predict tf.summary.image('img/gtboxes', gtboxes_in_img) tf.summary.image('img/faster_rcnn_predict', fast_rcnn_predict_boxes_in_imgs) # rpn loss and image tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss) tf.summary.scalar('rpn/rpn_classification_loss', rpn_classification_loss) tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_location_loss', fast_rcnn_location_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss', fast_rcnn_classification_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss', fast_rcnn_total_loss) tf.summary.scalar('loss/total_loss', total_loss) # # # tf.summary.image('C2', _concact_features(share_net['resnet_v1_50/block1/unit_2/bottleneck_v1'][:, :, :, 0:16]), 1) # tf.summary.image('C3', _concact_features(share_net['resnet_v1_50/block2/unit_3/bottleneck_v1'][:, :, :, 0:16]), 1) # tf.summary.image('C4', _concact_features(share_net['resnet_v1_50/block3/unit_5/bottleneck_v1'][:, :, :, 0:16]), 1) # tf.summary.image('C5', _concact_features(share_net['resnet_v1_50/block4'][:, :, :, 0:16]), 1) # tf.summary.image('P2', _concact_features(rpn.feature_pyramid['P2'][:, :, :, 0:16]),1) # tf.summary.image('P3', _concact_features(rpn.feature_pyramid['P3'][:, :, :, 0:16]),1) # tf.summary.image('P4', _concact_features(rpn.feature_pyramid['P4'][:, :, :, 0:16]),1) # tf.summary.image('P5', _concact_features(rpn.feature_pyramid['P5'][:, :, :, 0:16]), 1) # tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img) # tf.summary.image('rpn/rpn_object_boxes', rpn_proposals_objcet_boxes_in_img) # learning_rate # tf.summary.scalar('learning_rate', lr) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() saver = tf.train.Saver(max_to_keep=16) config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: if cfgs.NET_NAME == 'pvanet': sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) start = 0 if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') start = int("".join( list(restore_ckpt.split('/')[-1])[4:8])) + 1 else: # read_npy.load_initial_weights(sess) read_npy.load_ckpt_weights(sess) else: sess.run(init_op) # print(sess.run('resnet_v1_50/block4/unit_3/bottleneck_v1/conv3/BatchNorm/moving_variance')) # print(sess.run('vgg_16/block4/unit_3/bottleneck_v1/conv3/BatchNorm/moving_variance')) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) start = 0 if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') # start = int("".join(list(restore_ckpt.split('/')[-1])[4:8]))+1 summary_path = os.path.join(FLAGS.summary_path, cfgs.VERSION) mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) df = pd.DataFrame( [], columns=['Recall', 'Precision', 'mAP', 'F1_score'], index=[]) for step in range(0, cfgs.MAX_ITERATION): # print(img_name_batch.eval()) training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) start = time.time() _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \ _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \ _fast_rcnn_total_loss, _total_loss, _ = \ sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss, rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss, fast_rcnn_total_loss, total_loss, train_op]) end = time.time() # if step == 100: # save_dir = os.path.join(FLAGS.trained_checkpoint, cfgs.VERSION) # mkdir(save_dir) # # save_ckpt = os.path.join(save_dir, 'voc_' + str(_global_step) + 'model.ckpt') # saver.save(sess, save_ckpt) # print(' weights had been saved') # if step == 500: # save_dir = os.path.join(FLAGS.trained_checkpoint, cfgs.VERSION) # mkdir(save_dir) # # save_ckpt = os.path.join(save_dir, 'voc_' + str(_global_step) + 'model.ckpt') # saver.save(sess, save_ckpt) # print(' weights had been saved') if step % 50 == 0: print(""" {}: step{} image_name:{} |\t rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} | fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} | total_loss:{} |\t pre_cost_time:{}s""" \ .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss, _rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss, (end - start))) # print(""" {}: step{} image_name:{} |\t # rpn_loc_loss:{} |\t # fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} | # total_loss:{} |\t pre_cost_time:{}s""" \ # .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss, # _fast_rcnn_location_loss, # _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss, # (end - start))) if step % 250 == 0: summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, _global_step) summary_writer.flush() if (step > 0 and step % 2000 == 0) or ( step > 0 and (step == 1000)) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(FLAGS.trained_checkpoint, cfgs.VERSION) mkdir(save_dir) save_ckpt = os.path.join( save_dir, 'voc_' + str(_global_step) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') #保存验证集信息 if (step > 0 and step % 2000 == 0) or (step == cfgs.MAX_ITERATION - 1): save_excel = os.path.abspath( '../' ) + r'/Loss/' + cfgs.NET_NAME + r'_' + cfgs.VERSION mkdir(save_excel) new_index = np.append(df.index, [str(step)]) df2 = pd.DataFrame( [valval.val(is_val=True)], columns=['Recall', 'Precision', 'mAP', 'F1_score']) df = df.append(df2) df.index = new_index df.to_excel(save_excel + r'/validation.xls') print('validation result had been saved') coord.request_stop() coord.join(threads)
def train(): with tf.Graph().as_default(): with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=True) with tf.name_scope('draw_gtboxes'): gtboxes_in_img = draw_box_with_color(img_batch, tf.reshape(gtboxes_and_label_batch, [-1, 5])[:, :-1], text=tf.shape(gtboxes_and_label_batch)[1]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * rpn * # *********************************************************************************************** rpn = build_rpn.RPN(net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0), is_training=True, share_head=cfgs.SHARE_HEAD, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs.BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals() # rpn_score shape: [300, ] rpn_location_loss, rpn_classification_loss = rpn.rpn_losses() rpn_total_loss = 10*rpn_classification_loss + rpn_location_loss with tf.name_scope('draw_proposals'): # score > 0.5 is object rpn_object_boxes_indices = tf.reshape(tf.where(tf.greater(rpn_proposals_scores, 0.5)), [-1]) rpn_object_boxes = tf.gather(rpn_proposals_boxes, rpn_object_boxes_indices) rpn_proposals_objcet_boxes_in_img = draw_box_with_color(img_batch, rpn_object_boxes, text=tf.shape(rpn_object_boxes)[0]) rpn_proposals_boxes_in_img = draw_box_with_color(img_batch, rpn_proposals_boxes, text=tf.shape(rpn_proposals_boxes)[0]) # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN(img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=tf.squeeze(gtboxes_and_label_batch, 0), fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, # iou>0.5 is positive, iou<0.5 is negative fast_rcnn_positives_iou_threshold=cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD, use_dropout=False, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=True, level=cfgs.LEVEL) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() fast_rcnn_location_l, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss() fast_rcnn_total_loss = fast_rcnn_location_l + 10*fast_rcnn_classification_loss with tf.name_scope('draw_boxes_with_categories'): fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories(img_batch=img_batch, boxes=fast_rcnn_decode_boxes, labels=detection_category, scores=fast_rcnn_score) # train added_loss = rpn_total_loss + fast_rcnn_total_loss total_loss = tf.losses.get_total_loss() global_step = tf.train.get_or_create_global_step() lr = tf.train.piecewise_constant(global_step, boundaries=[np.int64(20000), np.int64(40000)], values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) # optimizer = tf.train.AdamOptimizer(lr) # train_op = slim.learning.create_train_op(total_loss, optimizer, global_step) # rpn_total_loss, train_op = slim.learning.create_train_op(added_loss, optimizer, global_step) # rpn_total_loss, # train_op = optimizer.minimize(second_classification_loss, global_step) # **********************************-************************************************************* # * Summary * # *********************************************************************************************** # ground truth and predict tf.summary.image('img/gtboxes', gtboxes_in_img) tf.summary.image('img/faster_rcnn_predict', fast_rcnn_predict_boxes_in_imgs) # rpn loss and image tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss) tf.summary.scalar('rpn/rpn_classification_loss', rpn_classification_loss) tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_location_lossum', fast_rcnn_location_l) tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss', fast_rcnn_classification_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss', fast_rcnn_total_loss) tf.summary.scalar('loss/added_loss', added_loss) tf.summary.scalar('loss/total_loss', total_loss) tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img) tf.summary.image('rpn/rpn_object_boxes', rpn_proposals_objcet_boxes_in_img) # learning_rate tf.summary.scalar('learning_rate', lr) summary_op = tf.compat.v1.summary.merge_all() init_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer() ) restorer, restore_ckpt = restore_model.get_restorer(test=False) saver = tf.train.Saver(max_to_keep=5) config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model', restore_ckpt) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join('output/{}'.format(cfgs.DATASET_NAME), FLAGS.summary_path, cfgs.VERSION) mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) start = time.time() # gtboxes = tf.cast( # tf.reshape(fast_rcnn.gtboxes_and_label[:, :-1], [-1, 4]), tf.float32) # ious = iou.iou_calculate(fast_rcnn.fast_rcnn_all_level_proposals, gtboxes) # _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \ # _rpn_total_loss, _fast_rcnn_location_l, _fast_rcnn_classification_loss, \ # _fast_rcnn_total_loss, _added_loss, _total_loss, rcnn_minibatch, find_samples,\ # gtbox1, iou1, _ = \ # sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss, # rpn_total_loss, fast_rcnn_location_l, fast_rcnn_classification_loss, # fast_rcnn_total_loss, added_loss, total_loss, # fast_rcnn.fast_rcnn_minibatch(fast_rcnn.fast_rcnn_all_level_proposals), # fast_rcnn.fast_rcnn_find_positive_negative_samples(fast_rcnn.fast_rcnn_all_level_proposals), # gtboxes, ious, # train_op]) _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \ _rpn_total_loss, _fast_rcnn_location_l, _fast_rcnn_classification_loss, \ _fast_rcnn_total_loss, _added_loss, _total_loss, \ _ = \ sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss, rpn_total_loss, fast_rcnn_location_l, fast_rcnn_classification_loss, fast_rcnn_total_loss, added_loss, total_loss, train_op]) # _global_step, _img_name_batch, _added_loss, _total_loss, \ # _ = \ # sess.run([global_step, img_name_batch, added_loss, total_loss, # train_op]) end = time.time() # a, b, c ,d = rcnn_minibatch # aa, bb, cc = find_samples # q = [] # for i in cc: # if i not in q: # q.append(i) # qq = [] # for i in c: # if i not in qq: # qq.append(i) # minibatch_encode_boxes = tf.gather(fast_rcnn.fast_rcnn_encode_boxes,a) # minibatch_reference_boxes = tf.gather(fast_rcnn.fast_rcnn_all_level_proposals, a) # minibatch_encode_gtboxes = \ # encode_and_decode.encode_boxes( # unencode_boxes=b, # reference_boxes=minibatch_reference_boxes, # scale_factors=fast_rcnn.scale_factors # ) # # [minibatch_size, num_classes*4] # minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes, [1, fast_rcnn.num_classes]) # # class_weights_list = [] # category_list = tf.unstack(d, axis=1) # for i in range(1, fast_rcnn.num_classes + 1): # tmp_class_weights = tf.ones( # shape=[tf.shape(minibatch_encode_boxes)[0], 4], dtype=tf.float32) # tmp_class_weights = tmp_class_weights * tf.expand_dims(category_list[i], axis=1) # class_weights_list.append(tmp_class_weights) # class_weights = tf.concat(class_weights_list, axis=1) # [minibatch_size, num_classes*4] # # loc_loss1 = losses.l1_smooth_losses(predict_boxes=minibatch_encode_boxes, # gtboxes=minibatch_encode_gtboxes, # object_weights=c, # classes_weights=class_weights) # print('loction-loss:', sess.run(loc_loss1)) if step % 50 == 0: print("""{}: step{} image_name:{} rpn_loc_loss:{:.4f} | rpn_cla_loss:{:.4f} | rpn_total_loss:{:.4f} fast_rcnn_loc_loss:{:.10f} | fast_rcnn_cla_loss:{:.10f} | fast_rcnn_total_loss:{:.4f} added_loss:{:.4f} | total_loss:{:.4f} | pre_cost_time:{:.4f}s""" .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss, _rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_l, _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _added_loss, _total_loss, (end - start))) # if step % 100 == 0: # print("""{}: step{} image_name:{} # added_loss:{:.4f} | total_loss:{:.4f} | pre_cost_time:{:.4f}s""" # .format(training_time, _global_step, str(_img_name_batch[0]), _added_loss, _total_loss, (end - start) # )) # print(q, '\n', qq) # print(gtbox1) # print(iou1) if step % 500000 == 0: summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, _global_step) summary_writer.flush() if (step > 0 and step % 10000 == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join('output/{}'.format(cfgs.DATASET_NAME), FLAGS.trained_checkpoint, cfgs.VERSION) mkdir(save_dir) save_ckpt = os.path.join(save_dir, '{}_'.format( cfgs.DATASET_NAME)+str(_global_step)+'model.ckpt') saver.save(sess, save_ckpt) print('Weights have been saved to {}.'.format(save_ckpt)) print('Training done.') coord.request_stop() coord.join(threads)
def test(img_num): with tf.Graph().as_default(): # img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8) img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=gtboxes_and_label, is_training=False, share_head=cfgs.SHARE_HEAD, share_net=share_net, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, anchor_angles=cfgs.ANCHOR_ANGLES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, anchor_stride=cfgs.ANCHOR_STRIDE, pool_stride=cfgs.POOL_STRIDE, top_k_nms=cfgs.RPN_TOP_K_NMS, kernel_size=cfgs.KERNEL_SIZE, use_angles_condition=True, anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD, nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], scope='') # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] _, _, rpn_top_k_boxes, rpn_top_k_scores = rpn.rpn_losses() # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN( img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, stop_gradient_for_proposals=False, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS, nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD, use_angle_condition=False, level=cfgs.LEVEL, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD, use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() ############################################################################################## if cfgs.NEED_AUXILIARY: predict_boxes = tf.concat( [fast_rcnn_decode_boxes, rpn_top_k_boxes], axis=0) predict_scores = tf.concat( [fast_rcnn_score, rpn_top_k_scores - 0.2], axis=0) rpn_top_k_label = tf.ones([ tf.shape(rpn_top_k_scores)[0], ], tf.int64) labels = tf.concat([detection_category, rpn_top_k_label], axis=0) # valid_indices = nms_rotate.nms_rotate(decode_boxes=predict_boxes, # scores=predict_scores, # iou_threshold=0.15, # max_output_size=30, # use_angle_condition=False, # angle_threshold=15, # use_gpu=True) valid_indices = tf.py_func(nms_rotate.nms_rotate_cpu, inp=[ predict_boxes, predict_scores, tf.constant(0.15, tf.float32), tf.constant(30, tf.float32) ], Tout=tf.int64) fast_rcnn_decode_boxes = tf.gather(predict_boxes, valid_indices) fast_rcnn_score = tf.gather(predict_scores, valid_indices) detection_category = tf.gather(labels, valid_indices) ############################################################################################## # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \ _fast_rcnn_score, _detection_category \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category]) end = time.time() _img_batch = np.squeeze(_img_batch, axis=0) _img_batch_fpn = help_utils.draw_box_cv( _img_batch, boxes=_fast_rcnn_decode_boxes, labels=_detection_category, scores=_fast_rcnn_score) mkdir(cfgs.TEST_SAVE_PATH) cv2.imwrite( cfgs.TEST_SAVE_PATH + '/{}_fpn.jpg'.format( str(_img_name_batch[0]).split('.tif')[0]), _img_batch_fpn) # _gtboxes_and_label_batch = np.squeeze(_gtboxes_and_label_batch, axis=0) temp_label = np.reshape(_gtboxes_and_label[:, -1:], [ -1, ]).astype(np.int64) _img_batch_gt = help_utils.draw_box_cv( _img_batch, boxes=_gtboxes_and_label[:, :-1], labels=temp_label, scores=None) cv2.imwrite( cfgs.TEST_SAVE_PATH + '/{}_gt.jpg'.format( str(_img_name_batch[0]).split('.tif')[0]), _img_batch_gt) #view_bar('{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i+1, img_num) print('{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)) + str(i + 1) + '/' + str(img_num)) coord.request_stop() coord.join(threads)
def eval_text(img_num, mode): with tf.Graph().as_default(): faster_rcnn = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=False) # 通过readtfrecord来读取文件 img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=False) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) final_boxes_h, final_scores_h, final_category_h, \ final_boxes_r, final_scores_r, final_category_r= faster_rcnn.build_whole_detection_network( input_img_batch=img_batch, gtboxes_r_batch=gtboxes_and_label, gtboxes_h_batch=gtboxes_and_label_minAreaRectangle) if mode == 0: final_boxes_r = get_horizen_minAreaRectangle(final_boxes_r, False) # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = faster_rcnn.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) gtboxes_horizontal_dict = {} predict_horizontal_dict = {} gtboxes_rotate_dict = {} predict_rotate_dict = {} for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \ _final_boxes_h, _final_scores_h, _final_category_h, _final_boxes_r, \ _final_scores_r, _final_category_r \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle, final_boxes_h, final_scores_h, final_category_h, final_boxes_r, final_scores_r, final_category_r]) end = time.time() # det_detections_h = draw_box_in_img.draw_box_cv(np.squeeze(_img_batch, 0), # boxes=_final_boxes_h, # labels=_final_category_h, # scores=final_scores_h) # det_detections_r = draw_box_in_img.draw_rotate_box_cv(np.squeeze(_img_batch, 0), # boxes=_final_boxes_r, # labels=_final_category_r, # scores= _final_scores_r) # save_dir = os.path.join('/home/sws/code/Tfrecord/', cfgs.VERSION) # tools.mkdir(save_dir) # cv2.imwrite(save_dir + '/' + _img_name_batch[0] + '_h.jpg', # det_detections_h) # cv2.imwrite(save_dir + '/' + _img_name_batch[0] + '_r.jpg', # det_detections_r) # gtboxes convert dict gtboxes_horizontal_dict[str(_img_name_batch[0])] = [] predict_horizontal_dict[str(_img_name_batch[0])] = [] gtboxes_rotate_dict[str(_img_name_batch[0])] = [] predict_rotate_dict[str(_img_name_batch[0])] = [] gtbox_horizontal_list, predict_horizontal_list = \ make_dict_packle(_gtboxes_and_label_minAreaRectangle, _final_boxes_h, _final_scores_h, _final_category_h) # if mode == 0: # gtbox_rotate_list, predict_rotate_list = \ # make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes_rotate, # _fast_rcnn_score_rotate, _detection_category_rotate) if mode == 0: gtbox_rotate_list, predict_rotate_list = \ make_dict_packle(_gtboxes_and_label_minAreaRectangle, _final_boxes_r, _final_scores_r, _final_category_r) else: gtbox_rotate_list, predict_rotate_list = \ make_dict_packle(_gtboxes_and_label, _final_boxes_r, _final_scores_r, _final_category_r) gtboxes_horizontal_dict[str( _img_name_batch[0])].extend(gtbox_horizontal_list) predict_horizontal_dict[str( _img_name_batch[0])].extend(predict_horizontal_list) gtboxes_rotate_dict[str( _img_name_batch[0])].extend(gtbox_rotate_list) predict_rotate_dict[str( _img_name_batch[0])].extend(predict_rotate_list) view_bar( '{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) print('\n') fw1 = open('gtboxes_horizontal_dict.pkl', 'w') fw2 = open('predict_horizontal_dict.pkl', 'w') fw3 = open('gtboxes_rotate_dict.pkl', 'w') fw4 = open('predict_rotate_dict.pkl', 'w') pickle.dump(gtboxes_horizontal_dict, fw1) pickle.dump(predict_horizontal_dict, fw2) pickle.dump(gtboxes_rotate_dict, fw3) pickle.dump(predict_rotate_dict, fw4) fw1.close() fw2.close() fw3.close() fw4.close() coord.request_stop() coord.join(threads)
def train(): with tf.Graph().as_default(), tf.device('/cpu:0'): global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[ np.int64(cfgs.DECAY_STEP[0]), np.int64(cfgs.DECAY_STEP[1]) ], values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) img_batch = tf.reshape(img_batch, [1, 720, 1280, 3]) batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue( [img_name_batch, img_batch, gtboxes_and_label, num_objects_batch], capacity=2 * len(cfgs.GPU_GROUP)) tower_grads = [] for i in range(len(cfgs.GPU_GROUP)): with tf.device('/gpu:%d' % i): with tf.name_scope('Gpu%d' % i) as scope: img_name_, img_, gtboxes_and_label_, num_objects_ = batch_queue.dequeue( ) loss, faster_rcnn, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss,\ fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss = \ tower_loss(scope, img_name_, img_, gtboxes_and_label_, num_objects_) tf.get_variable_scope().reuse_variables() grads = optimizer.compute_gradients(loss) if cfgs.MUTILPY_BIAS_GRADIENT: grads = faster_rcnn.enlarge_gradients_for_bias(grads) if cfgs.GRADIENT_CLIPPING_BY_NORM: with tf.name_scope('clip_gradients'): grads = slim.learning.clip_gradient_norms( grads, cfgs.GRADIENT_CLIPPING_BY_NORM) tower_grads.append(grads) grads = average_gradients(tower_grads) train_op = optimizer.apply_gradients(grads, global_step) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = faster_rcnn.get_restorer() saver = tf.train.Saver(max_to_keep=10) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) tools.mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0: _, global_stepnp = sess.run([train_op, global_step]) else: if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0: start = time.time() _, global_stepnp, img_name, rpnLocLoss, rpnClsLoss, rpnTotalLoss, \ fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss = \ sess.run( [train_op, global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss, fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss, loss]) end = time.time() print(""" {}: step{} image_name:{} |\t rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} | fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} | total_loss:{} |\t per_cost_time:{}s""" \ .format(training_time, global_stepnp, str(img_name[0]), rpnLocLoss, rpnClsLoss, rpnTotalLoss, fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss, (end - start))) else: if step % cfgs.SMRY_ITER == 0: _, global_stepnp, summary_str = sess.run( [train_op, global_step, summary_op]) summary_writer.add_summary(summary_str, global_stepnp) summary_writer.flush() if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.mkdir(save_dir) save_ckpt = os.path.join( save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)
def tower_loss(scope): with tf.name_scope(scope): faster_rcnn = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=True) with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5]) biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer( cfgs.WEIGHT_DECAY) # list as many types of layers as possible, even if they are not used now with slim.arg_scope([ slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected ], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, biases_initializer=tf.constant_initializer(0.0)): final_bbox, final_scores, final_category, loss_dict = faster_rcnn.build_whole_detection_network( input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label) # ----------------------------------------------------------------------------------------------------build loss weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses()) rpn_location_loss = loss_dict['rpn_loc_loss'] rpn_cls_loss = loss_dict['rpn_cls_loss'] rpn_total_loss = rpn_location_loss + rpn_cls_loss fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss'] fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss'] fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss # ____________________________________________________________________________________________________build loss # ---------------------------------------------------------------------------------------------------add summary tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss) tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss) tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss', fastrcnn_loc_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss) tf.summary.scalar('LOSS/total_loss', total_loss) tf.summary.scalar('LOSS/regular_weights', weight_decay_loss) gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories( img_batch=img_batch, boxes=gtboxes_and_label[:, :-1], labels=gtboxes_and_label[:, -1]) if cfgs.ADD_BOX_IN_TENSORBOARD: detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores( img_batch=img_batch, boxes=final_bbox, labels=final_category, scores=final_scores) tf.summary.image('Compare/final_detection', detections_in_img) tf.summary.image('Compare/gtboxes', gtboxes_in_img) return total_loss, faster_rcnn, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss,\ fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss
def eval_dict_convert(img_num, mode): with tf.Graph().as_default(): # img = tf.placeholder(shape=[None, None, 3], dtype=tf.uint8) img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) gtboxes_and_label_minAreaRectangle = get_horizon_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=gtboxes_and_label, is_training=False, share_head=False, share_net=share_net, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, anchor_angles=cfgs.ANCHOR_ANGLES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, anchor_stride=cfgs.ANCHOR_STRIDE, top_k_nms=cfgs.RPN_TOP_K_NMS, kernel_size=cfgs.KERNEL_SIZE, use_angles_condition=False, anchor_angle_threshold=cfgs.RPN_ANCHOR_ANGLES_THRESHOLD, nms_angle_threshold=cfgs.RPN_NMS_ANGLES_THRESHOLD, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=cfgs. IS_FILTER_OUTSIDE_BOXES, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], scope='') rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] _, _, rpn_predict_boxes, rpn_predict_scores = rpn.rpn_losses() # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN( img_batch=img_batch, feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, stop_gradient_for_proposals=False, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, top_k_nms=cfgs.FAST_RCNN_TOP_K_NMS, nms_angle_threshold=cfgs.FAST_RCNN_NMS_ANGLES_THRESHOLD, use_angle_condition=False, level=cfgs.LEVEL, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, boxes_angle_threshold=cfgs.FAST_RCNN_BOXES_ANGLES_THRESHOLD, use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() ############################################################################################## if cfgs.NEED_AUXILIARY: predict_boxes = tf.concat( [fast_rcnn_decode_boxes, rpn_predict_boxes], axis=0) predict_scores = tf.concat( [fast_rcnn_score, rpn_predict_scores - 0.2], axis=0) rpn_predict_label = tf.ones([ tf.shape(rpn_predict_scores)[0], ], tf.int64) labels = tf.concat([detection_category, rpn_predict_label], axis=0) # valid_indices = nms_rotate.nms_rotate(decode_boxes=predict_boxes, # scores=predict_scores, # iou_threshold=0.15, # max_output_size=30, # use_angle_condition=False, # angle_threshold=15, # use_gpu=True) valid_indices = tf.py_func(nms_rotate.nms_rotate_cpu, inp=[ predict_boxes, predict_scores, tf.constant(0.15, tf.float32), tf.constant(30, tf.float32) ], Tout=tf.int64) fast_rcnn_decode_boxes = tf.gather(predict_boxes, valid_indices) fast_rcnn_score = tf.gather(predict_scores, valid_indices) detection_category = tf.gather(labels, valid_indices) ############################################################################################## if mode == 0: fast_rcnn_decode_boxes = get_horizon_minAreaRectangle( fast_rcnn_decode_boxes, False) # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) gtboxes_dict = {} predict_dict = {} for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _fast_rcnn_decode_boxes, \ _gtboxes_and_label_minAreaRectangle, _fast_rcnn_score, _detection_category \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, fast_rcnn_decode_boxes, gtboxes_and_label_minAreaRectangle, fast_rcnn_score, detection_category]) end = time.time() # gtboxes convert dict gtboxes_dict[str(_img_name_batch[0])] = [] predict_dict[str(_img_name_batch[0])] = [] # for j, box in enumerate(_gtboxes_and_label): # bbox_dict = {} # bbox_dict['bbox'] = np.array(_gtboxes_and_label[j, :-1], np.float64) # bbox_dict['name'] = LABEl_NAME_MAP[int(_gtboxes_and_label[j, -1])] # gtbox_dict[str(_img_name_batch[0])].append(bbox_dict) # # for label in NAME_LABEL_MAP.keys(): # if label == 'back_ground': # continue # else: # temp_dict = {} # temp_dict['name'] = label # # ind = np.where(_detection_category == NAME_LABEL_MAP[label])[0] # temp_boxes = _fast_rcnn_decode_boxes[ind] # temp_score = np.reshape(_fast_rcnn_score[ind], [-1, 1]) # temp_dict['bbox'] = np.array(np.concatenate([temp_boxes, temp_score], axis=1), np.float64) # predict_dict[str(_img_name_batch[0])].append(temp_dict) if mode == 0: gtboxes_list, predict_list = \ make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category) else: gtboxes_list, predict_list = \ make_dict_packle(_gtboxes_and_label, _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category) gtboxes_dict[str(_img_name_batch[0])].extend(gtboxes_list) predict_dict[str(_img_name_batch[0])].extend(predict_list) view_bar( '{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) fw1 = open('gtboxes_dict.pkl', 'w') fw2 = open('predict_dict.pkl', 'w') pickle.dump(gtboxes_dict, fw1) pickle.dump(predict_dict, fw2) fw1.close() fw2.close() coord.request_stop() coord.join(threads)
def train(): with tf.Graph().as_default(): with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) with tf.name_scope('draw_gtboxes'): gtboxes_in_img = draw_box_with_color( img_batch, tf.reshape(gtboxes_and_label_minAreaRectangle, [-1, 5])[:, :-1], text=tf.shape(gtboxes_and_label_minAreaRectangle)[0]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * rpn * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=gtboxes_and_label_minAreaRectangle, is_training=True, share_head=cfgs.SHARE_HEAD, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs. RPN_IOU_NEGATIVE_THRESHOLD, # iou>=0.7 is positive box, iou< 0.3 is negative rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] rpn_location_loss, rpn_classification_loss = rpn.rpn_losses() rpn_total_loss = rpn_classification_loss + rpn_location_loss with tf.name_scope('draw_proposals'): # score > 0.5 is object rpn_object_boxes_indices = tf.reshape( tf.where(tf.greater(rpn_proposals_scores, 0.5)), [-1]) rpn_object_boxes = tf.gather(rpn_proposals_boxes, rpn_object_boxes_indices) rpn_proposals_objcet_boxes_in_img = draw_box_with_color( img_batch, rpn_object_boxes, text=tf.shape(rpn_object_boxes)[0]) rpn_proposals_boxes_in_img = draw_box_with_color( img_batch, rpn_proposals_boxes, text=tf.shape(rpn_proposals_boxes)[0]) # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN( feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=gtboxes_and_label, gtboxes_and_label_minAreaRectangle= gtboxes_and_label_minAreaRectangle, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs. FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=True, level=cfgs.LEVEL) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = \ fast_rcnn.fast_rcnn_predict() fast_rcnn_location_loss, fast_rcnn_classification_loss = fast_rcnn.fast_rcnn_loss( ) fast_rcnn_total_loss = fast_rcnn_location_loss + fast_rcnn_classification_loss with tf.name_scope('draw_boxes_with_categories'): fast_rcnn_predict_boxes_in_imgs = draw_boxes_with_categories( img_batch=img_batch, boxes=fast_rcnn_decode_boxes, labels=detection_category, scores=fast_rcnn_score) # train total_loss = slim.losses.get_total_loss() global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[np.int64(20000), np.int64(40000)], values=[cfgs.LR, cfgs.LR / 10, cfgs.LR / 100]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) train_op = slim.learning.create_train_op( total_loss, optimizer, global_step) # rpn_total_loss, # train_op = optimizer.minimize(second_classification_loss, global_step) # *********************************************************************************************** # * Summary * # *********************************************************************************************** # ground truth and predict tf.summary.image('img/gtboxes', gtboxes_in_img) tf.summary.image('img/faster_rcnn_predict', fast_rcnn_predict_boxes_in_imgs) # rpn loss and image tf.summary.scalar('rpn/rpn_location_loss', rpn_location_loss) tf.summary.scalar('rpn/rpn_classification_loss', rpn_classification_loss) tf.summary.scalar('rpn/rpn_total_loss', rpn_total_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_location_loss', fast_rcnn_location_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_classification_loss', fast_rcnn_classification_loss) tf.summary.scalar('fast_rcnn/fast_rcnn_total_loss', fast_rcnn_total_loss) tf.summary.scalar('loss/total_loss', total_loss) tf.summary.image('rpn/rpn_all_boxes', rpn_proposals_boxes_in_img) tf.summary.image('rpn/rpn_object_boxes', rpn_proposals_objcet_boxes_in_img) # learning_rate tf.summary.scalar('learning_rate', lr) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() saver = tf.train.Saver(max_to_keep=10) config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join(FLAGS.summary_path, cfgs.VERSION) mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) start = time.time() _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \ _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \ _fast_rcnn_total_loss, _total_loss, _ = \ sess.run([global_step, img_name_batch, rpn_location_loss, rpn_classification_loss, rpn_total_loss, fast_rcnn_location_loss, fast_rcnn_classification_loss, fast_rcnn_total_loss, total_loss, train_op]) end = time.time() if step % 10 == 0: print(""" {}: step{} image_name:{} |\t rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} | fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_total_loss:{} | total_loss:{} |\t pre_cost_time:{}s""" \ .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss, _rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, _fast_rcnn_total_loss, _total_loss, (end - start))) if step % 50 == 0: summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, _global_step) summary_writer.flush() if (step > 0 and step % 1000 == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(FLAGS.trained_checkpoint, cfgs.VERSION) mkdir(save_dir) save_ckpt = os.path.join( save_dir, 'voc_' + str(_global_step) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)
def test_rotate(): with tf.Graph().as_default(): with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=True) gtboxes_and_label, head = get_head( tf.squeeze(gtboxes_and_label_batch, 0)) gtboxes_and_label = tf.py_func(back_forward_convert, inp=[gtboxes_and_label], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) head_quadrant = tf.py_func(get_head_quadrant, inp=[head, gtboxes_and_label], Tout=tf.float32) head_quadrant = tf.reshape(head_quadrant, [-1, 1]) gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) with tf.name_scope('draw_gtboxes'): gtboxes_in_img = draw_box_with_color( img_batch, tf.reshape(gtboxes_and_label_minAreaRectangle, [-1, 5])[:, :-1], text=tf.shape(gtboxes_and_label_minAreaRectangle)[0]) gtboxes_rotate_in_img = draw_box_with_color_rotate( img_batch, tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1], text=tf.shape(gtboxes_and_label)[0], head=head_quadrant) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) for i in range(650): img_gtboxes, img_gtboxes_rotate, img_name = sess.run( [gtboxes_in_img, gtboxes_rotate_in_img, img_name_batch]) img_gtboxes = np.squeeze(img_gtboxes, axis=0) img_gtboxes_rotate = np.squeeze(img_gtboxes_rotate, axis=0) print(i) cv2.imwrite( cfgs.INFERENCE_SAVE_PATH + '/{}_horizontal_fpn.jpg'.format(str(img_name[0])), img_gtboxes) cv2.imwrite( cfgs.INFERENCE_SAVE_PATH + '/{}_rotate_fpn.jpg'.format(str(img_name[0])), img_gtboxes_rotate) coord.request_stop() coord.join(threads)
def train(): faster_rcnn = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=True) with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) mask_batch = tf.py_func( image_preprocess.get_mask, [tf.squeeze(img_batch, 0), tf.squeeze(gtboxes_and_label_batch, 0)], [tf.float32]) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) gtboxes_and_label_AreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_AreaRectangle = tf.reshape( gtboxes_and_label_AreaRectangle, [-1, 5]) with tf.name_scope('draw_gtboxes'): gtboxes_in_img = draw_box_with_color( img_batch, tf.reshape(gtboxes_and_label_AreaRectangle, [-1, 5])[:, :-1], text=tf.shape(gtboxes_and_label_AreaRectangle)[0]) gtboxes_rotate_in_img = draw_box_with_color_rotate( img_batch, tf.reshape(gtboxes_and_label, [-1, 6])[:, :-1], text=tf.shape(gtboxes_and_label)[0]) biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY) # list as many types of layers as possible, even if they are not used now with slim.arg_scope([ slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected ], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, biases_initializer=tf.constant_initializer(0.0)): final_boxes_h, final_scores_h, final_category_h, \ final_boxes_r, final_scores_r, final_category_r, loss_dict = faster_rcnn.build_whole_detection_network( input_img_batch=img_batch, gtboxes_r_batch=gtboxes_and_label, gtboxes_h_batch=gtboxes_and_label_AreaRectangle, mask_batch=mask_batch[0]) dets_in_img = draw_boxes_with_categories_and_scores( img_batch=img_batch, boxes=final_boxes_h, labels=final_category_h, scores=final_scores_h) dets_rotate_in_img = draw_boxes_with_categories_and_scores_rotate( img_batch=img_batch, boxes=final_boxes_r, labels=final_category_r, scores=final_scores_r) # ----------------------------------------------------------------------------------------------------build loss weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses()) rpn_location_loss = loss_dict['rpn_loc_loss'] rpn_cls_loss = loss_dict['rpn_cls_loss'] rpn_total_loss = rpn_location_loss + rpn_cls_loss fastrcnn_cls_loss_h = loss_dict['fastrcnn_cls_loss_h'] fastrcnn_loc_loss_h = loss_dict['fastrcnn_loc_loss_h'] fastrcnn_cls_loss_r = loss_dict['fastrcnn_cls_loss_r'] fastrcnn_loc_loss_r = loss_dict['fastrcnn_loc_loss_r'] fastrcnn_total_loss = fastrcnn_cls_loss_h + fastrcnn_loc_loss_h + fastrcnn_cls_loss_r + fastrcnn_loc_loss_r if cfgs.ADD_ATTENTION: attention_loss = loss_dict['attention_loss'] total_loss = rpn_total_loss + fastrcnn_total_loss + attention_loss + weight_decay_loss else: attention_loss = tf.convert_to_tensor(0) total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss # ---------------------------------------------------------------------------------------------------add summary tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss) tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss) tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss_h', fastrcnn_cls_loss_h) tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss_h', fastrcnn_loc_loss_h) tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss_r', fastrcnn_cls_loss_r) tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss_r', fastrcnn_loc_loss_r) tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss) if cfgs.ADD_ATTENTION: tf.summary.scalar('ATTENTION_LOSS/attention_loss', attention_loss) tf.summary.scalar('LOSS/total_loss', total_loss) tf.summary.scalar('LOSS/regular_weights', weight_decay_loss) tf.summary.image('img/gtboxes', gtboxes_in_img) tf.summary.image('img/gtboxes_rotate', gtboxes_rotate_in_img) tf.summary.image('img/mask', mask_batch) tf.summary.image('img/dets', dets_in_img) tf.summary.image('img/dets_rotate', dets_rotate_in_img) # ---------------------------------------------------------------------------------------------add summary global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[ np.int64(cfgs.DECAY_STEP[0]), np.int64(cfgs.DECAY_STEP[1]) ], values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) # ---------------------------------------------------------------------------------------------compute gradients gradients = faster_rcnn.get_gradients(optimizer, total_loss) # enlarge_gradients for bias if cfgs.MUTILPY_BIAS_GRADIENT: gradients = faster_rcnn.enlarge_gradients_for_bias(gradients) if cfgs.GRADIENT_CLIPPING_BY_NORM: with tf.name_scope('clip_gradients'): gradients = slim.learning.clip_gradient_norms( gradients, cfgs.GRADIENT_CLIPPING_BY_NORM) # ---------------------------------------------------------------------------------------------compute gradients # train_op train_op = optimizer.apply_gradients(grads_and_vars=gradients, global_step=global_step) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = faster_rcnn.get_restorer() saver = tf.train.Saver(max_to_keep=10) config = tf.ConfigProto() config.gpu_options.allow_growth = True # with tf.Session(config=config) as sess: # with tf.Session() as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) tools.mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0: _, global_stepnp = sess.run([train_op, global_step]) else: if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0: start = time.time() _global_step, _img_name_batch, _rpn_location_loss, _rpn_classification_loss, \ _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, \ _fast_rcnn_location_rotate_loss, _fast_rcnn_classification_rotate_loss, \ _fast_rcnn_total_loss, _attention_loss, _total_loss, _ = \ sess.run([global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss, fastrcnn_loc_loss_h, fastrcnn_cls_loss_h, fastrcnn_loc_loss_r, fastrcnn_cls_loss_r, fastrcnn_total_loss, attention_loss, total_loss, train_op]) end = time.time() print(""" {}: step{} image_name:{} |\t rpn_loc_loss:{} |\t rpn_cla_loss:{} |\t rpn_total_loss:{} | fast_rcnn_loc_loss:{} |\t fast_rcnn_cla_loss:{} |\t fast_rcnn_loc_rotate_loss:{} |\t fast_rcnn_cla_rotate_loss:{} |\t fast_rcnn_total_loss:{} |\t attention_loss:{} |\t total_loss:{} |\t pre_cost_time:{}s""" \ .format(training_time, _global_step, str(_img_name_batch[0]), _rpn_location_loss, _rpn_classification_loss, _rpn_total_loss, _fast_rcnn_location_loss, _fast_rcnn_classification_loss, _fast_rcnn_location_rotate_loss, _fast_rcnn_classification_rotate_loss, _fast_rcnn_total_loss, _attention_loss, _total_loss, (end - start))) else: if step % cfgs.SMRY_ITER == 0: _, global_stepnp, summary_str = sess.run( [train_op, global_step, summary_op]) summary_writer.add_summary(summary_str, global_stepnp) summary_writer.flush() if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.makedirs(save_dir) save_ckpt = os.path.join( save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)
def eval_ship(img_num): with tf.Graph().as_default(): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=True) gtboxes_and_label, head = get_head( tf.squeeze(gtboxes_and_label_batch, 0)) gtboxes_and_label = tf.py_func(back_forward_convert, inp=[gtboxes_and_label], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) head_quadrant = tf.py_func(get_head_quadrant, inp=[head, gtboxes_and_label], Tout=tf.float32) head_quadrant = tf.reshape(head_quadrant, [-1, 1]) gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=None, is_training=False, share_head=cfgs.SHARE_HEAD, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn.FastRCNN( feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), img_batch=img_batch, roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, gtboxes_and_label_minAreaRectangle= gtboxes_and_label_minAreaRectangle, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False, level=cfgs.LEVEL, head_quadrant=head_quadrant) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \ fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, fast_rcnn_head_quadrant, \ num_of_objects_rotate, detection_category_rotate = fast_rcnn.fast_rcnn_predict() # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) gtboxes_horizontal_dict = {} gtboxes_rotate_dict = {} all_boxes_h = [] all_boxes_r = [] all_img_names = [] for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \ _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \ _fast_rcnn_score_rotate, _detection_category_rotate \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, detection_category_rotate]) end = time.time() # gtboxes convert dict gtboxes_horizontal_dict[str(_img_name_batch[0])] = [] gtboxes_rotate_dict[str(_img_name_batch[0])] = [] gtbox_horizontal_list, gtbox_rotate_list = make_dict_packle( _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle) xmin, ymin, xmax, ymax = _fast_rcnn_decode_boxes[:, 1], _fast_rcnn_decode_boxes[:, 0], _fast_rcnn_decode_boxes[:, 3], _fast_rcnn_decode_boxes[:, 2] x_c, y_c, w, h, theta = _fast_rcnn_decode_boxes_rotate[:, 1], _fast_rcnn_decode_boxes_rotate[:, 0], _fast_rcnn_decode_boxes_rotate[:, 3], \ _fast_rcnn_decode_boxes_rotate[:, 2], _fast_rcnn_decode_boxes_rotate[:, 4] boxes_h = np.transpose(np.stack([xmin, ymin, xmax, ymax])) boxes_r = np.transpose(np.stack([x_c, y_c, w, h, theta])) dets_h = np.hstack((_detection_category.reshape(-1, 1), _fast_rcnn_score.reshape(-1, 1), boxes_h)) dets_r = np.hstack( (_detection_category_rotate.reshape(-1, 1), _fast_rcnn_score_rotate.reshape(-1, 1), boxes_r)) all_boxes_h.append(dets_h) all_boxes_r.append(dets_r) all_img_names.append(str(_img_name_batch[0])) gtboxes_horizontal_dict[str( _img_name_batch[0])].extend(gtbox_horizontal_list) gtboxes_rotate_dict[str( _img_name_batch[0])].extend(gtbox_rotate_list) print(str(_img_name_batch[0])) view_bar( '{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) write_voc_results_file(all_boxes_h, all_img_names, cfgs.EVALUATE_R_DIR, 0) write_voc_results_file(all_boxes_r, all_img_names, cfgs.EVALUATE_R_DIR, 1) fw1 = open('gtboxes_horizontal_dict.pkl', 'wb') fw2 = open('gtboxes_rotate_dict.pkl', 'wb') pickle.dump(gtboxes_horizontal_dict, fw1) pickle.dump(gtboxes_rotate_dict, fw2) fw1.close() fw2.close() coord.request_stop() coord.join(threads)
def eval_ship(img_num, mode): with tf.Graph().as_default(): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=None, is_training=False, share_head=False, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn1.FastRCNN( feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, gtboxes_and_label_minAreaRectangle= gtboxes_and_label_minAreaRectangle, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False, level=cfgs.LEVEL) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \ fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, num_of_objects_rotate, detection_category_rotate = \ fast_rcnn.fast_rcnn_predict() if mode == 0: fast_rcnn_decode_boxes_rotate = get_horizen_minAreaRectangle( fast_rcnn_decode_boxes_rotate, False) # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() with tf.Session() as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) gtboxes_horizontal_dict = {} predict_horizontal_dict = {} gtboxes_rotate_dict = {} predict_rotate_dict = {} for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \ _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \ _fast_rcnn_score_rotate, _detection_category_rotate \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, detection_category_rotate]) end = time.time() # gtboxes convert dict gtboxes_horizontal_dict[str(_img_name_batch[0])] = [] predict_horizontal_dict[str(_img_name_batch[0])] = [] gtboxes_rotate_dict[str(_img_name_batch[0])] = [] predict_rotate_dict[str(_img_name_batch[0])] = [] gtbox_horizontal_list, predict_horizontal_list = \ make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category) if mode == 0: gtbox_rotate_list, predict_rotate_list = \ make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes_rotate, _fast_rcnn_score_rotate, _detection_category_rotate) else: gtbox_rotate_list, predict_rotate_list = \ make_dict_packle(_gtboxes_and_label, _fast_rcnn_decode_boxes_rotate, _fast_rcnn_score_rotate, _detection_category_rotate) gtboxes_horizontal_dict[str( _img_name_batch[0])].extend(gtbox_horizontal_list) predict_horizontal_dict[str( _img_name_batch[0])].extend(predict_horizontal_list) gtboxes_rotate_dict[str( _img_name_batch[0])].extend(gtbox_rotate_list) predict_rotate_dict[str( _img_name_batch[0])].extend(predict_rotate_list) view_bar( '{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) fw1 = open('gtboxes_horizontal_dict.pkl', 'w') fw2 = open('predict_horizontal_dict.pkl', 'w') fw3 = open('gtboxes_rotate_dict.pkl', 'w') fw4 = open('predict_rotate_dict.pkl', 'w') pickle.dump(gtboxes_horizontal_dict, fw1) pickle.dump(predict_horizontal_dict, fw2) pickle.dump(gtboxes_rotate_dict, fw3) pickle.dump(predict_rotate_dict, fw4) fw1.close() fw2.close() fw3.close() fw4.close() coord.request_stop() coord.join(threads)
def test(img_num): with tf.Graph().as_default(): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.SHORT_SIDE_LEN, is_training=False) gtboxes_and_label = tf.py_func( back_forward_convert, inp=[tf.squeeze(gtboxes_and_label_batch, 0)], Tout=tf.float32) gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6]) gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle( gtboxes_and_label) gtboxes_and_label_minAreaRectangle = tf.reshape( gtboxes_and_label_minAreaRectangle, [-1, 5]) # *********************************************************************************************** # * share net * # *********************************************************************************************** _, share_net = get_network_byname(net_name=cfgs.NET_NAME, inputs=img_batch, num_classes=None, is_training=True, output_stride=None, global_pool=False, spatial_squeeze=False) # *********************************************************************************************** # * RPN * # *********************************************************************************************** rpn = build_rpn.RPN( net_name=cfgs.NET_NAME, inputs=img_batch, gtboxes_and_label=None, is_training=False, share_head=cfgs.SHARE_HEAD, share_net=share_net, stride=cfgs.STRIDE, anchor_ratios=cfgs.ANCHOR_RATIOS, anchor_scales=cfgs.ANCHOR_SCALES, scale_factors=cfgs.SCALE_FACTORS, base_anchor_size_list=cfgs. BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6 level=cfgs.LEVEL, top_k_nms=cfgs.RPN_TOP_K_NMS, rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD, max_proposals_num=cfgs.MAX_PROPOSAL_NUM, rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD, rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD, rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE, rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE, remove_outside_anchors=False, # whether remove anchors outside rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME]) # rpn predict proposals rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals( ) # rpn_score shape: [300, ] # *********************************************************************************************** # * Fast RCNN * # *********************************************************************************************** fast_rcnn = build_fast_rcnn1.FastRCNN( feature_pyramid=rpn.feature_pyramid, rpn_proposals_boxes=rpn_proposals_boxes, rpn_proposals_scores=rpn_proposals_scores, img_shape=tf.shape(img_batch), roi_size=cfgs.ROI_SIZE, roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE, scale_factors=cfgs.SCALE_FACTORS, gtboxes_and_label=None, gtboxes_and_label_minAreaRectangle= gtboxes_and_label_minAreaRectangle, fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD, fast_rcnn_maximum_boxes_per_img=100, fast_rcnn_nms_max_boxes_per_class=cfgs. FAST_RCNN_NMS_MAX_BOXES_PER_CLASS, show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD, # show detections which score >= 0.6 num_classes=cfgs.CLASS_NUM, fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE, fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE, fast_rcnn_positives_iou_threshold=cfgs. FAST_RCNN_IOU_POSITIVE_THRESHOLD, # iou>0.5 is positive, iou<0.5 is negative use_dropout=cfgs.USE_DROPOUT, weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME], is_training=False, level=cfgs.LEVEL) fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category, \ fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, num_of_objects_rotate, detection_category_rotate,_, _, _, _ = \ fast_rcnn.fast_rcnn_predict() # train init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = restore_model.get_restorer() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) TrueDetection = 0 FalseAlarm = 0 TotalObjects = 0 TotalDetections = 0 for i in range(img_num): start = time.time() _img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \ _fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category, _fast_rcnn_decode_boxes_rotate, \ _fast_rcnn_score_rotate, _detection_category_rotate \ = sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle, fast_rcnn_decode_boxes, fast_rcnn_score, detection_category, fast_rcnn_decode_boxes_rotate, fast_rcnn_score_rotate, detection_category_rotate]) end = time.time() _img_batch = np.squeeze(_img_batch, axis=0) _img_batch_fpn_horizonal = help_utils.draw_box_cv( _img_batch, boxes=_fast_rcnn_decode_boxes, labels=_detection_category, scores=_fast_rcnn_score) _img_batch_fpn_rotate = help_utils.draw_rotate_box_cv( _img_batch, boxes=_fast_rcnn_decode_boxes_rotate, labels=_detection_category_rotate, scores=_fast_rcnn_score_rotate) mkdir(cfgs.TEST_SAVE_PATH) cv2.imwrite( cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) + '_horizontal_fpn.jpg', _img_batch_fpn_horizonal) cv2.imwrite( cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) + '_rotate_fpn.jpg', _img_batch_fpn_rotate) temp_label_horizontal = np.reshape(_gtboxes_and_label[:, -1:], [ -1, ]).astype(np.int64) temp_label_rotate = np.reshape(_gtboxes_and_label[:, -1:], [ -1, ]).astype(np.int64) _img_batch_gt_horizontal = help_utils.draw_box_cv( _img_batch, boxes=_gtboxes_and_label_minAreaRectangle[:, :-1], labels=temp_label_horizontal, scores=None) _img_batch_gt_rotate = help_utils.draw_rotate_box_cv( _img_batch, boxes=_gtboxes_and_label[:, :-1], labels=temp_label_rotate, scores=None) cv2.imwrite( cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) + '_horizontal_gt.jpg', _img_batch_gt_horizontal) cv2.imwrite( cfgs.TEST_SAVE_PATH + str(_img_name_batch[0]) + '_rotate_gt.jpg', _img_batch_gt_rotate) Current_TrueDetection = 0 lock_list = [] for m in range(len(_gtboxes_and_label)): for n, box in enumerate(_fast_rcnn_decode_boxes_rotate): if n in lock_list: continue y_c, x_c, h, w, theta = box[0], box[1], box[2], box[ 3], box[4] r1 = ((x_c, y_c), (w, h), theta) gt_box = _gtboxes_and_label[m] gt_yc, gt_xc, gt_h, gt_w, gt_theta = gt_box[0], gt_box[ 1], gt_box[2], gt_box[3], gt_box[4] r2 = ((gt_xc, gt_yc), (gt_w, gt_h), gt_theta) int_pts = cv2.rotatedRectangleIntersection(r1, r2)[0] if int_pts > 0.5: Current_TrueDetection += 1 lock_list.append(n) TrueDetection += Current_TrueDetection if (len(_fast_rcnn_decode_boxes_rotate) > 0): FalseAlarm += len( _fast_rcnn_score_rotate) - Current_TrueDetection TotalDetections += len(_fast_rcnn_score_rotate) TotalObjects += len(_gtboxes_and_label) view_bar( '{} image cost {}s'.format(str(_img_name_batch[0]), (end - start)), i + 1, img_num) Pd = float(TrueDetection) / float(TotalDetections) Pf = float(FalseAlarm) / float(TotalDetections) F1 = 2 * float(Pd * (1 - Pf)) / float(Pd + (1 - Pf)) Op = float(TrueDetection) / float(TotalObjects) print("\nACC: " + str(Pd)) print("FalseAlarm: " + str(Pf)) print("F1: " + str(F1)) print("Op: " + str(TrueDetection) + "/" + str(TotalObjects) + "=" + str(Op)) coord.request_stop() coord.join(threads)
def train(): faster_rcnn = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=True) with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5]) biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY) # list as many types of layers as possible, even if they are not used now with slim.arg_scope([slim.conv2d, slim.conv2d_in_plane, \ slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, biases_initializer=tf.constant_initializer(0.0)): final_bbox, final_scores, final_category, loss_dict = faster_rcnn.build_whole_detection_network( input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label) # ----------------------------------------------------------------------------------------------------build loss weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses()) rpn_location_loss = loss_dict['rpn_loc_loss'] rpn_cls_loss = loss_dict['rpn_cls_loss'] rpn_total_loss = rpn_location_loss + rpn_cls_loss fastrcnn_cls_loss = loss_dict['fastrcnn_cls_loss'] fastrcnn_loc_loss = loss_dict['fastrcnn_loc_loss'] fastrcnn_total_loss = fastrcnn_cls_loss + fastrcnn_loc_loss total_loss = rpn_total_loss + fastrcnn_total_loss + weight_decay_loss # ____________________________________________________________________________________________________build loss # ---------------------------------------------------------------------------------------------------add summary tf.summary.scalar('RPN_LOSS/cls_loss', rpn_cls_loss) tf.summary.scalar('RPN_LOSS/location_loss', rpn_location_loss) tf.summary.scalar('RPN_LOSS/rpn_total_loss', rpn_total_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_cls_loss', fastrcnn_cls_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_location_loss', fastrcnn_loc_loss) tf.summary.scalar('FAST_LOSS/fastrcnn_total_loss', fastrcnn_total_loss) tf.summary.scalar('LOSS/total_loss', total_loss) tf.summary.scalar('LOSS/regular_weights', weight_decay_loss) gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories( img_batch=img_batch, boxes=gtboxes_and_label[:, :-1], labels=gtboxes_and_label[:, -1]) if cfgs.ADD_BOX_IN_TENSORBOARD: detections_in_img = show_box_in_tensor.draw_boxes_with_categories_and_scores( img_batch=img_batch, boxes=final_bbox, labels=final_category, scores=final_scores) tf.summary.image('Compare/final_detection', detections_in_img) tf.summary.image('Compare/gtboxes', gtboxes_in_img) # ___________________________________________________________________________________________________add summary global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[ np.int64(cfgs.DECAY_STEP[0]), np.int64(cfgs.DECAY_STEP[1]) ], values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) # optimizer = tf.train.AdamOptimizer(lr) # ---------------------------------------------------------------------------------------------compute gradients gradients = faster_rcnn.get_gradients(optimizer, total_loss) # enlarge_gradients for bias if cfgs.MUTILPY_BIAS_GRADIENT: gradients = faster_rcnn.enlarge_gradients_for_bias(gradients) if cfgs.GRADIENT_CLIPPING_BY_NORM: with tf.name_scope('clip_gradients_YJR'): gradients = slim.learning.clip_gradient_norms( gradients, cfgs.GRADIENT_CLIPPING_BY_NORM) # _____________________________________________________________________________________________compute gradients # train_op train_op = optimizer.apply_gradients(grads_and_vars=gradients, global_step=global_step) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = faster_rcnn.get_restorer() saver = tf.train.Saver(max_to_keep=30) # Create session #allow_soft_placement=True自动将无法放到GPU上的操作放回到CPU config = tf.ConfigProto(allow_soft_placement=True) #让GPU按需分配,不一定占用某个GPU的全部内存 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) #日志保存地址 summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) tools.mkdir(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) timer = Timer() for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0: _, global_stepnp = sess.run([train_op, global_step]) else: if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0: start = time.time() _, global_stepnp, img_name, rpnLocLoss, rpnClsLoss, rpnTotalLoss, \ fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss, totalLoss = \ sess.run( [train_op, global_step, img_name_batch, rpn_location_loss, rpn_cls_loss, rpn_total_loss, fastrcnn_loc_loss, fastrcnn_cls_loss, fastrcnn_total_loss, total_loss]) end = time.time() print('{}: step:{}/{}iter'.format(training_time, global_stepnp, cfgs.MAX_ITERATION)) print('>>> rpn_loc_loss:{:.4f} | rpn_cla_loss:{:.4f} | rpn_total_loss:{:.4f}\n' '>>> fast_rcnn_loc_loss:{:.4f} | fast_rcnn_cla_loss:{:.4f} | fast_rcnn_total_loss:{:.4f}\n' '>>> single_time:{:.4f}s | total_loss:{:.4f} '\ .format(rpnLocLoss, rpnClsLoss,rpnTotalLoss, fastrcnnLocLoss, fastrcnnClsLoss, fastrcnnTotalLoss,(end-start), totalLoss)) print( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' ) else: if step % cfgs.SMRY_ITER == 0: _, global_stepnp, summary_str = sess.run( [train_op, global_step, summary_op]) summary_writer.add_summary(summary_str, global_stepnp) summary_writer.flush() if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.mkdir(save_dir) save_ckpt = os.path.join( save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)
def train(): faster_rcnn = build_whole_network.DetectionNetwork( base_network_name=cfgs.NET_NAME, is_training=True) with tf.name_scope('get_batch'): img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \ next_batch(dataset_name=cfgs.DATASET_NAME, # 'pascal', 'coco' batch_size=cfgs.BATCH_SIZE, shortside_len=cfgs.IMG_SHORT_SIDE_LEN, is_training=True) gtboxes_and_label = tf.reshape(gtboxes_and_label_batch, [-1, 5]) biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY) # list as many types of layers as possible, even if they are not used now with slim.arg_scope([ slim.conv2d, slim.conv2d_in_plane, slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected ], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, biases_initializer=tf.constant_initializer(0.0)): # result_dict, losses_dict = faster_rcnn.build_whole_detection_network(input_img_batch=img_batch, # gtboxes_batch=gtboxes_and_label) result_dict, losses_dict = faster_rcnn.build_whole_detection_network( input_img_batch=img_batch, gtboxes_batch=gtboxes_and_label) # ----------------------------------------------------------------------------------------------------build loss weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses()) # weight_decay_loss = tf.add_n(tf.losses.get_regularization_losses()) bbox_loss_m1 = losses_dict['bbox_loss_m1'] cls_loss_m1 = losses_dict['cls_loss_m1'] total_loss_m1 = bbox_loss_m1 + cls_loss_m1 bbox_loss_m2 = losses_dict['bbox_loss_m2'] cls_loss_m2 = losses_dict['cls_loss_m2'] total_loss_m2 = bbox_loss_m2 + cls_loss_m2 bbox_loss_m3 = losses_dict['bbox_loss_m3'] cls_loss_m3 = losses_dict['cls_loss_m3'] total_loss_m3 = bbox_loss_m3 + cls_loss_m3 total_loss = total_loss_m1 + total_loss_m2 + total_loss_m3 + weight_decay_loss # ---------------------------------------------------------------------------------------------------add summary tf.summary.scalar('SSH_M1_LOSS/cls_loss_m1', cls_loss_m1) tf.summary.scalar('SSH_M1_LOSS/bbox_loss_m1', bbox_loss_m1) tf.summary.scalar('SSH_M1_LOSS/total_loss_m1', total_loss_m1) tf.summary.scalar('SSH_M2_LOSS/cls_loss_m2', cls_loss_m2) tf.summary.scalar('SSH_M2_LOSS/bbox_loss_m2', bbox_loss_m2) tf.summary.scalar('SSH_M2_LOSS/total_loss_m2', total_loss_m2) tf.summary.scalar('SSH_M3_LOSS/cls_loss_m3', cls_loss_m3) tf.summary.scalar('SSH_M3_LOSS/bbox_loss_m3', bbox_loss_m3) tf.summary.scalar('SSH_M3_LOSS/total_loss_m3', total_loss_m3) tf.summary.scalar('LOSS/total_loss', total_loss) tf.summary.scalar('LOSS/regular_weights', weight_decay_loss) gtboxes_in_img = show_box_in_tensor.draw_boxes_with_categories( img_batch=img_batch, boxes=gtboxes_and_label[:, :-1], labels=gtboxes_and_label[:, -1]) if cfgs.ADD_BOX_IN_TENSORBOARD: detections_in_img_m1 = \ show_box_in_tensor.draw_boxes_with_categories_and_scores(img_batch=img_batch, boxes=result_dict['final_bbox_m1'], labels=result_dict['final_category_m1'], scores=result_dict['final_scores_m1']) tf.summary.image('Compare/final_detection_m1', detections_in_img_m1) detections_in_img_m2 = \ show_box_in_tensor.draw_boxes_with_categories_and_scores(img_batch=img_batch, boxes=result_dict['final_bbox_m2'], labels=result_dict['final_category_m2'], scores=result_dict['final_scores_m2']) tf.summary.image('Compare/final_detection_m2', detections_in_img_m2) detections_in_img_m3 = \ show_box_in_tensor.draw_boxes_with_categories_and_scores(img_batch=img_batch, boxes=result_dict['final_bbox_m3'], labels=result_dict['final_category_m3'], scores=result_dict['final_scores_m3']) tf.summary.image('Compare/final_detection_m3', detections_in_img_m3) tf.summary.image('Compare/gtboxes', gtboxes_in_img) global_step = slim.get_or_create_global_step() lr = tf.train.piecewise_constant( global_step, boundaries=[ np.int64(cfgs.DECAY_STEP[0]), np.int64(cfgs.DECAY_STEP[1]) ], values=[cfgs.LR, cfgs.LR / 10., cfgs.LR / 100.]) tf.summary.scalar('lr', lr) optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM) # ---------------------------------------------------------------------------------------------compute gradients gradients = faster_rcnn.get_gradients(optimizer, total_loss) # enlarge_gradients for bias if cfgs.MUTILPY_BIAS_GRADIENT: gradients = faster_rcnn.enlarge_gradients_for_bias(gradients) if cfgs.GRADIENT_CLIPPING_BY_NORM: with tf.name_scope('clip_gradients'): gradients = slim.learning.clip_gradient_norms( gradients, cfgs.GRADIENT_CLIPPING_BY_NORM) # train_op train_op = optimizer.apply_gradients(grads_and_vars=gradients, global_step=global_step) summary_op = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) restorer, restore_ckpt = faster_rcnn.get_restorer() saver = tf.train.Saver(max_to_keep=30) config = tf.ConfigProto() # config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) if not restorer is None: restorer.restore(sess, restore_ckpt) print('restore model') coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION) if not os.path.exists(summary_path): os.makedirs(summary_path) summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph) for step in range(cfgs.MAX_ITERATION): training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) # start = time.time() # _, global_stepnp, img_name, totalLoss, summary_str = \ # sess.run( # [train_op, global_step, img_name_batch, total_loss, summary_op]) # # end = time.time() # # print(""" {}: step{} image_name:{} |\t total_loss:{} |\t per_cost_time:{}s""" \ # .format(training_time, global_stepnp, str(img_name[0]), totalLoss, # (end - start))) # summary_writer.add_summary(summary_str, global_stepnp) # summary_writer.flush() if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0: _, global_stepnp = sess.run([train_op, global_step]) else: if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0: start = time.time() _, global_stepnp, img_name, totalLoss = \ sess.run( [train_op, global_step, img_name_batch, total_loss]) end = time.time() print(""" {}: step{} image_name:{} |\t total_loss:{} |\t per_cost_time:{}s""" \ .format(training_time, global_stepnp, str(img_name[0]), totalLoss, (end - start))) else: if step % cfgs.SMRY_ITER == 0: _, global_stepnp, summary_str = sess.run( [train_op, global_step, summary_op]) summary_writer.add_summary(summary_str, global_stepnp) summary_writer.flush() if (step > 0 and step % cfgs.SAVE_WEIGHTS_INTE == 0) or (step == cfgs.MAX_ITERATION - 1): save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION) if not os.path.exists(save_dir): os.makedirs(save_dir) save_ckpt = os.path.join( save_dir, 'voc_' + str(global_stepnp) + 'model.ckpt') saver.save(sess, save_ckpt) print(' weights had been saved') coord.request_stop() coord.join(threads)