random.shuffle(all_imgs) num_imgs = len(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) from keras_frcnn import data_generators from keras import backend as K data_gen_train = data_generators.get_anchor_gt(train_imgs, class_mapping, classes_count, C, K.image_dim_ordering(), mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, class_mapping, classes_count, C, K.image_dim_ordering(), mode='val') from keras_frcnn import resnet as nn from keras.optimizers import Adam, SGD from keras.layers import Input from keras.models import Model from keras.callbacks import EarlyStopping, ModelCheckpoint from keras_frcnn import losses
## C : 训练配置 ## nn.get_img_output_length: return width//16, height//16 表示使用vgg16最后会缩小到原来的16倍 ## mode = ‘train' 表示训练集,’val‘同理 ## 理论返回四个值,但是这里只有一个,是这四个值的一个元组(x_img, y_rpn_cls, y_rpn_reg, img_data_aug) ## 分别是(x_img: 图片, y_rpn_cls: 包含y_rpn_valid, y_rpn_overlap 前者表示该像素位的某种格式的anchor是否有效,后者表示是pos还是neg ) ## y_rpn_reg包含:y_rpn_overlap 和 原本的y_rpn_reg 后者表示若有超过阈值的anchor的 四个loss里用到的值 ## img_data_aug 表示图片的一系列相关信息) #data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.get_img_output_length, mode = 'train') #data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.get_img_output_length, mode = 'val') import itertools data_gen_train = itertools.cycle( data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.get_img_output_length, mode='train')) data_gen_val = itertools.cycle( data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.get_img_output_length, mode='val')) ######################################## 4.1 ########################################################################### ## input for nn input_shape_img = (None, None, 3) ## Input用来初始化一个keras tensor img_input = Input(shape=input_shape_img)
# generate Config file for test phase config_output_filename = options.config_filename with open(config_output_filename, 'wb') as config_f: pickle.dump(C,config_f) print('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_output_filename)) # shuffle data randomlyl, and split them into two parts: train, val random.shuffle(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) # generate positive and negative rpn anchors for groungtruth bbox # K.image_dim_ordering() is to get the backend name string ('tf' for tensorflow) data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, K.image_dim_ordering(), mode='train') #data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, K.image_dim_ordering(), mode='val') # set input format according to the backend if K.image_dim_ordering() == 'th': input_shape_img = (3, None, None) else: # tensorflow: (W,H,C) input_shape_img = (None, None, 3) # set up two inputs img_input = Input(shape=input_shape_img) roi_input = Input(shape=(C.num_rois, 4)) # define the base network - resnet shared_layers = nn.nn_base(img_input, trainable=True)
def train_kitti(): # config for data argument cfg = config.Config() cfg.balanced_classes = True cfg.use_horizontal_flips = True cfg.use_vertical_flips = True cfg.rot_90 = True cfg.num_rois = 50 # 对于星图杯的光学遥感飞机检测,应该改为50+ cfg.anchor_box_scales = [41, 70, 120, 20, 90] cfg.anchor_box_ratios = [[1, 1.4], [1, 0.84], [1, 1.17], [1, 0.64], [1, 1]] cfg.base_net_weights = os.path.join('./model/', nn.get_weight_path()) # TODO: the only file should to be change for other data to train cfg.model_path = './model/kitti_frcnn_last.hdf5' cfg.simple_label_file = 'E:/Xingtubei/official_datas/OpticalAircraft/laptop_Chreoc_OpticalAircraft_bboxes.txt' # '/media/liuhuaqing/Elements/Xingtubei/official_datas/OpticalAircraft/Chreoc_OpticalAircraft_bboxes.txt'#'F:/Xingtubei/official_datas/OpticalAircraft/Chreoc_OpticalAircraft_bboxes.txt' # 'kitti_simple_label.txt' all_images, classes_count, class_mapping = get_data( cfg.simple_label_file) #读取数据集,cv2.imread()要求数据里不能有中文路径 if 'bg' not in classes_count: #'bg'应该是代表背景 classes_count['bg'] = 0 # =0表示训练数据中没有“背景”这一类别 class_mapping['bg'] = len(class_mapping) cfg.class_mapping = class_mapping with open(cfg.config_save_file, 'wb') as config_f: pickle.dump(cfg, config_f) print( 'Config has been written to {}, and can be loaded when testing to ensure correct results' .format(cfg.config_save_file)) inv_map = {v: k for k, v in class_mapping.items()} #class_mapping的逆向map print('Training images per class:') pprint.pprint(classes_count) print('Num classes (including bg) = {}'.format(len(classes_count))) random.shuffle(all_images) num_imgs = len(all_images) train_imgs = [s for s in all_images if s['imageset'] == 'trainval'] #训练集,列表形式,列表中的元素是字典 val_imgs = [s for s in all_images if s['imageset'] == 'test'] #验证集,列表形式,列表中的元素是字典 print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) data_gen_train = data_generators.get_anchor_gt( train_imgs, classes_count, cfg, nn.get_img_output_length, K.image_dim_ordering(), mode='train') #数据扩增,然后生成frcnn所需的训练数据(如:图片、rpn的梯度等等) data_gen_val = data_generators.get_anchor_gt( val_imgs, classes_count, cfg, nn.get_img_output_length, K.image_dim_ordering(), mode='val') #数据扩增,然后生成frcnn所需的验证数据(如:图片、rpn的梯度等等) # 根据keras实际用的后端,定义相应的输入数据维度,因为两类后端的维度顺序不一样 if K.image_dim_ordering() == 'th': input_shape_img = (3, None, None) #当后端是thaneo else: input_shape_img = (None, None, 3) #当后端是tensorflow img_input = Input(shape=input_shape_img) # 输入图片 roi_input = Input(shape=(None, 4)) # 输入人工标注的roi坐标,4表示x1,y1,x2,y2 # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn.nn_base( img_input, trainable=True) # shared_layers是frcnn网络底部那些共享的层,在这里是ResNet。由nn定义好 # define the RPN, built on the base layers num_anchors = len(cfg.anchor_box_scales) * len(cfg.anchor_box_ratios) rpn = nn.rpn(shared_layers, num_anchors) classifier = nn.classifier(shared_layers, roi_input, cfg.num_rois, nb_classes=len(classes_count), trainable=True) model_rpn = Model( img_input, rpn[:2]) #rpn网络由keras_frcnn/resnet定义好。rpn[:2]的前两个元素分别表示rpn网络的分类输出和回归输出 model_classifier = Model([img_input, roi_input], classifier) #Keras的函数式模型为Model,即广义的拥有输入和输出的模型 # this is a model that holds both the RPN and the classifier, used to load/save weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) #rpn[:2]+classifier的含义是?????? try: # 尝试载入与训练网络权值 print('loading weights from {}'.format(cfg.base_net_weights)) model_rpn.load_weights(cfg.model_path, by_name=True) model_classifier.load_weights(cfg.model_path, by_name=True) except Exception as e: print(e) print( 'Could not load pretrained model weights. Weights can be found in the keras application folder ' 'https://github.com/fchollet/keras/tree/master/keras/applications') optimizer = Adam(lr=1e-5) # 定义一个Adam求解器,学习率lr optimizer_classifier = Adam(lr=1e-5) # 定义一个Adam求解器,学习率lr # num_anchors等于9 model_rpn.compile(optimizer=optimizer, loss=[ losses_fn.rpn_loss_cls(num_anchors), losses_fn.rpn_loss_regr(num_anchors) ]) model_classifier.compile( optimizer=optimizer_classifier, loss=[ losses_fn.class_loss_cls, losses_fn.class_loss_regr(len(classes_count) - 1) ], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mae') epoch_length = 100 # 每迭代epoch_length次就检查一次是否要保存网络权值,然后重置iter_num = 0 num_epochs = int(cfg.num_epochs) iter_num = 0 # 迭代次数的初值 losses = np.zeros((epoch_length, 5)) # 初始化loss数组,记录每个周期的loss rpn_accuracy_rpn_monitor = [] # 初始化一个数组,记录rpn的训练过程中的精度变化 rpn_accuracy_for_epoch = [] # 初始化一个数组,记录rpn的每个训练周期的的精度变化 start_time = time.time() # 开始训练的时间 best_loss = np.Inf # 改变量纪律训练以来最小的loss class_mapping_inv = {v: k for k, v in class_mapping.items() } # class_mapping_inv是一个字典,key是目标类别编号,value是类别名称 print('Starting training') vis = True for epoch_num in range(num_epochs): progbar = generic_utils.Progbar(epoch_length) # 生成一个进度条对象 print('Epoch {}/{}'.format(epoch_num + 1, num_epochs)) # 输出当前训练周期数/总周期数 while True: # 什么时候才结束这个循环?答:第247行的break(每迭代epoch_length次) try: if len( rpn_accuracy_rpn_monitor ) == epoch_length and cfg.verbose: # 每epoch_length次训练周期就在窗口显示一次RPN平均精度 mean_overlapping_bboxes = float( sum(rpn_accuracy_rpn_monitor)) / len( rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( 'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations' .format(mean_overlapping_bboxes, epoch_length)) if mean_overlapping_bboxes == 0: print( 'RPN is not producing bounding boxes that overlap' ' the ground truth boxes. Check RPN settings or keep training.' ) #X应该是图像,如kitti尺寸是(1,600,1987,3)。Y是label,img_data是字典,包含文件名、尺寸、人工标记的roi和类别等 X, Y, img_data = next(data_gen_train) Y_1 = Y[0] Y_1 = Y_1[0, :, :, :] loss_rpn = model_rpn.train_on_batch( X, Y) #为什么Y的尺寸与P_rpn的尺寸不同?为什么loss_rpn的尺寸是3,含义是什么,在哪里定义的? P_rpn = model_rpn.predict_on_batch( X) #P_rpn的尺寸是(1, 124, 38, 9) (1, 124, 38, 36) result = roi_helpers.rpn_to_roi( P_rpn[0], P_rpn[1], cfg, K.image_dim_ordering(), use_regr=True, overlap_thresh=0.7, max_boxes=300) #result的尺寸是300*4 # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format # X2的尺寸是100*4,Y1的尺寸是1*100*8(8=训练集中目标类别总数),IouS尺寸是100 X2, Y1, Y2, IouS = roi_helpers.calc_iou( result, img_data, cfg, class_mapping ) #Y2的尺寸是1*1*56,56=28*2,(28=4*7)前28是coords,后28是labels(是该类别则标1) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where( Y1[0, :, -1] == 1) #Y1的尺寸是1*1*8表示分类预测结果,最后一个元素为1表示是背景 pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if cfg.num_rois > 1: if len(pos_samples) < cfg.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, cfg.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, cfg.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, cfg.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) loss_class = model_classifier.train_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :] ]) #用rpn输出的roi输入给classifier losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] iter_num += 1 progbar.update( iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])), ('detector_cls', np.mean(losses[:iter_num, 2])), ('detector_regr', np.mean(losses[:iter_num, 3]))]) if iter_num == epoch_length: # 每迭代epoch_length次就检查一次是否要保存网络权值,然后重置iter_num = 0 loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float(sum( rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if cfg.verbose: print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print( 'Classifier accuracy for bounding boxes from RPN: {}' .format(class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format( loss_class_cls)) print('Loss Detector regression: {}'.format( loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr iter_num = 0 start_time = time.time() if curr_loss < best_loss: if cfg.verbose: print( 'Total loss decreased from {} to {}, saving weights' .format(best_loss, curr_loss)) best_loss = curr_loss model_all.save_weights(cfg.model_path) break except Exception as e: print('Exception: {}'.format(e)) # save model model_all.save_weights(cfg.model_path) continue print('Training complete, exiting.')
f'Config has been written to {config_output_filename}, and can be loaded when testing to ensure correct results' ) random.shuffle(train_imgs) num_imgs = len(train_imgs) # train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] # val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print(f'Num train samples {len(train_imgs)}') # print(f'Num val samples {len(val_imgs)}') data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.get_img_output_length, K.image_data_format(), mode='train') # data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.get_img_output_length,K.common.image_dim_ordering(), mode='val') if K.image_data_format() == 'channels_first': input_shape_img = (3, None, None) else: input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) #img_input = tf.TensorShape(img_input) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn.nn_base(img_input, trainable=True)
def train(): # config for data argument cfg = config.Config() cfg.use_horizontal_flips = True cfg.use_vertical_flips = True cfg.rot_90 = True cfg.num_rois = 32 cfg.base_net_weights = os.path.join('./model/', nn.get_weight_path()) # TODO: the only file should to be change for other data to train cfg.model_path = './model/kitti_frcnn_last.hdf5' cfg.simple_label_file = 'kitti_simple_label.txt' all_images, classes_count, class_mapping = get_data(cfg.simple_label_file) if 'bg' not in classes_count: classes_count['bg'] = 0 class_mapping['bg'] = len(class_mapping) cfg.class_mapping = class_mapping with open(cfg.config_save_file, 'wb') as config_f: pickle.dump(cfg, config_f) print('Config has been written to {}, and can be loaded when testing to ensure correct results'.format( cfg.config_save_file)) inv_map = {v: k for k, v in class_mapping.items()} print('Training images per class:') pprint.pprint(classes_count) print('Num classes (including bg) = {}'.format(len(classes_count))) random.shuffle(all_images) num_imgs = len(all_images) train_imgs = [s for s in all_images if s['imageset'] == 'trainval'] val_imgs = [s for s in all_images if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, cfg, nn.get_img_output_length, K.image_dim_ordering(), mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, cfg, nn.get_img_output_length, K.image_dim_ordering(), mode='val') if K.image_dim_ordering() == 'th': input_shape_img = (3, None, None) else: input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn.nn_base(img_input, trainable=True) # define the RPN, built on the base layers num_anchors = len(cfg.anchor_box_scales) * len(cfg.anchor_box_ratios) rpn = nn.rpn(shared_layers, num_anchors) classifier = nn.classifier(shared_layers, roi_input, cfg.num_rois, nb_classes=len(classes_count), trainable=True) model_rpn = Model(img_input, rpn[:2]) #print(model_rpn.summary()) model_classifier = Model([img_input, roi_input], classifier) # this is a model that holds both the RPN and the classifier, used to load/save weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) try: print('loading weights from {}'.format(cfg.base_net_weights)) # first tryain should do that ?????? model_rpn.load_weights(cfg.base_net_weights, by_name=True) model_classifier.load_weights(cfg.base_net_weights, by_name=True) except Exception as e: print(e) print('Could not load pretrained model weights. Weights can be found in the keras application folder ' 'https://github.com/fchollet/keras/tree/master/keras/applications') # optimizer = Adam(lr=3e-5) # optimizer_classifier = Adam(lr=3e-5) # optimizer = Adam(lr=3e-4) # optimizer_classifier = Adam(lr=3e-4) # optimizer = Adam(lr=5e-3) # optimizer_classifier = Adam(lr=5e-3) optimizer = Adam(lr=1e-5) optimizer_classifier = Adam(lr=1e-5) model_rpn.compile(optimizer=optimizer, loss=[losses_fn.rpn_loss_cls(num_anchors), losses_fn.rpn_loss_regr(num_anchors)]) model_classifier.compile(optimizer=optimizer_classifier, loss=[losses_fn.class_loss_cls, losses_fn.class_loss_regr(len(classes_count) - 1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mae') epoch_length = 1000 num_epochs = int(cfg.num_epochs) iter_num = 0 losses = np.zeros((epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] start_time = time.time() best_loss = np.Inf class_mapping_inv = {v: k for k, v in class_mapping.items()} print('Starting training') vis = True for epoch_num in range(num_epochs): progbar = generic_utils.Progbar(epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, num_epochs)) while True: try: if len(rpn_accuracy_rpn_monitor) == epoch_length and cfg.verbose: mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor)) / len(rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( 'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'.format( mean_overlapping_bboxes, epoch_length)) if mean_overlapping_bboxes == 0: print('RPN is not producing bounding boxes that overlap' ' the ground truth boxes. Check RPN settings or keep training.') X, Y, img_data = next(data_gen_train) loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) result = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], cfg, K.image_dim_ordering(), use_regr=True, overlap_thresh=0.7, max_boxes=300) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = roi_helpers.calc_iou(result, img_data, cfg, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if cfg.num_rois > 1: if len(pos_samples) < cfg.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice(pos_samples, cfg.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice(neg_samples, cfg.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice(neg_samples, cfg.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) loss_class = model_classifier.train_on_batch([X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] iter_num += 1 progbar.update(iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])), ('detector_cls', np.mean(losses[:iter_num, 2])), ('detector_regr', np.mean(losses[:iter_num, 3]))]) if iter_num == epoch_length: loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if cfg.verbose: print('Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format( mean_overlapping_bboxes)) print('Classifier accuracy for bounding boxes from RPN: {}'.format(class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format(loss_class_cls)) print('Loss Detector regression: {}'.format(loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr iter_num = 0 start_time = time.time() if curr_loss < best_loss: if cfg.verbose: print('Total loss decreased from {} to {}, saving weights'.format(best_loss, curr_loss)) with open('res.txt','w') as ress: ress.write('epoch {} Total loss decreased from {} to {}, saving weights'.format(epoch_num + 1,best_loss, curr_loss)) best_loss = curr_loss model_all.save_weights(cfg.model_path) break except Exception as e: print('Exception: {}'.format(e)) # save model why???? # model_all.(csave_weightsfg.model_path) continue print('Training complete, exiting.')
) random.shuffle(train_imgs) num_imgs = len(train_imgs) #train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] #val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print(f'Num train samples {len(train_imgs)}') print(f'Num val samples {len(val_imgs)}') data_gen_train = data_generators.get_anchor_gt( train_imgs, classes_count, C, nn.get_img_output_length, keras.backend.image_data_format(), mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.get_img_output_length, keras.backend.image_data_format(), mode='val') if keras.backend.image_data_format() == 'th': input_shape_img = (3, None, None) else: input_shape_img = (None, None, 3)
def Train_frcnn(train_path = './data/FlickrLogos_47/train/', # path to the text file containing the data class_name = './data/FlickrLogos_47/className2ClassID.txt', network_arch = 'vgg', # the type of the base faster rcnn network architecture num_epochs = 50, # num of epochs output_weight_path = './models/model_frcnn_47.hdf5', # path to save the model_all.weights as hdf5 preprocessing_function = None, config_filename="config_47.pickle", input_weights_path = './models/vgg16_weights_tf_dim_ordering_tf_kernels.h5', train_rpn = True, train_final_classifier = True, train_base_nn = True, losses_to_watch = ['rpn_cls','rpn_reg','final_cls','final_reg'], tb_log_dir="log", num_rois=32, horizontal_flips=False, vertical_flips=False, rot_90=False, anchor_box_scales=[128, 256, 512], anchor_box_ratios=[[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]], im_size=600, rpn_stride=16, # depends on network architecture visualize_model = None, verify_trainable = True, optimizer_rpn = Adam(lr=1e-5), optimizer_classifier = Adam(lr=1e-5), validation_interval = 3, rpn_min_overlap = 0.3, rpn_max_overlap = 0.7, classifier_min_overlap = 0.1, classifier_max_overlap = 0.5, rpn_nms_threshold = 0.7, # original implementation seed=5000 ): """ Trains a Faster RCNN for object detection in keras NOTE: This trains 2 models namely model_rpn and model_classifer with the same shared base_nn (fixed feature extractor) Keyword Arguments train_path -- str: path to the text file or pascal_voc (no Default) network_arch --object: the full faster rcnn network .py file passed as an object (no default) num_epochs -- int: number of epochs to train (no Default) output_weight_path --str: path to save the frcnn weights (no Default) preprocessing_function --function: Optional preprocessing function (must be defined like given in keras docs) (Default None) config_filename --str: Path to save the config file. Used when testing (Default "config.pickle") input_weight_path --str: Path to hdf5 file containing weights for the model (Default None) you can pass path to both classification and detection checkpoints as long as the names dont' change train_rpn --bool: whether to train the rpn layer (Default True) train_final_classifier --bool:Whether to train the final_classifier (Fast Rcnn layer) (Default True) train_base_nn --bool:Whether to train the base_nn/fixed_feature_extractor (Default True) losses_to_watch --list: A list of losses to watch (Default ['rpn_cls','rpn_reg','final_cls','final_reg']). The losses in this list are added and then weights are saved wrt to that. The list can contain any combination of the above 4 only. tb_log_dir --str: path to log dir for tensorboard logging (Default 'log') num_rois --int: The number of rois to use at once (Default = 32) horizontal_flips --bool: augment training data by horizontal flips (Default False) vertical_flips --bool: augment training data by vertical flips (Default False) rot_90 --bool: augment training data by 90 deg rotations (Default False) anchor_box_scales --list: The list of anchor box scales to use (Default [128,256,512]) anchor_box ratios --list of list: The list of anchorbox aspect ratios to use (Default [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]) im_size --int: The size to resize the image (Default 600). This is the smallest side of Pascal VOC format rpn_stride --int: The stride for rpn (Default = 16) visualize_model --str: Path to save the model as .png file verify_trainable --bool: print layer wise names and prints if it is trainable or not (Default True) optimizer_rpn --keras.optimizer: The optimizer for rpn (Default Adam(lr=1e-5)) optimizer_classifier --keras.optimizer: The optimizer for classifier (Default Adam(lr=1e-5)) validation_interval --int: The frequency (in epochs) to do validation. supply 0 if no validation rpn_min_overlap --float: (0,1) The Min IOU in rpn layer (Default 0.3) (original implementation) rpn_max_overlap --float: (0,1) The max IOU in rpn layer (Default 0.7) (original implementation) classifier_min_overlap --float: (0,1) same as above but in final classifier (Default 0.1) (original implementation) classifier_max_overlap --float: (0,1) same as above (Default 0.5) (original implementation) rpn_nms_threshold --float :(0,1) The threshold above which to supress the bbox using Non max supression in rpn (Default 0.7)(from original implementation) seed --int: To seed the random shuffling of training data (Default = 5000) Performing alternating training: - Use the train_rpn,train_final_classifier and train_base_nn boolean arguments to accomplish alternating training. - While using the above arguments change the members of losses_to_watch = ['rpn_cls','rpn_reg','final_cls','final_reg'] accordingly else it will throw error - for eg if you are training only the base_nn and the rpn set: train_rpn = True train_base_nn = True train_final_classifier = False losses_to_watch = ['rpn_cls','rpn_reg'] (do not include 'final_cls', 'final_reg') OUTPUT: prints the training log. Does not return anything Save details: 1.saves the weights of the full FRCNN model as .h5 2.saves a tensorboard file 3.saves the history of weights saved in ./saving_log.txt so that it can be known at which epoch the model is saved 4.saves the model configuration as a .pickle file 5.optionally saves the full FRCNN architecture as .png NOTE: as of now the batch size = 1 Prints loss = 0 for losses from model which is not being trained TODO: The training is a bit slow because of the data generation step. Generate_data in multiple threads and queue them for faster training """ check_list = ['rpn_cls','rpn_reg','final_cls','final_reg'] for n in losses_to_watch: if n not in check_list: raise ValueError("unsupported loss the supported losses are: {}".format(check_list)) if not train_rpn: if "rpn_cls" in losses_to_watch or "rpn_reg" in losses_to_watch: raise ValueError("Cannot watch rpn_cls and rpn_reg when train_rpn == False") if not train_final_classifier: if "final_cls" in losses_to_watch or "final_reg" in losses_to_watch: raise ValueError("cannot watch final_cls and final_reg when train_final_classifier == False") if network_arch == 'vgg': from keras_frcnn import nn_arch_vgg16 as nn elif network_arch == 'resnet50': from keras_frcnn import nn_arch_resnet50 as nn else: print('Not a valid model') raise ValueError random.seed(seed) np.random.seed(seed) # pass the settings from the function call, and persist them in the config object C = config.Config() C.rpn_max_overlap = rpn_max_overlap C.rpn_min_overlap = rpn_min_overlap C.classifier_min_overlap = classifier_min_overlap C.classifier_max_overlap = classifier_max_overlap C.anchor_box_scales = anchor_box_scales C.anchor_box_ratios = anchor_box_ratios C.im_size = im_size C.use_horizontal_flips = bool(horizontal_flips) C.use_vertical_flips = bool(vertical_flips) C.rot_90 = bool(rot_90) C.rpn_stride=rpn_stride C.rpn_nms_threshold = rpn_nms_threshold C.weights_all_path = output_weight_path C.num_rois = int(num_rois) # check if weight path was passed via command line if input_weights_path: C.initial_weights = input_weights_path all_imgs, classes_count, class_mapping = get_data(train_path, class_name) print("The class mapping is:") print(class_mapping) if 'bg' not in classes_count: classes_count['bg'] = 0 class_mapping['bg'] = len(class_mapping) C.class_mapping = class_mapping print('Training images per class:') pprint.pprint(classes_count) print('Num classes (including bg) = {}'.format(len(classes_count))) with open(config_filename, 'wb') as config_f: pickle.dump(C,config_f) print('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_filename)) np.random.shuffle(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'train'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn.nn_base(img_input,trainable = train_base_nn) # define the RPN, built on the base layers num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios) rpn = nn.rpn(shared_layers, num_anchors,trainable = train_rpn) # define the classifier, built on base layers classifier = nn.classifier(shared_layers, roi_input, C.num_rois, len(classes_count),trainable = train_final_classifier) # create models model_base = Model(img_input,shared_layers) # for computing the output shape model_rpn = Model(img_input, rpn[:2]) # used for training model_classifier = Model([img_input, roi_input], classifier) # used for training # this is a model that holds both the RPN and the classifier, used to load/save and freeze/unfreeze weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) # tensorboard tbCallBack = TensorBoard(log_dir=tb_log_dir, histogram_freq=1,write_graph=False, write_images=False) tbCallBack.set_model(model_all) #NOTE: both model_rpn and model_classifer contains the base_nn try: print('loading weights from {}'.format(C.initial_weights)) model_all.load_weights(C.initial_weights, by_name=True) except: print('Could not load pretrained model weights') # number of trainable parameters trainable_count = int(np.sum([K.count_params(p) for p in set(model_all.trainable_weights)])) non_trainable_count = int(np.sum([K.count_params(p) for p in set(model_all.non_trainable_weights)])) print('Total params: {:,}'.format(trainable_count + non_trainable_count)) print('Trainable params: {:,}'.format(trainable_count)) print('Non-trainable params: {:,}'.format(non_trainable_count)) if verify_trainable: for layer in model_all.layers: print(layer.name,layer.trainable) model_rpn.compile(optimizer=optimizer_rpn, loss=[Losses.rpn_loss_cls(num_anchors), Losses.rpn_loss_regr(num_anchors)]) model_classifier.compile(optimizer=optimizer_classifier, loss=[Losses.class_loss_cls, Losses.class_loss_regr(len(classes_count)-1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mse') # save model_all as png for visualization if visualize_model != None: # from IPython.display import SVG # from keras.utils.vis_utils import model_to_dot # SVG(model_to_dot(model_all).create(prog='dot', format='svg')) plot_model(model=model_all,to_file=visualize_model,show_shapes=True,show_layer_names=True) epoch_length = len(train_imgs) validation_epoch_length=len(val_imgs) num_epochs = int(num_epochs) iter_num = 0 # train and valid data generator data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, model_base, K.image_dim_ordering(), preprocessing_function ,mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, model_base,K.image_dim_ordering(), preprocessing_function ,mode='val') losses_val=np.zeros((validation_epoch_length,5)) losses = np.zeros((epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] start_time = time.time() best_loss = np.Inf val_best_loss = np.Inf val_best_loss_epoch = 0 print('Starting training') def write_log(callback, names, logs, batch_no): for name, value in zip(names, logs): summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = value summary_value.tag = name callback.writer.add_summary(summary, batch_no) callback.writer.flush() train_names = ['train_loss_rpn_cls', 'train_loss_rpn_reg','train_loss_class_cls','train_loss_class_reg','train_total_loss','train_acc'] val_names = ['val_loss_rpn_cls', 'val_loss_rpn_reg','val_loss_class_cls','val_loss_class_reg','val_total_loss','val_acc'] for epoch_num in range(num_epochs): progbar = generic_utils.Progbar(epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, num_epochs)) while True: try: if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose: mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor))/len(rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print('Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'.format(mean_overlapping_bboxes, epoch_length)) if mean_overlapping_bboxes == 0: print('RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.') X, Y, img_data = next(data_gen_train) if train_rpn: loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], C, K.image_dim_ordering(), use_regr=True, overlap_thresh=C.rpn_nms_threshold,flag="train") # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = roi_helpers.calc_iou(R, img_data, C, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois//2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice(pos_samples, C.num_rois//2, replace=False).tolist() try: selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) if train_final_classifier: loss_class = model_classifier.train_on_batch([X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) # losses if train_rpn: losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] else: losses[iter_num, 0] = 0 losses[iter_num, 1] = 0 if train_final_classifier: losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] # accuracy else: losses[iter_num, 2] = 0 losses[iter_num, 3] = 0 losses[iter_num, 4] = 0 iter_num += 1 progbar.update(iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])), ('detector_cls', np.mean(losses[:iter_num, 2])), ('detector_regr', np.mean(losses[:iter_num, 3]))]) if iter_num == epoch_length: if train_rpn: loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) else: loss_rpn_cls = 0 loss_rpn_regr = 0 if train_final_classifier: loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) else: loss_class_cls = 0 loss_class_regr = 0 class_acc = 0 mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if C.verbose: print('Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes)) print('Classifier accuracy for bounding boxes from RPN: {}'.format(class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format(loss_class_cls)) print('Loss Detector regression: {}'.format(loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) loss_dict_train = {"rpn_cls":loss_rpn_cls,"rpn_reg":loss_rpn_regr,"final_cls":loss_class_cls,"final_reg":loss_class_regr} curr_loss = 0 for l in losses_to_watch: curr_loss += loss_dict_train[l] iter_num = 0 start_time = time.time() write_log(tbCallBack, train_names, [loss_rpn_cls,loss_rpn_regr,loss_class_cls,loss_class_regr,curr_loss,class_acc], epoch_num) if curr_loss < best_loss: if C.verbose: print('Total loss decreased from {} to {} in training, saving weights'.format(best_loss,curr_loss)) save_log_data = '\nTotal loss decreased from {} to {} in epoch {}/{} in training, saving weights'.format(best_loss,curr_loss,epoch_num + 1,num_epochs) with open("./saving_log.txt","a") as f: f.write(save_log_data) best_loss = curr_loss model_all.save_weights(C.weights_all_path) break except Exception as e: print('Exception: {}'.format(e)) continue if validation_interval > 0: # validation if (epoch_num+1)%validation_interval==0 : progbar = generic_utils.Progbar(validation_epoch_length) print("Validation... \n") while True: try: X, Y, img_data = next(data_gen_val) if train_rpn: val_loss_rpn = model_rpn.test_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], C, K.image_dim_ordering(), use_regr=True, overlap_thresh=C.rpn_nms_threshold,flag="train") # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = roi_helpers.calc_iou(R, img_data, C, class_mapping) neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois//2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice(pos_samples, C.num_rois//2, replace=False).tolist() try: selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) if train_final_classifier: val_loss_class = model_classifier.test_on_batch([X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) if train_rpn: losses_val[iter_num, 0] = val_loss_rpn[1] losses_val[iter_num, 1] = val_loss_rpn[2] else: losses_val[iter_num, 0] = 0 losses_val[iter_num, 1] = 0 if train_final_classifier: losses_val[iter_num, 2] = val_loss_class[1] losses_val[iter_num, 3] = val_loss_class[2] losses_val[iter_num, 4] = val_loss_class[3] else: losses_val[iter_num, 2] = 0 losses_val[iter_num, 3] = 0 losses_val[iter_num, 4] = 0 iter_num += 1 progbar.update(iter_num, [('rpn_cls', np.mean(losses_val[:iter_num, 0])), ('rpn_regr', np.mean(losses_val[:iter_num, 1])), ('detector_cls', np.mean(losses_val[:iter_num, 2])), ('detector_regr', np.mean(losses_val[:iter_num, 3]))]) if iter_num == validation_epoch_length: if train_rpn: val_loss_rpn_cls = np.mean(losses_val[:, 0]) val_loss_rpn_regr = np.mean(losses_val[:, 1]) else: val_loss_rpn_cls = 0 val_loss_rpn_regr = 0 if train_final_classifier: val_loss_class_cls = np.mean(losses_val[:, 2]) val_loss_class_regr = np.mean(losses_val[:, 3]) val_class_acc = np.mean(losses_val[:, 4]) else: val_loss_class_cls = 0 val_loss_class_regr = 0 val_class_acc = 0 mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] loss_dict_valid = {"rpn_cls":val_loss_rpn_cls,"rpn_reg":val_loss_rpn_regr,"final_cls":val_loss_class_cls,"final_reg":val_loss_class_regr} val_curr_loss = 0 for l in losses_to_watch: val_curr_loss += loss_dict_valid[l] write_log(tbCallBack, val_names, [val_loss_rpn_cls,val_loss_rpn_regr,val_loss_class_cls,val_loss_class_regr,val_curr_loss,val_class_acc], epoch_num) if C.verbose: print('[INFO VALIDATION]') print('Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes)) print('Classifier accuracy for bounding boxes from RPN: {}'.format(val_class_acc)) print('Loss RPN classifier: {}'.format(val_loss_rpn_cls)) print('Loss RPN regression: {}'.format(val_loss_rpn_regr)) print('Loss Detector classifier: {}'.format(val_loss_class_cls)) print('Loss Detector regression: {}'.format(val_loss_class_regr)) print("current loss: %.2f, best loss: %.2f at epoch: %d"%(val_curr_loss,val_best_loss,val_best_loss_epoch)) print('Elapsed time: {}'.format(time.time() - start_time)) if val_curr_loss < val_best_loss: if C.verbose: print('Total loss decreased from {} to {}, saving weights'.format(val_best_loss,val_curr_loss)) save_log_data = '\nTotal loss decreased from {} to {} in epoch {}/{} in validation, saving weights'.format(val_best_loss,val_curr_loss,epoch_num + 1 ,num_epochs) with open("./saving_log.txt","a") as f: f.write(save_log_data) val_best_loss = val_curr_loss val_best_loss_epoch=epoch_num model_all.save_weights(C.weights_all_path) start_time = time.time() iter_num = 0 break except: pass print('Training complete, exiting.')
print('Num classes (including bg) = {}'.format(len(classes_count))) config_output_filename = options.config_filename with open(config_output_filename, 'wb') as config_f: pickle.dump(C, config_f) print('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_output_filename)) random.shuffle(all_imgs) num_imgs = len(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'train'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.img_length_calc_function, K.image_dim_ordering(), mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.img_length_calc_function,K.image_dim_ordering(), mode='val') input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (fcnet) shared_layers = nn.nn_base(img_input, trainable=C.trainable_base_layer) # define the RPN, built on the base layers, rpn = [x_class, x_regr, shared_layers] rpn = nn.rpn(shared_layers, num_anchors) # classifier = [out_class, out_regr] classifier = nn.classifier(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True) model_rpn = Model(img_input, rpn[:2]) model_classifier = Model([img_input, roi_input], classifier)
optimizer = Adam(lr=1e-3, clipnorm=0.001) model_rpn.compile(optimizer=optimizer, loss=[ loss_func.rpn_loss_cls(num_anchors), loss_func.rpn_loss_regr(num_anchors) ]) #### load images here #### from keras_frcnn.simple_parser import get_data all_imgs, classes_count, class_mapping = get_data(options.test_path, test_only=False) train_imgs = [s for s in all_imgs if s['imageset'] == 'train'] data_gen_val = data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.get_img_output_length, K.common.image_dim_ordering(), mode='val') with open('log/losses.csv', 'w') as log: log_writer = csv.writer(log, delimiter=';') log_writer.writerow(['loss', 'loss_rpn_cls', 'loss_rpn_regr', 'img']) for img in train_imgs: X_gen, Y_gen, imgdata = next(data_gen_val) loss = model_rpn.test_on_batch(X_gen, Y_gen) log_writer.writerow([loss[0], loss[1], loss[2], imgdata['filepath']]) print( f'Row written with loss {loss[0]} for image {imgdata["filepath"]}')
if os.path.exists(sorted_path): print("loading sorted data") with open(sorted_path) as f: trip_data = pickle.load(f) im_file = [] ind = [] for ii in range(360): for jj in range(3): try: im_file.append(trip_data[test_cls_NN][ii][jj]) ind.append(ii) except: if jj == 0: print('no azimuth {}'.format(ii)) data_gen_train = data_generators.get_anchor_gt(im_file, [], C, K.image_dim_ordering(), mode='test') azimuth_dict = [] inner_NN = [] for tt in range(len(ind)): try: if tt % 100 == 0: print('worked on {}/{}'.format(tt, len(ind))) # print ('im num {}'.format(good_img)) X, Y, img_data = next(data_gen_train) [Y1, Y2, F] = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(Y1, Y2, C,
def train_model(dataset_directory: str, model_name: str, delete_and_recreate_dataset_directory: bool, configuration_name: str, output_weight_path: str, configuration_filename: str, number_of_epochs: int, early_stopping: int, learning_rate_reduction_patience: int, learning_rate_reduction_factor: float, non_max_suppression_overlap_threshold: float, non_max_suppression_max_boxes: int, input_weight_path: str = None): muscima_pp_raw_dataset_directory = os.path.join(dataset_directory, "muscima_pp_raw") muscima_image_directory = os.path.join(dataset_directory, "cvcmuscima_staff_removal") muscima_cropped_directory = os.path.join(dataset_directory, "muscima_pp_cropped_images") if not dataset_directory: # if filename is not given parser.error( 'Error: path to training data must be specified. Pass --path to command line' ) network = NetworkFactory.get_network_by_name(model_name) try: training_images, validation_images, test_images, classes_count, class_mapping = \ get_data(dataset_directory, os.path.join(dataset_directory, "Annotations.txt")) data_loaded = True except: print( "Could not load dataset. Automatically downloading and recreating dataset." ) data_loaded = False delete_and_recreate_dataset_directory = True if delete_and_recreate_dataset_directory: print("Deleting dataset directory {0}".format(dataset_directory)) if os.path.exists(dataset_directory): shutil.rmtree(dataset_directory) downloader = MuscimaPlusPlusDatasetDownloader( muscima_pp_raw_dataset_directory) downloader.download_and_extract_dataset() downloader = CvcMuscimaDatasetDownloader( muscima_image_directory, CvcMuscimaDataset.StaffRemoval) downloader.download_and_extract_dataset() delete_unused_images(muscima_image_directory) inverter = ImageInverter() # We would like to work with black-on-white images instead of white-on-black images inverter.invert_images(muscima_image_directory, "*.png") shutil.copy("Staff-Vertical-Positions.txt", dataset_directory) cut_images( muscima_image_directory, os.path.join(dataset_directory, "Staff-Vertical-Positions.txt"), muscima_cropped_directory, muscima_pp_raw_dataset_directory, os.path.join(dataset_directory, "Annotations.txt"), os.path.join(dataset_directory, "Annotations")) dataset_splitter = DatasetSplitter(muscima_cropped_directory, dataset_directory) dataset_splitter.split_images_into_training_validation_and_test_set() # pass the settings from the command line, and persist them in the config object C = ConfigurationFactory.get_configuration_by_name(configuration_name) C.model_path = output_weight_path start_time = time.time() if not data_loaded: training_images, validation_images, test_images, classes_count, class_mapping = \ get_data(dataset_directory, os.path.join(dataset_directory, "Annotations.txt")) if 'bg' not in classes_count: classes_count['bg'] = 0 class_mapping['bg'] = len(class_mapping) C.class_mapping = class_mapping s = "" object_classes = list(class_mapping.keys()) object_classes.sort() i = 1 for object_class in object_classes: if object_class is "bg": continue s += "item {\n" s += " id: {0}\n".format(i) s += " name: '{0}'\n".format(object_class) s += "}\n\n" i += 1 with open("mapping.txt", "w") as mapping: mapping.write(s) # inv_map = {v: k for k, v in class_mapping.items()} print('Training images per class:') pprint.pprint(classes_count) print('Num classes (including bg) = {}'.format(len(classes_count))) print( 'Hyperparameters: {0} RoIs generated per run with {1} boxes remaining from non-max suppression and using ' 'non-max suppression threshold of {2:.2f}'.format( C.num_rois, non_max_suppression_max_boxes, non_max_suppression_overlap_threshold)) config_output_filename = configuration_filename with open(config_output_filename, 'wb') as config_f: pickle.dump(C, config_f) print( 'Config has been written to {}, and can be loaded when testing to ensure correct results' .format(config_output_filename)) random.seed(1) random.shuffle(training_images) print('Number of training samples: {}'.format(len(training_images))) print('Number of validation samples: {}'.format(len(validation_images))) print('Number of test samples: {}'.format(len(test_images))) if not use_fast_data_generators: print("Using standard data_generator") data_gen_train = data_generators.get_anchor_gt( training_images, classes_count, C, network.get_img_output_length, mode='train') data_gen_val = data_generators.get_anchor_gt( validation_images, classes_count, C, network.get_img_output_length, mode='val') else: print("Using fast data_generator") data_gen_train = data_generators_fast.get_anchor_gt( training_images, classes_count, C, network.get_img_output_length, mode='train') data_gen_val = data_generators_fast.get_anchor_gt( validation_images, classes_count, C, network.get_img_output_length, mode='val') input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = network.nn_base(img_input, trainable=True) # define the RPN, built on the base layers num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios) rpn = network.rpn(shared_layers, num_anchors) classifier = network.classifier(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True) model_rpn = Model(img_input, rpn[:2]) model_classifier = Model([img_input, roi_input], classifier) # this is a model that holds both the RPN and the classifier, used to load/save weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) start_of_training = datetime.date.today() tensorboard_callback = TensorBoard(log_dir="./logs/{0}_{1}/".format( start_of_training, configuration_name)) tensorboard_callback.set_model(model_all) try: print('Loading weights from {0}'.format(input_weight_path)) model_rpn.load_weights(input_weight_path, by_name=True) model_classifier.load_weights(input_weight_path, by_name=True) except: print( 'Could not load pretrained model weights. Weights can be found in the keras application folder \ https://github.com/fchollet/keras/tree/master/keras/applications') optimizer = Adadelta() optimizer_classifier = Adadelta() model_rpn.compile(optimizer=optimizer, loss=[ faster_rcnn_losses.rpn_loss_cls(num_anchors), faster_rcnn_losses.rpn_loss_regr(num_anchors) ]) model_classifier.compile( optimizer=optimizer_classifier, loss=[ faster_rcnn_losses.class_loss_cls, faster_rcnn_losses.class_loss_regr(len(classes_count) - 1) ], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer=Adadelta(), loss='mae') epoch_length = 1000 validation_epoch_length = len(validation_images) validation_interval = 1 losses = np.zeros((epoch_length, 5)) losses_val = np.zeros((validation_epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] best_loss_training = np.inf best_loss_epoch = 0 best_total_loss_validation = np.Inf best_loss_rpn_cls = np.inf best_loss_rpn_regr = np.inf best_loss_class_cls = np.inf best_loss_class_regr = np.inf best_class_acc = 0.0 model_classifier.summary() print(C.summary()) print('Starting training') train_names = [ 'train_loss_rpn_cls', 'train_loss_rpn_reg', 'train_loss_class_cls', 'train_loss_class_reg', 'train_total_loss', 'train_acc' ] val_names = [ 'val_loss_rpn_cls', 'val_loss_rpn_reg', 'val_loss_class_cls', 'val_loss_class_reg', 'val_total_loss', 'val_acc' ] epochs_without_improvement = 0 for epoch_num in range(number_of_epochs): progbar = generic_utils.Progbar(epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, number_of_epochs)) for iter_num in range(epoch_length): try: if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose: mean_overlapping_bboxes = float( sum(rpn_accuracy_rpn_monitor)) / len( rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( '\nAverage number of overlapping bounding boxes from RPN = {} for {} previous iterations' .format(mean_overlapping_bboxes, epoch_length)) if mean_overlapping_bboxes == 0: print( 'RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.' ) X, Y, img_data = next(data_gen_train) loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi( P_rpn[0], P_rpn[1], C, use_regr=True, overlap_thresh=non_max_suppression_overlap_threshold, max_boxes=non_max_suppression_max_boxes) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = roi_helpers.calc_iou( R, img_data, C, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, C.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) loss_class = model_classifier.train_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] progbar.update( iter_num + 1, [('rpn_cls', np.mean(losses[:iter_num + 1, 0])), ('rpn_regr', np.mean(losses[:iter_num + 1, 1])), ('detector_cls', np.mean(losses[:iter_num + 1, 2])), ('detector_regr', np.mean(losses[:iter_num + 1, 3]))]) except Exception as e: print('Exception during training: {}'.format(e)) continue # Calculate losses after the specified number of iterations loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float( sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if C.verbose: print('[INFO TRAINING]') print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print('Classifier accuracy for bounding boxes from RPN: {}'.format( class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format(loss_class_cls)) print('Loss Detector regression: {}'.format(loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) print("Best loss for training: {0:.3f}".format(best_loss_training)) curr_total_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr val_start_time = time.time() write_log(tensorboard_callback, train_names, [ loss_rpn_cls, loss_rpn_regr, loss_class_cls, loss_class_regr, curr_total_loss, class_acc ], epoch_num) if curr_total_loss < best_loss_training: model_path = C.model_path[:-5] + "_training.hdf5" if C.verbose: print( 'Total training loss decreased from {0:.3f} to {1:.3f}, saving weights to {2}' .format(best_loss_training, curr_total_loss, model_path)) best_loss_training = curr_total_loss model_all.save_weights(model_path) ############# # VALIDATION ############# if (epoch_num + 1) % validation_interval != 0: continue progbar = generic_utils.Progbar(validation_epoch_length) for iter_num in range(validation_epoch_length): try: X, Y, img_data = next(data_gen_val) loss_rpn = model_rpn.test_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi( P_rpn[0], P_rpn[1], C, use_regr=True, overlap_thresh=non_max_suppression_overlap_threshold, max_boxes=non_max_suppression_max_boxes) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = roi_helpers.calc_iou( R, img_data, C, class_mapping) neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, C.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) loss_class = model_classifier.test_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) losses_val[iter_num, 0] = loss_rpn[1] losses_val[iter_num, 1] = loss_rpn[2] losses_val[iter_num, 2] = loss_class[1] losses_val[iter_num, 3] = loss_class[2] losses_val[iter_num, 4] = loss_class[3] progbar.update( iter_num + 1, [('rpn_cls', np.mean(losses_val[:iter_num + 1, 0])), ('rpn_regr', np.mean(losses_val[:iter_num + 1, 1])), ('detector_cls', np.mean(losses_val[:iter_num + 1, 2])), ('detector_regr', np.mean(losses_val[:iter_num + 1, 3]))]) except Exception as e: #print('Exception during validation: {}'.format(e)) continue # Computer aggregated losses loss_rpn_cls = np.mean(losses_val[:, 0]) loss_rpn_regr = np.mean(losses_val[:, 1]) loss_class_cls = np.mean(losses_val[:, 2]) loss_class_regr = np.mean(losses_val[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float( sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] curr_total_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr write_log(tensorboard_callback, val_names, [ loss_rpn_cls, loss_rpn_regr, loss_class_cls, loss_class_regr, curr_total_loss, class_acc ], epoch_num) if C.verbose: print('[INFO VALIDATION]') print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print('Classifier accuracy for bounding boxes from RPN: {}'.format( class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format(loss_class_cls)) print('Loss Detector regression: {}'.format(loss_class_regr)) print( "Current validation loss: {0:.3f}, Best validation loss: {1:.3f} at epoch: {2}" .format(curr_total_loss, best_total_loss_validation, best_loss_epoch)) print('Elapsed time: {}'.format(time.time() - val_start_time)) if curr_total_loss < best_total_loss_validation: if C.verbose: print( 'Total validation loss decreased from {0:.3f} to {1:.3f}, saving weights to {2}' .format(best_total_loss_validation, curr_total_loss, C.model_path)) best_total_loss_validation = curr_total_loss best_loss_rpn_cls = loss_rpn_cls best_loss_rpn_regr = loss_rpn_regr best_loss_class_cls = loss_class_cls best_loss_class_regr = loss_class_regr best_class_acc = class_acc best_loss_epoch = epoch_num model_all.save_weights(C.model_path) epochs_without_improvement = 0 else: epochs_without_improvement += validation_interval if epochs_without_improvement > early_stopping: print( "Early stopping training after {0} epochs without improvement on validation set" .format(epochs_without_improvement)) break if epochs_without_improvement > learning_rate_reduction_patience: current_learning_rate = K.get_value(model_classifier.optimizer.lr) new_learning_rate = current_learning_rate * learning_rate_reduction_factor print( "Not improved validation accuracy for {0} epochs. Reducing learning rate from {1} to {2}" .format(learning_rate_reduction_patience, current_learning_rate, new_learning_rate)) K.set_value(model_classifier.optimizer.lr, new_learning_rate) K.set_value(model_rpn.optimizer.lr, new_learning_rate) K.set_value(model_all.optimizer.lr, new_learning_rate) end_time = time.time() execution_time_in_seconds = round(end_time - start_time) print("Execution time: {0:.1f}s".format(end_time - start_time)) notification_message = "Training on {0} dataset with model {1} and configuration {2} finished. " \ "Val. accuracy: {3:0.5f}%".format("muscima_pp", model_name, configuration_name, best_class_acc * 100) TelegramNotifier.send_message_via_telegram(notification_message) today = "{0:02d}.{1:02d}.{2}".format(start_of_training.day, start_of_training.month, start_of_training.year) total_number_of_images = len(training_images) + len( validation_images) + len(test_images) GoogleSpreadsheetReporter.append_result_to_spreadsheet( dataset_size=total_number_of_images, model_name=model_name, configuration_name=configuration_name, data_augmentation="", early_stopping=early_stopping, reduction_patience=learning_rate_reduction_patience, learning_rate_reduction_factor=learning_rate_reduction_factor, optimizer="Adadelta", initial_learning_rate=1.0, non_max_suppression_overlap_threshold= non_max_suppression_overlap_threshold, non_max_suppression_max_boxes=non_max_suppression_max_boxes, validation_accuracy=best_class_acc, validation_total_loss=best_total_loss_validation, best_loss_rpn_cls=best_loss_rpn_cls, best_loss_rpn_regr=best_loss_rpn_regr, best_loss_class_cls=best_loss_class_cls, best_loss_class_regr=best_loss_class_regr, date=today, datasets="muscima_pp", execution_time_in_seconds=execution_time_in_seconds)
C.rot_90 = False class_mapping = C.class_mapping all_imgs, classes_count, class_mapping = get_data(options.test_path) if 'bg' not in classes_count: classes_count['bg'] = 0 class_mapping['bg'] = len(class_mapping) test_imgs = [s for s in all_imgs if s['imageset'] == 'trainval' or s['imageset'] == 'test'] print('Num test samples {}'.format(len(test_imgs))) data_gen_test = data_generators.get_anchor_gt(test_imgs, classes_count, C, nn.get_img_output_length, mode='val') C.num_rois = int(options.num_rois) if C.network == 'resnet50': num_features = 1024 elif C.network == 'vgg': num_features = 512 input_shape_img = (600,600, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # feature_map_input = Input(shape=input_shape_features)
random.shuffle(all_imgs) num_imgs = len(all_imgs) print(num_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'train'] val_imgs = [s for s in all_imgs if s['imageset'] == 'val'] test_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) print('Num test samples {}'.format(len(test_imgs))) # groundtruth anchor 데이터 가져오기 data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.get_img_output_length, K.image_dim_ordering(), mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.get_img_output_length, K.image_dim_ordering(), mode='val') data_gen_test = data_generators.get_anchor_gt(test_imgs, classes_count, C, nn.get_img_output_length, K.image_dim_ordering(), mode='val') if K.image_dim_ordering() == 'th': input_shape_img = (3, None, None) else: input_shape_img = (None, None, 3) # input placeholder 정의 img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # base network(feature extractor) 정의 (resnet, VGG, Inception, Inception Resnet V2, etc) shared_layers = nn.nn_base(img_input, trainable=True)
#cancel the K.image_dim_ordering() arg ## nn. ##输入参数为 train_imgs: all_imgs中的子集,包含图片路径,大小,bbox的类别,坐标 ## class_count: 各个类别的数目 ## C : 训练配置 ## nn.get_img_output_length: return width//16, height//16 表示使用vgg16最后会缩小到原来的16倍 ## mode = ‘train' 表示训练集,’val‘同理 ## 理论返回四个值,但是这里只有一个,是这四个值的一个元组(x_img, y_rpn_cls, y_rpn_reg, img_data_aug) ## 分别是(x_img: 图片, y_rpn_cls: 包含y_rpn_valid, y_rpn_overlap 前者表示该像素位的某种格式的anchor是否有效,后者表示是pos还是neg ) ## y_rpn_reg包含:y_rpn_overlap 和 原本的y_rpn_reg 后者表示若有超过阈值的anchor的 四个loss里用到的值 ## img_data_aug 表示图片的一系列相关信息) #data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.get_img_output_length, mode = 'train') #data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.get_img_output_length, mode = 'val') import itertools data_gen_train = itertools.cycle(data_generators.get_anchor_gt(train_imgs, classes_count, C, nn.get_img_output_length, mode = 'train')) data_gen_val = itertools.cycle(data_generators.get_anchor_gt(val_imgs, classes_count, C, nn.get_img_output_length, mode = 'val')) ######################################## 4.1 ########################################################################### ## input for nn input_shape_img = (None, None, 3) ## Input用来初始化一个keras tensor img_input = Input(shape = input_shape_img) roi_input = Input(shape = (None,4)) ## set the bone net only with the shape of input ## shared_layers is the output of VGG16 bone net shared_layers = nn.nn_base(img_input, trainable = True) # ## build division shared model of VGG16 #################### # VGG_model = Model(img_input, shared_layers) ##
pickle.dump(C, config_f) print('Config has been written to {}, and can be loaded when testing to ensure correct results'.format( config_output_filename)) random.shuffle(all_imgs) num_imgs = len(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) data_gen_train = data_generators.get_anchor_gt( train_imgs, classes_count, C, nn.get_img_output_length, K.image_dim_ordering(), mode='train') data_gen_val = data_generators.get_anchor_gt( val_imgs, classes_count, C, nn.get_img_output_length, K.image_dim_ordering(), mode='val') if K.image_dim_ordering() == 'th': input_shape_img = (3, None, None) else: input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn.nn_base(img_input, trainable=True) # define the RPN, built on the base layers
random.shuffle(all_imgs) num_imgs = len(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) from keras_frcnn import data_generators data_gen_train = data_generators.get_anchor_gt(train_imgs, class_mapping, classes_count, C, mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, class_mapping, classes_count, C, mode='train') from keras_frcnn import resnet as nn from keras import backend as K from keras.optimizers import Adam, SGD from keras.layers import Input from keras.callbacks import ModelCheckpoint from keras.models import Model from keras.callbacks import EarlyStopping, ModelCheckpoint