def prep_coco_dataset(type, config, load_class_ids=None, class_ids=None, generator=False, shuffle=True, augment=False, return_coco=False): # dataset_train, train_generator = coco_dataset(["train", "val35k"], mrcnn_config) # if args.command == "train": # Training dataset. Use the training set and 35K from the validation set, as as in the Mask RCNN paper. dataset = CocoDataset() # dataset_test.load_coco(COCO_DATASET_PATH, "train", class_ids=mrcnn_config.COCO_CLASSES) for i in type: dataset.load_coco(config.COCO_DATASET_PATH, i, class_ids=class_ids, load_class_ids=load_class_ids, return_coco=return_coco) # all datasets loaded, now prep the final dataset dataset.prepare() results = dataset if generator: generator = data_generator(dataset, config, batch_size=config.BATCH_SIZE, shuffle=shuffle, augment=augment) results = [dataset, generator] return results
def prep_oldshapes_train(init_with = None, FCN_layers = False, batch_sz = 5, epoch_steps = 4, training_folder= "mrcnn_oldshape_training_logs"): import mrcnn.shapes as shapes MODEL_DIR = os.path.join(TRAINING_DIR, training_folder) # Build configuration object ----------------------------------------------- config = shapes.ShapesConfig() config.BATCH_SIZE = batch_sz # Batch size is 2 (# GPUs * images/GPU). config.IMAGES_PER_GPU = batch_sz # Must match BATCH_SIZE config.STEPS_PER_EPOCH = epoch_steps config.FCN_INPUT_SHAPE = config.IMAGE_SHAPE[0:2] # Build shape dataset ----------------------------------------------- dataset_train = shapes.ShapesDataset(config) dataset_train.load_shapes(3000) dataset_train.prepare() # Validation dataset dataset_val = shapes.ShapesDataset(config) dataset_val.load_shapes(500) dataset_val.prepare() try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() model = modellib.MaskRCNN(mode="training", config=config, model_dir=MODEL_DIR, FCN_layers = FCN_layers) print(' COCO Model Path : ', COCO_TRAINING_DIR) print(' Checkpoint folder Path: ', MODEL_DIR) print(' Model Parent Path : ', TRAINING_DIR) print(' Resent Model Path : ', RESNET_TRAINING_DIR) model.load_model_weights(init_with = init_with) train_generator = data_generator(dataset_train, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment = False) val_generator = data_generator(dataset_val, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment=False) model.config.display() return [model, dataset_train, dataset_val, train_generator, val_generator, config]
def prep_newshapes_train(init_with = "last", FCN_layers= False, batch_sz =5, epoch_steps = 4, folder_name= "mrcnn_newshape_training_logs"): import mrcnn.new_shapes as new_shapes MODEL_DIR = os.path.join(MODEL_PATH, folder_name) # Build configuration object ----------------------------------------------- config = new_shapes.NewShapesConfig() config.BATCH_SIZE = batch_sz # Batch size is 2 (# GPUs * images/GPU). config.IMAGES_PER_GPU = batch_sz # Must match BATCH_SIZE config.STEPS_PER_EPOCH = epoch_steps config.FCN_INPUT_SHAPE = config.IMAGE_SHAPE[0:2] # Build shape dataset ----------------------------------------------- # Training dataset dataset_train = new_shapes.NewShapesDataset() dataset_train.load_shapes(3000, config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1]) dataset_train.prepare() # Validation dataset dataset_val = new_shapes.NewShapesDataset() dataset_val.load_shapes(500, config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1]) dataset_val.prepare() try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() model = modellib.MaskRCNN(mode="training", config=config, model_dir=MODEL_DIR,FCN_layers = FCN_layers) print('MODEL_PATH : ', MODEL_PATH) print('COCO_MODEL_PATH : ', COCO_MODEL_PATH) print('RESNET_MODEL_PATH : ', RESNET_MODEL_PATH) print('MODEL_DIR : ', MODEL_DIR) print('Last Saved Model : ', model.find_last()) load_model(model, init_with = 'last') train_generator = data_generator(dataset_train, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment = False) config.display() return [model, dataset_train, train_generator, config]
def prep_newshapes_test(init_with = 'last', FCN_layers = False, batch_sz = 5, epoch_steps = 4,folder_name= "mrcnn_newshape_test_logs"): import mrcnn.new_shapes as new_shapes MODEL_DIR = os.path.join(MODEL_PATH, folder_name) # Build configuration object ----------------------------------------------- config = new_shapes.NewShapesConfig() config.BATCH_SIZE = batch_sz # Batch size is 2 (# GPUs * images/GPU). config.IMAGES_PER_GPU = batch_sz # Must match BATCH_SIZE config.STEPS_PER_EPOCH = epoch_steps config.FCN_INPUT_SHAPE = config.IMAGE_SHAPE[0:2] # Build shape dataset ----------------------------------------------- # Training dataset dataset_test = new_shapes.NewShapesDataset() dataset_test.load_shapes(3000, config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1]) dataset_test.prepare() # Recreate the model in inference mode try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() model = modellib.MaskRCNN(mode="inference", config=config, model_dir=MODEL_DIR, FCN_layers = FCN_layers ) print(' COCO Model Path : ', COCO_MODEL_PATH) print(' Checkpoint folder Path: ', MODEL_DIR) print(' Model Parent Path : ', MODEL_PATH) print(' Resent Model Path : ', RESNET_MODEL_PATH) load_model(model, init_with = init_with) test_generator = data_generator(dataset_test, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment = False) model.config.display() return [model, dataset_test, test_generator, config]
def prep_newshapes2_dataset(config, image_count, shuffle=True, augment=False, generator=False): ''' ''' dataset = NewImagesDataset(config) dataset.load_images(image_count) dataset.prepare() results = dataset if generator: generator = data_generator(dataset, config, batch_size=config.BATCH_SIZE, shuffle=True, augment=False) return [dataset, generator] else: return results
def mrcnn_coco_train(init_weights = 'last', mode = 'train', FCN_layers = False, batch_sz = 5, epoch_steps = 4, training_folder = "mrcnn_coco_dev"): TRAINING_PATH = os.path.join(TRAINING_DIR, training_folder) ##------------------------------------------------------------------------------------ ## Build configuration object ##------------------------------------------------------------------------------------ mrcnn_config = CocoInferenceConfig() mrcnn_config.NAME = 'mrcnn' mrcnn_config.TRAINING_PATH = TRAINING_PATH mrcnn_config.COCO_MODEL_PATH = COCO_MODEL_PATH mrcnn_config.RESNET_MODEL_PATH = RESNET_MODEL_PATH mrcnn_config.VGG16_MODEL_PATH = VGG16_MODEL_PATH mrcnn_config.COCO_CLASSES = None # mrcnn_config.COCO_CLASSES = [1,2,3,4,5,6,7,8,9,10] # mrcnn_config.NUM_CLASSES = len(mrcnn_config.COCO_CLASSES) + 1 # config.BATCH_SIZE = batch_sz # Batch size is 2 (# GPUs * images/GPU). # config.IMAGES_PER_GPU = batch_sz # Must match BATCH_SIZE # config.STEPS_PER_EPOCH = epoch_steps # config.FCN_INPUT_SHAPE = config.IMAGE_SHAPE[0:2] # config.DETECTION_MIN_CONFIDENCE = 0.1 ##------------------------------------------------------------------------------------ ## Build shape dataset for Training and Validation ##------------------------------------------------------------------------------------ # if args.command == "train": # Training dataset. Use the training set and 35K from the # validation set, as as in the Mask RCNN paper. dataset_train = CocoDataset() dataset_train.load_coco(COCO_DATASET_PATH, "train", class_ids=mrcnn_config.COCO_CLASSES) dataset_train.load_coco(COCO_DATASET_PATH, "val35k", class_ids=mrcnn_config.COCO_CLASSES) dataset_train.prepare() # Validation dataset dataset_val = CocoDataset() dataset_val.load_coco(COCO_DATASET_PATH, "minival", class_ids=mrcnn_config.COCO_CLASSES) dataset_val.prepare() # Recreate the model in inference mode try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() mrcnn_model = mrcnn_modellib.MaskRCNN(mode=mode, config=mrcnn_config, model_dir=TRAINING_PATH) ##------------------------------------------------------------------------------------ ## Load Mask RCNN Model Weight file ##------------------------------------------------------------------------------------ mrcnn_model.load_model_weights( init_with = init_weights) print('==========================================') print(" MRCNN MODEL Load weight file COMPLETE ") print('==========================================') # print('\n\n\n') # print(' Checkpoint directory : ', TRAINING_DIR) # print(' Checkpoint folder : ', TRAINING_PATH) # print(' COCO Model Path : ', COCO_MODEL_PATH) # print(' ResNet Model Path : ', RESNET_MODEL_PATH) # print(' VGG16 Model Path : ', COCO_MODEL_PATH) # mrcnn_config.display() mrcnn_model.layer_info() print('\n Outputs: ') # pp.pprint(mrcnn_model.keras_model.outputs) trainable = mrcnn_model.get_trainable_layers() for i in trainable: print(' Layer:', i.name) train_generator = data_generator(dataset_train, mrcnn_model.config, shuffle=True, batch_size = mrcnn_model.config.BATCH_SIZE, augment = False) val_generator = data_generator(dataset_val, mrcnn_model.config, shuffle=True, batch_size = mrcnn_model.config.BATCH_SIZE, augment=False) mrcnn_config.display() return [mrcnn_model, dataset_train, dataset_val, train_generator, val_generator, mrcnn_config]
def prep_newshapes_train(init_with = "last", FCN_layers= False, batch_sz =5, epoch_steps = 4, training_folder= None): MODEL_DIR = os.path.join(TRAINING_DIR, training_folder) # Build configuration object ----------------------------------------------- config = new_shapes.NewShapesConfig() config.BATCH_SIZE = batch_sz # Batch size is 2 (# GPUs * images/GPU). config.IMAGES_PER_GPU = batch_sz # Must match BATCH_SIZE config.STEPS_PER_EPOCH = epoch_steps config.FCN_INPUT_SHAPE = config.IMAGE_SHAPE[0:2] # Build shape dataset ----------------------------------------------- # Training dataset dataset_train = new_shapes.NewShapesDataset(config) dataset_train.load_shapes(10000) dataset_train.prepare() # Validation dataset dataset_val = new_shapes.NewShapesDataset(config) dataset_val.load_shapes(2500) dataset_val.prepare() try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() model = modellib.MaskRCNN(mode="training", config=config, model_dir=MODEL_DIR,FCN_layers = FCN_layers) print('TRAINING_DIR : ', TRAINING_DIR) print('COCO_TRAINING_DIR : ', COCO_TRAINING_DIR) print('RESNET_TRAINING_DIR : ', RESNET_TRAINING_DIR) print('MODEL_DIR : ', MODEL_DIR) print('Last Saved Model : ', model.find_last()) # exclude_layers = \ # ['fcn_block1_conv1' # ,'fcn_block1_conv2' # ,'fcn_block1_pool' # ,'fcn_block2_conv1' # ,'fcn_block2_conv2' # ,'fcn_block2_pool' # ,'fcn_block3_conv1' # ,'fcn_block3_conv2' # ,'fcn_block3_conv3' # ,'fcn_block3_pool' # ,'fcn_block4_conv1' # ,'fcn_block4_conv2' # ,'fcn_block4_conv3' # ,'fcn_block4_pool' # ,'fcn_block5_conv1' # ,'fcn_block5_conv2' # ,'fcn_block5_conv3' # ,'fcn_block5_pool' # ,'fcn_fc1' # ,'dropout_1' # ,'fcn_fc2' # ,'dropout_2' # ,'fcn_classify' # ,'fcn_bilinear' # ,'fcn_heatmap_norm' # ,'fcn_scoring' # ,'fcn_heatmap' # ,'fcn_norm_loss'] # load_model(model, init_with = 'last', exclude = exclude_layers) model.load_model_weights(init_with = init_with) # print('=====================================') # print(" Load second weight file ?? ") # model.keras_model.load_weights('E:/Models/vgg16_weights_tf_dim_ordering_tf_kernels.h5', by_name= True) train_generator = data_generator(dataset_train, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment = False) val_generator = data_generator(dataset_val, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment=False) config.display() return [model, dataset_train, dataset_val, train_generator, val_generator, config]
def prep_newshapes_train2(init_with = "last", config=None): import mrcnn.new_shapes as new_shapes config.CHECKPOINT_FOLDER = os.path.join(TRAINING_DIR, config.CHECKPOINT_FOLDER) # Build shape dataset ----------------------------------------------- # Training dataset dataset_train = new_shapes.NewShapesDataset(config) dataset_train.load_shapes(config.TRAINING_IMAGES) dataset_train.prepare() # Validation dataset dataset_val = new_shapes.NewShapesDataset(config) dataset_val.load_shapes(config.VALIDATION_IMAGES) dataset_val.prepare() try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() model = modellib.MaskRCNN(mode="training", config=config, model_dir=config.CHECKPOINT_FOLDER, FCN_layers = config.FCN_LAYERS) print('TRAINING_DIR : ', TRAINING_DIR) print('COCO_TRAINING_DIR : ', COCO_TRAINING_DIR) print('RESNET_TRAINING_DIR : ', RESNET_TRAINING_DIR) print('CHECKPOINT_DIR : ', config.CHECKPOINT_FOLDER) print('Last Saved Model : ', model.find_last()) # exclude_layers = \ # ['fcn_block1_conv1' # ,'fcn_block1_conv2' # ,'fcn_block1_pool' # ,'fcn_block2_conv1' # ,'fcn_block2_conv2' # ,'fcn_block2_pool' # ,'fcn_block3_conv1' # ,'fcn_block3_conv2' # ,'fcn_block3_conv3' # ,'fcn_block3_pool' # ,'fcn_block4_conv1' # ,'fcn_block4_conv2' # ,'fcn_block4_conv3' # ,'fcn_block4_pool' # ,'fcn_block5_conv1' # ,'fcn_block5_conv2' # ,'fcn_block5_conv3' # ,'fcn_block5_pool' # ,'fcn_fc1' # ,'dropout_1' # ,'fcn_fc2' # ,'dropout_2' # ,'fcn_classify' # ,'fcn_bilinear' # ,'fcn_heatmap_norm' # ,'fcn_scoring' # ,'fcn_heatmap' # ,'fcn_norm_loss'] # load_model(model, init_with = 'last', exclude = exclude_layers) model.load_model_weights(init_with = init_with) # print('=====================================') # print(" Load second weight file ?? ") # model.keras_model.load_weights('E:/Models/vgg16_weights_tf_dim_ordering_tf_kernels.h5', by_name= True) train_generator = data_generator(dataset_train, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment = False) val_generator = data_generator(dataset_val, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment=False) config.display() return [model, dataset_train, dataset_val, train_generator, val_generator, config]
def prep_newshapes_test(init_with = 'last', FCN_layers = False, batch_sz = 5, epoch_steps = 4,training_folder= "mrcnn_newshape_test_logs"): MODEL_DIR = os.path.join(TRAINING_DIR, training_folder) # Build configuration object ----------------------------------------------- config = new_shapes.NewShapesConfig() config.BATCH_SIZE = batch_sz # Batch size is 2 (# GPUs * images/GPU). config.IMAGES_PER_GPU = batch_sz # Must match BATCH_SIZE config.STEPS_PER_EPOCH = epoch_steps config.FCN_INPUT_SHAPE = config.IMAGE_SHAPE[0:2] config.DETECTION_MIN_CONFIDENCE = 0.1 # Build shape dataset ----------------------------------------------- # Training dataset dataset_test = new_shapes.NewShapesDataset(config) dataset_test.load_shapes(3000) dataset_test.prepare() # Recreate the model in inference mode try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() model = modellib.MaskRCNN(mode="inference", config=config, model_dir=MODEL_DIR, FCN_layers = FCN_layers ) print(' COCO Model Path : ', COCO_TRAINING_DIR) print(' Checkpoint folder Path: ', MODEL_DIR) print(' Model Parent Path : ', TRAINING_DIR) print(' Resent Model Path : ', RESNET_TRAINING_DIR) # exclude_layers = \ # ['fcn_block1_conv1' # ,'fcn_block1_conv2' # ,'fcn_block1_pool' # ,'fcn_block2_conv1' # ,'fcn_block2_conv2' # ,'fcn_block2_pool' # ,'fcn_block3_conv1' # ,'fcn_block3_conv2' # ,'fcn_block3_conv3' # ,'fcn_block3_pool' # ,'fcn_block4_conv1' # ,'fcn_block4_conv2' # ,'fcn_block4_conv3' # ,'fcn_block4_pool' # ,'fcn_block5_conv1' # ,'fcn_block5_conv2' # ,'fcn_block5_conv3' # ,'fcn_block5_pool' # ,'fcn_fc1' # ,'dropout_1' # ,'fcn_fc2' # ,'dropout_2' # ,'fcn_classify' # ,'fcn_bilinear' # ,'fcn_heatmap_norm' # ,'fcn_scoring' # ,'fcn_heatmap' # ,'fcn_norm_loss'] # load_model(model, init_with = init_with, exclude = exclude_layers) model.load_model_weights(init_with = init_with) # print('=====================================') # print(" Load second weight file ?? ") # model.keras_model.load_weights('E:/Models/vgg16_weights_tf_dim_ordering_tf_kernels.h5', by_name= True ) test_generator = data_generator(dataset_test, model.config, shuffle=True, batch_size=model.config.BATCH_SIZE, augment = False) model.config.display() return [model, dataset_test, test_generator, config]
def mrcnn_coco_test(init_weights = 'last', mode = 'inference' , batch_sz = 5, epoch_steps = 4, training_folder = "mrcnn_coco_dev"): TRAINING_PATH = os.path.join(TRAINING_DIR, training_folder) mrcnn_config = CocoInferenceConfig() mrcnn_config.NAME = 'mrcnn' mrcnn_config.TRAINING_PATH = TRAINING_PATH mrcnn_config.COCO_MODEL_PATH = COCO_MODEL_PATH mrcnn_config.RESNET_MODEL_PATH = RESNET_MODEL_PATH mrcnn_config.VGG16_MODEL_PATH = VGG16_MODEL_PATH mrcnn_config.COCO_CLASSES = None # mrcnn_config.NUM_CLASSES = len(mrcnn_config.COCO_CLASSES) + 1 # mrcnn_config.NUM_CLASSES = len(mrcnn_config.COCO_CLASSES) + 1 ##------------------------------------------------------------------------------------ ## Build shape dataset for Training and Validation ##------------------------------------------------------------------------------------ # if args.command == "train": # Training dataset. Use the training set and 35K from the validation set, as as in the Mask RCNN paper. dataset_test = CocoDataset() # dataset_test.load_coco(COCO_DATASET_PATH, "train", class_ids=mrcnn_config.COCO_CLASSES) dataset_test.load_coco(COCO_DATASET_PATH, "val", class_ids=mrcnn_config.COCO_CLASSES) dataset_test.prepare() # Validation dataset # dataset_val = CocoDataset() # dataset_val.load_coco(COCO_DATASET_PATH, "minival", class_ids=mrcnn_config.COCO_CLASSES) # dataset_val.prepare() # Recreate the model in inference mode try : del model print('delete model is successful') gc.collect() except: pass KB.clear_session() mrcnn_model = mrcnn_modellib.MaskRCNN(mode=mode, config=mrcnn_config, model_dir=TRAINING_PATH) ##------------------------------------------------------------------------------------ ## Load Mask RCNN Model Weight file ##------------------------------------------------------------------------------------ # mrcnn_model.load_model_weights( init_with = init_weights) print('==========================================') print(" MRCNN MODEL Load weight file COMPLETE ") print('==========================================') # mrcnn_config.display() mrcnn_model.layer_info() print('\n Outputs: ') # pp.pprint(mrcnn_model.keras_model.outputs) for i in mrcnn_model.get_trainable_layers(): print(' Layer:', i.name) test_generator = data_generator(dataset_test, mrcnn_model.config, shuffle=True, batch_size = mrcnn_model.config.BATCH_SIZE, augment = False) mrcnn_config.display() return [mrcnn_model, dataset_test, test_generator, mrcnn_config]
def build_heatmap_files(mrcnn_model, dataset, iterations=5, start_from=0, dest_path=None): ''' train_dataset: Training Dataset objects. ''' assert mrcnn_model.mode == "trainfcn", "Create model in training mode." log("Starting for {} iterations - batch size of each iteration: {}". format(iterations, batch_size)) log(" Output destination: {}".format(dest_path)) tr_generator = data_generator(dataset, mrcnn_model.config, shuffle=False, augment=False, batch_size=mrcnn_model.config.BATCH_SIZE, image_index=start_from) ## Start main loop epoch_idx = 0 for epoch_idx in range(iterations): tm_start = time.time() train_batch_x, train_batch_y = next(tr_generator) print( ' ==> mrcnn_model: step {} of {} iterations, image_id: {} '.format( epoch_idx, iterations, train_batch_x[1][:, 0])) # print(' length of train_batch_x:', len(train_batch_x), ' number of things in batch x :', train_batch_x[1].shape) # for i in train_batch_x: # print(' ', i.shape) # print('length of train_batch_y:', len(train_batch_y)) # results = get_layer_output_1(mrcnn_model.keras_model, train_batch_x, [0,1,2,3], 1) results = mrcnn_model.keras_model.predict(train_batch_x) # pr_hm_norm, gt_hm_norm, pr_hm_scores, gt_hm_scores = results[:4] for i in range(batch_size): # print(' pr_hm_norm shape :', results[0][i].shape) # print(' pr_hm_scores shape :', results[1][i].shape) # print(' gt_hm_norm shape :', results[2][i].shape) # print(' gt_hm_scores shape :', results[3][i].shape) image_id = train_batch_x[1][i, 0] coco_image_id = dataset.image_info[image_id]['id'] coco_filename = os.path.basename( dataset.image_info[image_id]['path']) ## If we want to save the files with a sequence # 0,1,2,.... which is the index of dataset.image_info[index] use this: # filename = 'hm_{:012d}.npz'.format(image_id) ## If we want to use the coco_id as the file name, use the following: filename = 'hm_{:012d}.npz'.format(coco_image_id) print( ' output: {} image_id: {} coco_image_id: {} coco_filename: {} output file: {}' .format(i, image_id, coco_image_id, coco_filename, filename)) # print(' output file: ',os.path.join(dest_path, filename)) np.savez_compressed(os.path.join(dest_path, filename), input_image_meta=train_batch_x[1][i], pr_hm_norm=results[0][i], pr_hm_scores=results[1][i], gt_hm_norm=results[2][i], gt_hm_scores=results[3][i], coco_info=np.array( [coco_image_id, coco_filename])) tm_stop = time.time() print(' ==> Elapsed time {:.4f}s # of items in results: {} '. format(tm_stop - tm_start, len(train_batch_x))) print('Final : mrcnn_model epoch_idx{} iterations {}'.format( epoch_idx, iterations)) return
def train_in_batches(self, train_dataset, val_dataset, learning_rate, layers, losses=None, epochs_to_run=1, batch_size=0, steps_per_epoch=0): ''' Train the model. train_dataset, val_dataset: Training and validation Dataset objects. learning_rate: The learning rate to train with epochs: Number of training epochs. Note that previous training epochs are considered to be done already, so this actually determines the epochs to train in total rather than in this particaular call. layers: Allows selecting wich layers to train. It can be: - A regular expression to match layer names to train - One of these predefined values: heads: The RPN, classifier and mask heads of the network all: All the layers 3+: Train Resnet stage 3 and up 4+: Train Resnet stage 4 and up 5+: Train Resnet stage 5 and up ''' assert self.mode == "training", "Create model in training mode." # Use Pre-defined layer regular expressions # if layers in self.layer_regex.keys(): # layers = self.layer_regex[layers] print(layers) train_regex_list = [self.layer_regex[x] for x in layers] print(train_regex_list) layers = '|'.join(train_regex_list) print('layers regex :', layers) if batch_size == 0: batch_size = self.config.BATCH_SIZE if steps_per_epoch == 0: steps_per_epoch = self.config.STEPS_PER_EPOCH # Data generators train_generator = data_generator(train_dataset, self.config, shuffle=True, batch_size=batch_size) val_generator = data_generator(val_dataset, self.config, shuffle=True, batch_size=batch_size, augment=False) log(" Last epoch completed : {} ".format(self.epoch)) log(" Starting from epoch : {} for {} epochs".format( self.epoch, epochs_to_run)) log(" Learning Rate : {} ".format(learning_rate)) log(" Steps per epoch : {} ".format(steps_per_epoch)) log(" Batchsize : {} ".format(batch_size)) log(" Checkpoint Folder : {} ".format(self.checkpoint_path)) epochs = self.epoch + epochs_to_run from tensorflow.python.platform import gfile if not gfile.IsDirectory(self.log_dir): log('Creating checkpoint folder') gfile.MakeDirs(self.log_dir) else: log('Checkpoint folder already exists') self.set_trainable(layers) self.compile(learning_rate, self.config.LEARNING_MOMENTUM, losses) # copied from \keras\engine\training.py # def _get_deduped_metrics_names(self): ## get metrics from keras_model.metrics_names out_labels = self.get_deduped_metrics_names() print(' ====> out_labels : ', out_labels) ## setup Progress Bar callback callback_metrics = out_labels + ['val_' + n for n in out_labels] print(' Callback metrics monitored by progbar') pp.pprint(callback_metrics) progbar = keras.callbacks.ProgbarLogger(count_mode='steps') progbar.set_model(self.keras_model) progbar.set_params({ 'epochs': epochs, 'steps': steps_per_epoch, 'verbose': 1, 'do_validation': False, 'metrics': callback_metrics, }) progbar.set_model(self.keras_model) ## setup Checkpoint callback chkpoint = keras.callbacks.ModelCheckpoint(self.checkpoint_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True) chkpoint.set_model(self.keras_model) progbar.on_train_begin() epoch_idx = self.epoch if epoch_idx >= epochs: print( 'Final epoch {} has already completed - Training will not proceed' .format(epochs)) else: while epoch_idx < epochs: progbar.on_epoch_begin(epoch_idx) for steps_index in range(steps_per_epoch): batch_logs = {} # print(' self.epoch {} epochs {} step {} '.format(self.epoch, epochs, steps_index)) batch_logs['batch'] = steps_index batch_logs['size'] = batch_size progbar.on_batch_begin(steps_index, batch_logs) train_batch_x, train_batch_y = next(train_generator) outs = self.keras_model.train_on_batch( train_batch_x, train_batch_y) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o progbar.on_batch_end(steps_index, batch_logs) # print(outs) progbar.on_epoch_end(epoch_idx, {}) # if (epoch_idx % 10) == 0: chkpoint.on_epoch_end(epoch_idx, batch_logs) epoch_idx += 1 # if epoch_idx != self.epoch: # chkpoint.on_epoch_end(epoch_idx -1, batch_logs) self.epoch = max(epoch_idx - 1, epochs) print('Final : self.epoch {} epochs {}'.format( self.epoch, epochs))
def train(self, train_dataset, val_dataset, learning_rate, layers=None, losses=None, epochs=0, epochs_to_run=0, batch_size=0, steps_per_epoch=0): ''' Train the model. train_dataset, val_dataset: Training and validation Dataset objects. learning_rate: The learning rate to train with layers: Allows selecting wich layers to train. It can be: - A regular expression to match layer names to train - One of these predefined values: heads: The RPN, classifier and mask heads of the network all: All the layers 3+: Train Resnet stage 3 and up 4+: Train Resnet stage 4 and up 5+: Train Resnet stage 5 and up losses: List of losses to monitor. epochs: Number of training epochs. Note that previous training epochs are considered to be done already, so this actually determines the epochs to train in total rather than in this particaular call. epochs_to_run: Number of epochs to run, will update the 'epochs parm. ''' assert self.mode == "training", "Create model in training mode." if batch_size == 0: batch_size = self.config.BATCH_SIZE if epochs_to_run > 0: epochs = self.epoch + epochs_to_run if steps_per_epoch == 0: steps_per_epoch = self.config.STEPS_PER_EPOCH # use Pre-defined layer regular expressions # if layers in self.layer_regex.keys(): # layers = self.layer_regex[layers] print(layers) # train_regex_list = [] # for x in layers: # print( ' layers ias : ',x) # train_regex_list.append(x) train_regex_list = [self.layer_regex[x] for x in layers] print(train_regex_list) layers = '|'.join(train_regex_list) print('layers regex :', layers) # Data generators train_generator = data_generator(train_dataset, self.config, shuffle=True, batch_size=batch_size) val_generator = data_generator(val_dataset, self.config, shuffle=True, batch_size=batch_size, augment=False) # my_callback = MyCallback() # Callbacks ## call back for model checkpoint was originally (?) loss. chanegd to val_loss (which is default) 2-5-18 callbacks = [ keras.callbacks.TensorBoard(log_dir=self.log_dir, histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None), keras.callbacks.ModelCheckpoint(self.checkpoint_path, mode='auto', period=1, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True), keras.callbacks.ReduceLROnPlateau(monitor='val_loss', mode='auto', factor=0.3, cooldown=30, patience=50, min_lr=0.00001, verbose=1), keras.callbacks.EarlyStopping(monitor='val_loss', mode='auto', min_delta=1e-5, patience=200, verbose=1) ] # Train self.set_trainable(layers) self.compile(learning_rate, self.config.LEARNING_MOMENTUM, losses) log("Starting at epoch {} of {} epochs. LR={}\n".format( self.epoch, epochs, learning_rate)) log("Steps per epochs {} ".format(steps_per_epoch)) log("Batch size {} ".format(batch_size)) log("Checkpoint Path: {} ".format(self.checkpoint_path)) self.keras_model.fit_generator( train_generator, initial_epoch=self.epoch, epochs=epochs, steps_per_epoch=steps_per_epoch, callbacks=callbacks, validation_data=next(val_generator), validation_steps=self.config.VALIDATION_STEPS, max_queue_size=100, workers=1, # max(self.config.BATCH_SIZE // 2, 2), use_multiprocessing=False) self.epoch = max(self.epoch, epochs) print('Final : self.epoch {} epochs {}'.format(self.epoch, epochs))