def fTrainInner(cnn, modelName, X_train=None, y_train=None, X_valid=None, y_valid=None, X_test=None, y_test=None, sOutPath=None, patchSize=0, batchSize=None, learningRate=None, iEpochs=None, dlart_handle=None, dimExpandEnable=False): print('Training CNN') print('with lr = ' + str(learningRate) + ' , batchSize = ' + str(batchSize)) # save names _, sPath = os.path.splitdrive(sOutPath) sPath, sFilename = os.path.split(sPath) sFilename, sExt = os.path.splitext(sFilename) model_name = sOutPath + os.sep + sFilename weight_name = model_name + '_weights.h5' model_json = model_name + '.json' model_all = model_name + '_model.h5' model_mat = model_name + '.mat' if (os.path.isfile(model_mat)): # no training if output file exists print('------- already trained -> go to next') return # create optimizer if dlart_handle != None: if dlart_handle.getOptimizer() == SGD_OPTIMIZER: opti = keras.optimizers.SGD( lr=learningRate, momentum=dlart_handle.getMomentum(), decay=dlart_handle.getWeightDecay(), nesterov=dlart_handle.getNesterovEnabled()) elif dlart_handle.getOptimizer() == RMS_PROP_OPTIMIZER: opti = keras.optimizers.RMSprop( lr=learningRate, decay=dlart_handle.getWeightDecay()) elif dlart_handle.getOptimizer() == ADAGRAD_OPTIMIZER: opti = keras.optimizers.Adagrad( lr=learningRate, epsilon=None, decay=dlart_handle.getWeightDecay()) elif dlart_handle.getOptimizer() == ADADELTA_OPTIMIZER: opti = keras.optimizers.Adadelta( lr=learningRate, rho=0.95, epsilon=None, decay=dlart_handle.getWeightDecay()) elif dlart_handle.getOptimizer() == ADAM_OPTIMIZER: opti = keras.optimizers.Adam(lr=learningRate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=dlart_handle.getWeightDecay()) else: raise ValueError("Unknown Optimizer!") else: opti = keras.optimizers.Adam(lr=learningRate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) cnn.summary() # compile model loss_func = 'categorical_crossentropy' # change loss function cnn.compile(loss=loss_func, optimizer=opti, metrics=['accuracy']) # callbacks callback_earlyStopping = EarlyStopping(monitor='val_loss', patience=25, verbose=1) # callback_tensorBoard = keras.callbacks.TensorBoard(log_dir=dlart_handle.getLearningOutputPath() + '/logs', # histogram_freq=2, # batch_size=batchSize, # write_graph=True, # write_grads=True, # write_images=True, # embeddings_freq=0, # embeddings_layer_names=None, # embeddings_metadata=None) callbacks = [ callback_earlyStopping, ModelCheckpoint(sOutPath + os.sep + 'checkpoints/checker.hdf5', monitor='val_acc', verbose=0, period=5, save_best_only=True), LearningRateScheduler(schedule=step_decay, verbose=1), LivePlotCallback(dlart_handle) ] # callbacks.append(ReduceLROnPlateau(monitor='loss', factor=0.1, patience=5, min_lr=1e-4, verbose=1)) # data augmentation if dlart_handle.getDataAugmentationEnabled(): # Initialize Image Generator # all shifted and rotated images are filled with zero padded pixels datagen = ImageDataGenerator( featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=dlart_handle.getZCA_Whitening(), zca_epsilon=1e-6, rotation_range=dlart_handle.getRotation(), width_shift_range=dlart_handle.getWidthShift(), height_shift_range=dlart_handle.getHeightShift(), shear_range=0., zoom_range=dlart_handle.getZoom(), channel_shift_range=0., fill_mode='constant', cval=0., horizontal_flip=dlart_handle.getHorizontalFlip(), vertical_flip=dlart_handle.getVerticalFlip(), rescale=None, histogram_equalization=dlart_handle.getHistogramEqualization(), contrast_stretching=dlart_handle.getContrastStretching(), adaptive_equalization=dlart_handle.getAdaptiveEqualization(), preprocessing_function=None, data_format=K.image_data_format()) datagen_val = ImageDataGenerator( featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=dlart_handle.getZCA_Whitening(), zca_epsilon=1e-6, rotation_range=0., width_shift_range=0., height_shift_range=0., shear_range=0., zoom_range=0., channel_shift_range=0., fill_mode='constant', cval=0., horizontal_flip=False, vertical_flip=False, rescale=None, histogram_equalization=dlart_handle.getHistogramEqualization(), contrast_stretching=dlart_handle.getContrastStretching(), adaptive_equalization=dlart_handle.getAdaptiveEqualization(), preprocessing_function=None, data_format=K.image_data_format()) datagen_test = ImageDataGenerator( featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=dlart_handle.getZCA_Whitening(), zca_epsilon=1e-6, rotation_range=0., width_shift_range=0., height_shift_range=0., shear_range=0., zoom_range=0., channel_shift_range=0., fill_mode='constant', cval=0., horizontal_flip=False, vertical_flip=False, rescale=None, histogram_equalization=dlart_handle.getHistogramEqualization(), contrast_stretching=dlart_handle.getContrastStretching(), adaptive_equalization=dlart_handle.getAdaptiveEqualization(), preprocessing_function=None, data_format=K.image_data_format()) # fit parameters from dataset datagen.fit(X_train) datagen_test.fit(X_test) # configure batch size and get one batch of images for x_batch, y_batch in datagen.flow(X_train, y_train, batch_size=9): # display first 9 images for i in range(0, 9): plt.subplot(330 + 1 + i) plt.imshow(x_batch[i].reshape(x_batch.shape[1], x_batch.shape[2]), cmap='gray') plt.show() break if X_valid is not None and y_valid is not None: # fit model on data # use validation/test split datagen_val.fit(X_valid) result = cnn.fit_generator( datagen.flow(X_train, y_train, batch_size=batchSize), steps_per_epoch=X_train.shape[0] // batchSize, epochs=iEpochs, validation_data=datagen_val.flow(X_valid, y_valid, batch_size=batchSize), callbacks=callbacks, workers=1, use_multiprocessing=False) else: # fit model on data # use test data for validation and test result = cnn.fit_generator( datagen.flow(X_train, y_train, batch_size=batchSize), steps_per_epoch=X_train.shape[0] // batchSize, epochs=iEpochs, validation_data=datagen_test.flow(X_test, y_test, batch_size=batchSize), callbacks=callbacks, workers=1, use_multiprocessing=False) # return the loss value and metrics values for the model in test mode score_test, acc_test = cnn.evaluate_generator( datagen_test.flow(X_test, y_test, batch_size=batchSize), steps=None, max_queue_size=10, workers=1, use_multiprocessing=False) prob_test = cnn.predict_generator(datagen_test.flow( X_test, y_test, batch_size=batchSize), steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=1) else: if X_valid is not None and y_valid is not None: if dimExpandEnable: X_valid = np.expand_dims(X_valid, axis=-1) # use validation/test split result = cnn.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=iEpochs, batch_size=batchSize, callbacks=callbacks, verbose=1) else: if dimExpandEnable: X_train = np.expand_dims(X_train, axis=-1) X_test = np.expand_dims(X_test, axis=-1) # use test set for validation and test result = cnn.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=iEpochs, batch_size=batchSize, callbacks=callbacks, verbose=1) # return the loss value and metrics values for the model in test mode score_test, acc_test = cnn.evaluate(X_test, y_test, batch_size=batchSize, verbose=1) prob_test = cnn.predict(X_test, batchSize, 0) # save model json_string = cnn.to_json() with open(model_json, 'w') as jsonFile: jsonFile.write(json_string) # wei = cnn.get_weights() cnn.save_weights(weight_name, overwrite=True) cnn.save(model_all) # keras > v0.7 model_png_dir = sOutPath + os.sep + "model.png" from keras.utils import plot_model plot_model(cnn, to_file=model_png_dir, show_shapes=True, show_layer_names=True) # matlab acc = result.history['acc'] loss = result.history['loss'] val_acc = result.history['val_acc'] val_loss = result.history['val_loss'] print('Saving results: ' + model_name) sio.savemat( model_name, { 'model_settings': model_json, 'model': model_all, 'weights': weight_name, 'acc': acc, 'loss': loss, 'val_acc': val_acc, 'val_loss': val_loss, 'score_test': score_test, 'acc_test': acc_test, 'prob_test': prob_test })
def fTrainInner(cnn, modelName, X_train=None, y_train=None, Y_segMasks_train=None, X_valid=None, y_valid=None, Y_segMasks_valid=None, X_test=None, y_test=None, Y_segMasks_test=None, sOutPath=None, patchSize=0, batchSize=None, learningRate=None, iEpochs=None, dlart_handle=None, usingClassification=False): print('Training CNN') print('with lr = ' + str(learningRate) + ' , batchSize = ' + str(batchSize)) # sio.savemat('D:med_data/' + 'checkdata_voxel_and_mask.mat', # {'mask_train': Y_segMasks_train, # 'voxel_train': X_train, # 'mask_test': Y_segMasks_test, # 'voxel_test': X_test}) # save names _, sPath = os.path.splitdrive(sOutPath) sPath, sFilename = os.path.split(sPath) sFilename, sExt = os.path.splitext(sFilename) model_name = sOutPath + os.sep + sFilename weight_name = model_name + '_weights.h5' model_json = model_name + '.json' model_all = model_name + '_model.h5' model_mat = model_name + '.mat' if (os.path.isfile(model_mat)): # no training if output file exists print('------- already trained -> go to next') return # create optimizer if dlart_handle != None: if dlart_handle.getOptimizer() == SGD_OPTIMIZER: opti = keras.optimizers.SGD( lr=learningRate, momentum=dlart_handle.getMomentum(), decay=dlart_handle.getWeightDecay(), nesterov=dlart_handle.getNesterovEnabled()) elif dlart_handle.getOptimizer() == RMS_PROP_OPTIMIZER: opti = keras.optimizers.RMSprop( lr=learningRate, decay=dlart_handle.getWeightDecay()) elif dlart_handle.getOptimizer() == ADAGRAD_OPTIMIZER: opti = keras.optimizers.Adagrad( lr=learningRate, epsilon=None, decay=dlart_handle.getWeightDecay()) elif dlart_handle.getOptimizer() == ADADELTA_OPTIMIZER: opti = keras.optimizers.Adadelta( lr=learningRate, rho=0.95, epsilon=None, decay=dlart_handle.getWeightDecay()) elif dlart_handle.getOptimizer() == ADAM_OPTIMIZER: opti = keras.optimizers.Adam(lr=learningRate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=dlart_handle.getWeightDecay()) else: raise ValueError("Unknown Optimizer!") else: # opti = SGD(lr=learningRate, momentum=1e-8, decay=0.1, nesterov=True);#Adag(lr=0.01, epsilon=1e-06) opti = keras.optimizers.Adam(lr=learningRate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) cnn.summary() # compile model if usingClassification: cnn.compile(loss={ 'segmentation_output': dice_coef_loss, 'classification_output': 'categorical_crossentropy' }, optimizer=opti, metrics={ 'segmentation_output': dice_coef, 'classification_output': 'accuracy' }) else: cnn.compile(loss=dice_coef_loss, optimizer=opti, metrics=[dice_coef]) # callbacks # callback_earlyStopping = EarlyStopping(monitor='val_loss', patience=12, verbose=1) # callback_tensorBoard = keras.callbacks.TensorBoard(log_dir=dlart_handle.getLearningOutputPath() + '/logs', # histogram_freq=2, # batch_size=batchSize, # write_graph=True, # write_grads=True, # write_images=True, # embeddings_freq=0, # embeddings_layer_names=None, # embeddings_metadata=None) from utils.LivePlotCallback import LivePlotCallback callbacks = [ EarlyStopping(monitor='val_loss', patience=12, verbose=1), ModelCheckpoint(sOutPath + os.sep + 'checkpoints' + os.sep + 'checker.hdf5', monitor='val_acc', verbose=0, period=1, save_best_only=True), LearningRateScheduler(schedule=step_decay, verbose=1), LivePlotCallback(dlart_handle) ] # callbacks.append(ReduceLROnPlateau(monitor='loss', factor=0.1, patience=5, min_lr=1e-4, verbose=1)) if X_valid.size == 0 and y_valid.size == 0: # using test set for validation if usingClassification: result = cnn.fit(X_train, { 'segmentation_output': Y_segMasks_train, 'classification_output': y_train }, validation_data=(X_test, { 'segmentation_output': Y_segMasks_test, 'classification_output': y_test }), epochs=iEpochs, batch_size=batchSize, callbacks=callbacks, verbose=1) else: result = cnn.fit(X_train, Y_segMasks_train, validation_data=(X_test, Y_segMasks_test), epochs=iEpochs, batch_size=batchSize, callbacks=callbacks, verbose=1) else: # using validation set for validation if usingClassification: result = cnn.fit(X_train, { 'segmentation_output': Y_segMasks_train, 'classification_output': y_train }, validation_data=(X_valid, { 'segmentation_output': Y_segMasks_valid, 'classification_output': y_valid }), epochs=iEpochs, batch_size=batchSize, callbacks=callbacks, verbose=1) else: result = cnn.fit(X_train, Y_segMasks_train, validation_data=(X_valid, Y_segMasks_valid), epochs=iEpochs, batch_size=batchSize, callbacks=callbacks, verbose=1) # return the loss value and metrics values for the model in test mode if usingClassification: model_metrics = cnn.metrics_names loss_test, segmentation_output_loss_test, classification_output_loss_test, segmentation_output_dice_coef_test, classification_output_acc_test \ = cnn.evaluate(X_test, {'segmentation_output': Y_segMasks_test, 'classification_output': y_test}, batch_size=batchSize, verbose=1) else: score_test, dice_coef_test = cnn.evaluate(X_test, Y_segMasks_test, batch_size=batchSize, verbose=1) prob_test = cnn.predict(X_test, batchSize, 0) # save model json_string = cnn.to_json() with open(model_json, 'w') as jsonFile: jsonFile.write(json_string) # wei = cnn.get_weights() cnn.save_weights(weight_name, overwrite=True) cnn.save(model_all) # keras > v0.7 model_png_dir = sOutPath + os.sep + "model.png" from keras.utils import plot_model plot_model(cnn, to_file=model_png_dir, show_shapes=True, show_layer_names=True) if not usingClassification: # matlab dice_coef_training = result.history['dice_coef'] training_loss = result.history['loss'] val_dice_coef = result.history['val_dice_coef'] val_loss = result.history['val_loss'] print('Saving results: ' + model_name) sio.savemat( model_name, { 'model_settings': model_json, 'model': model_all, 'weights': weight_name, 'dice_coef': dice_coef_training, 'training_loss': training_loss, 'val_dice_coef': val_dice_coef, 'val_loss': val_loss, 'score_test': score_test, 'dice_coef_test': dice_coef_test, 'prob_test': prob_test }) else: # matlab segmentation_output_loss_training = result.history[ 'segmentation_output_loss'] classification_output_loss_training = result.history[ 'classification_output_loss'] segmentation_output_dice_coef_training = result.history[ 'segmentation_output_dice_coef'] classification_output_acc_training = result.history[ 'classification_output_acc'] val_segmentation_output_loss = result.history[ 'val_segmentation_output_loss'] val_classification_output_loss = result.history[ 'val_classification_output_loss'] val_segmentation_output_dice_coef = result.history[ 'val_segmentation_output_dice_coef'] val_classification_output_acc = result.history[ 'val_classification_output_acc'] print('Saving results: ' + model_name) sio.savemat( model_name, { 'model_settings': model_json, 'model': model_all, 'weights': weight_name, 'segmentation_output_loss_training': segmentation_output_loss_training, 'classification_output_loss_training': classification_output_loss_training, 'segmentation_output_dice_coef_training': segmentation_output_dice_coef_training, 'classification_output_acc_training': classification_output_acc_training, 'segmentation_output_loss_val': val_segmentation_output_loss, 'classification_output_loss_val': val_classification_output_loss, 'segmentation_output_dice_coef_val': val_segmentation_output_dice_coef, 'classification_output_acc_val': val_classification_output_acc, 'loss_test': loss_test, 'segmentation_output_loss_test': segmentation_output_loss_test, 'classification_output_loss_test': classification_output_loss_test, 'segmentation_output_dice_coef_test': segmentation_output_dice_coef_test, 'classification_output_acc_test': classification_output_acc_test, 'segmentation_predictions': prob_test[0], 'classification_predictions': prob_test[1] })