def fTrainInner(cnn,
modelName,
X_train=None,
y_train=None,
X_valid=None,
y_valid=None,
X_test=None,
y_test=None,
sOutPath=None,
patchSize=0,
batchSize=None,
learningRate=None,
iEpochs=None,
dlart_handle=None,
dimExpandEnable=False):
print('Training CNN')
print('with lr = ' + str(learningRate) + ' , batchSize = ' +
str(batchSize))
# save names
_, sPath = os.path.splitdrive(sOutPath)
sPath, sFilename = os.path.split(sPath)
sFilename, sExt = os.path.splitext(sFilename)
model_name = sOutPath + os.sep + sFilename
weight_name = model_name + '_weights.h5'
model_json = model_name + '.json'
model_all = model_name + '_model.h5'
model_mat = model_name + '.mat'
if (os.path.isfile(model_mat)): # no training if output file exists
print('------- already trained -> go to next')
return
# create optimizer
if dlart_handle != None:
if dlart_handle.getOptimizer() == SGD_OPTIMIZER:
opti = keras.optimizers.SGD(
lr=learningRate,
momentum=dlart_handle.getMomentum(),
decay=dlart_handle.getWeightDecay(),
nesterov=dlart_handle.getNesterovEnabled())
elif dlart_handle.getOptimizer() == RMS_PROP_OPTIMIZER:
opti = keras.optimizers.RMSprop(
lr=learningRate, decay=dlart_handle.getWeightDecay())
elif dlart_handle.getOptimizer() == ADAGRAD_OPTIMIZER:
opti = keras.optimizers.Adagrad(
lr=learningRate,
epsilon=None,
decay=dlart_handle.getWeightDecay())
elif dlart_handle.getOptimizer() == ADADELTA_OPTIMIZER:
opti = keras.optimizers.Adadelta(
lr=learningRate,
rho=0.95,
epsilon=None,
decay=dlart_handle.getWeightDecay())
elif dlart_handle.getOptimizer() == ADAM_OPTIMIZER:
opti = keras.optimizers.Adam(lr=learningRate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=dlart_handle.getWeightDecay())
else:
raise ValueError("Unknown Optimizer!")
else:
opti = keras.optimizers.Adam(lr=learningRate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
cnn.summary()
# compile model
loss_func = 'categorical_crossentropy' # change loss function
cnn.compile(loss=loss_func, optimizer=opti, metrics=['accuracy'])
# callbacks
callback_earlyStopping = EarlyStopping(monitor='val_loss',
patience=25,
verbose=1)
# callback_tensorBoard = keras.callbacks.TensorBoard(log_dir=dlart_handle.getLearningOutputPath() + '/logs',
# histogram_freq=2,
# batch_size=batchSize,
# write_graph=True,
# write_grads=True,
# write_images=True,
# embeddings_freq=0,
# embeddings_layer_names=None,
# embeddings_metadata=None)
callbacks = [
callback_earlyStopping,
ModelCheckpoint(sOutPath + os.sep + 'checkpoints/checker.hdf5',
monitor='val_acc',
verbose=0,
period=5,
save_best_only=True),
LearningRateScheduler(schedule=step_decay, verbose=1),
LivePlotCallback(dlart_handle)
]
# callbacks.append(ReduceLROnPlateau(monitor='loss', factor=0.1, patience=5, min_lr=1e-4, verbose=1))
# data augmentation
if dlart_handle.getDataAugmentationEnabled():
# Initialize Image Generator
# all shifted and rotated images are filled with zero padded pixels
datagen = ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=dlart_handle.getZCA_Whitening(),
zca_epsilon=1e-6,
rotation_range=dlart_handle.getRotation(),
width_shift_range=dlart_handle.getWidthShift(),
height_shift_range=dlart_handle.getHeightShift(),
shear_range=0.,
zoom_range=dlart_handle.getZoom(),
channel_shift_range=0.,
fill_mode='constant',
cval=0.,
horizontal_flip=dlart_handle.getHorizontalFlip(),
vertical_flip=dlart_handle.getVerticalFlip(),
rescale=None,
histogram_equalization=dlart_handle.getHistogramEqualization(),
contrast_stretching=dlart_handle.getContrastStretching(),
adaptive_equalization=dlart_handle.getAdaptiveEqualization(),
preprocessing_function=None,
data_format=K.image_data_format())
datagen_val = ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=dlart_handle.getZCA_Whitening(),
zca_epsilon=1e-6,
rotation_range=0.,
width_shift_range=0.,
height_shift_range=0.,
shear_range=0.,
zoom_range=0.,
channel_shift_range=0.,
fill_mode='constant',
cval=0.,
horizontal_flip=False,
vertical_flip=False,
rescale=None,
histogram_equalization=dlart_handle.getHistogramEqualization(),
contrast_stretching=dlart_handle.getContrastStretching(),
adaptive_equalization=dlart_handle.getAdaptiveEqualization(),
preprocessing_function=None,
data_format=K.image_data_format())
datagen_test = ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=dlart_handle.getZCA_Whitening(),
zca_epsilon=1e-6,
rotation_range=0.,
width_shift_range=0.,
height_shift_range=0.,
shear_range=0.,
zoom_range=0.,
channel_shift_range=0.,
fill_mode='constant',
cval=0.,
horizontal_flip=False,
vertical_flip=False,
rescale=None,
histogram_equalization=dlart_handle.getHistogramEqualization(),
contrast_stretching=dlart_handle.getContrastStretching(),
adaptive_equalization=dlart_handle.getAdaptiveEqualization(),
preprocessing_function=None,
data_format=K.image_data_format())
# fit parameters from dataset
datagen.fit(X_train)
datagen_test.fit(X_test)
# configure batch size and get one batch of images
for x_batch, y_batch in datagen.flow(X_train, y_train, batch_size=9):
# display first 9 images
for i in range(0, 9):
plt.subplot(330 + 1 + i)
plt.imshow(x_batch[i].reshape(x_batch.shape[1],
x_batch.shape[2]),
cmap='gray')
plt.show()
break
if X_valid is not None and y_valid is not None:
# fit model on data
# use validation/test split
datagen_val.fit(X_valid)
result = cnn.fit_generator(
datagen.flow(X_train, y_train, batch_size=batchSize),
steps_per_epoch=X_train.shape[0] // batchSize,
epochs=iEpochs,
validation_data=datagen_val.flow(X_valid,
y_valid,
batch_size=batchSize),
callbacks=callbacks,
workers=1,
use_multiprocessing=False)
else:
# fit model on data
# use test data for validation and test
result = cnn.fit_generator(
datagen.flow(X_train, y_train, batch_size=batchSize),
steps_per_epoch=X_train.shape[0] // batchSize,
epochs=iEpochs,
validation_data=datagen_test.flow(X_test,
y_test,
batch_size=batchSize),
callbacks=callbacks,
workers=1,
use_multiprocessing=False)
# return the loss value and metrics values for the model in test mode
score_test, acc_test = cnn.evaluate_generator(
datagen_test.flow(X_test, y_test, batch_size=batchSize),
steps=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False)
prob_test = cnn.predict_generator(datagen_test.flow(
X_test, y_test, batch_size=batchSize),
steps=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
verbose=1)
else:
if X_valid is not None and y_valid is not None:
if dimExpandEnable:
X_valid = np.expand_dims(X_valid, axis=-1)
# use validation/test split
result = cnn.fit(X_train,
y_train,
validation_data=(X_valid, y_valid),
epochs=iEpochs,
batch_size=batchSize,
callbacks=callbacks,
verbose=1)
else:
if dimExpandEnable:
X_train = np.expand_dims(X_train, axis=-1)
X_test = np.expand_dims(X_test, axis=-1)
# use test set for validation and test
result = cnn.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=iEpochs,
batch_size=batchSize,
callbacks=callbacks,
verbose=1)
# return the loss value and metrics values for the model in test mode
score_test, acc_test = cnn.evaluate(X_test,
y_test,
batch_size=batchSize,
verbose=1)
prob_test = cnn.predict(X_test, batchSize, 0)
# save model
json_string = cnn.to_json()
with open(model_json, 'w') as jsonFile:
jsonFile.write(json_string)
# wei = cnn.get_weights()
cnn.save_weights(weight_name, overwrite=True)
cnn.save(model_all) # keras > v0.7
model_png_dir = sOutPath + os.sep + "model.png"
from keras.utils import plot_model
plot_model(cnn,
to_file=model_png_dir,
show_shapes=True,
show_layer_names=True)
# matlab
acc = result.history['acc']
loss = result.history['loss']
val_acc = result.history['val_acc']
val_loss = result.history['val_loss']
print('Saving results: ' + model_name)
sio.savemat(
model_name, {
'model_settings': model_json,
'model': model_all,
'weights': weight_name,
'acc': acc,
'loss': loss,
'val_acc': val_acc,
'val_loss': val_loss,
'score_test': score_test,
'acc_test': acc_test,
'prob_test': prob_test
})
def fTrainInner(cnn,
modelName,
X_train=None,
y_train=None,
Y_segMasks_train=None,
X_valid=None,
y_valid=None,
Y_segMasks_valid=None,
X_test=None,
y_test=None,
Y_segMasks_test=None,
sOutPath=None,
patchSize=0,
batchSize=None,
learningRate=None,
iEpochs=None,
dlart_handle=None,
usingClassification=False):
print('Training CNN')
print('with lr = ' + str(learningRate) + ' , batchSize = ' +
str(batchSize))
# sio.savemat('D:med_data/' + 'checkdata_voxel_and_mask.mat',
# {'mask_train': Y_segMasks_train,
# 'voxel_train': X_train,
# 'mask_test': Y_segMasks_test,
# 'voxel_test': X_test})
# save names
_, sPath = os.path.splitdrive(sOutPath)
sPath, sFilename = os.path.split(sPath)
sFilename, sExt = os.path.splitext(sFilename)
model_name = sOutPath + os.sep + sFilename
weight_name = model_name + '_weights.h5'
model_json = model_name + '.json'
model_all = model_name + '_model.h5'
model_mat = model_name + '.mat'
if (os.path.isfile(model_mat)): # no training if output file exists
print('------- already trained -> go to next')
return
# create optimizer
if dlart_handle != None:
if dlart_handle.getOptimizer() == SGD_OPTIMIZER:
opti = keras.optimizers.SGD(
lr=learningRate,
momentum=dlart_handle.getMomentum(),
decay=dlart_handle.getWeightDecay(),
nesterov=dlart_handle.getNesterovEnabled())
elif dlart_handle.getOptimizer() == RMS_PROP_OPTIMIZER:
opti = keras.optimizers.RMSprop(
lr=learningRate, decay=dlart_handle.getWeightDecay())
elif dlart_handle.getOptimizer() == ADAGRAD_OPTIMIZER:
opti = keras.optimizers.Adagrad(
lr=learningRate,
epsilon=None,
decay=dlart_handle.getWeightDecay())
elif dlart_handle.getOptimizer() == ADADELTA_OPTIMIZER:
opti = keras.optimizers.Adadelta(
lr=learningRate,
rho=0.95,
epsilon=None,
decay=dlart_handle.getWeightDecay())
elif dlart_handle.getOptimizer() == ADAM_OPTIMIZER:
opti = keras.optimizers.Adam(lr=learningRate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=dlart_handle.getWeightDecay())
else:
raise ValueError("Unknown Optimizer!")
else:
# opti = SGD(lr=learningRate, momentum=1e-8, decay=0.1, nesterov=True);#Adag(lr=0.01, epsilon=1e-06)
opti = keras.optimizers.Adam(lr=learningRate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
cnn.summary()
# compile model
if usingClassification:
cnn.compile(loss={
'segmentation_output': dice_coef_loss,
'classification_output': 'categorical_crossentropy'
},
optimizer=opti,
metrics={
'segmentation_output': dice_coef,
'classification_output': 'accuracy'
})
else:
cnn.compile(loss=dice_coef_loss, optimizer=opti, metrics=[dice_coef])
# callbacks
# callback_earlyStopping = EarlyStopping(monitor='val_loss', patience=12, verbose=1)
# callback_tensorBoard = keras.callbacks.TensorBoard(log_dir=dlart_handle.getLearningOutputPath() + '/logs',
# histogram_freq=2,
# batch_size=batchSize,
# write_graph=True,
# write_grads=True,
# write_images=True,
# embeddings_freq=0,
# embeddings_layer_names=None,
# embeddings_metadata=None)
from utils.LivePlotCallback import LivePlotCallback
callbacks = [
EarlyStopping(monitor='val_loss', patience=12, verbose=1),
ModelCheckpoint(sOutPath + os.sep + 'checkpoints' + os.sep +
'checker.hdf5',
monitor='val_acc',
verbose=0,
period=1,
save_best_only=True),
LearningRateScheduler(schedule=step_decay, verbose=1),
LivePlotCallback(dlart_handle)
]
# callbacks.append(ReduceLROnPlateau(monitor='loss', factor=0.1, patience=5, min_lr=1e-4, verbose=1))
if X_valid.size == 0 and y_valid.size == 0:
# using test set for validation
if usingClassification:
result = cnn.fit(X_train, {
'segmentation_output': Y_segMasks_train,
'classification_output': y_train
},
validation_data=(X_test, {
'segmentation_output': Y_segMasks_test,
'classification_output': y_test
}),
epochs=iEpochs,
batch_size=batchSize,
callbacks=callbacks,
verbose=1)
else:
result = cnn.fit(X_train,
Y_segMasks_train,
validation_data=(X_test, Y_segMasks_test),
epochs=iEpochs,
batch_size=batchSize,
callbacks=callbacks,
verbose=1)
else:
# using validation set for validation
if usingClassification:
result = cnn.fit(X_train, {
'segmentation_output': Y_segMasks_train,
'classification_output': y_train
},
validation_data=(X_valid, {
'segmentation_output': Y_segMasks_valid,
'classification_output': y_valid
}),
epochs=iEpochs,
batch_size=batchSize,
callbacks=callbacks,
verbose=1)
else:
result = cnn.fit(X_train,
Y_segMasks_train,
validation_data=(X_valid, Y_segMasks_valid),
epochs=iEpochs,
batch_size=batchSize,
callbacks=callbacks,
verbose=1)
# return the loss value and metrics values for the model in test mode
if usingClassification:
model_metrics = cnn.metrics_names
loss_test, segmentation_output_loss_test, classification_output_loss_test, segmentation_output_dice_coef_test, classification_output_acc_test \
= cnn.evaluate(X_test, {'segmentation_output': Y_segMasks_test, 'classification_output': y_test},
batch_size=batchSize, verbose=1)
else:
score_test, dice_coef_test = cnn.evaluate(X_test,
Y_segMasks_test,
batch_size=batchSize,
verbose=1)
prob_test = cnn.predict(X_test, batchSize, 0)
# save model
json_string = cnn.to_json()
with open(model_json, 'w') as jsonFile:
jsonFile.write(json_string)
# wei = cnn.get_weights()
cnn.save_weights(weight_name, overwrite=True)
cnn.save(model_all) # keras > v0.7
model_png_dir = sOutPath + os.sep + "model.png"
from keras.utils import plot_model
plot_model(cnn,
to_file=model_png_dir,
show_shapes=True,
show_layer_names=True)
if not usingClassification:
# matlab
dice_coef_training = result.history['dice_coef']
training_loss = result.history['loss']
val_dice_coef = result.history['val_dice_coef']
val_loss = result.history['val_loss']
print('Saving results: ' + model_name)
sio.savemat(
model_name, {
'model_settings': model_json,
'model': model_all,
'weights': weight_name,
'dice_coef': dice_coef_training,
'training_loss': training_loss,
'val_dice_coef': val_dice_coef,
'val_loss': val_loss,
'score_test': score_test,
'dice_coef_test': dice_coef_test,
'prob_test': prob_test
})
else:
# matlab
segmentation_output_loss_training = result.history[
'segmentation_output_loss']
classification_output_loss_training = result.history[
'classification_output_loss']
segmentation_output_dice_coef_training = result.history[
'segmentation_output_dice_coef']
classification_output_acc_training = result.history[
'classification_output_acc']
val_segmentation_output_loss = result.history[
'val_segmentation_output_loss']
val_classification_output_loss = result.history[
'val_classification_output_loss']
val_segmentation_output_dice_coef = result.history[
'val_segmentation_output_dice_coef']
val_classification_output_acc = result.history[
'val_classification_output_acc']
print('Saving results: ' + model_name)
sio.savemat(
model_name, {
'model_settings': model_json,
'model': model_all,
'weights': weight_name,
'segmentation_output_loss_training':
segmentation_output_loss_training,
'classification_output_loss_training':
classification_output_loss_training,
'segmentation_output_dice_coef_training':
segmentation_output_dice_coef_training,
'classification_output_acc_training':
classification_output_acc_training,
'segmentation_output_loss_val': val_segmentation_output_loss,
'classification_output_loss_val':
val_classification_output_loss,
'segmentation_output_dice_coef_val':
val_segmentation_output_dice_coef,
'classification_output_acc_val': val_classification_output_acc,
'loss_test': loss_test,
'segmentation_output_loss_test': segmentation_output_loss_test,
'classification_output_loss_test':
classification_output_loss_test,
'segmentation_output_dice_coef_test':
segmentation_output_dice_coef_test,
'classification_output_acc_test':
classification_output_acc_test,
'segmentation_predictions': prob_test[0],
'classification_predictions': prob_test[1]
})