}
hist_dict[f'{model}_{depth_level}_{opt}'] = {
'train_acc': history.history['acc'],
'val_acc': history.history['val_acc'],
'train_loss': history.history['loss'],
'val_loss': history.history['val_loss'],
'epoch': np.arange(1,
len(history.history['acc']) + 1),
'model': model,
'depth': depth_level,
'optmizer': opt
}
print(f'{model}_{depth_level}_{opt} okay')
dict_counter += 1
reset_keras()
# print accuracy results
for k in acc_dict:
print(acc_dict[k])
# convert accuracy to dataframe
df_acc = pd.DataFrame.from_dict(
acc_dict,
orient='index',
)
# change column names to facilitate legends:
df_acc = df_acc.rename(columns={
"opt": "optmizer parameters",
"opt_mode": "optmizer"
})
def fine_tune(model_filename, model_name, depth_level, data_in, train_dir,
valid_dir, test_dir, image_dir, model_dir, epochs, opt):
"""
function that fine tunes a pre-trained cnn model using a small learning rate
or trains a model from scracth
:param model_filename: filename of hdf5 file (Keras model) to be loaded
:param model_name: name of the original cnn model - necessary for preprocessing
(currently only supports InceptionV3 and VGG19)
:param depth_level: one of [shallow, intermediate, deep]
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:model_dir: model output location
:epochs: number of epochs to train
:opt: string that maps to keras optmizer to be used for training
:return:
the entire trained model
the test set accuracy
the test set confusion matrix
also saves a plot of the training loss/accuracy
"""
########
# common trainig parameters:
########
batch_size = 16
loss = 'categorical_crossentropy'
# save the number of classes:
num_classes = len(os.listdir(train_dir))
opts = {
'SGD (1e-2)':
optimizers.SGD(lr=0.01, momentum=0.0, clipvalue=5.),
'SGD (1e-2) momentum 0.5':
optimizers.SGD(lr=0.01, momentum=0.5, clipvalue=5.),
'SGD (1e-2) momentum 0.9':
optimizers.SGD(lr=0.01, momentum=0.9, clipvalue=5.),
'SGD (1e-3)':
optimizers.SGD(lr=0.001, momentum=0.0, clipvalue=5.),
'SGD (1e-3) momentum 0.5':
optimizers.SGD(lr=0.001, momentum=0.5, clipvalue=5.),
'SGD (1e-3) momentum 0.9':
optimizers.SGD(lr=0.001, momentum=0.9, clipvalue=5.),
'RMSprop (1e-3) ':
optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0),
'Adam (1e-2)':
optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False),
'Adamax (2e-3)':
optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
}
opt = opts[opt]
if model_filename:
# user provided a filename of a model saved with weights
tag = 'fine_tune'
# load the model:
model = load_model(os.path.join(model_dir, model_filename))
else:
print('starting new model with random weights')
tag = 'randomly_initialized_weights'
# we start a model from scratch
base_model = load_part_model(model_name, depth_level, None)
top_model = Sequential()
top_model.add(
GlobalAveragePooling2D(input_shape=base_model.output_shape[1:4]))
top_model.add(Dense(512, activation='relu'))
top_model.add(Dropout(rate=0.5))
# the last layer depends on the number of classes
top_model.add(Dense(num_classes, activation='softmax'))
# set the entire model:
# build the network
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
# delete top model to release memory
del top_model
# make sure all layers are trainable
for layer in model.layers:
layer.trainable = True
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name))
# do the same thing for both training and validation:
train_generator = datagen.flow_from_directory(
train_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=True)
valid_generator = datagen.flow_from_directory(
valid_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
if valid_generator.num_classes != train_generator.num_classes:
print(
'Warning! Different number of classes in training and validation')
#################################
# generators set
#################################
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
print(f'model needs {get_model_memory_usage(batch_size, model)} Gb')
history = model.fit_generator(generator=train_generator,
validation_data=valid_generator,
shuffle=True,
steps_per_epoch=len(train_generator),
validation_steps=len(valid_generator),
epochs=epochs)
# plot and save the training history:
plot_history(history, model_name, depth_level, data_in, tag, image_dir)
# predict test values accuracy
generator = datagen.flow_from_directory(test_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# compute confusion matrix:
cfm = confusion_matrix(y_true=y_true, y_pred=y_pred)
cf_matrix(
y_true,
y_pred,
image_dir,
plot_name=f"conf_matrix_{data_in}_{model_name}_{depth_level}_{tag}.png",
dpi=1000)
# clear memory
reset_keras()
return acc, cfm, history
def fine_tune(model_filename, model_name, data_in, train_dir, valid_dir,
test_dir, image_dir, model_dir, epochs):
"""
function that fine tunes a pre-trained cnn model using a small learning rate
:param model_filename: filename of hdf5 file (Keras model) to be loaded
:param model_name: name of the original cnn model - necessary for preprocessing
(currently only supports InceptionV3 and VGG19)
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:model_dir: model output location
:epochs: number of epochs to train
:return:
the entire trained model
the test set accuracy
the test set confusion matrix
also saves a plot of the training loss/accuracy
"""
########
# common trainig parameters:
########
batch_size = 32
lrate = 5 * 1e-5
loss = 'categorical_crossentropy'
opt = SGD(lr=lrate, momentum=0.0, clipvalue=5.)
# load the model:
model = load_model(os.path.join(model_dir, model_filename))
# make sure all layers are trainable
for layer in model.layers:
layer.trainable = True
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name))
# do the same thing for both training and validation:
train_generator = datagen.flow_from_directory(
train_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=True)
valid_generator = datagen.flow_from_directory(
valid_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
if valid_generator.num_classes != train_generator.num_classes:
print(
'Warning! Different number of classes in training and validation')
#################################
# generators set
#################################
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
print(f'model needs {get_model_memory_usage(batch_size, model)} Gb')
history = model.fit_generator(generator=train_generator,
validation_data=valid_generator,
shuffle=True,
epochs=epochs)
# plot and save the training history:
plot_history(history, model_name, depth_level, data_in, 'fine_tune',
image_dir)
# predict test values accuracy
generator = datagen.flow_from_directory(test_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# compute confusion matrix:
cfm = confusion_matrix(y_true=y_true, y_pred=y_pred)
cf_matrix(
y_true,
y_pred,
image_dir,
plot_name=
f"conf_matrix_{data_in}_{model_name}_{depth_level}_fine_tune.png",
dpi=200)
# clear memory
reset_keras()
return acc, cfm
def train_frozen(model_name, weights, data_in, train_dir, valid_dir, test_dir,
image_dir, model_dir, epochs, opt):
"""
function that freezes a cnn model to extract features and train a small
classification NN on top of the extracted features.
:param model_name: name of the cnn model to be loaded (currently only
supports InceptionV3 and VGG19)
:param
:weights: one of ['imagenet', None]
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:model_dir: model output location
:epochs: number of epochs to train
:opt: string that maps to keras optimizer to be used for training
:return:
the test set accuracy
the test set [y_true, y_pred]
the training history
saves the model
"""
########
# common trainig parameters:
########
batch_size = BATCH_SIZE
loss = 'categorical_crossentropy'
opts = {
'SGD (1e-2)':
optimizers.SGD(lr=0.01, momentum=0.0, clipvalue=5.),
'SGD (1e-2) momentum 0.5':
optimizers.SGD(lr=0.01, momentum=0.5, clipvalue=5.),
'SGD (1e-2) momentum 0.9':
optimizers.SGD(lr=0.01, momentum=0.9, clipvalue=5.),
'SGD (1e-3)':
optimizers.SGD(lr=0.001, momentum=0.0, clipvalue=5.),
'SGD (1e-3) momentum 0.5':
optimizers.SGD(lr=0.001, momentum=0.5, clipvalue=5.),
'SGD (1e-3) momentum 0.9':
optimizers.SGD(lr=0.001, momentum=0.9, clipvalue=5.),
'RMSprop (1e-3) ':
optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0),
'Adam (1e-3)':
optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False),
'Adamax (2e-3)':
optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
}
opt = opts[opt]
# load the base model:
base_model = model_app(model_name, weights)
# freeze layers (layers will not be updated during the first training process)
for layer in base_model.layers:
layer.trainable = False
# save the number of classes:
num_classes = len(os.listdir(train_dir))
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name),
zoom_range=0.05,
channel_shift_range=0.5,
rotation_range=5,
horizontal_flip=True,
vertical_flip=True,
)
# do the same thing for both training and validation:
train_generator = datagen.flow_from_directory(
train_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=True)
valid_generator = datagen.flow_from_directory(
valid_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
if valid_generator.num_classes != train_generator.num_classes:
print(
'Warning! Different number of classes in training and validation')
#################################
# generators set
#################################
# create the top model:
top_model = Sequential()
top_model.add(
GlobalAveragePooling2D(input_shape=base_model.output_shape[1:4]))
top_model.add(Dense(512, activation='relu'))
top_model.add(Dropout(rate=0.5))
# the last layer depends on the number of classes
top_model.add(Dense(num_classes, activation='softmax'))
# set the entire model:
# build the network
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
#print(model.summary())
print(f'model needs {get_model_memory_usage(batch_size, model)} Gb')
history = model.fit_generator(generator=train_generator,
validation_data=valid_generator,
shuffle=True,
steps_per_epoch=len(train_generator),
validation_steps=len(valid_generator),
epochs=epochs)
# save model:
model.save(os.path.join(model_dir, f"{model_name}_{data_in}_frozen.hdf5"))
# predict test values accuracy
generator = datagen.flow_from_directory(
test_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# clear memory
reset_keras()
return acc, [y_true, y_pred], history
def train_frozen(model_name, depth_level, weights, data_in, train_dir,
valid_dir, test_dir, image_dir, model_dir, epochs):
"""
function that freezes a cnn model to extract features and train a small
classification NN on top of the extracted features.
:param model_name: name of the cnn model to be loaded (currently only
supports InceptionV3 and VGG19)
:param depth_level: one of [shallow, intermediate, deep]
:weights: one of ['imagenet', None]
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:model_dir: model output location
:epochs: number of epochs to train
:return:
the test set accuracy
the test set confusion matrix
also saves a plot of the training loss/accuracy
saves the model
saves a plot of the confusion matrix
"""
########
# common trainig parameters:
########
batch_size = 32
lrate = 1e-3
loss = 'categorical_crossentropy'
opt = SGD(lr=lrate, momentum=0.0, clipvalue=5.)
# load the base model:
base_model = load_part_model(model_name, depth_level, weights)
# freeze layers (layers will not be updated during the first training process)
for layer in base_model.layers:
layer.trainable = False
# save the number of classes:
num_classes = len(os.listdir(train_dir))
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name))
# do the same thing for both training and validation:
train_generator = datagen.flow_from_directory(
train_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=True)
valid_generator = datagen.flow_from_directory(
valid_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
if valid_generator.num_classes != train_generator.num_classes:
print(
'Warning! Different number of classes in training and validation')
#################################
# generators set
#################################
# create the top model:
top_model = Sequential()
top_model.add(
GlobalAveragePooling2D(input_shape=base_model.output_shape[1:4]))
top_model.add(Dense(512, activation='relu'))
top_model.add(Dropout(rate=0.5))
# the last layer depends on the number of classes
top_model.add(Dense(num_classes, activation='softmax'))
# set the entire model:
# build the network
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
#print(model.summary())
print(f'model needs {get_model_memory_usage(batch_size, model)} Gb')
history = model.fit_generator(generator=train_generator,
validation_data=valid_generator,
shuffle=True,
epochs=epochs)
# save model:
model.save(
os.path.join(model_dir,
f"{model_name}_{depth_level}_{data_in}_frozen.hdf5"))
# plot and save the training history:
plot_history(history, model_name, depth_level, data_in, 'feat_extr',
image_dir)
# predict test values accuracy
generator = datagen.flow_from_directory(
test_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# compute confusion matrix:
cfm = confusion_matrix(y_true=y_true, y_pred=y_pred)
# plot confusion matrix:
cf_matrix(
y_true,
y_pred,
image_dir,
plot_name=
f"conf_matrix_{data_in}_{model_name}_{depth_level}_feat_extr.png",
dpi=200)
# clear memory
reset_keras()
return acc, cfm