def train_experiment():
normal_train, normal_test = get_data(data_cnt=1200,
split_ratio=0.5,
file_name='roomselection_1800.csv')
vae, summ, saver = make_model(encoded_size=3,
input_shape=5,
dense_layers_dims=[])
history = vae_fit(vae,
summ,
saver,
normal_train,
normal_test,
10000,
200,
tag='5x3_bern_b')
plts.plot_history(12, 4, history)
# saver.restore(sess, LOGDIR + '/headless_train.ckpt')
# output of TFP layer should be a distribution object
history = vae.fit(
x=normal_train,
y=normal_train,
epochs=10000,
batch_size=normal_train.shape[0],
steps_per_epoch=200, # 3*400 = 1200
validation_data=(normal_test, normal_test),
validation_steps=1,
callbacks=[tensorboard_callback])
#, distribute=strategy)
plts.plot_history(12, 4, history)
save_path = saver.save(sess, LOGDIR + '/headless_train.ckpt')
vae.summary()
# # with tf.Session() as sess:
# # sess.run(init)
# '''
# Sample
# '''
# # We'll just examine ten random digits.
#
# # assert tf.executing_eagerly()
# # x = np.random.uniform(size=(333, input_shape))
#
# # z = vae.predict(x=x)
def main(_):
print('Configuration = (PP: {}, R: {}, C: {}, B: {}, RT: {})'.format(
FLAGS.preprocess,
FLAGS.regenerate,
FLAGS.clahe,
FLAGS.blur,
FLAGS.random_transform
))
data_loader = DataLoader(TRAIN_FILE, LOG_FILE, IMG_DIR,
angle_correction = FLAGS.angle_correction,
mirror_min_angle = FLAGS.mirror_min_angle,
normalize_factor = FLAGS.normalize_factor,
normalize_bins = FLAGS.normalize_bins)
images, measurements = data_loader.load_dataset(regenerate = FLAGS.regenerate)
print('Total samples: {}'.format(images.shape[0]))
# Split in training and validation
X_train, X_valid, Y_train, Y_valid = data_loader.split_train_test(images, measurements)
print('Training samples: {}'.format(X_train.shape[0]))
print('Validation samples: {}'.format(X_valid.shape[0]))
plots.plot_distribution(Y_train[:,0], 'Training set distribution', save_path = os.path.join('images', 'train_distribution'))
plots.plot_distribution(Y_valid[:,0], 'Validation set distribution', save_path = os.path.join('images', 'valid_distribution'))
train_generator = data_loader.generator(X_train, Y_train, FLAGS.batch_size,
preprocess = FLAGS.preprocess,
random_transform = FLAGS.random_transform)
valid_generator = data_loader.generator(X_valid, Y_valid, FLAGS.batch_size,
preprocess = FLAGS.preprocess,
random_transform = False)
# The image processor gives us the input shape for the model (e.g. after cropping and resizing)
model = build_model(ip.output_shape())
print(model.summary())
model.compile(optimizer = Adam(lr = FLAGS.learning_rate), loss = FLAGS.loss)
date_time_str = time.strftime('%Y%m%d-%H%M%S')
callbacks = [
# To be used with tensorboard, creates the logs for the losses in the logs dir
TensorBoard(log_dir = os.path.join(LOGS_DIR, date_time_str),
histogram_freq = 0,
write_graph = False,
write_images = False),
# Early stopping guard
EarlyStopping(monitor='val_loss',
patience = 3,
verbose = 0,
mode = 'min')
]
model_name = 'model_{}'.format(date_time_str)
print('Training {} on {} samples (EP: {}, BS: {}, LR: {}, DO: {}, BN: {}, A: {}, L: {})...'.format(
model_name, X_train.shape[0], FLAGS.epochs, FLAGS.batch_size, FLAGS.learning_rate, FLAGS.dropout,
'{}'.format(FLAGS.batch_norm if FLAGS.batch_norm > 0 else 'OFF'),
FLAGS.activation, FLAGS.loss
))
# Train the model
history = model.fit_generator(train_generator,
nb_epoch = FLAGS.epochs,
samples_per_epoch = X_train.shape[0],
validation_data = valid_generator,
nb_val_samples = X_valid.shape[0],
callbacks = callbacks)
model.save(os.path.join(MODELS_DIR, model_name + '.h5'))
plots.plot_history(model_name, history)
def fine_tune(model_filename, model_name, depth_level, data_in, train_dir,
valid_dir, test_dir, image_dir, model_dir, epochs, opt):
"""
function that fine tunes a pre-trained cnn model using a small learning rate
or trains a model from scracth
:param model_filename: filename of hdf5 file (Keras model) to be loaded
:param model_name: name of the original cnn model - necessary for preprocessing
(currently only supports InceptionV3 and VGG19)
:param depth_level: one of [shallow, intermediate, deep]
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:model_dir: model output location
:epochs: number of epochs to train
:opt: string that maps to keras optmizer to be used for training
:return:
the entire trained model
the test set accuracy
the test set confusion matrix
also saves a plot of the training loss/accuracy
"""
########
# common trainig parameters:
########
batch_size = 16
loss = 'categorical_crossentropy'
# save the number of classes:
num_classes = len(os.listdir(train_dir))
opts = {
'SGD (1e-2)':
optimizers.SGD(lr=0.01, momentum=0.0, clipvalue=5.),
'SGD (1e-2) momentum 0.5':
optimizers.SGD(lr=0.01, momentum=0.5, clipvalue=5.),
'SGD (1e-2) momentum 0.9':
optimizers.SGD(lr=0.01, momentum=0.9, clipvalue=5.),
'SGD (1e-3)':
optimizers.SGD(lr=0.001, momentum=0.0, clipvalue=5.),
'SGD (1e-3) momentum 0.5':
optimizers.SGD(lr=0.001, momentum=0.5, clipvalue=5.),
'SGD (1e-3) momentum 0.9':
optimizers.SGD(lr=0.001, momentum=0.9, clipvalue=5.),
'RMSprop (1e-3) ':
optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0),
'Adam (1e-2)':
optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False),
'Adamax (2e-3)':
optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
}
opt = opts[opt]
if model_filename:
# user provided a filename of a model saved with weights
tag = 'fine_tune'
# load the model:
model = load_model(os.path.join(model_dir, model_filename))
else:
print('starting new model with random weights')
tag = 'randomly_initialized_weights'
# we start a model from scratch
base_model = load_part_model(model_name, depth_level, None)
top_model = Sequential()
top_model.add(
GlobalAveragePooling2D(input_shape=base_model.output_shape[1:4]))
top_model.add(Dense(512, activation='relu'))
top_model.add(Dropout(rate=0.5))
# the last layer depends on the number of classes
top_model.add(Dense(num_classes, activation='softmax'))
# set the entire model:
# build the network
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
# delete top model to release memory
del top_model
# make sure all layers are trainable
for layer in model.layers:
layer.trainable = True
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name))
# do the same thing for both training and validation:
train_generator = datagen.flow_from_directory(
train_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=True)
valid_generator = datagen.flow_from_directory(
valid_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
if valid_generator.num_classes != train_generator.num_classes:
print(
'Warning! Different number of classes in training and validation')
#################################
# generators set
#################################
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
print(f'model needs {get_model_memory_usage(batch_size, model)} Gb')
history = model.fit_generator(generator=train_generator,
validation_data=valid_generator,
shuffle=True,
steps_per_epoch=len(train_generator),
validation_steps=len(valid_generator),
epochs=epochs)
# plot and save the training history:
plot_history(history, model_name, depth_level, data_in, tag, image_dir)
# predict test values accuracy
generator = datagen.flow_from_directory(test_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# compute confusion matrix:
cfm = confusion_matrix(y_true=y_true, y_pred=y_pred)
cf_matrix(
y_true,
y_pred,
image_dir,
plot_name=f"conf_matrix_{data_in}_{model_name}_{depth_level}_{tag}.png",
dpi=1000)
# clear memory
reset_keras()
return acc, cfm, history
# now, fine tune the model for the "second half" of the
# epochs:
model_filename = f"{model}_{depth_level}_{data_in}_frozen.hdf5"
acc, _, h2 = fine_tune(model_filename, model, depth_level,
data_in, train_dir, valid_dir, test_dir,
image_dir, model_dir, epochs // 2, opt)
for metr in h2.history:
print(metr)
for jj in h2.history[metr]:
hist.history[metr].append(jj)
# plot the history:
plot_history(hist, model, depth_level, data_in, 'fine_tune',
image_dir)
print(f'{acc:.2f}')
acc_dict[dict_counter] = {
"model": model,
"depth": depth_level,
"depth_sw":
depths_dict[depth_level] + data_mod_dict[data_in],
"dataset": data_in,
"mode": 'fine tune',
"accuracy": acc
}
dict_counter += 1
reset_keras()
###############################################################
# random initialization:
"model": model,
"dataset": data_in,
"mode": 'feature extraction',
"accuracy": acc
}
dict_counter += 1
reset_keras()
# plot the confusion matrix
cf_matrix(y_true,
y_pred,
image_dir,
plot_name=f"conf_matrix_{data_in}_{model}_feat_extract",
title=f'Confusion Matrix for {model} - feature extract')
# plot the history:
plot_history(hist, model, data_in, 'feat_extract', image_dir,
f'{model} \nfeature extract')
###############################################################
# fine tune:
# to mantain the same overall epochs used in training,
# first fine tune using half of the total epochs (save the history):
_, _, hist = train_frozen(model, weights, data_in, train_dir,
valid_dir, test_dir, image_dir,
model_dir, epochs // 2, opt)
# now, fine tune the model for the "second half" of the
# epochs:
model_filename = f"{model}_{data_in}_frozen.hdf5"
acc, [y_true,
y_pred], h2 = fine_tune(model_filename, model, data_in,
# init datagenerator
train_generator = DataGen(data_path=patchespath, n_patches = n_patches_train,
shuffle=True, augment=True, indices=index_train ,
batch_size=batch_size, patch_size=patch_size_padded,
n_classes=n_classes, channels=channels, max_size=max_size,pretrained_resnet50=pretrained_resnet50)
val_generator = DataGen(data_path = patchespath, n_patches = n_patches_val,
shuffle=True, augment=False, indices=index_val ,
batch_size=batch_size, patch_size=patch_size_padded,
n_classes=n_classes, channels=channels, max_size=max_size,pretrained_resnet50=pretrained_resnet50)
# run
result = model.fit_generator(generator=train_generator, validation_data=val_generator,
epochs=epochs,callbacks=[checkpoint,tensorboard, stop])
# plot training history
plot_history(result)
#%% Test
"""
Test the keras model on independent test set
"""
from dataset import load_test_indices
from datagenerator import DataGen
from tensorflow.keras.models import load_model
from models.BilinearUpSampling import BilinearUpSampling2D
# init
batch_size = 125
# load model
def fine_tune(model_filename, model_name, data_in, train_dir, valid_dir,
test_dir, image_dir, model_dir, epochs):
"""
function that fine tunes a pre-trained cnn model using a small learning rate
:param model_filename: filename of hdf5 file (Keras model) to be loaded
:param model_name: name of the original cnn model - necessary for preprocessing
(currently only supports InceptionV3 and VGG19)
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:model_dir: model output location
:epochs: number of epochs to train
:return:
the entire trained model
the test set accuracy
the test set confusion matrix
also saves a plot of the training loss/accuracy
"""
########
# common trainig parameters:
########
batch_size = 32
lrate = 5 * 1e-5
loss = 'categorical_crossentropy'
opt = SGD(lr=lrate, momentum=0.0, clipvalue=5.)
# load the model:
model = load_model(os.path.join(model_dir, model_filename))
# make sure all layers are trainable
for layer in model.layers:
layer.trainable = True
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name))
# do the same thing for both training and validation:
train_generator = datagen.flow_from_directory(
train_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=True)
valid_generator = datagen.flow_from_directory(
valid_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
if valid_generator.num_classes != train_generator.num_classes:
print(
'Warning! Different number of classes in training and validation')
#################################
# generators set
#################################
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
print(f'model needs {get_model_memory_usage(batch_size, model)} Gb')
history = model.fit_generator(generator=train_generator,
validation_data=valid_generator,
shuffle=True,
epochs=epochs)
# plot and save the training history:
plot_history(history, model_name, depth_level, data_in, 'fine_tune',
image_dir)
# predict test values accuracy
generator = datagen.flow_from_directory(test_dir,
target_size=model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# compute confusion matrix:
cfm = confusion_matrix(y_true=y_true, y_pred=y_pred)
cf_matrix(
y_true,
y_pred,
image_dir,
plot_name=
f"conf_matrix_{data_in}_{model_name}_{depth_level}_fine_tune.png",
dpi=200)
# clear memory
reset_keras()
return acc, cfm
def train_frozen(model_name, depth_level, weights, data_in, train_dir,
valid_dir, test_dir, image_dir, model_dir, epochs):
"""
function that freezes a cnn model to extract features and train a small
classification NN on top of the extracted features.
:param model_name: name of the cnn model to be loaded (currently only
supports InceptionV3 and VGG19)
:param depth_level: one of [shallow, intermediate, deep]
:weights: one of ['imagenet', None]
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:model_dir: model output location
:epochs: number of epochs to train
:return:
the test set accuracy
the test set confusion matrix
also saves a plot of the training loss/accuracy
saves the model
saves a plot of the confusion matrix
"""
########
# common trainig parameters:
########
batch_size = 32
lrate = 1e-3
loss = 'categorical_crossentropy'
opt = SGD(lr=lrate, momentum=0.0, clipvalue=5.)
# load the base model:
base_model = load_part_model(model_name, depth_level, weights)
# freeze layers (layers will not be updated during the first training process)
for layer in base_model.layers:
layer.trainable = False
# save the number of classes:
num_classes = len(os.listdir(train_dir))
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name))
# do the same thing for both training and validation:
train_generator = datagen.flow_from_directory(
train_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=True)
valid_generator = datagen.flow_from_directory(
valid_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
if valid_generator.num_classes != train_generator.num_classes:
print(
'Warning! Different number of classes in training and validation')
#################################
# generators set
#################################
# create the top model:
top_model = Sequential()
top_model.add(
GlobalAveragePooling2D(input_shape=base_model.output_shape[1:4]))
top_model.add(Dense(512, activation='relu'))
top_model.add(Dropout(rate=0.5))
# the last layer depends on the number of classes
top_model.add(Dense(num_classes, activation='softmax'))
# set the entire model:
# build the network
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
#print(model.summary())
print(f'model needs {get_model_memory_usage(batch_size, model)} Gb')
history = model.fit_generator(generator=train_generator,
validation_data=valid_generator,
shuffle=True,
epochs=epochs)
# save model:
model.save(
os.path.join(model_dir,
f"{model_name}_{depth_level}_{data_in}_frozen.hdf5"))
# plot and save the training history:
plot_history(history, model_name, depth_level, data_in, 'feat_extr',
image_dir)
# predict test values accuracy
generator = datagen.flow_from_directory(
test_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# compute confusion matrix:
cfm = confusion_matrix(y_true=y_true, y_pred=y_pred)
# plot confusion matrix:
cf_matrix(
y_true,
y_pred,
image_dir,
plot_name=
f"conf_matrix_{data_in}_{model_name}_{depth_level}_feat_extr.png",
dpi=200)
# clear memory
reset_keras()
return acc, cfm
def feature_extract_fit(model_name, depth_level, weights, data_in, train_dir,
valid_dir, test_dir, image_dir):
"""
function that uses a cnn model to extract features and train a small
classification NN on top of the extracted features.
:param model_name: name of the cnn model to be loaded (currently only
supports InceptionV3 and VGG19)
:param depth_level: one of [shallow, intermediate, deep]
:weights: one of ['imagenet', None]
:data_in: a tag to keep track of input data used
:train_dir: training folder
:valid_dir: validation folder
:test_dir: test folder
:image_dir: image output location
:return:
the entire trained model
the test set accuracy
the test set confusion matrix
also saves a plot of the training loss/accuracy
"""
########
# common trainig parameters:
########
batch_size = 8
epochs = 8
lrate = 1e-4
loss = 'categorical_crossentropy'
opt = SGD(lr=lrate, momentum=0.0, clipvalue=5.)
train_val_dict = {'train': train_dir, 'validation': valid_dir}
# load the base model:
base_model = load_part_model(model_name, depth_level, weights)
# save the number of classes:
num_classes = len(os.listdir(train_dir))
#################################
# Features can be too large to fit in memory.
# Save the features as npy files (x, y in the same npy file)
#################################
# set the generator
datagen = ImageDataGenerator(
preprocessing_function=model_preprocess(model_name))
# dictionary that will hold the folder names - to be populated inside loop
feat_train_val_dict = {}
# do the same thing for both training and validation:
for dset in tqdm(train_val_dict):
# extract training features, shuffling now as (x,y) will be combined in the
# same file. Each file contains a single image feature (X) and its
# label (y) will be part of the name
generator = datagen.flow_from_directory(
train_val_dict[dset],
target_size=base_model.input_shape[1:3],
batch_size=1,
class_mode='sparse',
shuffle=False)
if num_classes != generator.num_classes:
print(
'Warning! Different number of classes in training and validation'
)
# create output folder
folder_out = os.path.join(os.path.dirname(train_val_dict[dset]),
f"feat_{dset}")
rmtree(folder_out, ignore_errors=True)
os.mkdir(folder_out)
# save location into dictionary:
feat_train_val_dict[dset] = folder_out
# loop through all the samples, saving them in the appropriate folder
sample = 0
for X_set, y_set in generator:
X_feat = base_model.predict(X_set)
np.save(
os.path.join(folder_out,
f'sample_{sample}_label_{np.int(y_set[0])}.npy'),
X_feat)
sample += 1
if sample == len(generator):
break
# generator parameters
params = {
'dim': base_model.output_shape[1:3],
'batch_size': batch_size,
'n_classes': num_classes,
'n_channels': base_model.output_shape[3]
}
# Generators
train_generator = feats_generator(data_dir=feat_train_val_dict['train'],
shuffle=True,
**params)
validation_generator = feats_generator(
data_dir=feat_train_val_dict['validation'], shuffle=False, **params)
#################################
# Features saved, generators set
#################################
# create the top model:
top_model = Sequential()
top_model.add(
GlobalAveragePooling2D(input_shape=base_model.output_shape[1:4]))
#top_model.add(Conv2D(512))
#top_model.add(GlobalAveragePooling2D())
top_model.add(Dense(512, activation='relu'))
top_model.add(Dropout(rate=0.5))
# the last layer depends on the number of classes
top_model.add(Dense(num_classes, activation='softmax'))
# train model with parameters specified at the top of the function
top_model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
history = top_model.fit_generator(generator=train_generator,
validation_data=validation_generator,
epochs=epochs)
# plot and save the training history:
plot_history(history, model_name, depth_level, data_in, 'feat_extr',
image_dir)
# set the entire model:
# build the network
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])
# predict test values accuracy
generator = datagen.flow_from_directory(
test_dir,
target_size=base_model.input_shape[1:3],
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
pred = model.predict_generator(generator, verbose=0, steps=len(generator))
# save results as dataframe
df = pd.DataFrame(pred, columns=generator.class_indices.keys())
df['file'] = generator.filenames
df['true_label'] = df['file'].apply(os.path.dirname).apply(str.lower)
df['pred_idx'] = np.argmax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
# save as the label (dictionary comprehension because generator.class_indices has the
# key,values inverted to what we want
df['pred_label'] = df['pred_idx'].map(
{value: key
for key, value in generator.class_indices.items()}).apply(str.lower)
# save the maximum probability for easier reference:
df['max_prob'] = np.amax(df[generator.class_indices.keys()].to_numpy(),
axis=1)
y_true = df['true_label']
y_pred = df['pred_label']
# compute accuracy:
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
# compute confusion matrix:
cfm = confusion_matrix(y_true=y_true, y_pred=y_pred)
# clear Keras/backend session (freeing memory and preventing slowdown)
K.clear_session()
return top_model, acc, cfm