def main(adjustable, all_ranking, names, model):
confusion_matrices = []
ranking_matrices = []
gregor_matrices = []
for item in range(len(all_ranking)):
name = names[item]
this_ranking = all_ranking[item]
h5_dataset = dp.load_datasets_from_h5([name])
if adjustable.only_test:
full_predictions = only_test(model, h5_dataset, this_ranking)
else:
full_predictions = train_and_test(adjustable, name, this_ranking,
model, h5_dataset)
# putting it all together
final_testing_labels = [
int(this_ranking[item].strip().split(',')[-1])
for item in range(len(this_ranking))
]
final_testing_labels = keras.utils.to_categorical(
final_testing_labels, pc.NUM_CLASSES)
matrix = pu.make_confusion_matrix(adjustable, full_predictions,
final_testing_labels)
accuracy = (matrix[0] + matrix[2]) * 1.0 / (sum(matrix) * 1.0)
if not matrix[0] == 0:
precision = (matrix[0] * 1.0 / (matrix[0] + matrix[1] * 1.0))
else:
precision = 0
confusion_matrices.append(matrix)
gregor_matrix = pu.make_gregor_matrix(adjustable, full_predictions,
final_testing_labels)
gregor_matrices.append(gregor_matrix)
detection_rate, false_alarm = pu.calculate_TPR_FPR(matrix)
ranking = pu.calculate_CMC(adjustable, full_predictions)
ranking_matrices.append(ranking)
print(
'%s accuracy: %0.2f precision: %0.2f confusion matrix: %s \nCMC: \n%s \nDetection rate: %s False alarm: %s'
% (name, accuracy, precision, str(matrix), str(ranking),
str(detection_rate), str(false_alarm)))
return confusion_matrices, ranking_matrices, gregor_matrices
def main(adjustable, training_h5, testing_h5, all_ranking, merged_training_pos,
merged_training_neg):
"""
Runs a the whole training and testing phase
:param adjustable: object of class ProjectVariable
:param training_h5: list of h5py object(s) containing training datasets
:param testing_h5: list of h5py object containing test/rank dataset
:param all_ranking: list of ranking pair string paths to images
:param merged_training_pos: list of training pos pair string paths to images
:param merged_training_neg: list of training neg pair string paths to images
:return: array of dataset names, array containing the confusion matrix for each dataset, array containing the
ranking for each dataset
"""
############################################################################################################
# Set GPU
############################################################################################################
os.environ["CUDA_VISIBLE_DEVICES"] = adjustable.use_gpu
############################################################################################################
# Create model
############################################################################################################
model = get_model(adjustable)
############################################################################################################
# Training phase
############################################################################################################
if adjustable.only_test == False:
for epoch in range(adjustable.epochs):
print('Epoch %d/%d' % (epoch, adjustable.epochs))
############################################################################################################
# Prepare the training data
############################################################################################################
training_neg_sample = get_negative_sample(adjustable,
merged_training_pos,
merged_training_neg)
final_training_data = get_final_training_data(
adjustable, merged_training_pos, training_neg_sample)
final_training_labels = [
int(final_training_data[item].strip().split(',')[-1])
for item in range(len(final_training_data))
]
if adjustable.cost_module_type == 'neural_network' or adjustable.cost_module_type == 'euclidean_fc':
final_training_labels = keras.utils.to_categorical(
final_training_labels, pc.NUM_CLASSES)
############################################################################################################
# Train the network
############################################################################################################
train_network(adjustable, model, final_training_data,
final_training_labels, training_h5, testing_h5)
############################################################################################################
# Save the model + weights (if specified with adjustable.save_inbetween and adjustable.save_points)
############################################################################################################
time_stamp = time.strftime('scnn_%d%m%Y_%H%M')
# TODO: fix dataset name
# note: for now we will just specify the name of the saved file ourselves
if adjustable.save_inbetween and adjustable.iterations == 1:
if epoch + 1 in adjustable.save_points:
if adjustable.name_of_saved_file is not None:
model_name = '%s_epoch_%d_model.h5' % (
adjustable.name_of_saved_file, epoch + 1)
weights_name = '%s_epoch_%d_weights.h5' % (
adjustable.name_of_saved_file, epoch + 1)
else:
print('Error: name of file to be saved not specified.')
return
# if adjustable.name_indication == 'epoch':
# model_name = time_stamp + '_epoch_%s_model.h5' % str(epoch + 1)
# weights_name = time_stamp + '_epoch_%s_weigths.h5' % str(epoch + 1)
# elif adjustable.name_indication == 'dataset_name' and len(adjustable.datasets) == 1:
# model_name = '%s_model_%s.h5' % (adjustable.datasets[0], adjustable.use_gpu)
# weights_name = '%s_weigths_%s.h5' % (adjustable.datasets[0], adjustable.use_gpu)
# else:
# model_name = None
# weights_name = None
model.save(
os.path.join(pc.SAVE_LOCATION_MODEL_WEIGHTS,
model_name))
model.save_weights(
os.path.join(pc.SAVE_LOCATION_MODEL_WEIGHTS,
weights_name))
print('MODEL SAVED at epoch %d' % (epoch + 1))
############################################################################################################
# Testing phase
############################################################################################################
if all_ranking is not None:
this_ranking = all_ranking[-1]
del all_ranking
################################################################################################################
# Prepare the testing/ranking data
################################################################################################################
# test_data = dp.grab_em_by_the_keys(this_ranking, training_h5, testing_h5)
#
# # prepare for testing the model
# final_testing_labels = [int(this_ranking[item].strip().split(',')[-1]) for item in range(len(this_ranking))]
#
# if adjustable.cost_module_type == 'neural_network' or adjustable.cost_module_type == 'euclidean_fc':
# final_testing_labels = keras.utils.to_categorical(final_testing_labels, pc.NUM_CLASSES)
################################################################################################################
# Test
################################################################################################################
print('Testing...')
print('Testing...')
# here we have a bottle neck when using large ranking numbers. Because the list is big we will load a shitton
# of data at once.
# modified to make this lighter on the memory usage so we can use big ranking numbers.
# so instead of passing the list in one go, we split the list in manageable pieces.
max_list_size = 500
len_test_data = max(np.shape(this_ranking))
print('length list test data: ', len_test_data)
# partitions = len_test_data / max_list_size + 1
partitions = int(math.ceil(len_test_data / (max_list_size * 1.0)))
# FIXME: issue when euclidean and cosine.
if adjustable.cost_module_type in ['euclidean', 'cosine']:
total_predictions = np.zeros(len_test_data)
total_final_labels = np.zeros(len_test_data)
else:
total_predictions = np.zeros((len_test_data, 2))
total_final_labels = np.zeros((len_test_data, 2))
for part in range(partitions):
b = part * max_list_size
if (1 + part) * max_list_size > len_test_data:
e = len_test_data
else:
e = (1 + part) * max_list_size
ranking_partition = this_ranking[b:e]
test_data = dp.grab_em_by_the_keys(ranking_partition, training_h5,
testing_h5)
# prepare for testing the model
final_testing_labels = [
int(ranking_partition[item].strip().split(',')[-1])
for item in range(len(ranking_partition))
]
if adjustable.cost_module_type == 'neural_network' or adjustable.cost_module_type == 'euclidean_fc':
final_testing_labels = keras.utils.to_categorical(
final_testing_labels, pc.NUM_CLASSES)
predictions = model.predict([test_data[0, :], test_data[1, :]])
if adjustable.cost_module_type in ['euclidean', 'cosine']:
predictions = predictions.ravel()
total_predictions[b:e] = predictions
total_final_labels[b:e] = final_testing_labels
# if adjustable.cost_module_type in ['euclidean', 'cosine']:
# total_predictions.append(predictions)
# total_final_labels.append(final_testing_labels)
# else:
# total_predictions[b:e] = predictions
# total_final_labels[b:e] = final_testing_labels
predictions = total_predictions
final_testing_labels = total_final_labels
del total_predictions
del total_final_labels
################################################################################################################
# Process the results
################################################################################################################
print('Processing results...')
if adjustable.cost_module_type == 'euclidean' or adjustable.cost_module_type == 'cosine':
new_thing = zip(predictions, final_testing_labels)
print(new_thing[0:50])
# create confusion matrix
matrix = pu.make_confusion_matrix(adjustable, predictions,
final_testing_labels)
accuracy = (matrix[0] + matrix[2]) * 1.0 / (sum(matrix) * 1.0)
if not matrix[0] == 0:
precision = (matrix[0] * 1.0 / (matrix[0] + matrix[1] * 1.0))
else:
precision = 0
# [upon Gregor's request] create a 0.1 ratio version of the confusion matrix where for each positive instance
# there are 9 negative instances
gregor_matrix = pu.make_gregor_matrix(adjustable, predictions,
final_testing_labels)
print(gregor_matrix)
if (gregor_matrix[0] * 1.0 + gregor_matrix[3] * 1.0) == 0:
detection_rate = 0
else:
detection_rate = (
gregor_matrix[0] * 1.0 /
(gregor_matrix[0] * 1.0 + gregor_matrix[3] * 1.0))
if (gregor_matrix[1] * 1.0 + gregor_matrix[2] * 1.0) == 0:
false_alarm = 0
else:
false_alarm = (gregor_matrix[1] * 1.0 /
(gregor_matrix[1] * 1.0 + gregor_matrix[2] * 1.0))
# calculate the Cumulative Matching Characteristic
ranking = pu.calculate_CMC(adjustable, predictions)
print(
'%s accuracy: %0.2f precision: %0.2f confusion matrix: %s \nCMC: \n%s \nDetection rate: %s False alarm: %s'
% (adjustable.dataset_test, accuracy, precision, str(matrix),
str(ranking), str(detection_rate), str(false_alarm)))
else:
matrix = None
ranking = None
gregor_matrix = None
del model
return matrix, ranking, gregor_matrix
def main(adjustable, training_h5, testing_h5, all_ranking, merged_training_pos,
merged_training_neg):
"""
Runs a the whole training and testing phase
:param adjustable: object of class ProjectVariable
:param training_h5: list of h5py object(s) containing training datasets
:param testing_h5: list of h5py object containing test/rank dataset
:param all_ranking: list of ranking pair string paths to images
:param merged_training_pos: list of training pos pair string paths to images
:param merged_training_neg: list of training neg pair string paths to images
:return: array of dataset names, array containing the confusion matrix for each dataset, array containing the
ranking for each dataset
"""
############################################################################################################
# Set GPU
############################################################################################################
os.environ["CUDA_VISIBLE_DEVICES"] = adjustable.use_gpu
############################################################################################################
# Create model
############################################################################################################
model = get_model(adjustable)
############################################################################################################
# Training phase
############################################################################################################
if adjustable.only_test == False:
for epoch in range(adjustable.epochs):
print('Epoch %d/%d' % (epoch, adjustable.epochs))
############################################################################################################
# Prepare the training data
############################################################################################################
training_neg_sample = get_negative_sample(adjustable,
merged_training_pos,
merged_training_neg)
final_training_data = get_final_training_data(
adjustable, merged_training_pos, training_neg_sample)
final_training_labels = [
int(final_training_data[item].strip().split(',')[-1])
for item in range(len(final_training_data))
]
if adjustable.cost_module_type == 'neural_network' or adjustable.cost_module_type == 'euclidean_fc':
final_training_labels = keras.utils.to_categorical(
final_training_labels, pc.NUM_CLASSES)
############################################################################################################
# Train the network
############################################################################################################
train_network(adjustable, model, final_training_data,
final_training_labels, training_h5, testing_h5)
############################################################################################################
# Save the model + weights (if specified with adjustable.save_inbetween and adjustable.save_points)
############################################################################################################
time_stamp = time.strftime('scnn_%d%m%Y_%H%M')
# TODO: fix dataset name
# note: for now we will just specify the name of the saved file ourselves
if adjustable.save_inbetween and adjustable.iterations == 1:
if epoch + 1 in adjustable.save_points:
if adjustable.name_of_saved_file is not None:
# model_name = '%s_model.h5' % adjustable.name_of_saved_file
# weights_name = '%s_weights.h5' % adjustable.name_of_saved_file
model_name = '%s_epoch_%d_model.h5' % (
adjustable.name_of_saved_file, epoch + 1)
weights_name = '%s_epoch_%d_weights.h5' % (
adjustable.name_of_saved_file, epoch + 1)
else:
print('Error: name of file to be saved not specified.')
return
# if adjustable.name_indication == 'epoch':
# model_name = time_stamp + '_epoch_%s_model.h5' % str(epoch + 1)
# weights_name = time_stamp + '_epoch_%s_weigths.h5' % str(epoch + 1)
# elif adjustable.name_indication == 'dataset_name' and len(adjustable.datasets) == 1:
# model_name = '%s_model_%s.h5' % (adjustable.datasets[0], adjustable.use_gpu)
# weights_name = '%s_weigths_%s.h5' % (adjustable.datasets[0], adjustable.use_gpu)
# else:
# model_name = None
# weights_name = None
model.save(
os.path.join(pc.SAVE_LOCATION_MODEL_WEIGHTS,
model_name))
model.save_weights(
os.path.join(pc.SAVE_LOCATION_MODEL_WEIGHTS,
weights_name))
print('MODEL SAVED at epoch %d' % (epoch + 1))
############################################################################################################
# Testing phase
############################################################################################################
if all_ranking is not None:
this_ranking = all_ranking[-1]
del all_ranking
################################################################################################################
# Prepare the testing/ranking data
################################################################################################################
test_data = ddl.grab_em_by_the_keys(this_ranking, training_h5,
testing_h5)
# prepare for testing the model
final_testing_labels = [
int(this_ranking[item].strip().split(',')[-1])
for item in range(len(this_ranking))
]
if adjustable.cost_module_type == 'neural_network' or adjustable.cost_module_type == 'euclidean_fc':
final_testing_labels = keras.utils.to_categorical(
final_testing_labels, pc.NUM_CLASSES)
################################################################################################################
# Test
################################################################################################################
print('Testing...')
predictions = model.predict([test_data[0, :], test_data[1, :]])
################################################################################################################
# Process the results
################################################################################################################
print('Processing results...')
if adjustable.cost_module_type == 'euclidean' or adjustable.cost_module_type == 'cosine':
new_thing = zip(predictions, final_testing_labels)
print(new_thing[0:50])
# create confusion matrix
matrix = pu.make_confusion_matrix(adjustable, predictions,
final_testing_labels)
accuracy = (matrix[0] + matrix[2]) * 1.0 / (sum(matrix) * 1.0)
if not matrix[0] == 0:
precision = (matrix[0] * 1.0 / (matrix[0] + matrix[1] * 1.0))
else:
precision = 0
# [upon Gregor's request] create a 0.1 ratio version of the confusion matrix where for each positive instance
# there are 9 negative instances
gregor_matrix = pu.make_gregor_matrix(adjustable, predictions,
final_testing_labels)
print(gregor_matrix)
if (gregor_matrix[0] * 1.0 + gregor_matrix[3] * 1.0) == 0:
detection_rate = 0
else:
detection_rate = (
gregor_matrix[0] * 1.0 /
(gregor_matrix[0] * 1.0 + gregor_matrix[3] * 1.0))
if (gregor_matrix[1] * 1.0 + gregor_matrix[2] * 1.0) == 0:
false_alarm = 0
else:
false_alarm = (gregor_matrix[1] * 1.0 /
(gregor_matrix[1] * 1.0 + gregor_matrix[2] * 1.0))
# calculate the Cumulative Matching Characteristic
ranking = pu.calculate_CMC(adjustable, predictions)
print(
'%s accuracy: %0.2f precision: %0.2f confusion matrix: %s \nCMC: \n%s \nDetection rate: %s False alarm: %s'
% (adjustable.dataset_test, accuracy, precision, str(matrix),
str(ranking), str(detection_rate), str(false_alarm)))
else:
matrix = None
ranking = None
gregor_matrix = None
del model
return matrix, ranking, gregor_matrix