def test_recalling(domain,
mem_size,
experiment,
occlusion=None,
bars_type=None,
tolerance=0):
n_memories = constants.n_labels
all_recalls = {}
all_entropies = {}
all_mprecision = {}
all_mrecall = {}
total_precisions = np.zeros(
(constants.training_stages, len(constants.memory_fills)))
total_recalls = np.zeros(
(constants.training_stages, len(constants.memory_fills)))
total_mismatches = np.zeros(
(constants.training_stages, len(constants.memory_fills)))
xlabels = constants.memory_fills
list_results = Parallel(n_jobs=constants.n_jobs, verbose=50)(
delayed(
test_recalling_fold)(n_memories, mem_size, domain, fold,
experiment, occlusion, bars_type, tolerance)
for fold in range(constants.training_stages))
for fold, stage_recalls, stage_entropies, stage_mprecision, stage_mrecall,\
total_precision, total_recall, mismatches in list_results:
all_recalls[fold] = stage_recalls
for msize in stage_entropies:
all_entropies[msize] = all_entropies[msize] + [stage_entropies[msize]] \
if msize in all_entropies.keys() else [stage_entropies[msize]]
all_mprecision[msize] = all_mprecision[msize] + [stage_mprecision[msize]] \
if msize in all_mprecision.keys() else [stage_mprecision[msize]]
all_mrecall[msize] = all_mrecall[msize] + [stage_mrecall[msize]] \
if msize in all_mrecall.keys() else [stage_mrecall[msize]]
total_precisions[fold] = total_precision
total_recalls[fold] = total_recall
total_mismatches[fold] = mismatches
for fold in all_recalls:
list_tups = all_recalls[fold]
tags = []
memories = []
for (idx, label, features) in list_tups:
tags.append((idx, label))
memories.append(np.array(features))
tags = np.array(tags)
memories = np.array(memories)
memories_filename = constants.memories_name(experiment, occlusion,
bars_type, tolerance)
memories_filename = constants.data_filename(memories_filename, fold)
np.save(memories_filename, memories)
tags_filename = constants.labels_name + constants.memory_suffix
tags_filename = constants.data_filename(tags_filename, fold)
np.save(tags_filename, tags)
main_avrge_entropies = get_means(all_entropies)
main_stdev_entropies = get_stdev(all_entropies)
main_avrge_mprecision = get_means(all_mprecision)
main_stdev_mprecision = get_stdev(all_mprecision)
main_avrge_mrecall = get_means(all_mrecall)
main_stdev_mrecall = get_stdev(all_mrecall)
np.savetxt(constants.csv_filename('main_average_precision', experiment,
occlusion, bars_type, tolerance),
main_avrge_mprecision,
delimiter=',')
np.savetxt(constants.csv_filename('main_average_recall', experiment,
occlusion, bars_type, tolerance),
main_avrge_mrecall,
delimiter=',')
np.savetxt(constants.csv_filename('main_average_entropy', experiment,
occlusion, bars_type, tolerance),
main_avrge_entropies,
delimiter=',')
np.savetxt(constants.csv_filename('main_stdev_precision', experiment,
occlusion, bars_type, tolerance),
main_stdev_mprecision,
delimiter=',')
np.savetxt(constants.csv_filename('main_stdev_recall', experiment,
occlusion, bars_type, tolerance),
main_stdev_mrecall,
delimiter=',')
np.savetxt(constants.csv_filename('main_stdev_entropy', experiment,
occlusion, bars_type, tolerance),
main_stdev_entropies,
delimiter=',')
np.savetxt(constants.csv_filename('main_total_recalls', experiment,
occlusion, bars_type, tolerance),
total_recalls,
delimiter=',')
np.savetxt(constants.csv_filename('main_total_mismatches', experiment,
occlusion, bars_type, tolerance),
total_mismatches,
delimiter=',')
plot_pre_graph(main_avrge_mprecision * 100,
main_avrge_mrecall * 100,
main_avrge_entropies,
main_stdev_mprecision * 100,
main_stdev_mrecall * 100,
main_stdev_entropies,
'recall-',
xlabels=xlabels,
xtitle=_('Percentage of memory corpus'),
action=experiment,
occlusion=occlusion,
bars_type=bars_type,
tolerance=tolerance)
plot_pre_graph(np.average(total_precisions, axis=0) * 100,
np.average(total_recalls, axis=0) * 100,
main_avrge_entropies,
np.std(total_precisions, axis=0) * 100,
np.std(total_recalls, axis=0) * 100,
main_stdev_entropies,
'total_recall-',
xlabels=xlabels,
xtitle=_('Percentage of memory corpus'),
action=experiment,
occlusion=occlusion,
bars_type=bars_type,
tolerance=tolerance)
print('Test completed')
def test_recalling(domain,
mem_size,
experiment,
occlusion=None,
bars_type=None,
tolerance=0):
n_memories = constants.n_labels
memory_fills = constants.memory_fills
training_stages = constants.training_stages
# All recalls, per memory fill and fold.
all_recalls = {}
# All entropies, precision, and recall, per fold, fill, and memory.
all_mfill_entropies = \
np.zeros((training_stages, len(memory_fills), n_memories))
all_mfill_precision = \
np.zeros((training_stages, len(memory_fills), n_memories))
all_mfill_recall = \
np.zeros((training_stages, len(memory_fills), n_memories))
# Store the matrix of stages x memory fills.
total_precisions = np.zeros((training_stages, len(memory_fills)))
total_recalls = np.zeros((training_stages, len(memory_fills)))
total_mismatches = np.zeros((training_stages, len(memory_fills)))
list_results = Parallel(n_jobs=constants.n_jobs, verbose=50)(
delayed(test_recalling_fold)(n_memories, mem_size, domain, fold, experiment, occlusion, bars_type, tolerance) \
for fold in range(constants.training_stages))
for fold, recalls, fill_mem_entropies, fill_mem_precision, fill_mem_recall,\
fold_precision, fold_recall, fold_mismatches in list_results:
all_recalls[fold] = recalls
total_precisions[fold] = fold_precision
total_recalls[fold] = fold_recall
total_mismatches[fold] = fold_mismatches
all_mfill_entropies[fold] = fill_mem_entropies
all_mfill_precision[fold] = fill_mem_precision
all_mfill_recall[fold] = fill_mem_recall
for fold in all_recalls:
list_tups = all_recalls[fold]
tags = []
memories = []
for (idx, label, features) in list_tups:
tags.append((idx, label))
memories.append(np.array(features))
tags = np.array(tags)
memories = np.array(memories)
memories_filename = constants.memories_name(experiment, occlusion,
bars_type, tolerance)
memories_filename = constants.data_filename(memories_filename, fold)
np.save(memories_filename, memories)
tags_filename = constants.labels_name + constants.memory_suffix
tags_filename = constants.data_filename(tags_filename, fold)
np.save(tags_filename, tags)
main_avrge_entropies = np.mean(all_mfill_entropies, axis=(0, 2))
main_stdev_entropies = np.std(all_mfill_entropies, axis=(0, 2))
main_avrge_mprecision = np.mean(all_mfill_precision, axis=(0, 2))
main_stdev_mprecision = np.std(all_mfill_precision, axis=(0, 2))
main_avrge_mrecall = np.mean(all_mfill_recall, axis=(0, 2))
main_stdev_mrecall = np.std(all_mfill_recall, axis=(0, 2))
np.savetxt(constants.csv_filename('main_average_precision',experiment, occlusion, bars_type, tolerance), \
main_avrge_mprecision, delimiter=',')
np.savetxt(constants.csv_filename('main_average_recall',experiment, occlusion, bars_type, tolerance), \
main_avrge_mrecall, delimiter=',')
np.savetxt(constants.csv_filename('main_average_entropy',experiment, occlusion, bars_type, tolerance), \
main_avrge_entropies, delimiter=',')
np.savetxt(constants.csv_filename('main_stdev_precision',experiment, occlusion, bars_type, tolerance), \
main_stdev_mprecision, delimiter=',')
np.savetxt(constants.csv_filename('main_stdev_recall',experiment, occlusion, bars_type, tolerance), \
main_stdev_mrecall, delimiter=',')
np.savetxt(constants.csv_filename('main_stdev_entropy',experiment, occlusion, bars_type, tolerance), \
main_stdev_entropies, delimiter=',')
np.savetxt(constants.csv_filename('main_total_recalls',experiment, occlusion, bars_type, tolerance), \
total_recalls, delimiter=',')
np.savetxt(constants.csv_filename('main_total_mismatches',experiment, occlusion, bars_type, tolerance), \
total_mismatches, delimiter=',')
plot_pre_graph(main_avrge_mprecision*100, main_avrge_mrecall*100, main_avrge_entropies,\
main_stdev_mprecision*100, main_stdev_mrecall*100, main_stdev_entropies, 'recall-', \
xlabels = constants.memory_fills, xtitle = _('Percentage of memory corpus'), action = experiment,
occlusion = occlusion, bars_type = bars_type, tolerance = tolerance)
plot_pre_graph(np.average(total_precisions, axis=0)*100, np.average(total_recalls, axis=0)*100, \
main_avrge_entropies, np.std(total_precisions, axis=0)*100, np.std(total_recalls, axis=0)*100, \
main_stdev_entropies, 'total_recall-', \
xlabels = constants.memory_fills, xtitle = _('Percentage of memory corpus'), action=experiment,
occlusion = occlusion, bars_type = bars_type, tolerance = tolerance)
print('Test completed')
def test_memories(domain, experiment):
average_entropy = []
stdev_entropy = []
average_precision = []
stdev_precision = []
average_recall = []
stdev_recall = []
all_precision = []
all_recall = []
no_response = []
no_correct_response = []
no_correct_chosen = []
correct_chosen = []
total_responses = []
labels_x_memory = constants.labels_per_memory[experiment]
n_memories = int(constants.n_labels / labels_x_memory)
for i in range(constants.training_stages):
gc.collect()
suffix = constants.filling_suffix
training_features_filename = constants.features_name(
experiment) + suffix
training_features_filename = constants.data_filename(
training_features_filename, i)
training_labels_filename = constants.labels_name + suffix
training_labels_filename = constants.data_filename(
training_labels_filename, i)
suffix = constants.testing_suffix
testing_features_filename = constants.features_name(
experiment) + suffix
testing_features_filename = constants.data_filename(
testing_features_filename, i)
testing_labels_filename = constants.labels_name + suffix
testing_labels_filename = constants.data_filename(
testing_labels_filename, i)
training_features = np.load(training_features_filename)
training_labels = np.load(training_labels_filename)
testing_features = np.load(testing_features_filename)
testing_labels = np.load(testing_labels_filename)
measures_per_size = np.zeros(
(len(constants.memory_sizes), n_memories, constants.n_measures),
dtype=np.float64)
# An entropy value per memory size and memory.
entropies = np.zeros((len(constants.memory_sizes), n_memories),
dtype=np.float64)
behaviours = np.zeros(
(len(constants.memory_sizes), constants.n_behaviours))
print('Train the different co-domain memories -- NxM: ', experiment,
' run: ', i)
# Processes running in parallel.
list_measures_entropies = Parallel(
n_jobs=constants.n_jobs,
verbose=50)(delayed(get_ams_results)(
midx, msize, domain, labels_x_memory, training_features,
testing_features, training_labels, testing_labels)
for midx, msize in enumerate(constants.memory_sizes))
for j, measures, entropy, behaviour in list_measures_entropies:
measures_per_size[j, :, :] = measures.T
entropies[j, :] = entropy
behaviours[j, :] = behaviour
##########################################################################################
# Calculate precision and recall
precision = np.zeros((len(constants.memory_sizes), n_memories + 2),
dtype=np.float64)
recall = np.zeros((len(constants.memory_sizes), n_memories + 2),
dtype=np.float64)
for j, s in enumerate(constants.memory_sizes):
precision[j, 0:n_memories] = measures_per_size[
j, :, constants.precision_idx]
precision[j, constants.mean_idx(n_memories)] = measures_per_size[
j, :, constants.precision_idx].mean()
precision[j, constants.std_idx(n_memories)] = measures_per_size[
j, :, constants.precision_idx].std()
recall[j, 0:n_memories] = measures_per_size[j, :,
constants.recall_idx]
recall[j, constants.mean_idx(n_memories)] = measures_per_size[
j, :, constants.recall_idx].mean()
recall[j, constants.std_idx(n_memories)] = measures_per_size[
j, :, constants.recall_idx].std()
###################################################################3##
# Measures by memory size
# Average entropy among al digits.
average_entropy.append(entropies.mean(axis=1))
stdev_entropy.append(entropies.std(axis=1))
# Average precision as percentage
average_precision.append(
precision[:, constants.mean_idx(n_memories)] * 100)
stdev_precision.append(precision[:, constants.std_idx(n_memories)] *
100)
# Average recall as percentage
average_recall.append(recall[:, constants.mean_idx(n_memories)] * 100)
stdev_recall.append(recall[:, constants.std_idx(n_memories)] * 100)
all_precision.append(behaviours[:, constants.precision_idx] * 100)
all_recall.append(behaviours[:, constants.recall_idx] * 100)
no_response.append(behaviours[:, constants.no_response_idx])
no_correct_response.append(
behaviours[:, constants.no_correct_response_idx])
no_correct_chosen.append(behaviours[:,
constants.no_correct_chosen_idx])
correct_chosen.append(behaviours[:, constants.correct_response_idx])
total_responses.append(behaviours[:, constants.mean_responses_idx])
average_precision = np.array(average_precision)
stdev_precision = np.array(stdev_precision)
main_average_precision = []
main_stdev_precision = []
average_recall = np.array(average_recall)
stdev_recall = np.array(stdev_recall)
main_average_recall = []
main_stdev_recall = []
all_precision = np.array(all_precision)
main_all_average_precision = []
main_all_stdev_precision = []
all_recall = np.array(all_recall)
main_all_average_recall = []
main_all_stdev_recall = []
average_entropy = np.array(average_entropy)
stdev_entropy = np.array(stdev_entropy)
main_average_entropy = []
main_stdev_entropy = []
no_response = np.array(no_response)
no_correct_response = np.array(no_correct_response)
no_correct_chosen = np.array(no_correct_chosen)
correct_chosen = np.array(correct_chosen)
total_responses = np.array(total_responses)
main_no_response = []
main_no_correct_response = []
main_no_correct_chosen = []
main_correct_chosen = []
main_total_responses = []
main_total_responses_stdev = []
for i in range(len(constants.memory_sizes)):
main_average_precision.append(average_precision[:, i].mean())
main_average_recall.append(average_recall[:, i].mean())
main_average_entropy.append(average_entropy[:, i].mean())
main_stdev_precision.append(stdev_precision[:, i].mean())
main_stdev_recall.append(stdev_recall[:, i].mean())
main_stdev_entropy.append(stdev_entropy[:, i].mean())
main_all_average_precision.append(all_precision[:, i].mean())
main_all_stdev_precision.append(all_precision[:, i].std())
main_all_average_recall.append(all_recall[:, i].mean())
main_all_stdev_recall.append(all_recall[:, i].std())
main_no_response.append(no_response[:, i].mean())
main_no_correct_response.append(no_correct_response[:, i].mean())
main_no_correct_chosen.append(no_correct_chosen[:, i].mean())
main_correct_chosen.append(correct_chosen[:, i].mean())
main_total_responses.append(total_responses[:, i].mean())
main_total_responses_stdev.append(total_responses[:, i].std())
main_behaviours = [
main_no_response, main_no_correct_response, main_no_correct_chosen,
main_correct_chosen, main_total_responses
]
np.savetxt(constants.csv_filename(
'main_average_precision--{0}'.format(experiment)),
main_average_precision,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_all_average_precision--{0}'.format(experiment)),
main_all_average_precision,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_average_recall--{0}'.format(experiment)),
main_average_recall,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_all_average_recall--{0}'.format(experiment)),
main_all_average_recall,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_average_entropy--{0}'.format(experiment)),
main_average_entropy,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_stdev_precision--{0}'.format(experiment)),
main_stdev_precision,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_all_stdev_precision--{0}'.format(experiment)),
main_all_stdev_precision,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_stdev_recall--{0}'.format(experiment)),
main_stdev_recall,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_all_stdev_recall--{0}'.format(experiment)),
main_all_stdev_recall,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_stdev_entropy--{0}'.format(experiment)),
main_stdev_entropy,
delimiter=',')
np.savetxt(constants.csv_filename(
'main_behaviours--{0}'.format(experiment)),
main_behaviours,
delimiter=',')
plot_pre_graph(main_average_precision,
main_average_recall,
main_average_entropy,
main_stdev_precision,
main_stdev_recall,
main_stdev_entropy,
action=experiment)
plot_pre_graph(main_all_average_precision,
main_all_average_recall,
main_average_entropy,
main_all_stdev_precision,
main_all_stdev_recall,
main_stdev_entropy,
'overall',
action=experiment)
plot_size_graph(main_total_responses,
main_total_responses_stdev,
action=experiment)
plot_behs_graph(main_no_response,
main_no_correct_response,
main_no_correct_chosen,
main_correct_chosen,
action=experiment)
print('Test complete')
counts_cols = np.zeros((labels.size, img_columns))
counts_rows = np.zeros((labels.size, img_rows))
for i in range(labels.size):
image = data[i]
counts_cols[i] = image.sum(axis=0)
counts_rows[i] = image.sum(axis=1)
means_cols = counts_cols.mean(axis=0)
stdev_cols = counts_cols.std(axis=0)
means_rows = counts_rows.mean(axis=0)
stdev_rows = counts_rows.std(axis=0)
pixels = 255 * means_cols.reshape(
(means_cols.size, 1)).dot(means_rows.reshape((1, means_rows.size)))
pixels = pixels.round().astype(np.uint8)
image_filename = constants.run_path + '/average_image.png'
png.from_array(pixels, 'L;8').save(image_filename)
labels = labels.reshape((labels.size, 1))
counts_cols = np.concatenate((labels, counts_cols), axis=1)
counts_rows = np.concatenate((labels, counts_rows), axis=1)
cols_filename = constants.csv_filename('counts_cols')
rows_filename = constants.csv_filename('counts_rows')
np.savetxt(cols_filename, counts_cols, delimiter=',')
np.savetxt(rows_filename, counts_rows, delimiter=',')