def get_endd_measures(n_models_base_names, n_models_list, endd_base_model,
dataset_name, test_images, test_labels):
endd_measures_list = []
for base_name in n_models_base_names:
endd_measures = defaultdict(list)
for n_models in n_models_list:
#print("{}/{}".format(base_name, n_models))
#print(n_models)
endd_model_name = base_name + '_N_MODELS={}'.format(n_models)
print(endd_model_name)
uncompiled_model = saveload.load_tf_model(endd_model_name,
compile=False)
endd_model = endd.get_model(uncompiled_model,
dataset_name=dataset_name,
compile=True)
evaluation_result = evaluation.calc_classification_measures(
endd_model,
test_images,
test_labels,
wrapper_type='individual')
#print("############# Measures")
for measure, value in evaluation_result.items():
endd_measures[measure].append(value)
#print("{}={}".format(measure, value))
#print()
endd_measures_list.append(endd_measures)
return endd_measures_list
def get_ensm_measures(model_names, n_models_list, test_images, test_labels):
ensm_measures = defaultdict(list)
for n_models in n_models_list:
print("############ ensm {}".format(n_models))
model_name_subset = model_names[:n_models]
print(model_name_subset)
wrapped_models = [
ensemble.KerasLoadsWhole(name, pop_last=True)
for name in model_name_subset
]
ensm_model = ensemble.Ensemble(wrapped_models)
evaluation_result = evaluation.calc_classification_measures(
ensm_model, test_images, test_labels, wrapper_type='ensemble')
for measure, value in evaluation_result.items():
ensm_measures[measure].append(value)
return ensm_measures
DATASET_NAME = 'cifar10'
# Load ensemble model
ensemble_model_names = saveload.get_ensemble_model_names()
model_names = ensemble_model_names[ENSEMBLE_NAME][DATASET_NAME][:3]
models = [ensemble.KerasLoadsWhole(name) for name in model_names]
ensm = ensemble.Ensemble(models)
ensm_wrapper_type = 'ensemble'
# Load individual model
ind = saveload.load_tf_model(model_names[0])
ind_wrapper_type = 'individual'
# Load data
_, (test_images, test_labels) = datasets.get_dataset(DATASET_NAME)
# Preprocess data
test_labels = test_labels.reshape(-1)
# Calculate measures
ensm_measures = evaluation.calc_classification_measures(
ensm, test_images, test_labels, wrapper_type=ensm_wrapper_type)
ind_measures = evaluation.calc_classification_measures(
ind, test_images, test_labels, wrapper_type=ind_wrapper_type)
# Format and print results
summary = evaluation.format_results(['ENSM', 'IND'],
[ensm_measures, ind_measures])
print(summary)
# test_images,
# test_labels,
# wrapper_type=ensm_wrapper_type)
# print("Evaluating ENDD...")
# endd_measures = evaluation.calc_classification_measures(endd_model,
# test_images,
# test_labels,
# wrapper_type=endd_wrapper_type)
# print("Evaluating ENDD+AUX...")
# endd_aux_measures = evaluation.calc_classification_measures(endd_aux_model,
# test_images,
# test_labels,
# wrapper_type=endd_aux_wrapper_type)
print("Evaluating PN+AUX...")
pn_aux_measures = evaluation.calc_classification_measures(pn_aux_model,
test_images,
test_labels,
wrapper_type=pn_aux_wrapper_type)
print("Evaluations complete.")
# # Format and print results
# summary = evaluation.format_results(
# ['IND', 'ENSM', 'ENDD', 'ENDD+AUX', 'PN_AUX'],
# [ind_measures, ensm_measures, endd_measures, endd_aux_measures, pn_aux_measures],
# dataset_name=DATASET_NAME)
# Format and print results
summary = evaluation.format_results(
['PN_AUX'],
[pn_aux_measures],
dataset_name=DATASET_NAME)
dataset_name=DATASET_NAME,
compile=True,
weights=ENDD_AUX_MODEL_NAME)
endd_aux_wrapper_type = 'individual'
# Load data
_, (test_images, test_labels) = datasets.get_dataset(DATASET_NAME)
# Preprocess data
test_images = preprocessing.normalize_minus_one_to_one(test_images,
min=0,
max=255)
# Calculate measures
print("Evaluating IND...")
ind_measures = evaluation.calc_classification_measures(
ind_model, test_images, test_labels, wrapper_type=ind_wrapper_type)
print("Evaluating ENSM...")
ensm_measures = evaluation.calc_classification_measures(
ensm_model, test_images, test_labels, wrapper_type=ensm_wrapper_type)
print("Evaluating END...")
end_measures = evaluation.calc_classification_measures(
end_model, test_images, test_labels, wrapper_type=end_wrapper_type)
print("Evaluating END+AUX...")
end_aux_measures = evaluation.calc_classification_measures(
end_aux_model, test_images, test_labels, wrapper_type=end_aux_wrapper_type)
print("Evaluating ENDD...")
endd_measures = evaluation.calc_classification_measures(
endd_model, test_images, test_labels, wrapper_type=endd_wrapper_type)
endd_model = training.train_vgg_endd(
train_images=train_images,
ensemble_model=ensm_model,
dataset_name=DATASET_NAME,
batch_size=BATCH_SIZE,
n_epochs=N_EPOCHS,
one_cycle_lr_policy=ONE_CYCLE_LR_POLICY,
init_lr=INIT_LR,
cycle_length=CYCLE_LENGTH,
temp_annealing=TEMP_ANNEALING,
init_temp=INIT_TEMP,
dropout_rate=DROPOUT_RATE,
save_endd_dataset=save,
load_previous_endd_dataset=load,
repetition=rep)
endd_measures = evaluation.calc_classification_measures(
endd_model, test_images, test_labels, wrapper_type='individual')
print("############# ENDD Measures")
for measure, value in endd_measures.items():
measures['endd'][measure].append(value)
print("{}={}".format(measure, value))
print()
if MODEL_BASE_SAVE_NAME:
saveload.save_tf_model(
endd_model, MODEL_BASE_SAVE_NAME +
"_{}".format(nr_repetition) + '_N_MODELS={}'.format(n_models))
saveload.save_weights(
endd_model, MODEL_BASE_SAVE_NAME +
"_{}".format(nr_repetition) + '_N_MODELS={}'.format(n_models))
print(measures)