def get_ensm_measures(model_names, n_models_list, test_images, test_labels):
ensm_measures = defaultdict(list)
for n_models in n_models_list:
print("############ ensm {}".format(n_models))
model_name_subset = model_names[:n_models]
print(model_name_subset)
wrapped_models = [
ensemble.KerasLoadsWhole(name, pop_last=True)
for name in model_name_subset
]
ensm_model = ensemble.Ensemble(wrapped_models)
evaluation_result = evaluation.calc_classification_measures(
ensm_model, test_images, test_labels, wrapper_type='ensemble')
for measure, value in evaluation_result.items():
ensm_measures[measure].append(value)
return ensm_measures
from models import ensemble
from tensorflow.keras import datasets
# Need these settings for GPU to work on my computer /Einar
import tensorflow as tf
#physical_devices = tf.config.experimental.list_physical_devices('GPU')
#tf.config.experimental.set_memory_growth(physical_devices[0], True)
# Example usage
if __name__ == "__main__":
# Note: in below example, it is assumed that there is a trained Keras model
# saved with saveload.save_model, saved using the name 'cnn'
# Wrap models
model1 = ensemble.KerasLoadsWhole(model_load_name="cnn", name="cnn_1")
model2 = ensemble.KerasLoadsWhole(model_load_name="cnn", name="cnn_2")
model3 = ensemble.KerasLoadsWhole(model_load_name="cnn", name="cnn_3")
model4 = ensemble.KerasLoadsWhole(model_load_name="cnn", name="cnn_4")
# Build ensemble
cnn_models = [model1, model2, model3, model4]
cnn_ensemble = ensemble.Ensemble(cnn_models)
print(cnn_ensemble)
# Load data
(train_images, train_labels), (test_images,
test_labels) = datasets.cifar10.load_data()
# Predict with ensemble
ensemble_preds = cnn_ensemble.predict(test_images)
# Choose models
ENSM_MODEL_NAME, ENSM_N_MODELS = 'vgg_a', 100
ENDD_MODEL_NAME, ENDD_BASE_MODEL = 'endd_vgg_cifar10_a', 'vgg'
ENDD_AUX_MODEL_NAME, ENDD_AUX_BASE_MODEL = 'new_cifar10_vgg_endd_aux_0_TEMP=10', 'vgg'
# Choose dataset
DATASET_NAME = 'cifar10'
OUT_DATASET_NAME = 'lsun'
# Prepare ENSM model
ensemble_model_names = saveload.get_ensemble_model_names()
model_names = ensemble_model_names[ENSM_MODEL_NAME][
DATASET_NAME][:ENSM_N_MODELS]
models = [
ensemble.KerasLoadsWhole(name, pop_last=True) for name in model_names
]
ensm_model = ensemble.Ensemble(models)
# Prepare ENDD model
endd_model = endd.get_model(ENDD_BASE_MODEL,
dataset_name=DATASET_NAME,
compile=True,
weights=ENDD_MODEL_NAME)
# Prepare ENDD+AUX model
endd_aux_model = endd.get_model(ENDD_AUX_BASE_MODEL,
dataset_name=DATASET_NAME,
compile=True,
weights=ENDD_AUX_MODEL_NAME)
import tensorflow as tf
physical_devices = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
from utils import evaluation
from utils import datasets
from utils import saveload
from models import ensemble
ENSEMBLE_NAME = 'basic_cnn'
DATASET_NAME = 'cifar10'
# Load ensemble model
ensemble_model_names = saveload.get_ensemble_model_names()
model_names = ensemble_model_names[ENSEMBLE_NAME][DATASET_NAME][:3]
models = [ensemble.KerasLoadsWhole(name) for name in model_names]
ensm = ensemble.Ensemble(models)
ensm_wrapper_type = 'ensemble'
# Load individual model
ind = saveload.load_tf_model(model_names[0])
ind_wrapper_type = 'individual'
# Load data
_, (test_images, test_labels) = datasets.get_dataset(DATASET_NAME)
# Preprocess data
test_labels = test_labels.reshape(-1)
# Calculate measures
ensm_measures = evaluation.calc_classification_measures(
# Load ensemble models
ensemble_model_names = saveload.get_ensemble_model_names()
model_names = ensemble_model_names[ENSEMBLE_LOAD_NAME][DATASET_NAME]
print(model_names)
measures = {'endd': defaultdict(list), 'ensm': defaultdict(list)}
for n_models in N_MODELS_LIST:
# Get model names
if SAMPLE_ENSEMBLE_MODELS:
model_name_subset = np.random.choice(model_names, n_models)
else:
model_name_subset = model_names[:n_models]
#model_name_subset = ['vgg_cifar10_cifar10_25']
print("##############", model_name_subset)
wrapped_models = [
ensemble.KerasLoadsWhole(name, pop_last=True)
for name in model_name_subset
]
# Build ensemble
ensm_model = ensemble.Ensemble(wrapped_models)
#import pdb; pdb.set_trace();
#ensm_measures = evaluation.calc_classification_measures(ensm_model,
# test_images,
# test_labels,
# wrapper_type='ensemble')
#print("############# Ensemble Measures")
#for measure, value in ensm_measures.items():
# print("{}={}".format(measure, value))
# measures['ensm'][measure].append(value)
#print()