def get_endd_measures(n_models_base_names, n_models_list, endd_base_model,
dataset_name, test_images, test_labels):
endd_measures_list = []
for base_name in n_models_base_names:
endd_measures = defaultdict(list)
for n_models in n_models_list:
#print("{}/{}".format(base_name, n_models))
#print(n_models)
endd_model_name = base_name + '_N_MODELS={}'.format(n_models)
print(endd_model_name)
uncompiled_model = saveload.load_tf_model(endd_model_name,
compile=False)
endd_model = endd.get_model(uncompiled_model,
dataset_name=dataset_name,
compile=True)
evaluation_result = evaluation.calc_classification_measures(
endd_model,
test_images,
test_labels,
wrapper_type='individual')
#print("############# Measures")
for measure, value in evaluation_result.items():
endd_measures[measure].append(value)
#print("{}={}".format(measure, value))
#print()
endd_measures_list.append(endd_measures)
return endd_measures_list
def predict_endd():
"""Predicts and saves the predictions of the ENDD-models to file"""
# Load model
MODEL_SAVE_NAMES = [
"endd_small_net_spiral", "endd_AUX_small_net_spiral",
"endd_AUX_20_small_net_spiral", "endd_AUX_ANN_small_net_spiral",
"endd_AUX_T25_small_net_spiral"
]
PREDICT_SAVE_NAMES = [
"endd", "endd_AUX", "endd_AUX_20", "endd_AUX_ANN", "endd_AUX_T25"
]
# Loop for aux or no aux
for i in range(len(MODEL_SAVE_NAMES)):
print(i)
MODEL_SAVE_NAME = MODEL_SAVE_NAMES[i]
PREDICT_SAVE_NAME = PREDICT_SAVE_NAMES[i]
endd_model = saveload.load_tf_model(MODEL_SAVE_NAME, compile=False)
endd_model = endd.get_model(endd_model,
init_temp=1,
teacher_epsilon=1e-4)
# Load data
(x_train, y_train), (x_test, y_test) = datasets.get_dataset("spiral")
grid = get_grid(size=2000, steps=1000)
# Predict
endd_logits_train = endd_model.predict(x_train)
endd_logits_test = endd_model.predict(x_test)
endd_logits_grid = endd_model.predict(grid)
with open('train_small_net_spiral_{}.pkl'.format(PREDICT_SAVE_NAME),
'wb') as file:
pickle.dump((x_train, y_train, endd_logits_train), file)
with open('test_small_net_spiral_{}.pkl'.format(PREDICT_SAVE_NAME),
'wb') as file:
pickle.dump((x_test, y_test, endd_logits_test), file)
with open('grid_small_net_spiral_{}.pkl'.format(PREDICT_SAVE_NAME),
'wb') as file:
pickle.dump((grid, 0, endd_logits_grid), file)
# Choose dataset
DATASET_NAME = 'cifar10'
OUT_DATASET_NAME = 'lsun'
# Prepare ENSM model
ensemble_model_names = saveload.get_ensemble_model_names()
model_names = ensemble_model_names[ENSM_MODEL_NAME][
DATASET_NAME][:ENSM_N_MODELS]
models = [
ensemble.KerasLoadsWhole(name, pop_last=True) for name in model_names
]
ensm_model = ensemble.Ensemble(models)
# Prepare ENDD model
endd_model = endd.get_model(ENDD_BASE_MODEL,
dataset_name=DATASET_NAME,
compile=True,
weights=ENDD_MODEL_NAME)
# Prepare ENDD+AUX model
endd_aux_model = endd.get_model(ENDD_AUX_BASE_MODEL,
dataset_name=DATASET_NAME,
compile=True,
weights=ENDD_AUX_MODEL_NAME)
# Load data
_, (in_images, _) = datasets.get_dataset(DATASET_NAME)
_, out_images = datasets.get_dataset(OUT_DATASET_NAME)
# Preprocess data
in_images = preprocessing.normalize_minus_one_to_one(in_images, min=0, max=255)
out_images = preprocessing.normalize_minus_one_to_one(out_images,
with open("ensm_preds.pkl", 'wb') as file:
pickle.dump((ensm_preds), file)
ensm_preds_noise = ensm_model.predict(noise_img)
with open("ensm_preds_noise.pkl", 'wb') as file:
pickle.dump((ensm_preds_noise), file)
if LOAD_PREVIOUS_ENDD_PREDS:
with open("endd_preds.pkl", 'rb') as file:
endd_preds = pickle.load(file)
with open("endd_preds_noise.pkl", 'rb') as file:
endd_preds_noise = pickle.load(file)
else:
# Load endd
endd_base_model = saveload.load_tf_model(ENDD_MODEL_NAME, compile=False)
endd_model = endd.get_model(endd_base_model,
init_temp=1,
teacher_epsilon=1e-4)
# Predict endd
endd_preds = endd_model.predict(test_images)
with open("endd_preds.pkl", 'wb') as file:
pickle.dump((endd_preds), file)
endd_preds_noise = endd_model.predict(noise_img)
with open("endd_preds_noise.pkl", 'wb') as file:
pickle.dump((endd_preds_noise), file)
# Plot random images
if PLOT_COLLAGE:
in_indices = np.where((test_labels == 4) | (test_labels == 5)
| (test_labels == 7))[0]
out_indices = np.where((test_labels != 4) & (test_labels != 5)
def train_endd():
"""Trains an ENDD and and ENDD_AUX model on the ensemble predictions"""
# Name
ENSEMBLE_SAVE_NAME = 'small_net' # Name that the ensemble models will be saved with
DATASET_NAME = 'spiral' # Name of dataset models were trained with
MODEL_SAVE_NAME = "endd_small_net_spiral"
MODEL_SAVE_NAME_AUX = "endd_AUX_small_net_spiral"
MODEL_SAVE_NAME_AUX_20 = "endd_AUX_20_small_net_spiral"
MODEL_SAVE_NAME_AUX_ANN = "endd_AUX_ANN_small_net_spiral"
MODEL_SAVE_NAME_AUX_T25 = "endd_AUX_T25_small_net_spiral"
# Load data
with open('train_small_net_spiral.pkl', 'rb') as file:
x_train, y_train, ensemble_logits_train = pickle.load(file)
with open('train_aux_small_net_spiral.pkl', 'rb') as file:
x_train_aux, y_train_aux, ensemble_logits_train_aux = pickle.load(file)
with open('train_aux_20_small_net_spiral.pkl', 'rb') as file:
x_train_aux_20, y_train_aux_20, ensemble_logits_train_aux_20 = pickle.load(
file)
# Build ENDD model
base_model = get_model(DATASET_NAME, compile=False)
endd_model = endd.get_model(base_model, init_temp=1, teacher_epsilon=1e-4)
base_model_AUX = get_model(DATASET_NAME, compile=False)
base_model_AUX_20 = get_model(DATASET_NAME, compile=False)
base_model_AUX_ANN = get_model(DATASET_NAME, compile=False)
base_model_AUX_T25 = get_model(DATASET_NAME, compile=False)
endd_model_AUX = endd.get_model(base_model_AUX,
init_temp=1,
teacher_epsilon=1e-4)
endd_model_AUX_20 = endd.get_model(base_model_AUX_20,
init_temp=1,
teacher_epsilon=1e-4)
endd_model_AUX_ANN = endd.get_model(base_model_AUX_ANN,
init_temp=2.5,
teacher_epsilon=1e-4)
endd_model_AUX_T25 = endd.get_model(base_model_AUX_T25,
init_temp=2.5,
teacher_epsilon=1e-4)
# Train model
#endd_model.fit(x_train, np.transpose(ensemble_logits_train, (1, 0, 2)), epochs=500)
#endd_model_AUX.fit(x_train_aux, np.transpose(ensemble_logits_train_aux, (1, 0, 2)), epochs=500)
#endd_model_AUX_20.fit(x_train_aux_20, np.transpose(ensemble_logits_train_aux_20, (1, 0, 2)), epochs=500)
endd_model_AUX_ANN.fit(x_train_aux,
np.transpose(ensemble_logits_train_aux, (1, 0, 2)),
epochs=500,
callbacks=[
callbacks.TemperatureAnnealing(init_temp=2.5,
cycle_length=400,
epochs=500)
])
endd_model_AUX_T25.fit(x_train_aux,
np.transpose(ensemble_logits_train_aux, (1, 0, 2)),
epochs=500)
# Save model
saveload.save_tf_model(endd_model, MODEL_SAVE_NAME)
saveload.save_tf_model(endd_model_AUX, MODEL_SAVE_NAME_AUX)
saveload.save_tf_model(endd_model_AUX_20, MODEL_SAVE_NAME_AUX_20)
saveload.save_tf_model(endd_model_AUX_ANN, MODEL_SAVE_NAME_AUX_ANN)
saveload.save_tf_model(endd_model_AUX_T25, MODEL_SAVE_NAME_AUX_T25)
# Evaluate model
_, (x_test, y_test) = datasets.get_dataset("spiral")
logits = endd_model_AUX.predict(x_test)
print(np.argmax(logits, axis=1))
print(y_test)
print(sklearn.metrics.accuracy_score(y_test, np.argmax(logits, axis=1)))
def train_vgg_endd(train_images,
ensemble_model,
dataset_name,
batch_size=128,
n_epochs=90,
one_cycle_lr_policy=True,
init_lr=0.001,
cycle_length=60,
temp_annealing=True,
init_temp=10,
dropout_rate=0.3,
save_endd_dataset=False,
load_previous_endd_dataset=False,
repetition = None):
"""Return a trained VGG ENDD model.
The save_endd_dataset and load_previous_endd_dataset arguments are useful to avoid having to
re-create the ensemble predictions.
Args:
train_images (np.array): Normalized train images, potentially including AUX data.
ensemble_model (models.ensemble.Ensemble): Ensemble to distill.
dataset_name (str): Name of dataset (required for loading correct model settings).
batch_size (int): Batch size to use while training. Default 128,
n_epochs (int): Number of epochs to train. Default 90.
one_cycle_lr_policy (bool): True if one cycle LR policy should be used. Default True.
init_lr (float): Initial learning rate for one cycle LR. Default 0.001.
cycle_length (int): Epoch length in number of cycles. Default 60.
temp_annealing (bool): True if temperature annealing should be used. Default True.
init_temp (float): Initial temperature. Default 10.
dropout_rate (float): Probability to drop node. Default 0.3.
save_endd_dataset (bool): True if ENDD dataset should be saved (useful for speeding up
repeated training with the same ensemble. Default False.
load_previous_endd_dataset (bool): True if ENDD dataset should be loaded. The dataset loaded
is the one saved the last time the function was run with
save_endd_dataset=True.
Returns:
(keras.Model): Trained VGG ENDD model.
"""
nr_models = len(ensemble_model.models)
if repetition is None:
save_str = 'train_endd_dataset_{}.pkl'.format(nr_models)
else:
save_str = 'train_endd_dataset_rep={}_{}'.format(reptition, nr_models)
if load_previous_endd_dataset:
with open('train_endd_dataset_100.pkl', 'rb') as file:
train_images, train_ensemble_preds = pickle.load(file)
# Load the particular ammount only
train_ensemble_preds = train_ensemble_preds[:, :nr_models, :]
print("loaded")
else:
# Get ensemble preds
print("Evaluating")
train_ensemble_preds = datasets.get_ensemble_preds(ensemble_model, train_images)
print("Evaluated")
# Save / Load pickled data. Generating ensemble preds takes a long time, so saving and
# loading can make testing much more efficient.
if save_endd_dataset:
with open('train_endd_dataset.pkl', 'wb') as file:
pickle.dump((train_images, train_ensemble_preds), file)
print("saved")
# Image augmentation
data_generator = preprocessing.make_augmented_generator(train_images, train_ensemble_preds,
batch_size)
# Callbacks
endd_callbacks = []
if one_cycle_lr_policy:
olp_callback = callbacks.OneCycleLRPolicy(init_lr=init_lr,
max_lr=init_lr * 10,
min_lr=init_lr / 1000,
cycle_length=cycle_length,
epochs=n_epochs)
endd_callbacks.append(olp_callback)
if temp_annealing:
temp_callback = callbacks.TemperatureAnnealing(init_temp=init_temp,
cycle_length=cycle_length,
epochs=n_epochs)
endd_callbacks.append(temp_callback)
if not endd_callbacks:
endd_callbacks = None
# Build ENDD model
base_model = vgg.get_model(dataset_name,
compile=False,
dropout_rate=dropout_rate,
softmax=False)
endd_model = endd.get_model(base_model, init_temp=init_temp, teacher_epsilon=1e-4)
# Train model
endd_model.fit(data_generator, epochs=n_epochs, callbacks=endd_callbacks)
return endd_model