def train_models():
'''Trains an ensemble of models on the spiral dataset.'''
MODEL_TYPE = "small_net"
ENSEMBLE_SAVE_NAME = "small_net"
DATASET_NAME = "spiral"
NAME_START_NUMBER = 0
N_MODELS = 100
N_EPOCHS = 85
# Import data
(x_train, y_train), (x_test, y_test) = datasets.get_dataset(DATASET_NAME)
y_train_one_hot = tf.one_hot(y_train.reshape((-1, )),
settings.DATASET_N_CLASSES[DATASET_NAME])
y_test_one_hot = tf.one_hot(y_test.reshape((-1, )),
settings.DATASET_N_CLASSES[DATASET_NAME])
# Train models
model_module = settings.MODEL_MODULES[MODEL_TYPE]
saved_model_names = []
try:
for i in range(N_MODELS):
# Get model
model = model_module.get_model(dataset_name=DATASET_NAME,
compile=True)
# Train model
model.fit(x_train,
y_train_one_hot,
validation_data=(x_test, y_test_one_hot),
epochs=N_EPOCHS,
verbose=2)
print("Model {} finished training.".format(i))
# Save model
model_name = "{}_{}_{}".format(ENSEMBLE_SAVE_NAME, DATASET_NAME, i)
saveload.save_tf_model(model, model_name)
saved_model_names.append(model_name)
finally:
append_model_names = NAME_START_NUMBER > 0
saveload.update_ensemble_names(ENSEMBLE_SAVE_NAME,
DATASET_NAME,
saved_model_names,
append=append_model_names)
def train_end():
"""Trains an END and and END_AUX model on the ensemble predictions"""
# Name
ENSEMBLE_SAVE_NAME = 'small_net' # Name that the ensemble models will be saved with
DATASET_NAME = 'spiral' # Name of dataset models were trained with
MODEL_SAVE_NAME = "end_small_net_spiral"
MODEL_SAVE_NAME_AUX = "end_AUX_small_net_spiral"
# Load data
with open('train_small_net_spiral.pkl', 'rb') as file:
x_train, y_train, ensemble_logits_train = pickle.load(file)
with open('train_aux_small_net_spiral.pkl', 'rb') as file:
x_train_aux, y_train_aux, ensemble_logits_train_aux = pickle.load(file)
# Build ENDD model
base_model = get_model(DATASET_NAME, compile=False)
end_model = end.get_model(base_model, init_temp=1)
base_model_AUX = get_model(DATASET_NAME, compile=False)
end_model_AUX = end.get_model(base_model_AUX, init_temp=1)
# Train model
end_model.fit(x_train,
np.transpose(ensemble_logits_train, (1, 0, 2)),
epochs=150)
end_model_AUX.fit(x_train_aux,
np.transpose(ensemble_logits_train_aux, (1, 0, 2)),
epochs=500)
# Save model
saveload.save_tf_model(end_model, MODEL_SAVE_NAME)
saveload.save_tf_model(end_model_AUX, MODEL_SAVE_NAME_AUX)
# Evaluate model
_, (x_test, y_test) = datasets.get_dataset("spiral")
logits = end_model.predict(x_test)
logits_aux = end_model_AUX.predict(x_test)
print(np.argmax(logits, axis=1))
print(np.argmax(logits_aux, axis=1))
print(y_test)
print(sklearn.metrics.accuracy_score(y_test, np.argmax(logits, axis=1)))
print(sklearn.metrics.accuracy_score(y_test, np.argmax(logits_aux,
axis=1)))
import settings
from models import cnn
from utils import saveload
from utils import datasets
# Need these settings for GPU to work on my computer /Einar
physical_devices = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
# Load data
(train_images,
train_labels), (test_images, test_labels) = datasets.get_dataset(DATASET_NAME)
# Preprocess
train_labels = tf.one_hot(train_labels.reshape((-1, )),
settings.DATASET_N_CLASSES[DATASET_NAME])
test_labels = tf.one_hot(test_labels.reshape((-1, )),
settings.DATASET_N_CLASSES[DATASET_NAME])
# Get model
model = cnn.get_model(dataset_name=DATASET_NAME, compile=True)
# Train
model.fit(train_images,
train_labels,
epochs=N_EPOCHS,
validation_data=(test_images, test_labels))
# Save weights
saveload.save_tf_model(model, "cnn")
model_module = settings.MODEL_MODULES[MODEL_TYPE]
# Train and save ensemble
# Try-finally construct ensures that the list of trained models in the ensemble is
# update correctly even if script fails before all models have been trained and saved.
try:
saved_model_names = []
for i in range(NAME_START_NUMBER, N_MODELS + NAME_START_NUMBER):
print("Training model {}...".format(i))
# Get model
model = model_module.get_model(dataset_name=DATASET_NAME, compile=True)
# Train model
model.fit(train_images,
train_labels,
epochs=N_EPOCHS,
validation_data=(test_images, test_labels))
print("Model {} finished training.".format(i))
# Save model
model_name = "{}_{}_{}".format(ENSEMBLE_SAVE_NAME, DATASET_NAME, i)
saveload.save_tf_model(model, model_name)
saved_model_names.append(model_name)
finally:
# Updates ensemble record with list of saved models
append_model_names = NAME_START_NUMBER > 0
saveload.update_ensemble_names(ENSEMBLE_SAVE_NAME,
DATASET_NAME,
saved_model_names,
append=append_model_names)
init_temp=INIT_TEMP,
dropout_rate=DROPOUT_RATE,
save_endd_dataset=save,
load_previous_endd_dataset=load,
repetition=rep)
endd_measures = evaluation.calc_classification_measures(
endd_model, test_images, test_labels, wrapper_type='individual')
print("############# ENDD Measures")
for measure, value in endd_measures.items():
measures['endd'][measure].append(value)
print("{}={}".format(measure, value))
print()
if MODEL_BASE_SAVE_NAME:
saveload.save_tf_model(
endd_model, MODEL_BASE_SAVE_NAME +
"_{}".format(nr_repetition) + '_N_MODELS={}'.format(n_models))
saveload.save_weights(
endd_model, MODEL_BASE_SAVE_NAME +
"_{}".format(nr_repetition) + '_N_MODELS={}'.format(n_models))
print(measures)
'''
# Plot results
plt.subplot(2, 2, 1)
plt.plot(N_MODELS_LIST, measures['endd']['err'], label='ENDD+AUX')
plt.plot(N_MODELS_LIST, measures['ensm']['err'], label='ENSM')
plt.xlabel("Number of models")
plt.ylabel("Prediction Error")
plt.legend()
def train_endd():
"""Trains an ENDD and and ENDD_AUX model on the ensemble predictions"""
# Name
ENSEMBLE_SAVE_NAME = 'small_net' # Name that the ensemble models will be saved with
DATASET_NAME = 'spiral' # Name of dataset models were trained with
MODEL_SAVE_NAME = "endd_small_net_spiral"
MODEL_SAVE_NAME_AUX = "endd_AUX_small_net_spiral"
MODEL_SAVE_NAME_AUX_20 = "endd_AUX_20_small_net_spiral"
MODEL_SAVE_NAME_AUX_ANN = "endd_AUX_ANN_small_net_spiral"
MODEL_SAVE_NAME_AUX_T25 = "endd_AUX_T25_small_net_spiral"
# Load data
with open('train_small_net_spiral.pkl', 'rb') as file:
x_train, y_train, ensemble_logits_train = pickle.load(file)
with open('train_aux_small_net_spiral.pkl', 'rb') as file:
x_train_aux, y_train_aux, ensemble_logits_train_aux = pickle.load(file)
with open('train_aux_20_small_net_spiral.pkl', 'rb') as file:
x_train_aux_20, y_train_aux_20, ensemble_logits_train_aux_20 = pickle.load(
file)
# Build ENDD model
base_model = get_model(DATASET_NAME, compile=False)
endd_model = endd.get_model(base_model, init_temp=1, teacher_epsilon=1e-4)
base_model_AUX = get_model(DATASET_NAME, compile=False)
base_model_AUX_20 = get_model(DATASET_NAME, compile=False)
base_model_AUX_ANN = get_model(DATASET_NAME, compile=False)
base_model_AUX_T25 = get_model(DATASET_NAME, compile=False)
endd_model_AUX = endd.get_model(base_model_AUX,
init_temp=1,
teacher_epsilon=1e-4)
endd_model_AUX_20 = endd.get_model(base_model_AUX_20,
init_temp=1,
teacher_epsilon=1e-4)
endd_model_AUX_ANN = endd.get_model(base_model_AUX_ANN,
init_temp=2.5,
teacher_epsilon=1e-4)
endd_model_AUX_T25 = endd.get_model(base_model_AUX_T25,
init_temp=2.5,
teacher_epsilon=1e-4)
# Train model
#endd_model.fit(x_train, np.transpose(ensemble_logits_train, (1, 0, 2)), epochs=500)
#endd_model_AUX.fit(x_train_aux, np.transpose(ensemble_logits_train_aux, (1, 0, 2)), epochs=500)
#endd_model_AUX_20.fit(x_train_aux_20, np.transpose(ensemble_logits_train_aux_20, (1, 0, 2)), epochs=500)
endd_model_AUX_ANN.fit(x_train_aux,
np.transpose(ensemble_logits_train_aux, (1, 0, 2)),
epochs=500,
callbacks=[
callbacks.TemperatureAnnealing(init_temp=2.5,
cycle_length=400,
epochs=500)
])
endd_model_AUX_T25.fit(x_train_aux,
np.transpose(ensemble_logits_train_aux, (1, 0, 2)),
epochs=500)
# Save model
saveload.save_tf_model(endd_model, MODEL_SAVE_NAME)
saveload.save_tf_model(endd_model_AUX, MODEL_SAVE_NAME_AUX)
saveload.save_tf_model(endd_model_AUX_20, MODEL_SAVE_NAME_AUX_20)
saveload.save_tf_model(endd_model_AUX_ANN, MODEL_SAVE_NAME_AUX_ANN)
saveload.save_tf_model(endd_model_AUX_T25, MODEL_SAVE_NAME_AUX_T25)
# Evaluate model
_, (x_test, y_test) = datasets.get_dataset("spiral")
logits = endd_model_AUX.predict(x_test)
print(np.argmax(logits, axis=1))
print(y_test)
print(sklearn.metrics.accuracy_score(y_test, np.argmax(logits, axis=1)))
elif NORMALIZATION == 'gaussian':
train_images, mean, std = preprocessing.normalize_gaussian(train_images)
test_images = preprocessing.normalize_gaussian(test_images, mean, std)
end_model = training.train_vgg_end(train_images=train_images,
ensemble_model=ensemble_model,
dataset_name=DATASET_NAME,
batch_size=BATCH_SIZE,
n_epochs=N_EPOCHS,
one_cycle_lr_policy=ONE_CYCLE_LR_POLICY,
init_lr=INIT_LR,
cycle_length=CYCLE_LENGTH,
temp_annealing=TEMP_ANNEALING,
init_temp=INIT_TEMP,
dropout_rate=DROPOUT_RATE,
save_end_dataset=False,
load_previous_end_dataset=True)
measures = evaluation.calc_classification_measures(end_model,
test_images,
test_labels,
wrapper_type='individual')
if MODEL_SAVE_NAME:
# Note: There seems to be some difficulties when trying to load whole model with custom loss
saveload.save_tf_model(end_model, MODEL_SAVE_NAME)
saveload.save_weights(end_model, MODEL_SAVE_NAME)
results = evaluation.format_results(['end'], [measures])
print(results)
import settings
from utils import preprocessing, saveload, simplex, losses, training
from models import cnn_priorNet
MODEL = 'vgg'
DATASET = 'cifar10'
SAVE_WEIGHTS = True
NORMALIZATION = "-1to1"
(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
(OOD_images, _), (_, _) = datasets.cifar100.load_data()
train_images, train_alphas, test_images, test_alphas = preprocessing.preprocess_cifar_for_priornet(
train_images,
train_labels,
test_images,
test_labels,
normalization=NORMALIZATION,
OOD_images=OOD_images)
model = training.train_pn(train_images, train_alphas, DATASET, MODEL)
if SAVE_WEIGHTS:
saveload.save_tf_model(model, "PN_vgg_cifar10_aux_c")
alphas = tf.math.exp(model.predict(test_images))
predictions = tf.math.argmax(tf.squeeze(alphas), axis=1)
real = tf.math.argmax(tf.squeeze(test_alphas), axis=1)
score = tf.math.reduce_sum(tf.cast(predictions == real, tf.float32)) / len(real)
init_lr=INIT_LR,
cycle_length=CYCLE_LENGTH,
temp_annealing=TEMP_ANNEALING,
init_temp=init_temp,
dropout_rate=DROPOUT_RATE,
save_endd_dataset=(not load_previous_dataset),
load_previous_endd_dataset=load_previous_dataset)
load_previous_dataset = True # Load previous to make training faster
endd_measures = evaluation.calc_classification_measures(
endd_model, test_images, test_labels, wrapper_type='individual')
for measure, value in endd_measures.items():
measures[measure].append(value)
if MODEL_BASE_SAVE_NAME:
saveload.save_tf_model(
endd_model, MODEL_BASE_SAVE_NAME +
"_{}".format(nr_repetition) + '_TEMP={}'.format(init_temp))
saveload.save_weights(
endd_model, MODEL_BASE_SAVE_NAME +
"_{}".format(nr_repetition) + '_TEMP={}'.format(init_temp))
print(measures)
'''
# Plot results
plt.subplot(2, 2, 1)
plt.plot(INIT_TEMP_LIST, measures['err'], label='ENDD+AUX')
plt.xlabel("Initial temperature")
plt.ylabel("Prediction Error")
plt.legend()
from models.dense_priornet import get_model
from utils.create_toy_data import create_mixed_data
import utils.saveload as saveload
dataset = "spiral"
BATCH_SIZE = 100
N_EPOCHS = 20
PLOT_SIMPLEX = False
SAVE_WEIGHTS = False
X, Y = create_mixed_data(settings.SAMPLES_PER_CLASS_PN,
settings.SAMPLES_OOD_PN,
settings.DATASET_N_CLASSES[dataset],
radius=settings.RADIUS_PN,
ID_noise=settings.ID_NOISE_PN,
OOD_noise=settings.OOD_NOISE_PN)
x_train, logits_train, x_test, logits_test = preprocessing.preprocess_toy_dataset(
X, Y, 0.8)
model = get_model(dataset, compile=True)
model.fit(x=x_train, y=logits_train, batch_size=BATCH_SIZE, epochs=N_EPOCHS)
if SAVE_WEIGHTS:
saveload.save_tf_model(model, "dense_priornet")
logits = model.predict(x_test)
predictions = tf.math.argmax(logits, axis=1)
real = tf.math.argmax(logits_test, axis=1)
if PLOT_SIMPLEX and settings.DATASET_N_CLASSES[dataset] == 3:
plot_simplex(logits)
