def evaluate_model(config, save_name, legacy=False):
"""
This function runs and evaluates the model with given config.
The model is saved to models/saved/config['save_name']/save_name.
If already calculated the result is loaded instead of calculated.
:param config:
:param save_name:
:param legacy: set to True to use old version
:return:
"""
if not os.path.exists(os.path.join("models/saved", config['save_name'])):
os.mkdir(os.path.join("models/saved", config['save_name']))
# folder for save and load
save_folder = os.path.join("models/saved", config['save_name'], save_name)
if not os.path.exists(save_folder):
os.mkdir(save_folder)
if not legacy:
try:
# load results if available
accuracy = np.load(os.path.join(save_folder, "accuracy.npy"))
it_resp = np.load(os.path.join(save_folder, "it_resp.npy"))
labels = np.load(os.path.join(save_folder, "labels.npy"))
except IOError:
try:
norm_base = NormBase(config,
input_shape=(224, 224, 3),
save_name=save_name)
except IOError:
norm_base = NormBase(config, input_shape=(224, 224, 3))
data_train = load_data(config)
norm_base.fit(data_train)
norm_base.save_model(config, save_name)
data_test = load_data(config, train=False)
accuracy, it_resp, labels = norm_base.evaluate(data_test)
return accuracy, it_resp, labels
else:
try:
# load results if available
accuracy = np.load(os.path.join(save_folder, "accuracy.npy"))
it_resp = np.load(os.path.join(save_folder, "it_resp.npy"))
labels = np.load(os.path.join(save_folder, "labels.npy"))
print("[MODEL] it_resp is available and is loaded from {}".format(
save_folder))
# load vectors if available
ref_vector = np.load(os.path.join(save_folder, "ref_vector.npy"))
tun_vector = np.load(os.path.join(save_folder,
"tuning_vector.npy"))
except IOError:
# calculate results if not available
print("[LOOP] start training")
# create model
norm_base = NormBase(config, input_shape=(224, 224, 3))
try:
# load vectors if available
ref_vector = np.load(
os.path.join(save_folder, "ref_vector.npy"))
tun_vector = np.load(
os.path.join(save_folder, "tuning_vector.npy"))
print(
"[MODEL] ref_vector and tun_vector are available and loaded from {}"
.format(save_folder))
norm_base.set_ref_vector(ref_vector)
norm_base.set_tuning_vector(tun_vector)
print("[MODEL] Set ref vector", np.shape(ref_vector))
print("[MODEL] Set tuning vector", np.shape(tun_vector))
except IOError:
# calculate vectors if not available
# load train data
data_train = load_data(config)
print("[Data] -- Data loaded --")
# train model
norm_base.fit(data_train, batch_size=config['batch_size'])
ref_vector = norm_base.r
tun_vector = norm_base.t
# save model
np.save(os.path.join(save_folder, "ref_vector"), ref_vector)
np.save(os.path.join(save_folder, "tuning_vector"), tun_vector)
print("[LOOP] start prediction")
# load test data
data_test = load_data(config, train=False, sort_by=['image'])
print("[Data] -- Data loaded --")
# evaluate
accuracy, it_resp, labels = norm_base.evaluate(data_test)
np.save(os.path.join(save_folder, "accuracy"), accuracy)
np.save(os.path.join(save_folder, "it_resp"), it_resp)
np.save(os.path.join(save_folder, "labels"), labels)
return accuracy, it_resp, labels, ref_vector, tun_vector
config_name = 'norm_base_affectNet_sub8_4000.json'
config_file_path = os.path.join(congig_path, config_name)
print("config_file_path", config_file_path)
# load norm_base_config file
with open(config_file_path) as json_file:
config = json.load(json_file)
# load data
data = load_data(config, train=False, sort_by=['image'])
print("[Data] -- Data loaded --")
# create model
norm_base = NormBase(config, input_shape=(224, 224, 3))
# "load" model
load_folder = os.path.join("../../models/saved", config['save_name'])
r = np.load(os.path.join(os.path.join(load_folder, "ref_vector.npy")))
t = np.load(os.path.join(os.path.join(load_folder, "tuning_vector.npy")))
norm_base.set_ref_vector(r)
norm_base.set_tuning_vector(t)
print("[MODEL] Set ref vector", np.shape(r))
print("[MODEL] Set tuning vector", np.shape(t))
# predict tuning vector
# it_resp = norm_base.predict(data)
_, it_resp, _ = norm_base.evaluate(data)
print("shape it_resp", np.shape(it_resp))
plt.plot(it_resp)
plt.show()
input_shape=tuple(config['input_shape']),
load_NB_model=True)
# train model
if train_model:
print("[FIT] Train model]")
data = load_data(config, train=True)
model.fit(data, fit_dim_red=True, fit_ref=True, fit_tun=True)
model.save()
# --------------------------------------------------------------------------------------------------------------------
# predict model
if predict_it_resp:
data = load_data(config, train=False)
print("[Test] Predict model")
accuracy, it_resp, labels = model.evaluate(data)
print("[Test] model predicted")
np.save(
os.path.join("models/saved", config['config_name'], "NormBase",
"it_resp"), it_resp)
print("[Test] it_resp saved!")
else:
it_resp = np.load(
os.path.join("models/saved", config['config_name'], "NormBase",
"it_resp.npy"))
print("[Test] it_resp loaded!")
print("[Test] shape it_resp", np.shape(it_resp))
print()
# --------------------------------------------------------------------------------------------------------------------
# plot