def get_Exp1_info(models, batch_sizes, model_names):
acc_val = []
acc_train = []
acc_test = []
models_list = []
batch_size_list = []
for model_name, batch_size, model in zip(model_names, batch_sizes, models):
if os.path.exists(f"../trained_models/{model_name}"):
val = helper.load_dict(
f"../trained_models/{model_name}/performance_val.csv")
train = helper.load_dict(
f"../trained_models/{model_name}/performance_train.csv")
test = helper.load_dict(
f"../trained_models/{model_name}/performance_test.csv")
acc_val.append(round(val["accuracy"], 4))
acc_train.append(round(train["accuracy"], 4))
acc_test.append(round(test["accuracy"], 4))
batch_size_list.append(batch_size)
models_list.append(model)
else:
print(model_name, "does not exists")
return {
"Model": models_list,
"Batch size": batch_size_list,
"acc train": acc_train,
"acc val": acc_val,
}
def get_Exp6_info(models, word_methods, min_counts, model_names, model_names2):
acc_train = []
acc_val = []
acc_val2 = []
val_acc_avg_list = []
acc_test = []
models_list = []
word_method_list = []
min_count_list = []
for model_name, model_name2, word_method, min_count, model in zip(
model_names, model_names2, word_methods, min_counts, models):
if os.path.exists(f"../trained_models/{model_name}"):
if os.path.exists(
f"../trained_models/{model_name}/performance_train.csv"):
train = helper.load_dict(
f"../trained_models/{model_name}/performance_train.csv")
val = helper.load_dict(
f"../trained_models/{model_name}/performance_val.csv")
test = helper.load_dict(
f"../trained_models/{model_name}/performance_test.csv")
val_acc_1 = val["accuracy"]
val_acc_2 = 0
if os.path.exists(f"../trained_models/{model_name2}"):
if os.path.exists(
f"../trained_models/{model_name2}/performance_train.csv"
):
val2 = helper.load_dict(
f"../trained_models/{model_name2}/performance_val.csv"
)
val_acc_2 = val2["accuracy"]
else:
print(model_name2, "does not have performance file")
else:
print(model_name2, "does not exists")
acc_val2.append(round(val_acc_2, 4))
val_acc_avg_list.append(round((val_acc_1 + val_acc_2) / 2, 4))
acc_train.append(round(train["accuracy"], 4))
acc_val.append(round(val_acc_1, 4))
acc_test.append(round(test["accuracy"], 4))
word_method_list.append(word_method)
min_count_list.append(min_count)
models_list.append(model)
else:
print(model_name, "does not have performance file")
else:
print(model_name, "does not exists")
return {
"Model": models_list,
"Word Embedding Method": word_method_list,
"Min count": min_count_list,
"Train Acc": acc_train,
"Val Acc Run 1": acc_val,
"Val Acc Run 2": acc_val2,
"Val Acc Avg": val_acc_avg_list,
"Test Acc": acc_test,
}
def get_fmri(fmri_dir, ROI):
"""This function loads fMRI data into a numpy array for to a given ROI.
Parameters
----------
fmri_dir : str
path to fMRI data.
ROI : str
name of ROI.
Returns
-------
np.array
matrix of dimensions #train_vids x #repetitions x #voxels
containing fMRI responses to train videos of a given ROI
"""
# Loading ROI data
ROI_file = os.path.join(fmri_dir, ROI + ".pkl")
ROI_data = load_dict(ROI_file)
# averaging ROI data across repetitions
ROI_data_train = np.mean(ROI_data["train"], axis=1)
if ROI == "WB":
voxel_mask = ROI_data['voxel_mask']
return ROI_data_train, voxel_mask
return ROI_data_train
def get_exp_speed_info(model, model_names):
best_time_list = []
best_epoch_list = []
total_time_list = []
total_epoch_list = []
for model_name in model_names:
if os.path.exists(f"../trained_models/{model_name}"):
if os.path.exists(
f"../trained_models/{model_name}/performance_train.csv"):
speed = helper.load_dict(
f"../trained_models/{model_name}/speed.csv")
best_time_list.append(speed["best_time"] / 60 / 60)
best_epoch_list.append(speed["best_epoch"])
total_time_list.append(speed["total_time"] / 60 / 60)
total_epoch_list.append(speed["epoch"])
else:
print(model_name, "does not have performance file")
else:
print(model_name, "does not exists")
return {
"Model": [model],
"Avg best epochs": [np.average(best_epoch_list)],
"Max best epochs": [np.max(best_epoch_list)],
"Min best epochs": [np.min(best_epoch_list)],
"Avg total epochs": [np.average(total_epoch_list)],
"Max total epochs": [np.max(total_epoch_list)],
"Min total epochs": [np.min(total_epoch_list)],
"Avg best time": [np.average(best_time_list)],
"Max best time": [np.max(best_time_list)],
"Min best time": [np.min(best_time_list)],
"Avg total time": [np.average(total_time_list)],
"Max total time": [np.max(total_time_list)],
"Min total time": [np.min(total_time_list)]
}
def get_models_info(models, batch_sizes, lr_decay, model_names):
acc_val = []
acc_test = []
for model_name in model_names:
val = helper.load_dict(
f"../trained_models/{model_name}/performance_val.csv")
test = helper.load_dict(
f"../trained_models/{model_name}/performance_test.csv")
acc_val.append(round(val["accuracy"], 4))
acc_test.append(round(test["accuracy"], 4))
return {
"Model": models,
"Batch size": batch_sizes,
"Lr decay": lr_decay,
"acc val": acc_val,
"acc test": acc_test,
}
def get_Exp5_info(models, loss_types, dropouts, l2s, model_names):
acc_train = []
acc_val = []
acc_test = []
models_list = []
loss_type_list = []
dropout_list = []
l2_list = []
for model_name, loss_type, dropout, l2, model in zip(
model_names, loss_types, dropouts, l2s, models):
if os.path.exists(f"../trained_models/{model_name}"):
if os.path.exists(
f"../trained_models/{model_name}/performance_train.csv"):
train = helper.load_dict(
f"../trained_models/{model_name}/performance_train.csv")
val = helper.load_dict(
f"../trained_models/{model_name}/performance_val.csv")
test = helper.load_dict(
f"../trained_models/{model_name}/performance_test.csv")
acc_train.append(round(train["accuracy"], 4))
acc_val.append(round(val["accuracy"], 4))
acc_test.append(round(test["accuracy"], 4))
loss_type_list.append(loss_type)
dropout_list.append(dropout)
l2_list.append(l2)
models_list.append(model)
else:
print(model_name, "does not have performance file")
else:
print(model_name, "does not exists")
return {
"Model": models_list,
"Loss type": loss_type_list,
"Dropout Rate": dropout_list,
"L2 Scalar": l2_list,
"Train Acc": acc_train,
"Val Acc": acc_val,
"Test Acc": acc_test,
}
def get_Exp3_info(models, optimizers, lrs, lr_decays, model_names):
acc_train = []
acc_val = []
acc_test = []
models_list = []
optimizer_list = []
lr_list = []
lr_decay_list = []
for model_name, optimizer, lr, lr_decays, model in zip(
model_names, optimizers, lrs, lr_decays, models):
if os.path.exists(f"../trained_models/{model_name}"):
if os.path.exists(
f"../trained_models/{model_name}/performance_train.csv"):
train = helper.load_dict(
f"../trained_models/{model_name}/performance_train.csv")
val = helper.load_dict(
f"../trained_models/{model_name}/performance_val.csv")
test = helper.load_dict(
f"../trained_models/{model_name}/performance_test.csv")
acc_train.append(round(train["accuracy"], 4))
acc_val.append(round(val["accuracy"], 4))
acc_test.append(round(test["accuracy"], 4))
optimizer_list.append(optimizer)
lr_list.append(lr)
lr_decay_list.append(lr_decays)
models_list.append(model)
else:
print(model_name, "does not have performance file")
else:
print(model_name, "does not exists")
return {
"Model": models_list,
"Optimizer": optimizer_list,
"Learning rate": lr_list,
"Learning rate decay": lr_decay_list,
"Train Acc": acc_train,
"Val Acc": acc_val,
"Test Acc": acc_test,
}
def load(self):
if os.path.exists(directories.PARAMETERS_FILE(self.model_name)):
self.parameters = helper.load_dict(directories.PARAMETERS_FILE(self.model_name))
if os.path.exists(directories.PERFORMANCE_TEST_FILE(self.model_name)):
self.performance_test = helper.load_dict(directories.PERFORMANCE_TEST_FILE(self.model_name))
if os.path.exists(directories.PERFORMANCE_VAL_FILE(self.model_name)):
self.performance_val = helper.load_dict(directories.PERFORMANCE_VAL_FILE(self.model_name))
if os.path.exists(directories.PERFORMANCE_TRAIN_FILE(self.model_name)):
self.performance_train = helper.load_dict(directories.PERFORMANCE_TRAIN_FILE(self.model_name))
if os.path.exists(directories.SPEED_FILE(self.model_name)):
self.speed = helper.load_dict(directories.SPEED_FILE(self.model_name))
if os.path.exists(directories.BEST_ACC_FILE(self.model_name)):
self.best_acc = helper.load_dict(directories.BEST_ACC_FILE(self.model_name))
if os.path.exists(directories.BEST_LOSS_FILE(self.model_name)):
self.best_loss = helper.load_dict(directories.BEST_LOSS_FILE(self.model_name))
if os.path.exists(directories.HISTORIES_FILE(self.model_name)):
tmp = np.load(directories.HISTORIES_FILE(self.model_name))
for data_set in self.all_data_sets:
self.history[data_set] = tmp[data_set].tolist()
self.construct_writers()