# Create a training-validation-test split, for example by leave-1-out
# This splitter requires the DataReader object and the number of elements to holdout
dataSplitter = DataSplitter_leave_k_out(data_reader,
k_out_value=1,
use_validation_set=True)
# The load_data function will split the data and save it in the desired folder.
# Once the split is saved, further calls to the load_data will load the splitted data ensuring you always use the same split
dataSplitter.load_data(
save_folder_path="result_experiments/usage_example/data/")
# We can access the three URMs with this function and the ICMs (if present in the data Reader)
URM_train, URM_validation, URM_test = dataSplitter.get_holdout_split()
ICM_dict = dataSplitter.get_loaded_ICM_dict()
# Now that we have the split, we can create the evaluators.
# The constructor of the evaluator allows you to specify the evaluation conditions (data, recommendation list length,
# excluding already seen items). Whenever you want to evaluate a model, use the evaluateRecommender function of the evaluator object
evaluator_validation = EvaluatorHoldout(URM_validation,
cutoff_list=[5],
exclude_seen=False)
evaluator_test = EvaluatorHoldout(URM_test,
cutoff_list=[5, 10, 20],
exclude_seen=False)
# We now fit and evaluate a non personalized algorithm
recommender = TopPop(URM_train)
recommender.fit()
def run_recommender(recommender_class):
temp_save_file_folder = "./result_experiments/__temp_model/"
if not os.path.isdir(temp_save_file_folder):
os.makedirs(temp_save_file_folder)
try:
dataset_object = Movielens1MReader()
dataSplitter = DataSplitter_leave_k_out(dataset_object, k_out_value=2)
dataSplitter.load_data()
URM_train, URM_validation, URM_test = dataSplitter.get_holdout_split()
ICM_all = dataSplitter.get_loaded_ICM_dict()["ICM_genres"]
UCM_all = dataSplitter.get_loaded_UCM_dict()["UCM_all"]
write_log_string(log_file, "On Recommender {}\n".format(recommender_class))
recommender_object = _get_instance(recommender_class, URM_train, ICM_all, UCM_all)
if isinstance(recommender_object, Incremental_Training_Early_Stopping):
fit_params = {"epochs": 15}
else:
fit_params = {}
recommender_object.fit(**fit_params)
write_log_string(log_file, "Fit OK, ")
evaluator = EvaluatorHoldout(URM_test, [5], exclude_seen = True)
results_df, results_run_string = evaluator.evaluateRecommender(recommender_object)
write_log_string(log_file, "EvaluatorHoldout OK, ")
evaluator = EvaluatorNegativeItemSample(URM_test, URM_train, [5], exclude_seen = True)
_, _ = evaluator.evaluateRecommender(recommender_object)
write_log_string(log_file, "EvaluatorNegativeItemSample OK, ")
recommender_object.save_model(temp_save_file_folder, file_name="temp_model")
write_log_string(log_file, "save_model OK, ")
recommender_object = _get_instance(recommender_class, URM_train, ICM_all, UCM_all)
recommender_object.load_model(temp_save_file_folder, file_name="temp_model")
evaluator = EvaluatorHoldout(URM_test, [5], exclude_seen = True)
result_df_load, results_run_string_2 = evaluator.evaluateRecommender(recommender_object)
assert results_df.equals(result_df_load), "The results of the original model should be equal to that of the loaded one"
write_log_string(log_file, "load_model OK, ")
shutil.rmtree(temp_save_file_folder, ignore_errors = True)
write_log_string(log_file, " PASS\n")
write_log_string(log_file, results_run_string + "\n\n")
except Exception as e:
print("On Recommender {} Exception {}".format(recommender_class, str(e)))
log_file.write("On Recommender {} Exception {}\n\n\n".format(recommender_class, str(e)))
log_file.flush()
traceback.print_exc()
recommender_object = recommender_class(URM_train, UCM_all)
else:
recommender_object = recommender_class(URM_train)
return recommender_object
if __name__ == '__main__':
dataset_object = Movielens1MReader()
dataSplitter = DataSplitter_leave_k_out(dataset_object, k_out_value=2)
dataSplitter.load_data()
URM_train, URM_validation, URM_test = dataSplitter.get_holdout_split()
ICM_all = dataSplitter.get_loaded_ICM_dict()["ICM_genres"]
UCM_all = dataSplitter.get_loaded_UCM_dict()["UCM_all"]
recommender_class_list = [
Random,
TopPop,
GlobalEffects,
SLIMElasticNetRecommender,
UserKNNCFRecommender,
IALSRecommender,
MatrixFactorization_BPR_Cython,
MatrixFactorization_FunkSVD_Cython,
MatrixFactorization_AsySVD_Cython,
EASE_R_Recommender,
ItemKNNCFRecommender,
P3alphaRecommender,