def evaluate(model, train, test, hybrid=False, features=None):
if hybrid:
auc_train = np.mean(auc_score(model, train, item_features=features))
pre_train = np.mean(precision_at_k(model, train, item_features=features))
mrr_train = np.mean(reciprocal_rank(model, train, item_features=features))
auc_test = np.mean(auc_score(model, test, item_features=features))
pre_test = np.mean(precision_at_k(model, test, item_features=features))
mrr_test = np.mean(reciprocal_rank(model, test, item_features=features))
else:
auc_train = np.mean(auc_score(model, train))
pre_train = np.mean(precision_at_k(model, train))
mrr_train = np.mean(reciprocal_rank(model, train))
auc_test = np.mean(auc_score(model, test))
pre_test = np.mean(precision_at_k(model, test))
mrr_test = np.mean(reciprocal_rank(model, test))
res_dict = {'auc_train': auc_train,
'pre_train': pre_train,
'mrr_train': mrr_train,
'auc_test': auc_test,
'pre_test': pre_test,
'mrr_test': mrr_test}
print('The AUC Score is in training/validation: ',
auc_train,' / ', auc_test)
print('The mean precision at k Score in training/validation is: ',
pre_train, ' / ', pre_test)
print('The mean reciprocal rank in training/validation is: ',
mrr_train, ' / ', mrr_test)
print('_________________________________________________________')
return res_dict
def resultados_colaborativo(self):
"""
Método resultados_colaboraivo. Obtiene los resultados del modelo colaborativo.
Este método solo se utiliza en la interfaz de texto.
"""
global train, test, modelo
# Se obtienen los resultados
precision = precision_at_k(modelo,
test,
train_interactions=train,
k=10,
num_threads=self.CPU_THREADS).mean()
auc = auc_score(modelo,
test,
train_interactions=train,
num_threads=self.CPU_THREADS).mean()
recall = recall_at_k(modelo,
test,
train_interactions=train,
k=10,
num_threads=self.CPU_THREADS).mean()
reciprocal = reciprocal_rank(modelo,
test,
train_interactions=train,
num_threads=self.CPU_THREADS).mean()
# Se imprimen los resultados
imprimir_resultados_clasico(precision, auc, recall, reciprocal)
def best_reccomendation():
#define variables
best = 0.0
best_model = ''
for model in models:
score = 0.0
pak_score = evaluation.precision_at_k(model, data2['test'])
score += np.mean(pak_score)
rak_score = evaluation.recall_at_k(model, data2['test'])
score += np.mean(rak_score)
auc_score = evaluation.auc_score(model, data2['test'])
score += np.mean(auc_score)
rr_score = evaluation.reciprocal_rank(model, data2['test'])
score += np.mean(rr_score)
print(score)
if score >= best:
best = score
best_model = model
return best_model
def reciprocal_rank_on_ranks(ranks,
test_interactions,
train_interactions=None,
preserve_rows=False):
return reciprocal_rank(
model=ModelMockRanksCacher(ranks.copy()),
test_interactions=test_interactions,
train_interactions=train_interactions,
preserve_rows=preserve_rows,
)
def test_intersections_check():
no_users, no_items = (10, 100)
train, test = _generate_data(no_users, no_items)
model = LightFM(loss="bpr")
model.fit_partial(train)
# check error is raised when train and test have interactions in common
with pytest.raises(ValueError):
evaluation.auc_score(model, train, train_interactions=train, check_intersections=True)
with pytest.raises(ValueError):
evaluation.recall_at_k(model, train, train_interactions=train, check_intersections=True)
with pytest.raises(ValueError):
evaluation.precision_at_k(model, train, train_interactions=train, check_intersections=True)
with pytest.raises(ValueError):
evaluation.reciprocal_rank(model, train, train_interactions=train, check_intersections=True)
# check no errors raised when train and test have no interactions in common
evaluation.auc_score(model, test, train_interactions=train, check_intersections=True)
evaluation.recall_at_k(model, test, train_interactions=train, check_intersections=True)
evaluation.precision_at_k(model, test, train_interactions=train, check_intersections=True)
evaluation.reciprocal_rank(model, test, train_interactions=train, check_intersections=True)
# check no error is raised when there are intersections but flag is False
evaluation.auc_score(model, train, train_interactions=train, check_intersections=False)
evaluation.recall_at_k(model, train, train_interactions=train, check_intersections=False)
evaluation.precision_at_k(model, train, train_interactions=train, check_intersections=False)
evaluation.reciprocal_rank(model, train, train_interactions=train, check_intersections=False)
def mrr():
"""Evaluates models on their Mean Reciprocal Rank.
Measure the reciprocal rank metric for a model: 1 / the rank of the highest ranked positive example.
A perfect score is 1.0.
"""
mrr = reciprocal_rank(model=model,
test_interactions=test,
train_interactions=train,
item_features=item_features).mean()
logger.info(model_name + ' MRR: %s' % (mrr))
def test_intersections_check():
no_users, no_items = (10, 100)
train, test = _generate_data(no_users, no_items)
model = LightFM(loss="bpr")
model.fit_partial(train)
# check error is raised when train and test have interactions in common
with pytest.raises(ValueError):
evaluation.auc_score(
model, train, train_interactions=train, check_intersections=True
)
with pytest.raises(ValueError):
evaluation.recall_at_k(
model, train, train_interactions=train, check_intersections=True
)
with pytest.raises(ValueError):
evaluation.precision_at_k(
model, train, train_interactions=train, check_intersections=True
)
with pytest.raises(ValueError):
evaluation.reciprocal_rank(
model, train, train_interactions=train, check_intersections=True
)
# check no errors raised when train and test have no interactions in common
evaluation.auc_score(
model, test, train_interactions=train, check_intersections=True
)
evaluation.recall_at_k(
model, test, train_interactions=train, check_intersections=True
)
evaluation.precision_at_k(
model, test, train_interactions=train, check_intersections=True
)
evaluation.reciprocal_rank(
model, test, train_interactions=train, check_intersections=True
)
# check no error is raised when there are intersections but flag is False
evaluation.auc_score(
model, train, train_interactions=train, check_intersections=False
)
evaluation.recall_at_k(
model, train, train_interactions=train, check_intersections=False
)
evaluation.precision_at_k(
model, train, train_interactions=train, check_intersections=False
)
evaluation.reciprocal_rank(
model, train, train_interactions=train, check_intersections=False
)
def reciprocal_rank(model,
test_interactions,
train_interactions=None,
user_features=None,
item_features=None,
preserve_rows=False,
num_threads=1,
check_intersections=True):
return evaluation.reciprocal_rank(model,
test_interactions=test_interactions,
train_interactions=train_interactions,
user_features=user_features,
item_features=item_features,
preserve_rows=preserve_rows,
num_threads=num_threads,
check_intersections=check_intersections)
def resultados_por_contenido(self):
"""
Método resultados_por_contenido. Obtiene los resultados del modelo basado en contenido.
Este método solo se utiliza en la interfaz de texto.
"""
global train, test, modelo, item_features, user_features
# Se obtienen los resultados
precision = precision_at_k(modelo,
test,
train_interactions=train,
k=10,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
auc = auc_score(modelo,
test,
train_interactions=train,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
recall = recall_at_k(modelo,
test,
train_interactions=train,
k=10,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
reciprocal = reciprocal_rank(modelo,
test,
train_interactions=train,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
# Se imprimen los resultados
imprimir_resultados_clasico(precision, auc, recall, reciprocal)
print("Train precision@7: %.2f" %
precision_at_k(model,
train_interactions,
k=7,
user_features=users_features,
item_features=items_features).mean())
print("Test precision@7: %.2f" %
precision_at_k(model,
test_interactions,
train_interactions,
k=7,
user_features=users_features,
item_features=items_features).mean())
print("Train reciprocal rank: %.2f" %
reciprocal_rank(model,
train_interactions,
user_features=users_features,
item_features=items_features).mean())
print("Test reciprocal rank: %.2f" %
reciprocal_rank(model,
test_interactions,
train_interactions,
user_features=users_features,
item_features=items_features).mean())
def get_predict(user, model, list_item):
print("=========================")
print("User %d" % user)
scores = model.predict(user - 1,
list_item,
item_features=items_features,
def obtener_metricas_gui(self):
"""
Método obtener_metricas_gui. Obtiene las métricas del modelo escogido.
Este método solo se utiliza en la interfaz web.
Returns
-------
metricas_devueltas: dict
diccionario con las métricas del modelo
"""
global train, test, modelo, item_features, user_features
# Se guardan las métricas en un diccionario para su futura muestra en la interfaz web
metricas = dict()
# Se calculan las métricas
if self.opcion_modelo == 1:
precision = precision_at_k(modelo,
test,
train_interactions=train,
k=10,
num_threads=self.CPU_THREADS).mean()
auc = auc_score(modelo,
test,
train_interactions=train,
num_threads=self.CPU_THREADS).mean()
recall = recall_at_k(modelo,
test,
train_interactions=train,
k=10,
num_threads=self.CPU_THREADS).mean()
reciprocal = reciprocal_rank(modelo,
test,
train_interactions=train,
num_threads=self.CPU_THREADS).mean()
else:
precision = precision_at_k(modelo,
test,
train_interactions=train,
k=10,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
auc = auc_score(modelo,
test,
train_interactions=train,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
recall = recall_at_k(modelo,
test,
train_interactions=train,
k=10,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
reciprocal = reciprocal_rank(modelo,
test,
train_interactions=train,
user_features=user_features,
item_features=item_features,
num_threads=self.CPU_THREADS).mean()
# Se guardan las métricas en el diccionario y se formatea su salida
metricas_devueltas = {
"Precisión k": format(precision, '.4f'),
"AUC Score": format(auc, '.4f'),
"Recall k": format(recall, '.4f'),
"Ranking recíproco": format(reciprocal, '.4f')
}
metricas_a_guardar = {
"Precisión k": [format(precision, '.4f')],
"AUC Score": [format(auc, '.4f')],
"Recall k": [format(recall, '.4f')],
"Ranking recíproco": [format(reciprocal, '.4f')]
}
# Se guardan las métricas en un archivo .csv
guardar_resultados(metricas_a_guardar)
return metricas_devueltas
def model(df, params, u=None, i=None):
state = np.random.RandomState(params['seed'])
data = Dataset()
data.fit(df['userID'].unique(),
df['poiID'].unique(),
user_features=u[1] if u is not None else None,
item_features=i[1] if i is not None else None)
if u is not None:
user_features_iterable = map(lambda l: (l[0], l[1]), u[0].iteritems())
user_features = data.build_user_features(user_features_iterable,
normalize=False)
else:
user_features = None
if i is not None:
item_features_iterable = map(lambda l: (l[0], [l[1]]),
i[0].iteritems())
item_features = data.build_item_features(item_features_iterable,
normalize=False)
else:
item_features = None
ratings, weights = data.build_interactions(df[['userID', 'poiID'
]].itertuples(index=False,
name=None))
train, test = random_train_test_split(ratings,
test_percentage=params['test'],
random_state=state)
lfm = LightFM(no_components=params['f'],
learning_rate=params['lr'],
loss=params['loss'],
user_alpha=params['alpha'],
random_state=state)
lfm.fit(train,
epochs=params['epochs'],
user_features=user_features,
item_features=item_features)
return {
'pr-train':
100.0 * precision_at_k(lfm,
train,
k=params['k'],
user_features=user_features,
item_features=item_features).mean(),
'mrr-train':
100.0 * reciprocal_rank(lfm,
train,
user_features=user_features,
item_features=item_features).mean(),
'pr-test':
100.0 * precision_at_k(lfm,
test,
k=params['k'],
user_features=user_features,
item_features=item_features).mean(),
'mrr-test':
100.0 * reciprocal_rank(lfm,
test,
user_features=user_features,
item_features=item_features).mean()
}
model = LightFM(loss='warp', random_state=0)
model.fit(train_interactions, epochs=100, num_threads=1)
from lightfm.evaluation import recall_at_k
from lightfm.evaluation import precision_at_k
print("Train recall@7: %.2f" %
recall_at_k(model, train_interactions, k=7).mean())
print("Test recall@7: %.2f" %
recall_at_k(model, test_interactions, train_interactions, k=7).mean())
print("Train precision@7: %.2f" %
precision_at_k(model, train_interactions, k=7).mean())
print("Test precision@7: %.2f" %
precision_at_k(model, test_interactions, train_interactions, k=7).mean())
print("Train reciprocal rank: %.2f" %
reciprocal_rank(model, train_interactions).mean())
print("Test reciprocal rank: %.2f" %
reciprocal_rank(model, test_interactions, train_interactions).mean())
def get_predict(user, model, list_item):
print("=========================")
print("User %d" % user)
scores = model.predict(user - 1, list_item)
print("Liked items from history:")
known_result = train[train[3] == user][1].values
print(known_result)
print("Favorite items ( haven't seen ):")
# print(ratings2[ratings2[3] == 1][1].values)
print(test[test[3] == user][1].values)
print("Predict results")