def rank(self, user_ratings: pd.DataFrame, items_directory: str, recs_number: int = None,
filter_list: List[str] = None) -> pd.DataFrame:
"""
Rank the top-n recommended items for the user. If the recs_number parameter isn't specified,
All items will be ranked.
One can specify which items must be ranked with the filter_list parameter,
in this case ONLY items in the filter_list will be used to calculate the rank.
One can also pass items already seen by the user with the filter_list parameter.
Otherwise, ALL unrated items will be used to calculate the rank.
Args:
user_ratings (pd.DataFrame): DataFrame containing ratings of a single user
items_directory (str): path of the directory where the items are stored
recs_number (int): number of the top items that will be present in the ranking
filter_list (list): list of the items to rank, if None all unrated items will be used to
calculate the rank
Returns:
pd.DataFrame: DataFrame containing one column with the items name,
one column with the rating predicted, sorted in descending order by the 'rating' column
"""
# Load items to predict
if filter_list is None:
items_to_predict = get_unrated_items(items_directory, user_ratings)
else:
items_to_predict = get_chosen_items(items_directory, filter_list)
# Extract features of the items to predict
id_items_to_predict = []
features_items_to_predict = []
for item in items_to_predict:
if item is not None:
id_items_to_predict.append(item.content_id)
features_items_to_predict.append(self.extract_features_item(item))
recsys_logger.info("Calculating rank")
if len(id_items_to_predict) > 0:
# Fuse the input if there are dicts, multiple representation, etc.
fused_features_items_to_pred = self.fuse_representations(features_items_to_predict, self.__embedding_combiner)
score_labels = self.__classifier.predict_proba(fused_features_items_to_pred)
else:
score_labels = []
result = {'to_id': [], 'score': []}
for item_id, score in zip(id_items_to_predict, score_labels):
result['to_id'].append(item_id)
result['score'].append(score[1])
result = pd.DataFrame(result, columns=['to_id', 'score'])
result.sort_values(by=['score'], ascending=False, inplace=True)
rank = result[:recs_number]
return rank
def predict(self,
user_ratings: pd.DataFrame,
items_directory: str,
filter_list: List[str] = None) -> pd.DataFrame:
"""
Predicts how much a user will like unrated items.
One can specify which items must be predicted with the filter_list parameter,
in this case ONLY items in the filter_list will be predicted.
One can also pass items already seen by the user with the filter_list parameter.
Otherwise, ALL unrated items will be predicted.
Args:
user_ratings (pd.DataFrame): DataFrame containing ratings of a single user
items_directory (str): path of the directory where the items are stored
filter_list (list): list of the items to predict, if None all unrated items will be predicted
Returns:
pd.DataFrame: DataFrame containing one column with the items name,
one column with the score predicted
"""
# Load items to predict
if filter_list is None:
items_to_predict = get_unrated_items(items_directory, user_ratings)
else:
items_to_predict = get_chosen_items(items_directory, filter_list)
# Extract features of the items to predict
id_items_to_predict = []
features_items_to_predict = []
for item in items_to_predict:
if item is not None:
# raises AttributeError if items are not present locally
id_items_to_predict.append(item.content_id)
features_items_to_predict.append(
self.extract_features_item(item))
recsys_logger.info("Calculating score predictions")
if len(id_items_to_predict) > 0:
# Fuse the input if there are dicts, multiple representation, etc.
fused_features_items_to_pred = self.fuse_representations(
features_items_to_predict, self.__embedding_combiner)
score_labels = self.__regressor.predict(
fused_features_items_to_pred)
else:
score_labels = []
# Build the score_frame to return
columns = ["to_id", "score"]
score_frame = pd.DataFrame(columns=columns)
score_frame["to_id"] = id_items_to_predict
score_frame["score"] = score_labels
return score_frame
def rank(self, user_ratings: pd.DataFrame, items_directory: str, recs_number: int = None,
filter_list: List[str] = None) -> pd.DataFrame:
"""
Rank the top-n recommended items for the user. If the recs_number parameter isn't specified,
All unrated items will be ranked (or only items in the filter list, if specified).
One can specify which items must be ranked with the filter_list parameter,
in this case ONLY items in the filter_list parameter will be ranked.
One can also pass items already seen by the user with the filter_list parameter.
Otherwise, ALL unrated items will be ranked.
Args:
user_ratings (pd.DataFrame): DataFrame containing ratings of a single user
items_directory (str): path of the directory where the items are stored
recs_number (int): number of the top items that will be present in the ranking, if None
all unrated items will be ranked
filter_list (list): list of the items to rank, if None all unrated items will be ranked
Returns:
pd.DataFrame: DataFrame containing one column with the items name,
one column with the rating predicted, sorted in descending order by the 'rating' column
"""
# Load items to predict
if filter_list is None:
items_to_predict = get_unrated_items(items_directory, user_ratings)
else:
items_to_predict = get_chosen_items(items_directory, filter_list)
# Extract features of the items to predict
id_items_to_predict = []
features_items_to_predict = []
for item in items_to_predict:
if item is not None:
id_items_to_predict.append(item.content_id)
features_items_to_predict.append(self.extract_features_item(item))
recsys_logger.info("Calculating rank")
if len(id_items_to_predict) > 0:
# Calculate predictions, they are the similarity of the new items with the centroid vector
features_fused = self.fuse_representations(features_items_to_predict, self.__embedding_combiner)
similarities = [self.__similarity.perform(self.__centroid, item) for item in features_fused]
else:
similarities = []
# Build the score frame
result = {'to_id': id_items_to_predict, 'score': similarities}
result = pd.DataFrame(result, columns=['to_id', 'score'])
# Sort them in descending order
result.sort_values(by=['score'], ascending=False, inplace=True)
rank = result[:recs_number]
return rank
def __get_item_list(self, item_to_predict_id_list, user_ratings):
if item_to_predict_id_list is None:
# all items without rating if the list is not set
item_to_predict_list = get_unrated_items(
self.__config.items_directory, user_ratings)
else:
item_to_predict_list = [
load_content_instance(self.__config.items_directory,
re.sub(r'[^\w\s]', '', item_id))
for item_id in item_to_predict_id_list
]
return item_to_predict_list
def preprocessing(self, items_directory: str, ratings: pd.DataFrame, candidate_item_id_list: list = None):
"""
Function used to retrieve data that will be used in the computation of the ranking.
It loads the rated and unrated items, computes the threshold if it was set to -1 and
extracts the features from the unrated items.
This method can throw two exceptions. The first one is thrown if the threshold value specified
in the constructor of the class it's not in the range [-1, 1], the second one is thrown if,
while considering a candidate_item_id_list passed as an argument, there are no valid
items to consider (example: ['test', 'test2'] but neither test nor test2 are items in the
items directory)
Args:
items_directory (str): directory where the items are stored
ratings (Dataframe): dataframe which contains ratings given by the user
candidate_item_id_list (list): list of the items that can be recommended, if None
all unrated items will be used
Returns:
rated_items (list): list containing the instances of the rated items
unrated_items (list): list containing the instances of the unrated items
unrated_features_baglist (list): list containing the features extracted from the unrated items
"""
# If threshold is the min possible (range is [-1, 1]), we calculate the mean value
# of all the ratings and set it as the threshold. Also an exception is thrown if the
# threshold value is not in the range
if not -1 <= self.__threshold <= 1:
raise ValueError("Threshold value must be in the range [-1, 1]")
if self.__threshold == -1:
self.__threshold = pd.to_numeric(ratings["score"], downcast="float").mean()
# Load unrated items from the path
if candidate_item_id_list is None or len(candidate_item_id_list) == 0:
unrated_items = get_unrated_items(items_directory, ratings)
else:
# If a candidate list is specified, it loads only items that are valid (it doesn't add None to the list)
unrated_items = [load_content_instance(items_directory, item_id) for item_id in candidate_item_id_list
if load_content_instance(items_directory, item_id) is not None]
if len(unrated_items) == 0:
raise ValueError("No valid unrated items found")
# Load rated items from the path
rated_items = get_rated_items(items_directory, ratings)
return rated_items, unrated_items, self.__calc_unrated_baglist(unrated_items)
def predict(self, user_id: str, ratings: pd.DataFrame, recs_number: int, items_directory: str, candidate_item_id_list: List = None) -> pd.DataFrame:
"""
1) Goes into items_directory and for each item takes the values corresponding to the field_representation of
the item_field. For example, if item_field == "Plot" and field_representation == "tf-idf", the function will
take the "tf-idf" representation of each "Plot" field for every rated item, the tf-idf representation of rated items
and items to classify will be parsed to dense arrays;
2) Define target features, items with rating greater (lower) than threshold will be used as positive(negative) examples;
3) Creates an object Classifier, uses the method fit and predicts the class of the new items
Args:
candidate_item_id_list: list of the items that can be recommended, if None
all unrated items will be used
user_id: user for which recommendations will be computed
recs_number (list[Content]): How long the ranking will be
ratings (pd.DataFrame): ratings of the user with id equal to user_id
items_directory (str): Name of the directory where the items are stored.
Returns:
The predicted classes, or the predict values.
"""
if candidate_item_id_list is None:
unrated_items = get_unrated_items(items_directory, ratings)
else:
unrated_items = [load_content_instance(items_directory, item_id) for item_id in candidate_item_id_list]
rated_features_bag_list = []
unrated_features_bag_list = []
logger.info("Retrieving rated items")
rated_items = get_rated_items(items_directory, ratings)
if self.__threshold == -1:
threshold = pd.to_numeric(ratings["score"], downcast="float").mean()
else:
threshold = self.__threshold
labels = []
for item in rated_items:
if item is not None:
rated_features_bag_list.append(item.get_field(self.get_item_field()).get_representation(self.get_item_field_representation()).get_value())
labels.append(1 if float(ratings[ratings['to_id'] == item.get_content_id()].score) >= threshold else 0)
logger.info("Labeling examples")
for item in unrated_items:
if item is not None:
unrated_features_bag_list.append(item.get_field(self.get_item_field()).get_representation(self.get_item_field_representation()).get_value())
clf = None
if self.__classifier.lower() == "random_forest":
clf = RandomForestClassifier(n_estimators=400, random_state=42)
elif self.__classifier.lower() == "svm":
clf = CalibratedClassifierCV(LinearSVC(random_state=42))
elif self.__classifier.lower() == "log_regr":
clf = LogisticRegression(random_state=42)
elif self.__classifier.lower() == "knn":
clf = neighbors.KNeighborsClassifier()
elif self.__classifier.lower() == "decision_tree":
clf = DecisionTreeClassifier(random_state=42)
elif self.__classifier.lower() == "gaussian_process":
clf = GaussianProcessClassifier(random_state=42)
logger.info("Fitting classifier")
if self.__classifier.lower() == "gaussian_process":
pipe = make_pipeline(DictVectorizer(sparse=True), FunctionTransformer(lambda x: x.todense(), accept_sparse=True), clf)
else:
pipe = make_pipeline(DictVectorizer(sparse=True), clf)
pipe = pipe.fit(rated_features_bag_list, labels)
columns = ["to_id", "rating"]
score_frame = pd.DataFrame(columns=columns)
logger.info("Predicting scores")
score_labels = pipe.predict_proba(unrated_features_bag_list)
for score, item in zip(score_labels, unrated_items):
if item is not None:
score_frame = pd.concat([score_frame, pd.DataFrame.from_records([(item.get_content_id(), score[1])], columns=columns)], ignore_index=True)
score_frame = score_frame.sort_values(['rating'], ascending=False).reset_index(drop=True)
score_frame = score_frame[:recs_number]
return score_frame
def predict(self,
user_id: str,
ratings: pd.DataFrame,
recs_number: int,
items_directory: str,
candidate_item_id_list: List = None) -> pd.DataFrame:
"""
Checks:
1) Checks if the representation corresponding to field_representation exists
2) Checks if the field representation is a document embedding (whose shape equals 1)
Example: item_field == "Plot" and field_representation == "1", the function will check if the "01"
representation of each "Plot" field is a document embedding or a tf-idf words bag, and then use the embedding
or the frequency vector for algorithm computation.
Computes the centroid of the positive rated items representations
For each candidate item:
1) Takes the embedding arrays
2) Determines the similarity between the centroid and the field_representation of the item_field in candidate item.
Args:
candidate_item_id_list: list of the items that can be recommended, if None
all unrated items will be used
user_id: user for which recommendations will be computed
recs_number (list[Content]): How long the ranking will be
ratings (pd.DataFrame): ratings of the user with id equal to user_id
items_directory (str): Name of the directory where the items are stored.
Returns:
scores (pd.DataFrame): DataFrame whose columns are the ids of the items (to_id), and the similarities between the
items and the centroid (rating)
"""
# try:
logger.info("Retrieving candidate items")
if candidate_item_id_list is None:
unrated_items = get_unrated_items(items_directory, ratings)
else:
unrated_items = [
load_content_instance(items_directory, item_id)
for item_id in candidate_item_id_list
]
logger.info("Retrieving rated items")
rated_items = get_rated_items(items_directory, ratings)
if len(rated_items) == 0:
columns = ["to_id", "rating"]
scores = pd.DataFrame(columns=columns)
return scores
first_item = rated_items[0]
need_vectorizer = False
if self.item_field not in first_item.field_dict:
raise ValueError("The field name specified could not be found!")
else:
try:
representation = first_item.get_field(
self.item_field).get_representation(
self.item_field_representation)
except KeyError:
raise ValueError(
"The given representation id wasn't found for the specified field"
)
if not isinstance(representation,
EmbeddingField) and not isinstance(
representation, FeaturesBagField):
raise ValueError(
"The given representation must be an embedding or a tf-idf vector"
)
if isinstance(representation, EmbeddingField):
if len(representation.value.shape) != 1:
raise ValueError(
"The specified representation is not a document embedding, so the centroid"
" can not be calculated")
if isinstance(representation, FeaturesBagField):
need_vectorizer = True
columns = ["to_id", "rating"]
scores = pd.DataFrame(columns=columns)
if not need_vectorizer:
logger.info("Computing centroid")
centroid = self.__get_centroid_without_vectorizer(
ratings, rated_items)
logger.info("Computing similarities")
for item in unrated_items:
item_id = item.content_id
item_field_representation = item.get_field(
self.item_field).get_representation(
self.item_field_representation).value
logger.info("Computing similarity with %s" % item_id)
similarity = self.__similarity.perform(
DenseVector(centroid),
DenseVector(item_field_representation))
scores = pd.concat([
scores,
pd.DataFrame.from_records([(item_id, similarity)],
columns=columns)
],
ignore_index=True)
else:
logger.info("Computing centroid")
centroid, unrated_matrix = self.__get_centroid_with_vectorizer(
ratings, rated_items, unrated_items)
logger.info("Computing similarities")
a = []
for x in unrated_items:
if x is not None:
a.append(x)
unrated_items = a
for item, item_array in zip(unrated_items, unrated_matrix):
item_id = item.content_id
logger.info("Computing similarity with %s" % item_id)
similarity = self.__similarity.perform(
SparseVector(centroid), SparseVector(item_array))
scores = pd.concat([
scores,
pd.DataFrame.from_records([(item_id, similarity)],
columns=columns)
],
ignore_index=True)
scores = scores.sort_values(['rating'],
ascending=False).reset_index(drop=True)
scores = scores[:recs_number]
return scores