class TagMovieRatingTensor(object):
def __init__(self):
self.conf = ParseConfig()
self.data_set_loc = self.conf.config_section_mapper("filePath").get(
"data_set_loc")
self.data_extractor = DataExtractor(self.data_set_loc)
self.max_ratings = 5
self.ordered_ratings = [0, 1, 2, 3, 4, 5]
self.ordered_movie_names = []
self.ordered_tag_names = []
self.print_list = [
"\n\nFor Tags:", "\n\nFor Movies:", "\n\nFor Ratings:"
]
self.util = Util()
self.tensor = self.fetchTagMovieRatingTensor()
self.factors = self.util.CPDecomposition(self.tensor, 5)
def fetchTagMovieRatingTensor(self):
"""
Create tag movie rating tensor
:return: tensor
"""
mltags_df = self.data_extractor.get_mltags_data()
tag_id_list = mltags_df["tagid"]
tag_id_count = 0
tag_id_dict = {}
for element in tag_id_list:
if element in tag_id_dict.keys():
continue
tag_id_dict[element] = tag_id_count
tag_id_count += 1
name = self.util.get_tag_name_for_id(element)
self.ordered_tag_names.append(name)
movieid_list = mltags_df["movieid"]
movieid_count = 0
movieid_dict = {}
for element in movieid_list:
if element in movieid_dict.keys():
continue
movieid_dict[element] = movieid_count
movieid_count += 1
name = self.util.get_movie_name_for_id(element)
self.ordered_movie_names.append(name)
tensor = np.zeros((tag_id_count, movieid_count, self.max_ratings + 1))
for index, row in mltags_df.iterrows():
tagid = row["tagid"]
movieid = row["movieid"]
avg_movie_rating = self.util.get_average_ratings_for_movie(movieid)
for rating in range(0, int(avg_movie_rating) + 1):
tagid_id = tag_id_dict[tagid]
movieid_id = movieid_dict[movieid]
tensor[tagid_id][movieid_id][rating] = 1
return tensor
def print_latent_semantics(self, r):
"""
Pretty print latent semantics
:param r:
"""
i = 0
for factor in self.factors:
print(self.print_list[i])
latent_semantics = self.util.get_latent_semantics(
r, factor.transpose())
self.util.print_latent_semantics(latent_semantics,
self.get_factor_names(i))
i += 1
def get_factor_names(self, i):
"""
Obtain factor names
:param i:
:return: factor names
"""
if i == 0:
return self.ordered_tag_names
elif i == 1:
return self.ordered_movie_names
elif i == 2:
return self.ordered_ratings
def get_partitions(self, no_of_partitions):
"""
Partition factor matrices
:param no_of_partitions:
:return: list of groupings
"""
i = 0
groupings_list = []
for factor in self.factors:
groupings = self.util.partition_factor_matrix(
factor, no_of_partitions, self.get_factor_names(i))
groupings_list.append(groupings)
i += 1
return groupings_list
def print_partitioned_entities(self, no_of_partitions):
"""
Pretty print groupings
:param no_of_partitions:
"""
groupings_list = self.get_partitions(no_of_partitions)
i = 0
for groupings in groupings_list:
print(self.print_list[i])
self.util.print_partitioned_entities(groupings)
i += 1
class ActorTag(object):
"""
Class to relate actors and tags.
"""
def __init__(self):
"""
Initializing the data extractor object to get data from the csv files
"""
self.data_set_loc = conf.config_section_mapper("filePath").get(
"data_set_loc")
self.data_extractor = DataExtractor(self.data_set_loc)
def assign_idf_weight(self, data_series, unique_tags):
"""
This function computes the idf weight for all tags in a data frame,
considering each movie as a document
:param data_frame:
:param unique_tags:
:return: dictionary of tags and idf weights
"""
idf_counter = {tag: 0 for tag in unique_tags}
for tag_list in data_series:
for tag in tag_list:
idf_counter[tag] += 1
for tag, count in list(idf_counter.items()):
idf_counter[tag] = math.log(len(data_series.index) / count, 2)
return idf_counter
def assign_tf_weight(self, tag_series):
"""
This function computes the tf weight for all tags for a movie
:param tag_series:
:return: dictionary of tags and tf weights
"""
counter = Counter()
for each in tag_series:
counter[each] += 1
total = sum(counter.values())
for each in counter:
counter[each] = (counter[each] / total)
return dict(counter)
def assign_rank_weight(self, data_frame):
"""
This function assigns a value for all the actors in a movie on a scale of 100,
based on their rank in the movie.
:param tag_series:
:return: dictionary of (movieid, actor_rank) to the computed rank_weight
"""
groupby_movies = data_frame.groupby("movieid")
movie_rank_weight_dict = {}
for movieid, info_df in groupby_movies:
max_rank = info_df.actor_movie_rank.max()
for rank in info_df.actor_movie_rank.unique():
movie_rank_weight_dict[(
movieid, rank)] = (max_rank - rank + 1) / max_rank * 100
return movie_rank_weight_dict
def get_model_weight(self, tf_weight_dict, idf_weight_dict,
rank_weight_dict, tag_df, model):
"""
This function combines tf_weight on a scale of 100, idf_weight on a scale of 100,
actor_rank for each tag on scale of 100 and timestamp_weight on a scale of 10 , based on the model.
:param tf_weight_dict, idf_weight_dict, rank_weight_dict, tag_df, model
:return: data_frame with column of the combined weight
"""
if model == "TF":
tag_df["value"] = pd.Series(
[(tf_weight_dict.get(movieid, 0).get(tag, 0) * 100) +
rank_weight_dict.get((movieid, rank), 0)
for index, ts_weight, tag, movieid, rank in zip(
tag_df.index, tag_df.timestamp_weight, tag_df.tag,
tag_df.movieid, tag_df.actor_movie_rank)],
index=tag_df.index)
else:
tag_df["value"] = pd.Series(
[(ts_weight +
(tf_weight_dict.get(movieid, 0).get(tag, 0) *
(idf_weight_dict.get(tag, 0)) * 100) + rank_weight_dict.get(
(movieid, rank), 0))
for index, ts_weight, tag, movieid, rank in zip(
tag_df.index, tag_df.timestamp_weight, tag_df.tag,
tag_df.movieid, tag_df.actor_movie_rank)],
index=tag_df.index)
return tag_df
def combine_computed_weights(self, data_frame, rank_weight_dict,
idf_weight_dict, model):
"""
Triggers the weighing process and sums up all the calculated weights for each tag
:param data_frame:
:param rank_weight_dict:
:param model:
:return: dictionary of tags and weights
"""
tag_df = data_frame.reset_index()
temp_df = tag_df.groupby(
['movieid'])['tag'].apply(lambda x: ','.join(x)).reset_index()
movie_tag_dict = dict(zip(temp_df.movieid, temp_df.tag))
tf_weight_dict = {
movie: self.assign_tf_weight(tags.split(','))
for movie, tags in list(movie_tag_dict.items())
}
tag_df = self.get_model_weight(tf_weight_dict, idf_weight_dict,
rank_weight_dict, tag_df, model)
tag_df["total"] = tag_df.groupby(['tag'])['value'].transform('sum')
tag_df = tag_df.drop_duplicates("tag").sort_values("total",
ascending=False)
actor_tag_dict = dict(zip(tag_df.tag, tag_df.total))
return actor_tag_dict
def merge_movie_actor_and_tag(self, actorid, model):
"""
Merges data from different csv files necessary to compute the tag weights for each actor,
assigns weights to timestamp.
:param actorid:
:param model:
:return: returns a dictionary of Actors to dictionary of tags and weights.
"""
mov_act = self.data_extractor.get_movie_actor_data()
ml_tag = self.data_extractor.get_mltags_data()
genome_tag = self.data_extractor.get_genome_tags_data()
actor_info = self.data_extractor.get_imdb_actor_info_data()
actor_movie_info = mov_act.merge(actor_info,
how="left",
left_on="actorid",
right_on="id")
tag_data_frame = ml_tag.merge(genome_tag,
how="left",
left_on="tagid",
right_on="tagId")
merged_data_frame = actor_movie_info.merge(tag_data_frame,
how="left",
on="movieid")
merged_data_frame = merged_data_frame[
merged_data_frame['timestamp'].notnull()]
merged_data_frame = merged_data_frame.drop(["userid"], axis=1)
rank_weight_dict = self.assign_rank_weight(
merged_data_frame[['movieid', 'actor_movie_rank']])
merged_data_frame = merged_data_frame.sort_values(
"timestamp", ascending=True).reset_index()
data_frame_len = len(merged_data_frame.index)
merged_data_frame["timestamp_weight"] = pd.Series(
[(index + 1) / data_frame_len * 10
for index in merged_data_frame.index],
index=merged_data_frame.index)
if model == 'TFIDF':
idf_weight_dict = self.assign_idf_weight(
merged_data_frame.groupby('movieid')['tag'].apply(set),
merged_data_frame.tag.unique())
tag_dict = self.combine_computed_weights(
merged_data_frame[merged_data_frame['actorid'] == actorid],
rank_weight_dict, idf_weight_dict, model)
else:
tag_dict = self.combine_computed_weights(
merged_data_frame[merged_data_frame['actorid'] == actorid],
rank_weight_dict, {}, model)
return tag_dict
class SimilarActorsFromDiffMoviesLda(object):
def __init__(self):
super().__init__()
self.data_set_loc = conf.config_section_mapper("filePath").get("data_set_loc")
self.data_extractor = DataExtractor(self.data_set_loc)
self.util = Util()
self.sim_act_diff_mov_tf = SimilarActorsFromDiffMovies()
def most_similar_actors_lda(self, moviename):
"""
Function to find related actors from related movies(movie_movie_similarity_matrix using lda)
corresponding to the given movie
:param moviename:
:return: actors
"""
data_frame = self.data_extractor.get_mlmovies_data()
tag_data_frame = self.data_extractor.get_genome_tags_data()
movie_data_frame = self.data_extractor.get_mltags_data()
movie_tag_data_frame = movie_data_frame.merge(tag_data_frame, how="left", left_on="tagid", right_on="tagId")
movie_tag_data_frame = movie_tag_data_frame.merge(data_frame, how="left", left_on="movieid", right_on="movieid")
tag_df = movie_tag_data_frame.groupby(['movieid'])['tag'].apply(list).reset_index()
tag_df = tag_df.sort_values('movieid')
movies = tag_df.movieid.tolist()
tag_df = list(tag_df.iloc[:, 1])
input_movieid = self.util.get_movie_id(moviename)
(U, Vh) = self.util.LDA(tag_df, num_topics=5, num_features=1000)
movie_topic_matrix = self.util.get_doc_topic_matrix(U, num_docs=len(movies), num_topics=5)
topic_movie_matrix = movie_topic_matrix.transpose()
movie_movie_matrix = numpy.dot(movie_topic_matrix, topic_movie_matrix)
index_movie = None
for i, j in enumerate(movies):
if j == input_movieid:
index_movie = i
break
if index_movie == None:
print("Movie Id not found.")
return None
movie_row = movie_movie_matrix[index_movie].tolist()
movie_movie_dict = dict(zip(movies, movie_row))
del movie_movie_dict[input_movieid]
for key in movie_movie_dict.keys():
movie_movie_dict[key] = abs(movie_movie_dict[key])
movie_movie_dict = sorted(movie_movie_dict.items(), key=operator.itemgetter(1), reverse=True)
if movie_movie_dict == None:
return None
actors = []
for (movie, val) in movie_movie_dict:
if val <= 0:
break
actors = actors + self.sim_act_diff_mov_tf.get_actors_of_movie(self.util.get_movie_name_for_id(movie))
if len(actors) >= 10:
break
actors_of_given_movie = self.sim_act_diff_mov_tf.get_actors_of_movie(moviename)
actorsFinal = [x for x in actors if x not in actors_of_given_movie]
actornames = []
for actorid in actorsFinal:
actor = self.util.get_actor_name_for_id(actorid)
actornames.append(actor)
return actornames
class LdaActorTag(object):
def __init__(self):
super().__init__()
self.data_set_loc = conf.config_section_mapper("filePath").get(
"data_set_loc")
self.data_extractor = DataExtractor(self.data_set_loc)
self.util = Util()
def get_related_actors_lda(self, actorid):
"""
Function to find similarity between actors using actor-actor similarity vector in tag space using lda
:param actorid:
:return:
"""
mov_act = self.data_extractor.get_movie_actor_data()
ml_tag = self.data_extractor.get_mltags_data()
genome_tag = self.data_extractor.get_genome_tags_data()
actor_info = self.data_extractor.get_imdb_actor_info_data()
actor_movie_info = mov_act.merge(actor_info,
how="left",
left_on="actorid",
right_on="id")
tag_data_frame = ml_tag.merge(genome_tag,
how="left",
left_on="tagid",
right_on="tagId")
merged_data_frame = tag_data_frame.merge(actor_movie_info,
how="left",
on="movieid")
merged_data_frame = merged_data_frame.fillna('')
tag_df = merged_data_frame.groupby(
['actorid'])['tag'].apply(list).reset_index()
tag_df = tag_df.sort_values('actorid')
actorid_list = tag_df.actorid.tolist()
tag_df = list(tag_df.iloc[:, 1])
(U, Vh) = self.util.LDA(tag_df, num_topics=5, num_features=100000)
actor_topic_matrix = self.util.get_doc_topic_matrix(
U, num_docs=len(actorid_list), num_topics=5)
topic_actor_matrix = actor_topic_matrix.transpose()
actor_actor_matrix = numpy.dot(actor_topic_matrix, topic_actor_matrix)
numpy.savetxt("actor_actor_matrix_with_svd_latent_values.csv",
actor_actor_matrix,
delimiter=",")
df = pd.DataFrame(
pd.read_csv('actor_actor_matrix_with_svd_latent_values.csv',
header=None))
matrix = df.values
actorids = self.util.get_sorted_actor_ids()
index_actor = None
for i, j in enumerate(actorids):
if j == actorid:
index_actor = i
break
if index_actor == None:
print("Actor Id not found.")
return None
actor_names = []
for actor_id in actorids:
actor_name = self.util.get_actor_name_for_id(int(actor_id))
actor_names = actor_names + [actor_name]
actor_row = matrix[index_actor].tolist()
actor_actor_dict = dict(zip(actor_names, actor_row))
del actor_actor_dict[self.util.get_actor_name_for_id(int(actorid))]
# for key in actor_actor_dict.keys():
# actor_actor_dict[key] = abs(actor_actor_dict[key])
actor_actor_dict = sorted(actor_actor_dict.items(),
key=operator.itemgetter(1),
reverse=True)
print(actor_actor_dict[0:10])
return actor_actor_dict[0:10]
