class TagMovieRatingTensor(object): def __init__(self): self.conf = ParseConfig() self.data_set_loc = self.conf.config_section_mapper("filePath").get( "data_set_loc") self.data_extractor = DataExtractor(self.data_set_loc) self.max_ratings = 5 self.ordered_ratings = [0, 1, 2, 3, 4, 5] self.ordered_movie_names = [] self.ordered_tag_names = [] self.print_list = [ "\n\nFor Tags:", "\n\nFor Movies:", "\n\nFor Ratings:" ] self.util = Util() self.tensor = self.fetchTagMovieRatingTensor() self.factors = self.util.CPDecomposition(self.tensor, 5) def fetchTagMovieRatingTensor(self): """ Create tag movie rating tensor :return: tensor """ mltags_df = self.data_extractor.get_mltags_data() tag_id_list = mltags_df["tagid"] tag_id_count = 0 tag_id_dict = {} for element in tag_id_list: if element in tag_id_dict.keys(): continue tag_id_dict[element] = tag_id_count tag_id_count += 1 name = self.util.get_tag_name_for_id(element) self.ordered_tag_names.append(name) movieid_list = mltags_df["movieid"] movieid_count = 0 movieid_dict = {} for element in movieid_list: if element in movieid_dict.keys(): continue movieid_dict[element] = movieid_count movieid_count += 1 name = self.util.get_movie_name_for_id(element) self.ordered_movie_names.append(name) tensor = np.zeros((tag_id_count, movieid_count, self.max_ratings + 1)) for index, row in mltags_df.iterrows(): tagid = row["tagid"] movieid = row["movieid"] avg_movie_rating = self.util.get_average_ratings_for_movie(movieid) for rating in range(0, int(avg_movie_rating) + 1): tagid_id = tag_id_dict[tagid] movieid_id = movieid_dict[movieid] tensor[tagid_id][movieid_id][rating] = 1 return tensor def print_latent_semantics(self, r): """ Pretty print latent semantics :param r: """ i = 0 for factor in self.factors: print(self.print_list[i]) latent_semantics = self.util.get_latent_semantics( r, factor.transpose()) self.util.print_latent_semantics(latent_semantics, self.get_factor_names(i)) i += 1 def get_factor_names(self, i): """ Obtain factor names :param i: :return: factor names """ if i == 0: return self.ordered_tag_names elif i == 1: return self.ordered_movie_names elif i == 2: return self.ordered_ratings def get_partitions(self, no_of_partitions): """ Partition factor matrices :param no_of_partitions: :return: list of groupings """ i = 0 groupings_list = [] for factor in self.factors: groupings = self.util.partition_factor_matrix( factor, no_of_partitions, self.get_factor_names(i)) groupings_list.append(groupings) i += 1 return groupings_list def print_partitioned_entities(self, no_of_partitions): """ Pretty print groupings :param no_of_partitions: """ groupings_list = self.get_partitions(no_of_partitions) i = 0 for groupings in groupings_list: print(self.print_list[i]) self.util.print_partitioned_entities(groupings) i += 1
class ActorTag(object): """ Class to relate actors and tags. """ def __init__(self): """ Initializing the data extractor object to get data from the csv files """ self.data_set_loc = conf.config_section_mapper("filePath").get( "data_set_loc") self.data_extractor = DataExtractor(self.data_set_loc) def assign_idf_weight(self, data_series, unique_tags): """ This function computes the idf weight for all tags in a data frame, considering each movie as a document :param data_frame: :param unique_tags: :return: dictionary of tags and idf weights """ idf_counter = {tag: 0 for tag in unique_tags} for tag_list in data_series: for tag in tag_list: idf_counter[tag] += 1 for tag, count in list(idf_counter.items()): idf_counter[tag] = math.log(len(data_series.index) / count, 2) return idf_counter def assign_tf_weight(self, tag_series): """ This function computes the tf weight for all tags for a movie :param tag_series: :return: dictionary of tags and tf weights """ counter = Counter() for each in tag_series: counter[each] += 1 total = sum(counter.values()) for each in counter: counter[each] = (counter[each] / total) return dict(counter) def assign_rank_weight(self, data_frame): """ This function assigns a value for all the actors in a movie on a scale of 100, based on their rank in the movie. :param tag_series: :return: dictionary of (movieid, actor_rank) to the computed rank_weight """ groupby_movies = data_frame.groupby("movieid") movie_rank_weight_dict = {} for movieid, info_df in groupby_movies: max_rank = info_df.actor_movie_rank.max() for rank in info_df.actor_movie_rank.unique(): movie_rank_weight_dict[( movieid, rank)] = (max_rank - rank + 1) / max_rank * 100 return movie_rank_weight_dict def get_model_weight(self, tf_weight_dict, idf_weight_dict, rank_weight_dict, tag_df, model): """ This function combines tf_weight on a scale of 100, idf_weight on a scale of 100, actor_rank for each tag on scale of 100 and timestamp_weight on a scale of 10 , based on the model. :param tf_weight_dict, idf_weight_dict, rank_weight_dict, tag_df, model :return: data_frame with column of the combined weight """ if model == "TF": tag_df["value"] = pd.Series( [(tf_weight_dict.get(movieid, 0).get(tag, 0) * 100) + rank_weight_dict.get((movieid, rank), 0) for index, ts_weight, tag, movieid, rank in zip( tag_df.index, tag_df.timestamp_weight, tag_df.tag, tag_df.movieid, tag_df.actor_movie_rank)], index=tag_df.index) else: tag_df["value"] = pd.Series( [(ts_weight + (tf_weight_dict.get(movieid, 0).get(tag, 0) * (idf_weight_dict.get(tag, 0)) * 100) + rank_weight_dict.get( (movieid, rank), 0)) for index, ts_weight, tag, movieid, rank in zip( tag_df.index, tag_df.timestamp_weight, tag_df.tag, tag_df.movieid, tag_df.actor_movie_rank)], index=tag_df.index) return tag_df def combine_computed_weights(self, data_frame, rank_weight_dict, idf_weight_dict, model): """ Triggers the weighing process and sums up all the calculated weights for each tag :param data_frame: :param rank_weight_dict: :param model: :return: dictionary of tags and weights """ tag_df = data_frame.reset_index() temp_df = tag_df.groupby( ['movieid'])['tag'].apply(lambda x: ','.join(x)).reset_index() movie_tag_dict = dict(zip(temp_df.movieid, temp_df.tag)) tf_weight_dict = { movie: self.assign_tf_weight(tags.split(',')) for movie, tags in list(movie_tag_dict.items()) } tag_df = self.get_model_weight(tf_weight_dict, idf_weight_dict, rank_weight_dict, tag_df, model) tag_df["total"] = tag_df.groupby(['tag'])['value'].transform('sum') tag_df = tag_df.drop_duplicates("tag").sort_values("total", ascending=False) actor_tag_dict = dict(zip(tag_df.tag, tag_df.total)) return actor_tag_dict def merge_movie_actor_and_tag(self, actorid, model): """ Merges data from different csv files necessary to compute the tag weights for each actor, assigns weights to timestamp. :param actorid: :param model: :return: returns a dictionary of Actors to dictionary of tags and weights. """ mov_act = self.data_extractor.get_movie_actor_data() ml_tag = self.data_extractor.get_mltags_data() genome_tag = self.data_extractor.get_genome_tags_data() actor_info = self.data_extractor.get_imdb_actor_info_data() actor_movie_info = mov_act.merge(actor_info, how="left", left_on="actorid", right_on="id") tag_data_frame = ml_tag.merge(genome_tag, how="left", left_on="tagid", right_on="tagId") merged_data_frame = actor_movie_info.merge(tag_data_frame, how="left", on="movieid") merged_data_frame = merged_data_frame[ merged_data_frame['timestamp'].notnull()] merged_data_frame = merged_data_frame.drop(["userid"], axis=1) rank_weight_dict = self.assign_rank_weight( merged_data_frame[['movieid', 'actor_movie_rank']]) merged_data_frame = merged_data_frame.sort_values( "timestamp", ascending=True).reset_index() data_frame_len = len(merged_data_frame.index) merged_data_frame["timestamp_weight"] = pd.Series( [(index + 1) / data_frame_len * 10 for index in merged_data_frame.index], index=merged_data_frame.index) if model == 'TFIDF': idf_weight_dict = self.assign_idf_weight( merged_data_frame.groupby('movieid')['tag'].apply(set), merged_data_frame.tag.unique()) tag_dict = self.combine_computed_weights( merged_data_frame[merged_data_frame['actorid'] == actorid], rank_weight_dict, idf_weight_dict, model) else: tag_dict = self.combine_computed_weights( merged_data_frame[merged_data_frame['actorid'] == actorid], rank_weight_dict, {}, model) return tag_dict
class SimilarActorsFromDiffMoviesLda(object): def __init__(self): super().__init__() self.data_set_loc = conf.config_section_mapper("filePath").get("data_set_loc") self.data_extractor = DataExtractor(self.data_set_loc) self.util = Util() self.sim_act_diff_mov_tf = SimilarActorsFromDiffMovies() def most_similar_actors_lda(self, moviename): """ Function to find related actors from related movies(movie_movie_similarity_matrix using lda) corresponding to the given movie :param moviename: :return: actors """ data_frame = self.data_extractor.get_mlmovies_data() tag_data_frame = self.data_extractor.get_genome_tags_data() movie_data_frame = self.data_extractor.get_mltags_data() movie_tag_data_frame = movie_data_frame.merge(tag_data_frame, how="left", left_on="tagid", right_on="tagId") movie_tag_data_frame = movie_tag_data_frame.merge(data_frame, how="left", left_on="movieid", right_on="movieid") tag_df = movie_tag_data_frame.groupby(['movieid'])['tag'].apply(list).reset_index() tag_df = tag_df.sort_values('movieid') movies = tag_df.movieid.tolist() tag_df = list(tag_df.iloc[:, 1]) input_movieid = self.util.get_movie_id(moviename) (U, Vh) = self.util.LDA(tag_df, num_topics=5, num_features=1000) movie_topic_matrix = self.util.get_doc_topic_matrix(U, num_docs=len(movies), num_topics=5) topic_movie_matrix = movie_topic_matrix.transpose() movie_movie_matrix = numpy.dot(movie_topic_matrix, topic_movie_matrix) index_movie = None for i, j in enumerate(movies): if j == input_movieid: index_movie = i break if index_movie == None: print("Movie Id not found.") return None movie_row = movie_movie_matrix[index_movie].tolist() movie_movie_dict = dict(zip(movies, movie_row)) del movie_movie_dict[input_movieid] for key in movie_movie_dict.keys(): movie_movie_dict[key] = abs(movie_movie_dict[key]) movie_movie_dict = sorted(movie_movie_dict.items(), key=operator.itemgetter(1), reverse=True) if movie_movie_dict == None: return None actors = [] for (movie, val) in movie_movie_dict: if val <= 0: break actors = actors + self.sim_act_diff_mov_tf.get_actors_of_movie(self.util.get_movie_name_for_id(movie)) if len(actors) >= 10: break actors_of_given_movie = self.sim_act_diff_mov_tf.get_actors_of_movie(moviename) actorsFinal = [x for x in actors if x not in actors_of_given_movie] actornames = [] for actorid in actorsFinal: actor = self.util.get_actor_name_for_id(actorid) actornames.append(actor) return actornames
class LdaActorTag(object): def __init__(self): super().__init__() self.data_set_loc = conf.config_section_mapper("filePath").get( "data_set_loc") self.data_extractor = DataExtractor(self.data_set_loc) self.util = Util() def get_related_actors_lda(self, actorid): """ Function to find similarity between actors using actor-actor similarity vector in tag space using lda :param actorid: :return: """ mov_act = self.data_extractor.get_movie_actor_data() ml_tag = self.data_extractor.get_mltags_data() genome_tag = self.data_extractor.get_genome_tags_data() actor_info = self.data_extractor.get_imdb_actor_info_data() actor_movie_info = mov_act.merge(actor_info, how="left", left_on="actorid", right_on="id") tag_data_frame = ml_tag.merge(genome_tag, how="left", left_on="tagid", right_on="tagId") merged_data_frame = tag_data_frame.merge(actor_movie_info, how="left", on="movieid") merged_data_frame = merged_data_frame.fillna('') tag_df = merged_data_frame.groupby( ['actorid'])['tag'].apply(list).reset_index() tag_df = tag_df.sort_values('actorid') actorid_list = tag_df.actorid.tolist() tag_df = list(tag_df.iloc[:, 1]) (U, Vh) = self.util.LDA(tag_df, num_topics=5, num_features=100000) actor_topic_matrix = self.util.get_doc_topic_matrix( U, num_docs=len(actorid_list), num_topics=5) topic_actor_matrix = actor_topic_matrix.transpose() actor_actor_matrix = numpy.dot(actor_topic_matrix, topic_actor_matrix) numpy.savetxt("actor_actor_matrix_with_svd_latent_values.csv", actor_actor_matrix, delimiter=",") df = pd.DataFrame( pd.read_csv('actor_actor_matrix_with_svd_latent_values.csv', header=None)) matrix = df.values actorids = self.util.get_sorted_actor_ids() index_actor = None for i, j in enumerate(actorids): if j == actorid: index_actor = i break if index_actor == None: print("Actor Id not found.") return None actor_names = [] for actor_id in actorids: actor_name = self.util.get_actor_name_for_id(int(actor_id)) actor_names = actor_names + [actor_name] actor_row = matrix[index_actor].tolist() actor_actor_dict = dict(zip(actor_names, actor_row)) del actor_actor_dict[self.util.get_actor_name_for_id(int(actorid))] # for key in actor_actor_dict.keys(): # actor_actor_dict[key] = abs(actor_actor_dict[key]) actor_actor_dict = sorted(actor_actor_dict.items(), key=operator.itemgetter(1), reverse=True) print(actor_actor_dict[0:10]) return actor_actor_dict[0:10]