def main(): err, input_movie_ids = util.parse_input(sys.argv) if err: return # *** Write your code here *** #process movie list to get matrix #matrix = util.get_matrix(movie_list) #perform PageRank #output = get_page_rank() # Remove this line pass the output here output_movie_ids = input_movie_ids #print output and log them util.process_output(input_movie_ids, output_movie_ids, output_file)
def main(): err, input_movie_ids = util.parse_input(sys.argv) if err: return #process movie list to get matrix matrix = util.get_movie_matrix_from_hd5() # *** Write your code here *** #perform LDA lda = LDA(n_components=no_of_components) lda.fit(matrix) lda_df = pd.DataFrame(lda.transform(matrix), index=matrix.index) input_movie_df = lda_df.loc[input_movie_ids] output_movies = [] for index, movie in lda_df.iterrows(): cosine_sum = 0 order = 1 for j, input_movie in input_movie_df.iterrows(): cosine_sum += (1 - cosine(movie, input_movie)) * order order -= order_factor output_movies.append((index, cosine_sum)) other_movies = list( filter(lambda tup: tup[0] not in input_movie_ids, output_movies)) other_movies.sort(key=lambda tup: tup[1], reverse=True) output_movie_ids = [t[0] for t in other_movies][:5] #print output and log them feedback = util.process_output(input_movie_ids, output_movie_ids, output_file) #process feedback to get relevant movies and movies to be excluded relevant_movies, movie_to_exclude = util.process_feedback( feedback, input_movie_ids) relevant_movie_count = len(relevant_movies) #if all recommended movies are relevant then return if relevant_movie_count == 5: print "\nAll the movies were relevant hence no modification to the suggestion" return #fetch data frames for relevant and feedback movies relevant_movies_df = lda_df.loc[relevant_movies] feedback_movies_df = lda_df.loc[feedback.keys()] modified_query = util.probabilistic_feedback_query(feedback_movies_df, relevant_movies_df, lda_df.index, relevant_movie_count) revised_movie_ids = util.get_revised_movies(lda_df, modified_query, movie_to_exclude) util.print_revised(revised_movie_ids, output_file)
def done(self, file): res = process_output(file) self.results.append(res) copy = "results/{}/runs/{}".format(self.id, file.split("/")[-1]) os.system(("cp {} {}").format(file, copy)) self.job_stats(file, res) self.fail(file)
def main(): err, input_movie_ids = util.parse_input(sys.argv) if err: return matrix = util.get_movie_matrix_from_hd5() svd_dict, svd_df = do_svd(matrix, input_movie_ids) print('SVD done') lda_dict, lda_df = do_lda(matrix, input_movie_ids) print('LDA done') tensor_dict, decomposed_movies_df, movies_list = do_tensor(input_movie_ids) print('Tensor done') page_rank_dict = do_page_rank(input_movie_ids) print('PageRank done') other_movies = [] for k in svd_dict: if k in input_movie_ids: continue total_weight = svd_dict[k] + lda_dict.get(k, 0) + tensor_dict.get( k, 0) + page_rank_dict.get(k, 0) other_movies.append((k, total_weight)) other_movies.sort(key=lambda tup: tup[1], reverse=True) output_movie_ids = [t[0] for t in other_movies][:5] feedback = util.process_output(input_movie_ids, output_movie_ids, output_file) similarity_svd, movie_to_exclude = process_feedback_movie_vector( feedback, input_movie_ids, svd_df.index, svd_df) similarity_lda, movie_to_exclude = process_feedback_movie_vector( feedback, input_movie_ids, lda_df.index, lda_df) similarity_tensor, movie_to_exclude = process_feedback_movie_vector( feedback, input_movie_ids, movies_list, decomposed_movies_df) page_rank_dict = do_page_rank_relevance(feedback, input_movie_ids) other_movies = [] for k in similarity_svd: if k in movie_to_exclude: continue total_weight = similarity_svd[k] + similarity_lda.get( k, 0) + similarity_tensor.get(k, 0) + page_rank_dict.get(k, 0) other_movies.append((k, total_weight)) other_movies.sort(key=lambda tup: tup[1], reverse=True) revised_movie_ids = [t[0] for t in other_movies][:5] util.print_revised(revised_movie_ids, output_file)
# NB, output is: # [batch, image_id, [x_center, y_center, width, height, objectness_score, class_score1, class_score2, ...]] if output.shape[0] > 0: # To later plot bboxes img_ = plt.imread(img_file) output = output.squeeze(0) for i in range(output.shape[0]): if output[i, -3] > float(args.plot_conf): logwriter.write( 'output,' + ','.join([str(x.item()) for x in output[i]]) + '\n') output = process_output(output, num_classes) # #Get rid of the zero entries for objectness # non_zero_ind = (torch.nonzero(output[:,4])) # image_pred_ = output[non_zero_ind.squeeze(),:].view(-1,7) # # Remove low confidence by class probs # image_pred_ = image_pred_[image_pred_[:, -1] >= args.plot_conf, :] # #Get the various classes detected in the image # try: # img_classes = unique(image_pred_[:,-2].int()) # print('img_classes ', img_classes) # except: # continue # #WE will do NMS classwise # for label in img_classes: # #get the detections with one particular class
def main(): err, input_movie_ids = util.parse_input(sys.argv) if err: return # read the pickle file that contains tensor actor_movie_year_3d_matrix = cPickle.load( open("actor_movie_genre_tensor.pkl", "rb")) actor_movie_year_array = np.array(actor_movie_year_3d_matrix) # perform cp decomposition decomposed = parafac(actor_movie_year_array, no_of_components, init='random') mlmovies = util.read_mlmovies() mlmovies = mlmovies.loc[mlmovies['year'] >= util.movie_year_for_tensor] movies_list = mlmovies.movieid.unique() # data frame for movie factor matrix from cp decomposition decomposed_movies_df = pd.DataFrame(decomposed[1], index=movies_list) # dataframe containing only input movies input_movie_df = decomposed_movies_df.loc[input_movie_ids] output_movies = [] # finding cosine similarity of each movie vector with the input movie vector and fetching the top 5 values for index, movie in decomposed_movies_df.iterrows(): cosine_sum = 0 order = 1 for j, input_movie in input_movie_df.iterrows(): cosine_sum += (1 - cosine(movie, input_movie)) * order order -= order_factor output_movies.append((index, cosine_sum)) other_movies = list( filter(lambda tup: tup[0] not in input_movie_ids, output_movies)) other_movies.sort(key=lambda tup: tup[1], reverse=True) output_movie_ids = [t[0] for t in other_movies][:5] #print output and log them feedback = util.process_output(input_movie_ids, output_movie_ids, output_file) #process feedback to get relevant movies and movies to be excluded relevant_movies, movie_to_exclude = util.process_feedback( feedback, input_movie_ids) relevant_movie_count = len(relevant_movies) #if all recommended movies are relevant then return if relevant_movie_count == 5: print "\nAll the movies were relevant hence no modification to the suggestion" return #fetch data frames for relevant and feedback movies relevant_movies_df = decomposed_movies_df.loc[relevant_movies] feedback_movies_df = decomposed_movies_df.loc[feedback.keys()] modified_query = util.probabilistic_feedback_query(feedback_movies_df, relevant_movies_df, movies_list, relevant_movie_count) revised_movie_ids = util.get_revised_movies(decomposed_movies_df, modified_query, movie_to_exclude) util.print_revised(revised_movie_ids, output_file)
def main(): # get user input, movies watched by the user seed_movies = sys.argv[1].split(',') seed_movies = list(map(int, seed_movies)) movie_matrix_svd = pd.read_pickle('../task3/movie_matrix_svd.pkl') #import movies movie_years = [] movies = pd.read_csv('../../phase3_dataset/mlmovies.csv') movie_ids = movies.movieid.unique() #Intialize PageRank values to 1.0 pr = pd.DataFrame(1.0, columns=movie_ids, index=('PageRank', )) # Calculate cosine distance between movies to movies in db M_sparse = sparse.csr_matrix(movie_matrix_svd) similarities = cosine_similarity(M_sparse) # create dataframe with headers and first colmns as movie ids df_movie = pd.DataFrame(similarities, columns=movie_ids, index=movie_ids) #normalize the matrix values norm_simil = df_movie / df_movie.sum(0) # Pagerank algorithm for j in range(0, 5): pr_new = pr.copy() #Update page rank for i in movie_ids: if i in seed_movies: pr_new[i] = (0.15 / len(seed_movies)) + ( 0.85 * norm_simil[i].dot(pr.loc['PageRank'])) + ( 0.01 / (seed_movies.index(i) + 1)) else: pr_new[i] = (0.85 * norm_simil[i].dot(pr.loc['PageRank'])) pr = pr_new.copy() #Remove seeded actors from list for i in seed_movies: try: pr_final = pr.drop(i, axis=1) pr = pr_final except: pass #Display top 5 related actors print("\nMovies user watched: ", seed_movies) O_file = 'task1d_task2_pagerank.out.txt' r_movies = [] r_movies = list(pr.loc['PageRank'].nlargest(5).index) feedback = util.process_output(seed_movies, r_movies, O_file) #process feedback to get relevant movies and movies to be excluded relevant_movies, movie_to_exclude = util.process_feedback( feedback, seed_movies) irrelevant_movies = np.setdiff1d(movie_to_exclude, seed_movies) relevant_movie_count = len(relevant_movies) #if all recommended movies are relevant then return if relevant_movie_count == 5: print( "\nAll the movies were relevant hence no modification to the suggestion" ) return pr = pd.DataFrame(1.0, columns=movie_ids, index=('PageRank', )) # Pagerank algorithm for j in range(0, 5): pr_new = pr.copy() #Update page rank for i in movie_ids: if i in seed_movies: pr_new[i] = (0.15 / len(seed_movies)) + ( 0.85 * norm_simil[i].dot(pr.loc['PageRank'])) + ( 0.01 / (seed_movies.index(i) + 1)) else: pr_new[i] = (0.85 * norm_simil[i].dot(pr.loc['PageRank'])) if i in irrelevant_movies: pr_new[i] = pr_new[i] - 0.02 elif i in relevant_movies: pr_new[i] = pr_new[i] + 0.02 pr = pr_new.copy() #Remove seeded actors from list for i in seed_movies: try: pr_final = pr.drop(i, axis=1) pr = pr_final except: pass #Display top 5 related actors r_movies = [] r_movies = list(pr.loc['PageRank'].nlargest(5).index) util.print_revised(r_movies, O_file)